From 81ccf399275f52d46ed5d06a4833298f471b7915 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 18 Mar 2016 13:34:04 -0400 Subject: [PATCH 001/184] now duplicate MPI communicators in init_intracomm --- src/clib/pioc.c | 149 +++++++++++++++++++++++++----------------------- 1 file changed, 79 insertions(+), 70 deletions(-) diff --git a/src/clib/pioc.c b/src/clib/pioc.c index eb9a790ffa2..506e6e642f5 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -357,87 +357,96 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, const int base,const int rearr, int *iosysidp) { iosystem_desc_t *iosys; - int ierr; + int ierr = PIO_NOERR; int ustride; int lbase; + int mpierr; + iosys = (iosystem_desc_t *) malloc(sizeof(iosystem_desc_t)); - iosys->union_comm = comp_comm; - iosys->comp_comm = comp_comm; - iosys->my_comm = comp_comm; - iosys->io_comm = MPI_COMM_NULL; - iosys->intercomm = MPI_COMM_NULL; - iosys->error_handler = PIO_INTERNAL_ERROR; - iosys->async_interface= false; - iosys->compmaster = false; - iosys->iomaster = false; - iosys->ioproc = false; - iosys->default_rearranger = rearr; - iosys->num_iotasks = num_iotasks; - - ustride = stride; - - CheckMPIReturn(MPI_Comm_rank(comp_comm, &(iosys->comp_rank)),__FILE__,__LINE__); - CheckMPIReturn(MPI_Comm_size(comp_comm, &(iosys->num_comptasks)),__FILE__,__LINE__); - if(iosys->comp_rank==0) - iosys->compmaster = true; - -#ifdef BGQxxx - lbase = base; - determineiotasks(comp_comm, &(iosys->num_iotasks), &lbase, &stride, &rearr, &(iosys->ioproc)); - if(iosys->comp_rank==0) - printf("%s %d %d\n",__FILE__,__LINE__,iosys->num_iotasks); - if(iosys->ioproc) - printf("%s %d %d\n",__FILE__,__LINE__,iosys->comp_rank); - -#else - if((iosys->num_comptasks == 1) && (num_iotasks*ustride > 1)) { - // This is a serial run with a bad configuration. Set up a single task. - fprintf(stderr, "PIO_TP PIOc_Init_Intracomm reset stride and tasks.\n"); - iosys->num_iotasks = 1; - ustride = 1; + /* Copy the computation communicator. */ + mpierr = MPI_Comm_dup(comp_comm, &iosys->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + if (!ierr) + { + mpierr = MPI_Comm_dup(comp_comm, &iosys->comp_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; } - if((iosys->num_iotasks < 1) || ((iosys->num_iotasks*ustride) > iosys->num_comptasks)){ - fprintf(stderr, "PIO_TP PIOc_Init_Intracomm error\n"); - fprintf(stderr, "num_iotasks=%d, ustride=%d, num_comptasks=%d\n", num_iotasks, ustride, iosys->num_comptasks); - return PIO_EBADID; - } - iosys->ioranks = (int *) calloc(sizeof(int), iosys->num_iotasks); - for(int i=0;i< iosys->num_iotasks; i++){ - iosys->ioranks[i] = (base + i*ustride) % iosys->num_comptasks; - if(iosys->ioranks[i] == iosys->comp_rank) - iosys->ioproc = true; - } - iosys->ioroot = iosys->ioranks[0]; -#endif - CheckMPIReturn(MPI_Info_create(&(iosys->info)),__FILE__,__LINE__); - iosys->info = MPI_INFO_NULL; + if (!ierr) + { + /* iosys->union_comm = comp_comm; */ + /* iosys->comp_comm = comp_comm; */ + iosys->my_comm = iosys->comp_comm; + iosys->io_comm = MPI_COMM_NULL; + iosys->intercomm = MPI_COMM_NULL; + iosys->error_handler = PIO_INTERNAL_ERROR; + iosys->async_interface= false; + iosys->compmaster = false; + iosys->iomaster = false; + iosys->ioproc = false; + iosys->default_rearranger = rearr; + iosys->num_iotasks = num_iotasks; + + ustride = stride; + + CheckMPIReturn(MPI_Comm_rank(iosys->comp_comm, &(iosys->comp_rank)),__FILE__,__LINE__); + CheckMPIReturn(MPI_Comm_size(iosys->comp_comm, &(iosys->num_comptasks)),__FILE__,__LINE__); + if(iosys->comp_rank==0) + iosys->compmaster = true; + + if((iosys->num_comptasks == 1) && (num_iotasks*ustride > 1)) { + // This is a serial run with a bad configuration. Set up a single task. + fprintf(stderr, "PIO_TP PIOc_Init_Intracomm reset stride and tasks.\n"); + iosys->num_iotasks = 1; + ustride = 1; + } + if((iosys->num_iotasks < 1) || ((iosys->num_iotasks*ustride) > iosys->num_comptasks)){ + fprintf(stderr, "PIO_TP PIOc_Init_Intracomm error\n"); + fprintf(stderr, "num_iotasks=%d, ustride=%d, num_comptasks=%d\n", num_iotasks, ustride, iosys->num_comptasks); + return PIO_EBADID; + } + iosys->ioranks = (int *) calloc(sizeof(int), iosys->num_iotasks); + for(int i=0;i< iosys->num_iotasks; i++){ + iosys->ioranks[i] = (base + i*ustride) % iosys->num_comptasks; + if(iosys->ioranks[i] == iosys->comp_rank) + iosys->ioproc = true; + } + iosys->ioroot = iosys->ioranks[0]; + + CheckMPIReturn(MPI_Info_create(&(iosys->info)),__FILE__,__LINE__); + iosys->info = MPI_INFO_NULL; - if(iosys->comp_rank == iosys->ioranks[0]) - iosys->iomaster = true; + if(iosys->comp_rank == iosys->ioranks[0]) + iosys->iomaster = true; - CheckMPIReturn(MPI_Comm_group(comp_comm, &(iosys->compgroup)),__FILE__,__LINE__); + CheckMPIReturn(MPI_Comm_group(iosys->comp_comm, &(iosys->compgroup)),__FILE__,__LINE__); - CheckMPIReturn(MPI_Group_incl(iosys->compgroup, iosys->num_iotasks, iosys->ioranks, - &(iosys->iogroup)),__FILE__,__LINE__); - - CheckMPIReturn(MPI_Comm_create(comp_comm, iosys->iogroup, &(iosys->io_comm)),__FILE__,__LINE__); - if(iosys->ioproc) - CheckMPIReturn(MPI_Comm_rank(iosys->io_comm, &(iosys->io_rank)),__FILE__,__LINE__); - else - iosys->io_rank = -1; + CheckMPIReturn(MPI_Group_incl(iosys->compgroup, iosys->num_iotasks, iosys->ioranks, + &(iosys->iogroup)),__FILE__,__LINE__); - iosys->union_rank = iosys->comp_rank; + CheckMPIReturn(MPI_Comm_create(iosys->comp_comm, iosys->iogroup, &(iosys->io_comm)),__FILE__,__LINE__); + if(iosys->ioproc) + CheckMPIReturn(MPI_Comm_rank(iosys->io_comm, &(iosys->io_rank)),__FILE__,__LINE__); + else + iosys->io_rank = -1; - *iosysidp = pio_add_to_iosystem_list(iosys); + iosys->union_rank = iosys->comp_rank; - pio_get_env(); + *iosysidp = pio_add_to_iosystem_list(iosys); - /* allocate buffer space for compute nodes */ - compute_buffer_init(*iosys); + pio_get_env(); - return PIO_NOERR; + /* allocate buffer space for compute nodes */ + compute_buffer_init(*iosys); + } + + return ierr; } /** @@ -507,9 +516,9 @@ int PIOc_finalize(const int iosysid) if(ios->io_comm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->io_comm)); } - /* if(ios->comp_comm != MPI_COMM_NULL){ */ - /* MPI_Comm_free(&(ios->comp_comm)); */ - /* } */ + if(ios->comp_comm != MPI_COMM_NULL){ + MPI_Comm_free(&(ios->comp_comm)); + } return pio_delete_iosystem_from_list(iosysid); From 238583a1f3731d513d19f0ca2505d062df96b7a4 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 18 Mar 2016 14:43:40 -0400 Subject: [PATCH 002/184] added comments --- src/clib/pioc.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 506e6e642f5..b2d66f41abe 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -364,12 +364,13 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, iosys = (iosystem_desc_t *) malloc(sizeof(iosystem_desc_t)); - /* Copy the computation communicator. */ + /* Copy the computation communicator into union_comm. */ mpierr = MPI_Comm_dup(comp_comm, &iosys->union_comm); CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; + /* Copy the computation communicator into comp_comm. */ if (!ierr) { mpierr = MPI_Comm_dup(comp_comm, &iosys->comp_comm); @@ -380,8 +381,6 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, if (!ierr) { - /* iosys->union_comm = comp_comm; */ - /* iosys->comp_comm = comp_comm; */ iosys->my_comm = iosys->comp_comm; iosys->io_comm = MPI_COMM_NULL; iosys->intercomm = MPI_COMM_NULL; @@ -394,12 +393,15 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, iosys->num_iotasks = num_iotasks; ustride = stride; - + + /* Find MPI rank and number of tasks in comp_comm communicator. */ CheckMPIReturn(MPI_Comm_rank(iosys->comp_comm, &(iosys->comp_rank)),__FILE__,__LINE__); CheckMPIReturn(MPI_Comm_size(iosys->comp_comm, &(iosys->num_comptasks)),__FILE__,__LINE__); if(iosys->comp_rank==0) iosys->compmaster = true; + /* Ensure that settings for number of computation tasks, number + * of IO tasks, and the stride are reasonable. */ if((iosys->num_comptasks == 1) && (num_iotasks*ustride > 1)) { // This is a serial run with a bad configuration. Set up a single task. fprintf(stderr, "PIO_TP PIOc_Init_Intracomm reset stride and tasks.\n"); @@ -411,6 +413,8 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, fprintf(stderr, "num_iotasks=%d, ustride=%d, num_comptasks=%d\n", num_iotasks, ustride, iosys->num_comptasks); return PIO_EBADID; } + + /* Create an array that holds the ranks of the tasks to be used for IO. */ iosys->ioranks = (int *) calloc(sizeof(int), iosys->num_iotasks); for(int i=0;i< iosys->num_iotasks; i++){ iosys->ioranks[i] = (base + i*ustride) % iosys->num_comptasks; @@ -419,18 +423,24 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, } iosys->ioroot = iosys->ioranks[0]; + /* Create an MPI info object. */ CheckMPIReturn(MPI_Info_create(&(iosys->info)),__FILE__,__LINE__); iosys->info = MPI_INFO_NULL; if(iosys->comp_rank == iosys->ioranks[0]) iosys->iomaster = true; + /* Create a group for the computation tasks. */ CheckMPIReturn(MPI_Comm_group(iosys->comp_comm, &(iosys->compgroup)),__FILE__,__LINE__); + /* Create a group for the IO tasks. */ CheckMPIReturn(MPI_Group_incl(iosys->compgroup, iosys->num_iotasks, iosys->ioranks, &(iosys->iogroup)),__FILE__,__LINE__); + /* Create an MPI communicator for the IO tasks. */ CheckMPIReturn(MPI_Comm_create(iosys->comp_comm, iosys->iogroup, &(iosys->io_comm)),__FILE__,__LINE__); + + /* For the tasks that are doing IO, get their rank. */ if(iosys->ioproc) CheckMPIReturn(MPI_Comm_rank(iosys->io_comm, &(iosys->io_rank)),__FILE__,__LINE__); else @@ -438,6 +448,7 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, iosys->union_rank = iosys->comp_rank; + /* Add this iosys struct to the list in the PIO library. */ *iosysidp = pio_add_to_iosystem_list(iosys); pio_get_env(); From 5712329f9922ae60a9b48fb2a307dccdda3efb63 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 29 Mar 2016 09:00:18 -0400 Subject: [PATCH 003/184] free MPI group --- src/clib/pioc.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/clib/pioc.c b/src/clib/pioc.c index b2d66f41abe..e81f0a5e243 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -530,6 +530,9 @@ int PIOc_finalize(const int iosysid) if(ios->comp_comm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->comp_comm)); } + if(ios->union_comm != MPI_COMM_NULL){ + MPI_Comm_free(&(ios->union_comm)); + } return pio_delete_iosystem_from_list(iosysid); From c3f5a1d161cc89db900644bee7276666c8dd0caa Mon Sep 17 00:00:00 2001 From: Katetc Date: Tue, 29 Mar 2016 09:51:32 -0600 Subject: [PATCH 004/184] changes to pioperformance.F90 to support the PIO1 library. --- tests/performance/pioperformance.F90 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/performance/pioperformance.F90 b/tests/performance/pioperformance.F90 index fb54f84f09f..89430c7c442 100644 --- a/tests/performance/pioperformance.F90 +++ b/tests/performance/pioperformance.F90 @@ -31,6 +31,11 @@ program pioperformance nvars, varsize, unlimdimindof #ifdef BGQTRY external :: print_memusage +#endif +#ifdef _PIO1 + integer, parameter :: PIO_FILL_INT = 02147483647 + real, parameter :: PIO_FILL_FLOAT = 9.969209968E+36 + double precision, parameter :: PIO_FILL_DOUBLE = 9.969209968E+36 #endif ! ! Initialize MPI @@ -167,7 +172,13 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & if(trim(filename) .eq. 'ROUNDROBIN' .or. trim(filename).eq.'BLOCK') then call init_ideal_dof(filename, mype, npe_base, ndims, gdims, compmap, varsize) else + ! Changed to support PIO1 as well +#ifdef _PIO1 + call pio_readdof(filename, compmap, MPI_COMM_WORLD) +#else call pio_readdof(filename, ndims, gdims, compmap, MPI_COMM_WORLD) +#endif + endif maplen = size(compmap) ! color = 0 From d5a24c00c769389249946422c53c3aefbe74b269 Mon Sep 17 00:00:00 2001 From: Katetc Date: Mon, 4 Apr 2016 15:34:49 -0600 Subject: [PATCH 005/184] Changes to pioperformance.F90 to get it to build and run with PIO1. Also adding the hacky makefile I built for my yelowstone work dir to build pioperf against PIO1 as an example and backup. --- tests/performance/kt.PIO1.perfmakefile | 19 +++++++++++++++++++ tests/performance/pioperformance.F90 | 12 +++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 tests/performance/kt.PIO1.perfmakefile diff --git a/tests/performance/kt.PIO1.perfmakefile b/tests/performance/kt.PIO1.perfmakefile new file mode 100644 index 00000000000..67f0d3be11f --- /dev/null +++ b/tests/performance/kt.PIO1.perfmakefile @@ -0,0 +1,19 @@ +all: pioperf pioperfp1 + +pioperf: pioperformance.o + mpif90 pioperformance.o -o pioperf ../pio_build_int/src/flib/libpiof.a ../pio_build_int/src/clib/libpioc.a ../pio_build_int/src/gptl/libgptl.a /glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/lib/libnetcdff.a /glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/lib/libnetcdf.so /glade/apps/opt/pnetcdf/1.6.1/intel/15.0.3/lib/libpnetcdf.a -lirng -ldecimal -lcilkrts -lstdc++ + +pioperformance.o: pioperformance.F90 + mpif90 -DCPRINTEL -DHAVE_MPI -DINCLUDE_CMAKE_FCI -DLINUX -DTIMING -DUSEMPIIO -DUSE_PNETCDF_VARN -DUSE_PNETCDF_VARN_ON_READ -D_NETCDF -D_NETCDF4 -D_PNETCDF -I/glade/p/work/katec/pio_work/ncar_pio2/src/flib -I/glade/p/work/katec/pio_work/pio_build_int/src/flib -I/glade/apps/opt/netcdf-mpi/4.3.2/intel/default/include -I/glade/apps/opt/pnetcdf/1.6.1/intel/15.0.3/include -I/glade/p/work/katec/pio_work/ncar_pio2/src/clib -I/glade/p/work/katec/pio_work/ncar_pio2/src/gptl -I/glade/p/work/katec/pio_work/pio_build_int/src/gptl -c pioperformance.F90 -o pioperformance.o + +pioperfp1: pioperformancep1.o + mpif90 pioperformancep1.o -o pioperfp1 ../PIO1_bld/pio/libpio.a ../pio_build_int/src/gptl/libgptl.a /glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/lib/libnetcdff.a /glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/lib/libnetcdf.so /glade/apps/opt/pnetcdf/1.6.1/intel/15.0.3/lib/libpnetcdf.a -lirng -ldecimal -lcilkrts -lstdc++ -openmp + +pioperformancep1.o: pioperformance.F90 + mpif90 -DPIO_GPFS_HINTS -DUSEMPIIO -D_NETCDF -D_NETCDF4 -D_NOUSEMCT -D_PNETCDF -D_USEBOX -D_PIO1 -no-opt-dynamic-align -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source -g -xHost -debug minimal -openmp -DLINUX -DMCT_INTERFACE -DHAVE_MPI -DTHREADED_OMP -DFORTRANUNDERSCORE -DNO_R16 -DINTEL_MKL -DHAVE_SSE2 -DLINUX -DCPRINTEL -DHAVE_SLASHPROC -I. -I/glade/p/work/katec/pio_work/PIO1_bld/pio -I/glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/include -I/glade/apps/opt/pnetcdf/1.6.1/intel/15.0.3/include -I/glade/p/work/katec/pio_work/PIO1/pio -I/glade/p/work/katec/pio_work/PIO1_bld/pio/timing -I/glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/include -I/glade/apps/opt/pnetcdf/1.6.1/intel/15.0.3/include -c pioperformance.F90 -o pioperformancep1.o + +cleanp1: + rm pioperformancep1.o pioperfp1 + +clean: + rm pioperformance.o pioperf \ No newline at end of file diff --git a/tests/performance/pioperformance.F90 b/tests/performance/pioperformance.F90 index 89430c7c442..16fe145a193 100644 --- a/tests/performance/pioperformance.F90 +++ b/tests/performance/pioperformance.F90 @@ -168,17 +168,18 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & character(len=*), parameter :: rearr_name(2) = (/' BOX','SUBSET'/) nullify(compmap) - if(mype.eq.0) print *,trim(filename) + if(trim(filename) .eq. 'ROUNDROBIN' .or. trim(filename).eq.'BLOCK') then call init_ideal_dof(filename, mype, npe_base, ndims, gdims, compmap, varsize) else ! Changed to support PIO1 as well #ifdef _PIO1 - call pio_readdof(filename, compmap, MPI_COMM_WORLD) + call pio_readdof(filename, compmap, MPI_COMM_WORLD, 81, ndims, gdims) #else call pio_readdof(filename, ndims, gdims, compmap, MPI_COMM_WORLD) #endif +! print *,__FILE__,__LINE__,' gdims=',ndims endif maplen = size(compmap) ! color = 0 @@ -260,8 +261,10 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & ierr = PIO_CreateFile(iosystem, File, iotype, trim(fname), mode) call WriteMetadata(File, gdims, vari, varr, vard, unlimdimindof) + call MPI_Barrier(comm,ierr) call t_stampf(wall(1), usr(1), sys(1)) + if(.not. unlimdimindof) then #ifdef VARINT call PIO_InitDecomp(iosystem, PIO_INT, gdims, compmap, iodesc_i4, rearr=rearr) @@ -273,7 +276,8 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & call PIO_InitDecomp(iosystem, PIO_DOUBLE, gdims, compmap, iodesc_r8, rearr=rearr) #endif endif -! print *,__FILE__,__LINE__,minval(dfld),maxval(dfld),minloc(dfld),maxloc(dfld) + + ! print *,__FILE__,__LINE__,minval(dfld),maxval(dfld),minloc(dfld),maxloc(dfld) do frame=1,nframes recnum = frame @@ -561,7 +565,9 @@ subroutine WriteMetadata(File, gdims, vari, varr, vard,unlimdimindof) ndims=ndims-1 endif allocate(dimid(ndims+1)) + do i=1,ndims + write(dimname,'(a,i6.6)') 'dim',i iostat = PIO_def_dim(File, trim(dimname), int(gdims(i),pio_offset_kind), dimid(i)) enddo From 17da322692844ac1fde6a649b57ba892431ef0ce Mon Sep 17 00:00:00 2001 From: Katetc Date: Fri, 15 Apr 2016 15:05:27 -0600 Subject: [PATCH 006/184] Added a catch for NC_EINVAL errors on file opening (in this case, try plain netcdf before giving up and throwing a total error). This addresses the runtime error in test ERP_Ln9.f19_f19.FW5.yellowstone_intel.cam-outfrq9s --- src/clib/pio_file.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 38cf64a8e2d..83bf9835a29 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -65,7 +65,7 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->compmaster, ios->intercomm); } - + if(ios->ioproc){ switch(file->iotype){ @@ -113,7 +113,7 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, // If we failed to open a file due to an incompatible type of NetCDF, try it // once with just plain old basic NetCDF #ifdef _NETCDF - if(ierr == NC_ENOTNC && (file->iotype != PIO_IOTYPE_NETCDF)) { + if((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF)) { if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); // reset ierr on all tasks ierr = PIO_NOERR; From 04db212a002133b09d3d8344c979ba723fe6c5d2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 4 May 2016 11:29:41 -0400 Subject: [PATCH 007/184] added documentation to iosystem_desc_t --- src/clib/pio.h | 63 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index 5b8446bf21e..5185a133b0e 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -139,43 +139,89 @@ typedef struct io_desc_t */ typedef struct iosystem_desc_t { + /** The ID of this iosystem_desc_t. */ int iosysid; + + /** This is an MPI intra communicator that includes all the tasks in + * both the IO and the computation communicators. */ MPI_Comm union_comm; + + /** This is an MPI intra communicator that includes all the tasks + * involved in IO. */ MPI_Comm io_comm; + + /** This is an MPI intra communicator that includes all the tasks + * involved in computation. */ MPI_Comm comp_comm; + + /** This is an MPI inter communicator between IO communicator and + * computation communicator. */ MPI_Comm intercomm; + + /** This is a copy (but not an MPI copy) of either the comp (for + * non-async) or the union (for async) communicator. */ MPI_Comm my_comm; /** This MPI group contains the processors involved in - * computation. It is created in PIOc_Init_Intracomm(), and freed my - * PIO_finalize(). */ + * computation. */ MPI_Group compgroup; - /** This MPI group contains the processors involved in I/O. It is - * created in PIOc_Init_Intracomm(), and freed my PIOc_finalize(). */ + /** This MPI group contains the processors involved in I/O. */ MPI_Group iogroup; - + + /** The number of tasks in the IO communicator. */ int num_iotasks; + + /** The number of tasks in the computation communicator. */ int num_comptasks; + /** Rank of this task in the union communicator. */ int union_rank; + + /** The rank of this process in the computation communicator, or -1 + * if this process is not part of the computation communicator. */ int comp_rank; + + /** The rank of this process in the IO communicator, or -1 if this + * process is not part of the IO communicator. */ int io_rank; - bool iomaster; - bool compmaster; + /** Set to MPI_ROOT if this task is the master of IO communicator, 0 + * otherwise. */ + int iomaster; + /** Set to MPI_ROOT if this task is the master of comp communicator, 0 + * otherwise. */ + int compmaster; + + /** Rank of IO root task (which is rank 0 in io_comm) in the union + * communicator. */ int ioroot; + + /** Rank of computation root task (which is rank 0 in + * comm_comms[cmp]) in the union communicator. */ int comproot; + + /** An array of the ranks of all IO tasks within the union + * communicator. */ int *ioranks; + /** Controls handling errors. */ int error_handler; + + /** ??? */ int default_rearranger; + /** True if asynchronous interface is in use. */ bool async_interface; + + /** True if this task is a member of the IO communicator. */ bool ioproc; - + + /** MPI Info object. */ MPI_Info info; + + /** Pointer to the next iosystem_desc_t in the list. */ struct iosystem_desc_t *next; } iosystem_desc_t; @@ -491,7 +537,6 @@ int PIOc_set_blocksize(const int newblocksize); int PIOc_put_var_short (int ncid, int varid, const short *op) ; int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], char *buf) ; int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op) ; - int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op) ; int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op); int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op); From 648b350cdb74f5bfde7f983d34782d90cf9245e7 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 4 May 2016 11:54:25 -0400 Subject: [PATCH 008/184] changed example data size, added valgrind suppression file --- examples/c/example2.c | 4 ++-- examples/c/valsupp_example1.supp | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 examples/c/valsupp_example1.supp diff --git a/examples/c/example2.c b/examples/c/example2.c index f4b75d075b7..d86af67774c 100644 --- a/examples/c/example2.c +++ b/examples/c/example2.c @@ -48,8 +48,8 @@ * responsibilty for writing and reading them will be spread between * all the processors used to run this example. */ /**@{*/ -#define X_DIM_LEN 400 -#define Y_DIM_LEN 400 +#define X_DIM_LEN 20 +#define Y_DIM_LEN 30 /**@}*/ /** The number of timesteps of data to write. */ diff --git a/examples/c/valsupp_example1.supp b/examples/c/valsupp_example1.supp new file mode 100644 index 00000000000..63f3e073836 --- /dev/null +++ b/examples/c/valsupp_example1.supp @@ -0,0 +1,15 @@ +{ + cond_jump_1 + Memcheck:Cond + fun:MPIC_Waitall + fun:MPIR_Alltoallw_intra + fun:MPIR_Alltoallw + fun:MPIR_Alltoallw_impl + fun:PMPI_Alltoallw + fun:pio_swapm + fun:rearrange_comp2io + fun:PIOc_write_darray_multi + fun:flush_buffer + fun:PIOc_sync + fun:main +} \ No newline at end of file From a4cdd068af0e3da06a8bc2bf1fb2e66f4d48e507 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 4 May 2016 17:47:13 -0400 Subject: [PATCH 009/184] changing in pio_file.c to support async --- src/clib/pio_file.c | 54 +++++++++++++++++++++++++---------------- src/clib/pio_internal.h | 12 ++++++++- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 83bf9835a29..37fa6285031 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -61,9 +61,10 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, if(ios->comp_rank==0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); len = strlen(filename); - mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); } if(ios->ioproc){ @@ -130,8 +131,10 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ierr==PIO_NOERR){ - mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm); + if (!ierr) { + mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm); + *ncidp = file->fh; pio_add_to_file_list(file); *ncidp = file->fh; } @@ -199,10 +202,11 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, if(ios->async_interface && ! ios->ioproc){ if(ios->comp_rank==0) - mpierr = MPI_Send( &msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); len = strlen(filename); - mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -248,8 +252,10 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, ierr = check_netcdf(file, ierr, __FILE__,__LINE__); if(ierr == PIO_NOERR){ - mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm); + mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm); file->mode = file->mode | PIO_WRITE; // This flag is implied by netcdf create functions but we need to know if its set + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm); + *ncidp = file->fh; pio_add_to_file_list(file); *ncidp = file->fh; } @@ -279,14 +285,20 @@ int PIOc_closefile(int ncid) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = 0; + msg = PIO_MSG_CLOSE_FILE; if((file->mode & PIO_WRITE)){ PIOc_sync(ncid); } - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + + /* If async is in use and this is a comp tasks, then the compmaster + * sends a msg to the pio_msg_handler running on the IO master and + * waiting for a message. Then broadcast the ncid over the intercomm + * to the IO tasks. */ + if(ios->async_interface && !ios->ioproc){ + if(ios->comp_rank==0) { + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + } + mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm); } if(ios->ioproc){ @@ -325,7 +337,6 @@ int PIOc_closefile(int ncid) int iret = pio_delete_file_from_list(ncid); - return ierr; } @@ -342,6 +353,7 @@ int PIOc_deletefile(const int iosysid, const char filename[]) int mpierr; int chkerr; iosystem_desc_t *ios; + size_t len; ierr = PIO_NOERR; ios = pio_get_iosystem_from_id(iosysid); @@ -349,12 +361,14 @@ int PIOc_deletefile(const int iosysid, const char filename[]) if(ios == NULL) return PIO_EBADID; - msg = 0; + msg = PIO_MSG_DELETE_FILE; if(ios->async_interface && ! ios->ioproc){ if(ios->comp_rank==0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - // mpierr = MPI_Bcast(iosysid,1, MPI_INT, ios->compmaster, ios->intercomm); + len = strlen(filename); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); } // The barriers are needed to assure that no task is trying to operate on the file while it is being deleted. if(ios->ioproc){ @@ -371,8 +385,6 @@ int PIOc_deletefile(const int iosysid, const char filename[]) } // Special case - always broadcast the return from the MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm); - - return ierr; } @@ -405,9 +417,9 @@ int PIOc_sync (int ncid) msg = PIO_MSG_SYNC; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(!ios->comp_rank) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } if((file->mode & PIO_WRITE)){ diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index f7a3c9ddf22..740b8fdf1a6 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -158,6 +158,8 @@ void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk); } #endif +/** These are the messages that can be sent over the intercomm when + * async is being used. */ enum PIO_MSG{ PIO_MSG_OPEN_FILE, PIO_MSG_CREATE_FILE, @@ -365,7 +367,15 @@ enum PIO_MSG{ PIO_MSG_SET_CHUNK_CACHE, PIO_MSG_GET_CHUNK_CACHE, PIO_MSG_SET_VAR_CHUNK_CACHE, - PIO_MSG_GET_VAR_CHUNK_CACHE + PIO_MSG_GET_VAR_CHUNK_CACHE, + PIO_MSG_INITDECOMP_DOF, + PIO_MSG_WRITEDARRAY, + PIO_MSG_READDARRAY, + PIO_MSG_SETERRORHANDLING, + PIO_MSG_FREEDECOMP, + PIO_MSG_CLOSE_FILE, + PIO_MSG_DELETE_FILE, + PIO_MSG_EXIT }; #endif From aebdcddf86ea42f0fda46ebfe85c70421d61bdf7 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 4 May 2016 18:02:40 -0400 Subject: [PATCH 010/184] change in response to review feedback --- src/clib/pio_file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 37fa6285031..461c8d2b4ca 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -417,7 +417,7 @@ int PIOc_sync (int ncid) msg = PIO_MSG_SYNC; if(ios->async_interface && ! ios->ioproc){ - if(!ios->comp_rank) + if(ios->comp_rank == 0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } From c854f3498c4d6675826b50e792b6b1a0cbeaf953 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 5 May 2016 07:18:06 -0400 Subject: [PATCH 011/184] changes to pioc.c to support async, also some temporary copies of code files for async development --- src/clib/pio_get_nc_async.c | 4988 ++++++++++++++++++++++++++++++++++ src/clib/pio_nc_async.c | 4534 +++++++++++++++++++++++++++++++ src/clib/pio_put_nc_async.c | 5073 +++++++++++++++++++++++++++++++++++ src/clib/pioc.c | 51 +- 4 files changed, 14626 insertions(+), 20 deletions(-) create mode 100644 src/clib/pio_get_nc_async.c create mode 100644 src/clib/pio_nc_async.c create mode 100644 src/clib/pio_put_nc_async.c diff --git a/src/clib/pio_get_nc_async.c b/src/clib/pio_get_nc_async.c new file mode 100644 index 00000000000..0da690aaf21 --- /dev/null +++ b/src/clib/pio_get_nc_async.c @@ -0,0 +1,4988 @@ +#include +#include + +int PIOc_get_var1_schar (int ncid, int varid, const PIO_Offset index[], signed char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_SCHAR; + ibuftype = MPI_CHAR; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_schar(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_schar_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_ulonglong(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_ulonglong_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_uchar(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_uchar_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], signed char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_SCHAR; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_schar(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_schar_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_SHORT; + ibuftype = MPI_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_short(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_short_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_double (int ncid, int varid, double *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_DOUBLE; + ibuftype = MPI_DOUBLE; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_double(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_double(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_double(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_double_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], double *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_DOUBLE; + ibuftype = MPI_DOUBLE; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_double(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_double_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_int (int ncid, int varid, int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_INT; + ibuftype = MPI_INT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_int(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_int(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_int(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_int_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_ushort (int ncid, int varid, unsigned short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_ushort(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_ushort(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_ushort(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_ushort_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_TEXT; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_text(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_text_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_INT; + ibuftype = MPI_INT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_int(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_int_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset index[], float *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_FLOAT; + ibuftype = MPI_FLOAT; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_float(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_float_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset index[], short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_SHORT; + ibuftype = MPI_SHORT; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_short(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_short_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_INT; + ibuftype = MPI_INT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_int(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_int_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_text (int ncid, int varid, char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_TEXT; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_text(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_text(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_text(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_text_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], double *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_DOUBLE; + ibuftype = MPI_DOUBLE; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_double(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_double_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], signed char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_SCHAR; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_schar(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_schar_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_ushort(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_ushort_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_ushort (int ncid, int varid, const PIO_Offset index[], unsigned short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_ushort(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_ushort_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_float (int ncid, int varid, float *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_FLOAT; + ibuftype = MPI_FLOAT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_float(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_float(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_float(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_float_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_uchar(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_uchar_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var(file->fh, varid, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_all(file->fh, varid, buf, bufcount, buftype);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset index[], long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_LONGLONG; + ibuftype = MPI_LONG_LONG; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_longlong(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_longlong_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_ushort(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_ushort_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_long (int ncid, int varid, long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_LONG; + ibuftype = MPI_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_long(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_long(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_long(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_long_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset index[], double *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_DOUBLE; + ibuftype = MPI_DOUBLE; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_double(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_double_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_UINT; + ibuftype = MPI_UNSIGNED; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_uint(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_uint_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_LONGLONG; + ibuftype = MPI_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_longlong(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_longlong_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_longlong (int ncid, int varid, long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_LONGLONG; + ibuftype = MPI_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_longlong(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_longlong(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_longlong(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_longlong_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_SHORT; + ibuftype = MPI_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_short(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_short_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_LONG; + ibuftype = MPI_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_long(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_long_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_int (int ncid, int varid, const PIO_Offset index[], int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_INT; + ibuftype = MPI_INT; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_int(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_int_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], unsigned long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_ulonglong(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_ulonglong_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_uchar (int ncid, int varid, unsigned char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_uchar(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_uchar(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_uchar(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_uchar_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_uchar(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_uchar_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], float *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_FLOAT; + ibuftype = MPI_FLOAT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_float(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_float_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_LONG; + ibuftype = MPI_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_long(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_long_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1 (int ncid, int varid, const PIO_Offset index[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1(file->fh, varid, index, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_all(file->fh, varid, index, buf, bufcount, buftype);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_UINT; + ibuftype = MPI_UNSIGNED; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_uint(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_uint(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_uint(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_uint_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara(file->fh, varid, start, count, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_all(file->fh, varid, start, count, buf, bufcount, buftype);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], signed char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_SCHAR; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_schar(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_schar_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset index[], unsigned int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_UINT; + ibuftype = MPI_UNSIGNED; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_uint(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_uint_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_UINT; + ibuftype = MPI_UNSIGNED; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_uint(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_uint_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], float *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_FLOAT; + ibuftype = MPI_FLOAT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_float(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_float_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_TEXT; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_text(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_text_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset index[], char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_TEXT; + ibuftype = MPI_CHAR; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_text(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_text_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_INT; + ibuftype = MPI_INT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_int(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_int_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_UINT; + ibuftype = MPI_UNSIGNED; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_uint(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_uint_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_all(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], double *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_DOUBLE; + ibuftype = MPI_DOUBLE; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_double(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_double_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_LONGLONG; + ibuftype = MPI_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_longlong(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_longlong_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_ulonglong(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_ulonglong(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_ulonglong(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_ulonglong_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_ulonglong(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vara_ulonglong_all(file->fh, varid, start, count, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_short (int ncid, int varid, short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_SHORT; + ibuftype = MPI_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_short(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_short(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_short(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_short_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], float *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_FLOAT; + ibuftype = MPI_FLOAT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_float(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_float_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset index[], long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_LONG; + ibuftype = MPI_LONG; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_long(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_long_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_LONG; + ibuftype = MPI_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_long(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_long_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_ushort(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_ushort_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_LONGLONG; + ibuftype = MPI_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_longlong(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_longlong_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_TEXT; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_text(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_text_all(file->fh, varid, start, count, stride, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset index[], unsigned char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_uchar(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var1_uchar_all(file->fh, varid, index, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_vars_all(file->fh, varid, start, count, stride, buf, bufcount, buftype);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_SHORT; + ibuftype = MPI_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_short(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_short_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_ulonglong(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_ulonglong_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_var_schar (int ncid, int varid, signed char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_SCHAR; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_schar(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_schar(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_schar(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_var_schar_all(file->fh, varid, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c new file mode 100644 index 00000000000..9e9f04a45db --- /dev/null +++ b/src/clib/pio_nc_async.c @@ -0,0 +1,4534 @@ +/** + * @file + * PIO interfaces to + * [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html) + * support functions + + * This file provides an interface to the + * [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html) + * support functions. Each subroutine calls the underlying netcdf or + * pnetcdf or netcdf4 functions from the appropriate subset of mpi + * tasks (io_comm). Each routine must be called collectively from + * union_comm. + * + * @author Jim Edwards (jedwards@ucar.edu), Ed Hartnett + * @date Feburary 2014, April 2016 + */ + +#include +#include + +/** Internal function to call nc_inq. Call this function only in I/O + * tasks. */ +int +pio_io_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, + int *unlimdimidp) +{ + file_desc_t *file; + char *errstr = NULL; + int ierr = PIO_NOERR; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d pio_io_inq ncid = %d\n", my_rank, ncid); + + if (!(file = pio_get_file_from_id(ncid))) + return PIO_EBADID; + + switch (file->iotype) + { +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + printf("%d pio_io_inq file->iosystem->io_rank = %d\n", my_rank, file->iosystem->io_rank); + if (!file->iosystem->io_rank) + { + /* Should not be necessary to do this - nc_inq should + * handle null pointers. This has been reported as a bug + * to netCDF developers. */ + int tmp_ndims, tmp_var, tmp_gatts, tmp_unl; + if (!ndimsp) + ndimsp = &tmp_ndims; + if (!nvarsp) + nvarsp = &tmp_var; + if (!ngattsp) + ngattsp = &tmp_gatts; + if (!unlimdimidp) + unlimdimidp = &tmp_unl; + ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + + if(ierr != PIO_NOERR) + { + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + + return ierr; +} + +/** + * @ingroup PIOc_inq + * The PIO-C interface for the NetCDF function nc_inq. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See + * PIOc_Set_File_Error_Handling + */ +int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, + int *unlimdimidp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_inq ncid = %d\n", my_rank, ncid); + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + if(ios->ioproc){ + ierr = pio_io_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp); + printf("%d PIOc_inq pio_io_inq returned %d\n", my_rank, ierr); + ierr = check_netcdf(file, ierr, errstr,__LINE__); + } + if(ndimsp != NULL) + { + printf("%d PIOc_inq Bcast ios->ioroot = %d *ndimsp = %d\n", my_rank, ios->ioroot, *ndimsp); + mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + } + if(nvarsp != NULL) + mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(ngattsp != NULL) + mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(unlimdimidp != NULL) + mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_dimname + * The PIO-C interface for the NetCDF function nc_inq_dimname. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_dimname (int ncid, int dimid, char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_DIMNAME; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_dimname(file->fh, dimid, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_dimname(file->fh, dimid, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_dimname(file->fh, dimid, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (name) + { + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_short + * The PIO-C interface for the NetCDF function nc_put_att_short. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_short(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_rename_dim + * The PIO-C interface for the NetCDF function nc_rename_dim. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_rename_dim (int ncid, int dimid, const char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_RENAME_DIM; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_rename_dim(file->fh, dimid, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_rename_dim(file->fh, dimid, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_rename_dim(file->fh, dimid, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_double + * The PIO-C interface for the NetCDF function nc_get_att_double. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_double(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_double(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_double(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_DOUBLE, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_set_fill + * The PIO-C interface for the NetCDF function nc_set_fill. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_set_fill (int ncid, int fillmode, int *old_modep) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_SET_FILL; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_set_fill(file->fh, fillmode, old_modep);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_set_fill(file->fh, fillmode, old_modep);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_set_fill(file->fh, fillmode, old_modep);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_def_var + * The PIO-C interface for the NetCDF function nc_def_var. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param varidp a pointer that will get the variable id + * @return PIO_NOERR for success, error code otherwise. See + * PIOc_Set_File_Error_Handling + */ +int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, + const int *dimidsp, int *varidp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + int namelen; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_DEF_VAR; + + /* If async is in use, and this is not an IO tasks, then bcast the + * function parameters to the intercomm, where it will be read by + * the IO root task. */ + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + namelen = strlen(name); + printf("bcasting namelen = %d name = %s\n", namelen, name); + if (!ios->compmaster) + ios->compmaster = MPI_PROC_NULL; + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmaster, ios->intercomm); + } + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; + break; + case PIO_IOTYPE_NETCDF4C: + if(ios->io_rank==0){ + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); + if(ierr == PIO_NOERR){ + ierr = nc_def_var_deflate(file->fh, *varidp, 0,1,1); + } + } + break; +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_double + * The PIO-C interface for the NetCDF function nc_put_att_double. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_double(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_dim + * The PIO-C interface for the NetCDF function nc_inq_dim. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param lenp a pointer that will get the number of values + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_DIM; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(name) + { + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(lenp != NULL) + mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_uchar + * The PIO-C interface for the NetCDF function nc_get_att_uchar. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_uchar(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_uchar(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_uchar(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_CHAR, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_var_fill + * The PIO-C interface for the NetCDF function nc_inq_var_fill. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VAR_FILL; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_value);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(fill_value, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_attid + * The PIO-C interface for the NetCDF function nc_inq_attid. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param idp a pointer that will get the id of the variable or attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_ATTID; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_attid(file->fh, varid, name, idp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_attid(file->fh, varid, name, idp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_attid(file->fh, varid, name, idp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_vartype + * The PIO-C interface for the NetCDF function nc_inq_vartype. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VARTYPE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh,1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_vartype(file->fh, varid, xtypep);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_vartype(file->fh, varid, xtypep);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_vartype(file->fh, varid, xtypep);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (xtypep) + mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_schar + * The PIO-C interface for the NetCDF function nc_put_att_schar. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_schar(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_vardimid + * The PIO-C interface for the NetCDF function nc_inq_vardimid. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VARDIMID; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_vardimid(file->fh, varid, dimidsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_vardimid(file->fh, varid, dimidsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_vardimid(file->fh, varid, dimidsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (!ierr && dimidsp) + { + int ndims; + PIOc_inq_varndims(file->fh, varid, &ndims); + mpierr = MPI_Bcast(dimidsp , ndims, MPI_INT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_ushort + * The PIO-C interface for the NetCDF function nc_get_att_ushort. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_ushort(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_ushort(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_ushort(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_SHORT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_varid + * The PIO-C interface for the NetCDF function nc_inq_varid. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param varidp a pointer that will get the variable id + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varid (int ncid, const char *name, int *varidp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VARID; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_inq_varid ncid = %d name = %s\n", my_rank, ncid, name); + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + printf("%d PIOc_inq_varid BCast ncid = %d\n", my_rank, ncid); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen; + namelen = strlen(name); + printf("%d PIOc_inq_varid BCast namelen = %d\n", my_rank, namelen); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq_varid BCast name = %s\n", my_rank, name); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq_varid BCast done\n", my_rank); + } + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_varid(file->fh, name, varidp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_varid(file->fh, name, varidp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_varid(file->fh, name, varidp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (varidp) + mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_attlen + * The PIO-C interface for the NetCDF function nc_inq_attlen. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param lenp a pointer that will get the number of values + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_ATTLEN; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_attlen(file->fh, varid, name, lenp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_atttype + * The PIO-C interface for the NetCDF function nc_inq_atttype. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_ATTTYPE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_atttype(file->fh, varid, name, xtypep);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_atttype(file->fh, varid, name, xtypep);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_atttype(file->fh, varid, name, xtypep);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_rename_var + * The PIO-C interface for the NetCDF function nc_rename_var. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_rename_var (int ncid, int varid, const char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_RENAME_VAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_rename_var(file->fh, varid, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_rename_var(file->fh, varid, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_rename_var(file->fh, varid, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_natts + * The PIO-C interface for the NetCDF function nc_inq_natts. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_natts (int ncid, int *ngattsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_NATTS; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_natts(file->fh, ngattsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_natts(file->fh, ngattsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_natts(file->fh, ngattsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (ngattsp) + mpierr = MPI_Bcast(ngattsp,1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_ulonglong + * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_ulonglong(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_var + * The PIO-C interface for the NetCDF function nc_inq_var. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @param nattsp a pointer that will get the number of attributes + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, + int *dimidsp, int *nattsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (xtypep) + mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); + if (ndimsp) + { + mpierr = MPI_Bcast(ndimsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + file->varlist[varid].ndims = (*ndimsp); + } + if (nattsp) + mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if (name) + { + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if (dimidsp) + { + int ndims; + PIOc_inq_varndims(file->fh, varid, &ndims); + mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_rename_att + * The PIO-C interface for the NetCDF function nc_rename_att. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_RENAME_ATT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_rename_att(file->fh, varid, name, newname);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_rename_att(file->fh, varid, name, newname);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_rename_att(file->fh, varid, name, newname);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_ushort + * The PIO-C interface for the NetCDF function nc_put_att_ushort. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_ushort(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_dimid + * The PIO-C interface for the NetCDF function nc_inq_dimid. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param idp a pointer that will get the id of the variable or attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_dimid (int ncid, const char *name, int *idp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_DIMID; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen; + if (ios->iomaster) + namelen = strlen(name); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_dimid(file->fh, name, idp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_dimid(file->fh, name, idp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_dimid(file->fh, name, idp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (idp) + mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_text + * The PIO-C interface for the NetCDF function nc_put_att_text. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_text(file->fh, varid, name, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_uint + * The PIO-C interface for the NetCDF function nc_get_att_uint. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_uint(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_uint(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_uint(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_format + * The PIO-C interface for the NetCDF function nc_inq_format. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param formatp a pointer that will get the file format + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_format (int ncid, int *formatp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_FORMAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_format(file->fh, formatp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_format(file->fh, formatp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_format(file->fh, formatp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_long + * The PIO-C interface for the NetCDF function nc_get_att_long. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_long(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_long(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_long(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_attname + * The PIO-C interface for the NetCDF function nc_inq_attname. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param attnum the attribute ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_ATTNAME; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_attname(file->fh, varid, attnum, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_attname(file->fh, varid, attnum, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(name != NULL){ + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_att + * The PIO-C interface for the NetCDF function nc_inq_att. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @param lenp a pointer that will get the number of values + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_ATT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(xtypep != NULL) mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(lenp != NULL) mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_long + * The PIO-C interface for the NetCDF function nc_put_att_long. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_long(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_unlimdim + * The PIO-C interface for the NetCDF function nc_inq_unlimdim. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_UNLIMDIM; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_unlimdim(file->fh, unlimdimidp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_unlimdim(file->fh, unlimdimidp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_unlimdim(file->fh, unlimdimidp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (unlimdimidp) + mpierr = MPI_Bcast(unlimdimidp,1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_float + * The PIO-C interface for the NetCDF function nc_get_att_float. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_float(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_float(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_float(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_FLOAT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_ndims + * The PIO-C interface for the NetCDF function nc_inq_ndims. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_ndims (int ncid, int *ndimsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_NDIMS; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_ndims(file->fh, ndimsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_ndims(file->fh, ndimsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_ndims(file->fh, ndimsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (ndimsp) + mpierr = MPI_Bcast(ndimsp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_int + * The PIO-C interface for the NetCDF function nc_put_att_int. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_int(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_nvars + * The PIO-C interface for the NetCDF function nc_inq_nvars. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_nvars (int ncid, int *nvarsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_NVARS; + + if(ios->async_interface && ! ios->ioproc){ + if(!ios->comp_rank) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_nvars(file->fh, nvarsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_nvars(file->fh, nvarsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_nvars(file->fh, nvarsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (nvarsp) + mpierr = MPI_Bcast(nvarsp,1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_enddef + * The PIO-C interface for the NetCDF function nc_enddef. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_enddef (int ncid) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_ENDDEF; + + if(ios->async_interface && ! ios->ioproc){ + if(!ios->comp_rank) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + printf("PIOc_enddef file->fh = %d\n", file->fh); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_enddef(file->fh);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_enddef(file->fh);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_enddef(file->fh);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_uchar + * The PIO-C interface for the NetCDF function nc_put_att_uchar. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_uchar(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_longlong + * The PIO-C interface for the NetCDF function nc_put_att_longlong. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_longlong(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_varnatts + * The PIO-C interface for the NetCDF function nc_inq_varnatts. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param nattsp a pointer that will get the number of attributes + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VARNATTS; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_varnatts(file->fh, varid, nattsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_varnatts(file->fh, varid, nattsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_varnatts(file->fh, varid, nattsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (nattsp) + mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_ubyte + * The PIO-C interface for the NetCDF function nc_get_att_ubyte. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_UBYTE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_ubyte(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_ubyte(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_ubyte(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_BYTE, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_text + * The PIO-C interface for the NetCDF function nc_get_att_text. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_text(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_text(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_text(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_del_att + * The PIO-C interface for the NetCDF function nc_del_att. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_del_att (int ncid, int varid, const char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_DEL_ATT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_del_att(file->fh, varid, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_del_att(file->fh, varid, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_del_att(file->fh, varid, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_dimlen + * The PIO-C interface for the NetCDF function nc_inq_dimlen. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param lenp a pointer that will get the number of values + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_DIMLEN; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_dimlen(file->fh, dimid, lenp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (lenp) + mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_schar + * The PIO-C interface for the NetCDF function nc_get_att_schar. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_schar(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_schar(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_schar(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_ulonglong + * The PIO-C interface for the NetCDF function nc_get_att_ulonglong. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_ulonglong(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_LONG_LONG, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_varndims + * The PIO-C interface for the NetCDF function nc_inq_varndims. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VARNDIMS; + if(file->varlist[varid].ndims > 0){ + (*ndimsp) = file->varlist[varid].ndims; + return PIO_NOERR; + } + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh,1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid,1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_varndims(file->fh, varid, ndimsp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_varndims(file->fh, varid, ndimsp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_varndims(file->fh, varid, ndimsp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (ndimsp) + { + mpierr = MPI_Bcast(ndimsp,1, MPI_INT, ios->ioroot, ios->my_comm); + file->varlist[varid].ndims = (*ndimsp); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_inq_varname + * The PIO-C interface for the NetCDF function nc_inq_varname. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varname (int ncid, int varid, char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_VARNAME; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh,1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_varname(file->fh, varid, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_varname(file->fh, varid, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_varname(file->fh, varid, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(name != NULL){ + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_def_dim + * The PIO-C interface for the NetCDF function nc_def_dim. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param idp a pointer that will get the id of the variable or attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + int namelen; + + errstr = NULL; + ierr = PIO_NOERR; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_def_dim ncid = %d name = %s len = %d\n", my_rank, + ncid, name, len); + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_DEF_DIM; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + namelen = strlen(name); + printf("bcasting namelen = %d name = %s len = %d\n", namelen, name, len); + if (!ios->compmaster) + ios->compmaster = MPI_PROC_NULL; + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_def_dim(file->fh, name, len, idp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_uint + * The PIO-C interface for the NetCDF function nc_put_att_uint. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_uint(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_short + * The PIO-C interface for the NetCDF function nc_get_att_short. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_short(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_short(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_short(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_SHORT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_redef + * The PIO-C interface for the NetCDF function nc_redef. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_redef (int ncid) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_REDEF; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_redef(file->fh);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_redef(file->fh);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_redef(file->fh);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_ubyte + * The PIO-C interface for the NetCDF function nc_put_att_ubyte. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_ubyte (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_UBYTE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_ubyte(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_ubyte(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_ubyte(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_int + * The PIO-C interface for the NetCDF function nc_get_att_int. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_int(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_int(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_int(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_INT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_get_att_longlong + * The PIO-C interface for the NetCDF function nc_get_att_longlong. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_longlong(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_longlong(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_longlong(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG_LONG, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_put_att_float + * The PIO-C interface for the NetCDF function nc_put_att_float. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_float(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c new file mode 100644 index 00000000000..c0b621eb903 --- /dev/null +++ b/src/clib/pio_put_nc_async.c @@ -0,0 +1,5073 @@ +#include +#include + +/// +/// PIO interface to nc_put_vars_uchar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_uchar(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_ushort +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_ushort(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_ulonglong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_ulonglong(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_uint +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_uint(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_uchar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_uchar(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_ushort +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_ushort (int ncid, int varid, const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_ushort(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_ushort(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_ushort(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_longlong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_longlong(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_uchar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_uchar(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_short +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_short(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_long +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_long (int ncid, int varid, const PIO_Offset index[], const long *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_long(file->fh, varid, index, ip, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_long +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_short +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_short (int ncid, int varid, const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_short(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_short(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_short(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_int +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_int(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_ushort +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_ushort(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_text +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_text(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_text +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_text(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_ushort +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_ushort(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_ulonglong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_ulonglong(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_ulonglong(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_ulonglong(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_int +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_int (int ncid, int varid, const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_int(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_int(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_int(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_longlong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_longlong (int ncid, int varid, const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_longlong(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_longlong(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_longlong(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_schar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_schar (int ncid, int varid, const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_schar(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_schar(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_schar(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_uint +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_uint (int ncid, int varid, const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_uint(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_uint(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_uint(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var (int ncid, int varid, const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var(file->fh, varid, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var(file->fh, varid, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_ushort +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_ushort(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_short +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_uint +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_uint(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_schar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_schar(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_ulonglong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_ulonglong(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_uchar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_uchar(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_int +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_int(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_schar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1 +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1(file->fh, varid, index, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_float +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_float(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_float +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_float(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_float +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_float(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_text +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_text(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_text +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_long +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_long(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_double +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_longlong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_longlong(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_double +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_double (int ncid, int varid, const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_double(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_double(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_double(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_float +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_float (int ncid, int varid, const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_float(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_float(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_float(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_ulonglong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_ulonglong(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_uint +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_uint(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_uint +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_uint(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_int +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_int(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_float +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_short +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_short(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_schar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_schar(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_ulonglong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_ulonglong(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_double +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_double(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara(file->fh, varid, start, count, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_long +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_long(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_double +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_double(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_schar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_schar(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_text +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_text (int ncid, int varid, const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_text(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_text(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_text(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_int +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var1_short +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_short (int ncid, int varid, const PIO_Offset index[], const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_short(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars_longlong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars_longlong(file->fh, varid, start, count, stride, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vara_double +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara_double(file->fh, varid, start, count, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_vars +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_uchar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_uchar (int ncid, int varid, const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_uchar(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_uchar(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_uchar(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_var_long +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_long (int ncid, int varid, const long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var_long(file->fh, varid, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var_long(file->fh, varid, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var_long(file->fh, varid, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_longlong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_longlong(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + diff --git a/src/clib/pioc.c b/src/clib/pioc.c index e81f0a5e243..22d6ea3307f 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -386,8 +386,8 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, iosys->intercomm = MPI_COMM_NULL; iosys->error_handler = PIO_INTERNAL_ERROR; iosys->async_interface= false; - iosys->compmaster = false; - iosys->iomaster = false; + iosys->compmaster = 0; + iosys->iomaster = 0; iosys->ioproc = false; iosys->default_rearranger = rearr; iosys->num_iotasks = num_iotasks; @@ -398,7 +398,7 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, CheckMPIReturn(MPI_Comm_rank(iosys->comp_comm, &(iosys->comp_rank)),__FILE__,__LINE__); CheckMPIReturn(MPI_Comm_size(iosys->comp_comm, &(iosys->num_comptasks)),__FILE__,__LINE__); if(iosys->comp_rank==0) - iosys->compmaster = true; + iosys->compmaster = MPI_ROOT; /* Ensure that settings for number of computation tasks, number * of IO tasks, and the stride are reasonable. */ @@ -428,7 +428,7 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, iosys->info = MPI_INFO_NULL; if(iosys->comp_rank == iosys->ioranks[0]) - iosys->iomaster = true; + iosys->iomaster = MPI_ROOT; /* Create a group for the computation tasks. */ CheckMPIReturn(MPI_Comm_group(iosys->comp_comm, &(iosys->compgroup)),__FILE__,__LINE__); @@ -489,8 +489,9 @@ int PIOc_set_hint(const int iosysid, char hint[], const char hintval[]) } -/** @ingroup PIO_finalize - * @brief Clean up data structures and exit the pio library. +/** @ingroup PIO_finalize + * Clean up internal data structures, free MPI resources, and exit the + * pio library. * * @param iosysid: the io system ID provided by PIOc_Init_Intracomm(). * @@ -500,27 +501,38 @@ int PIOc_set_hint(const int iosysid, char hint[], const char hintval[]) int PIOc_finalize(const int iosysid) { iosystem_desc_t *ios, *nios; + int msg; + int mpierr; ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) - return PIO_EBADID; - /* FIXME: The memory for ioranks is allocated in C only for intracomms - * Remove this check once mem allocs for ioranks completely moves to the - * C code - */ - if(ios->intercomm == MPI_COMM_NULL){ - if(ios->ioranks != NULL){ - free(ios->ioranks); - } + return PIO_EBADID; + + /* If asynch IO is in use, send the PIO_MSG_EXIT message from the + * comp master to the IO processes. */ + if (ios->async_interface && !ios->comp_rank) + { + msg = PIO_MSG_EXIT; + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); } + /* Free this memory that was allocated in init_intracomm. */ + if (ios->ioranks) + free(ios->ioranks); + + /* Free the buffer pool. */ free_cn_buffer_pool(*ios); /* Free the MPI groups. */ - MPI_Group_free(&(ios->compgroup)); - MPI_Group_free(&(ios->iogroup)); + if (ios->compgroup != MPI_GROUP_NULL) + MPI_Group_free(&ios->compgroup); + + if (ios->iogroup != MPI_GROUP_NULL) + MPI_Group_free(&(ios->iogroup)); - /* Free the MPI communicators. */ + /* Free the MPI communicators. my_comm is just a copy (but not an + * MPI copy), so does not have to have an MPI_Comm_free() call. */ if(ios->intercomm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->intercomm)); } @@ -534,9 +546,8 @@ int PIOc_finalize(const int iosysid) MPI_Comm_free(&(ios->union_comm)); } + /* Delete the iosystem_desc_t data associated with this id. */ return pio_delete_iosystem_from_list(iosysid); - - } /** From 458a3fae18f0795854fbdadc1a3b801646bdcea9 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 5 May 2016 07:52:48 -0400 Subject: [PATCH 012/184] removed extra index for async ncids --- src/clib/pio_lists.c | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/src/clib/pio_lists.c b/src/clib/pio_lists.c index 511918b52b5..efa0868c444 100644 --- a/src/clib/pio_lists.c +++ b/src/clib/pio_lists.c @@ -9,37 +9,34 @@ static iosystem_desc_t *pio_iosystem_list=NULL; static file_desc_t *pio_file_list = NULL; static file_desc_t *current_file=NULL; +/** Add a new entry to the global list of open files. + * + * @param file pointer to the file_desc_t struct for the new file. +*/ void pio_add_to_file_list(file_desc_t *file) { file_desc_t *cfile; - int cnt=-1; - // on iotasks the fh returned from netcdf should be unique, on non-iotasks we - // need to generate a unique fh, we do this with cnt, its a negative index + /* This file will be at the end of the list, and have no next. */ file->next = NULL; + + /* Get a pointer to the global list of files. */ cfile = pio_file_list; + + /* Keep a global pointer to the current file. */ current_file = file; - if(cfile==NULL){ - pio_file_list = file; - }else{ - cnt = min(cnt,cfile->fh-1); - while(cfile->next != NULL) - { - cfile=cfile->next; - cnt = min(cnt,cfile->fh-1); - } - cfile->next = file; - } - if(! file->iosystem->ioproc || ((file->iotype != PIO_IOTYPE_PNETCDF && - file->iotype != PIO_IOTYPE_NETCDF4P) && - file->iosystem->io_rank>0)) - file->fh = cnt; - - cfile = pio_file_list; + /* If there is nothing in the list, then file will be the first + * entry. Otherwise, move to end of the list. */ + if (!cfile) + pio_file_list = file; + else + { + while (cfile->next) + cfile = cfile->next; + cfile->next = file; + } } - - file_desc_t *pio_get_file_from_id(int ncid) { From 8e34e307c9a63dc67f94d7036f1c190ac9829fc5 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 5 May 2016 10:32:04 -0400 Subject: [PATCH 013/184] manually merged async changes from ejh_27 --- CMakeLists.txt | 1 + src/clib/CMakeLists.txt | 10 +- src/clib/pio.h | 2 + src/clib/pio_msg.c | 968 ++++++++++++++++++++++++++++++++++++ src/clib/pio_put_nc_async.c | 118 +++-- src/clib/pioc.c | 457 +++++++++++++++++ tests/unit/CMakeLists.txt | 15 + tests/unit/test_intercomm.c | 399 +++++++++++++++ 8 files changed, 1915 insertions(+), 55 deletions(-) create mode 100644 src/clib/pio_msg.c create mode 100644 tests/unit/test_intercomm.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ddaf9a15b8..b284128d2da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,7 @@ mark_as_advanced(VERSION_MAJOR VERSION_MINOR VERSION_PATCH) #===== Library Options ===== option (PIO_ENABLE_FORTRAN "Enable the Fortran library builds" ON) option (PIO_ENABLE_TIMING "Enable the use of the GPTL timing library" ON) +option (PIO_ENABLE_ASYNC "Enable the use of asychronus IO operations" OFF) option (PIO_TEST_BIG_ENDIAN "Enable test to see if machine is big endian" ON) option (PIO_USE_MPIIO "Enable support for MPI-IO auto detect" ON) option (PIO_USE_MPISERIAL "Enable mpi-serial support (instead of MPI)" OFF) diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt index 093aeac70c1..f5448d4d5c2 100644 --- a/src/clib/CMakeLists.txt +++ b/src/clib/CMakeLists.txt @@ -17,12 +17,18 @@ set (PIO_C_SRCS topology.c pio_darray.c bget.c) -set (PIO_GENNC_SRCS ${CMAKE_CURRENT_BINARY_DIR}/pio_nc.c +set (PIO_GENNC_SRCS ${CMAKE_CURRENT_BINARY_DIR}/pio_nc4.c ${CMAKE_CURRENT_BINARY_DIR}/pio_put_nc.c ${CMAKE_CURRENT_BINARY_DIR}/pio_get_nc.c) -add_library (pioc ${PIO_C_SRCS} ${PIO_GENNC_SRCS}) +if (PIO_ENABLE_ASYNC) + set (PIO_ADDL_SRCS pio_nc2.c pio_msg.c) +else () + set (PIO_ADDL_SRCS pio_nc.c) +endif () + +add_library (pioc ${PIO_C_SRCS} ${PIO_GENNC_SRCS} ${PIO_ADDL_SRCS}) # set up include-directories include_directories( diff --git a/src/clib/pio.h b/src/clib/pio.h index 5185a133b0e..e96f754e57e 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -481,6 +481,8 @@ int PIOc_InitDecomp(const int iosysid, const int basetype,const int ndims, const int PIOc_Init_Intracomm(const MPI_Comm comp_comm, const int num_iotasks, const int stride, const int base, const int rearr, int *iosysidp); +int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, MPI_Comm *comp_comms, + MPI_Comm io_comm, int *iosysidp); int PIOc_closefile(int ncid); int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, const char *fname, const int mode); diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c new file mode 100644 index 00000000000..a2bc45e955e --- /dev/null +++ b/src/clib/pio_msg.c @@ -0,0 +1,968 @@ +/** + * @file + * @author Ed Hartnett + * @date 2016 + * @brief PIO async msg handling + * + * @see http://code.google.com/p/parallelio/ + */ + +#include +#include + +/** This function is run on the IO tasks to create a netCDF file. */ +int create_file_handler(iosystem_desc_t *ios) +{ + int ncid; + int len; + int iotype; + char *filename; + int mode; + int mpierr; + int ret; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d create_file_handler comproot = %d\n", my_rank, ios->comproot); + + /* Get the parameters for this function that the he comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d create_file_handler got parameter len = %d\n", my_rank, len); + if (!(filename = malloc(len + 1 * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, 0, + ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d create_file_handler got parameters len = %d " + "filename = %s iotype = %d mode = %d\n", + my_rank, len, filename, iotype, mode); + + /* Call the create file function. */ + if ((ret = PIOc_createfile(ios->iosysid, &ncid, &iotype, filename, mode))) + return ret; + + /* Free resources. */ + free(filename); + + printf("%d create_file_handler succeeded!\n", my_rank); + return PIO_NOERR; +} + +/** This function is run on the IO tasks to close a netCDF file. It is + * only ever run on the IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. +*/ +int close_file_handler(iosystem_desc_t *ios) +{ + int ncid; + int mpierr; + int ret; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d close_file_handler\n", my_rank); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d create_file_handler got parameter ncid = %d\n", my_rank, ncid); + + /* Call the close file function. */ + if ((ret = PIOc_closefile(ncid))) + return ret; + + printf("%d close_file_handler succeeded!\n", my_rank); + return PIO_NOERR; +} + +/** This function is run on the IO tasks to inq a netCDF file. It is + * only ever run on the IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @param msg the message sent my the comp root task. + * @return PIO_NOERR for success, error code otherwise. +*/ +int inq_handler(iosystem_desc_t *ios, int msg) +{ + int ncid; + int mpierr; + int ret; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d inq_handler msg = %d\n", my_rank, msg); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + + /* Call the inq file function. */ + int ndims, nvars, ngatts, unlimdimid; + int *ndimsp = NULL, *nvarsp = NULL, *ngattsp = NULL, *unlimdimidp = NULL; + switch (msg) + { + case PIO_MSG_INQ: + ndimsp = &ndims; + nvarsp = &nvars; + ngattsp = &ngatts; + unlimdimidp = &unlimdimid; + break; + case PIO_MSG_INQ_NVARS: + nvarsp = &nvars; + break; + case PIO_MSG_INQ_NDIMS: + ndimsp = &ndims; + break; + case PIO_MSG_INQ_NATTS: + ngattsp = &ngatts; + break; + case PIO_MSG_INQ_UNLIMDIM: + unlimdimidp = &unlimdimid; + break; + default: + return PIO_EINVAL; + } + + /* Call the inq function to get the values. */ + if ((ret = PIOc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp))) + return ret; + + return PIO_NOERR; +} + +/** Do an inq_dim on a netCDF dimension. This function is only run on + * IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @param msg the message sent my the comp root task. + * @return PIO_NOERR for success, error code otherwise. +*/ +int inq_dim_handler(iosystem_desc_t *ios, int msg) +{ + int ncid; + int dimid; + int mpierr; + int ret; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d inq_handler msg = %d\n", my_rank, msg); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&dimid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + + /* Call the inq_dim function. */ + char *dimnamep = NULL; + PIO_Offset *dimlenp = NULL; + char dimname[NC_MAX_NAME + 1]; + PIO_Offset dimlen; + switch (msg) + { + case PIO_MSG_INQ_DIM: + dimnamep = dimname; + dimlenp = &dimlen; + break; + case PIO_MSG_INQ_DIMLEN: + dimlenp = &dimlen; + break; + case PIO_MSG_INQ_DIMNAME: + dimnamep = dimname; + break; + default: + return PIO_EINVAL; + } + + /* Call the inq function to get the values. */ + if ((ret = PIOc_inq_dim(ncid, dimid, dimnamep, dimlenp))) + return ret; + + return PIO_NOERR; +} + +/** Do an inq_dimid on a netCDF dimension name. This function is only + * run on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. +*/ +int inq_dimid_handler(iosystem_desc_t *ios) +{ + int ncid; + int dimid; + int mpierr; + int ret; + int namelen; + char *name; + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if (!(name = malloc((namelen + 1) * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + + /* Call the inq_dimid function. */ + if ((ret = PIOc_inq_dimid(ncid, name, &dimid))) + return ret; + + /* Free resources. */ + free(name); + + return PIO_NOERR; +} + +/** Handle attribute operations. This code only runs on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @param msg the message sent my the comp root task. + * @return PIO_NOERR for success, error code otherwise. +*/ +int att_handler(iosystem_desc_t *ios, int msg) +{ + int ncid; + int varid; + int mpierr; + int ret; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d inv_var_handler ncid = %d varid = %d\n", my_rank, ncid, varid); + + /* Call the inq_var function. */ + char name[NC_MAX_NAME + 1], *namep; + nc_type xtype, *xtypep = NULL; + int *ndimsp = NULL, *dimidsp = NULL, *nattsp = NULL; + int ndims, dimids[NC_MAX_DIMS], natts; + switch (msg) + { + case PIO_MSG_INQ_VAR: + namep = name; + xtypep = &xtype; + ndimsp = &ndims; + dimidsp = dimids; + nattsp = &natts; + break; + case PIO_MSG_INQ_VARNATTS: + nattsp = &natts; + break; + case PIO_MSG_INQ_VARNAME: + namep = name; + break; + case PIO_MSG_INQ_VARNDIMS: + ndimsp = &ndims; + break; + case PIO_MSG_INQ_VARDIMID: + dimidsp = dimids; + break; + case PIO_MSG_INQ_VARTYPE: + xtypep = &xtype; + break; + default: + return PIO_EINVAL; + } + + /* Call the inq function to get the values. */ + if ((ret = PIOc_inq_var(ncid, varid, namep, xtypep, ndimsp, dimidsp, nattsp))) + return ret; + + return PIO_NOERR; +} + +/** Do an inq_var on a netCDF variable. This function is only run on + * IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @param msg the message sent my the comp root task. + * @return PIO_NOERR for success, error code otherwise. +*/ +int inq_var_handler(iosystem_desc_t *ios, int msg) +{ + int ncid; + int varid; + int mpierr; + int ret; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d inv_var_handler ncid = %d varid = %d\n", my_rank, ncid, varid); + + /* Call the inq_var function. */ + char name[NC_MAX_NAME + 1], *namep; + nc_type xtype, *xtypep = NULL; + int *ndimsp = NULL, *dimidsp = NULL, *nattsp = NULL; + int ndims, dimids[NC_MAX_DIMS], natts; + switch (msg) + { + case PIO_MSG_INQ_VAR: + namep = name; + xtypep = &xtype; + ndimsp = &ndims; + dimidsp = dimids; + nattsp = &natts; + break; + case PIO_MSG_INQ_VARNATTS: + nattsp = &natts; + break; + case PIO_MSG_INQ_VARNAME: + namep = name; + break; + case PIO_MSG_INQ_VARNDIMS: + ndimsp = &ndims; + break; + case PIO_MSG_INQ_VARDIMID: + dimidsp = dimids; + break; + case PIO_MSG_INQ_VARTYPE: + xtypep = &xtype; + break; + default: + return PIO_EINVAL; + } + + /* Call the inq function to get the values. */ + if ((ret = PIOc_inq_var(ncid, varid, namep, xtypep, ndimsp, dimidsp, nattsp))) + return ret; + + return PIO_NOERR; +} + +/** Do an inq_varid on a netCDF variable name. This function is only + * run on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. +*/ +int inq_varid_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int mpierr; + int ret; + int namelen; + char *name; + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if (!(name = malloc((namelen + 1) * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + + /* Call the inq_dimid function. */ + if ((ret = PIOc_inq_varid(ncid, name, &varid))) + return ret; + + /* Free resources. */ + free(name); + + return PIO_NOERR; +} + +/** This function is run on the IO tasks to sync a netCDF file. + * + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. +*/ +int sync_file_handler(iosystem_desc_t *ios) +{ + int ncid; + int mpierr; + int ret; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d sync_file_handler\n", my_rank); + + /* Get the parameters for this function that the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d sync_file_handler got parameter ncid = %d\n", my_rank, ncid); + + /* Call the sync file function. */ + if ((ret = PIOc_sync(ncid))) + return ret; + + printf("%d sync_file_handler succeeded!\n", my_rank); + return PIO_NOERR; +} + +/** This function is run on the IO tasks to enddef a netCDF file. + * + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. +*/ +int enddef_file_handler(iosystem_desc_t *ios) +{ + int ncid; + int mpierr; + int ret; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d enddef_file_handler\n", my_rank); + + /* Get the parameters for this function that the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d enddef_file_handler got parameter ncid = %d\n", my_rank, ncid); + + /* Call the sync file function. */ + if ((ret = PIOc_enddef(ncid))) + return ret; + + printf("%d enddef_file_handler succeeded!\n", my_rank); + return PIO_NOERR; +} + +/** This function is run on the IO tasks to define a netCDF + * variable. */ +int def_var_handler(iosystem_desc_t *ios) +{ + int ncid; + int len, namelen; + int iotype; + char *name; + int mode; + int mpierr; + int ret; + int varid; + nc_type xtype; + int ndims; + int *dimids; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d def_var_handler comproot = %d\n", my_rank, ios->comproot); + + /* Get the parameters for this function that the he comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if (!(name = malloc(namelen + 1 * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, 0, + ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if (!(dimids = malloc(ndims * sizeof(int)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast(dimids, ndims, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d def_var_handler got parameters namelen = %d " + "name = %s len = %d ncid = %d\n", + my_rank, namelen, name, len, ncid); + + /* Call the create file function. */ + if ((ret = PIOc_def_var(ncid, name, xtype, ndims, dimids, &varid))) + return ret; + + /* Free resources. */ + free(name); + free(dimids); + + printf("%d def_var_handler succeeded!\n", my_rank); + return PIO_NOERR; +} + +/** This function is run on the IO tasks to define a netCDF + * dimension. */ +int def_dim_handler(iosystem_desc_t *ios) +{ + int ncid; + int len, namelen; + int iotype; + char *name; + int mode; + int mpierr; + int ret; + int dimid; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d def_dim_handler comproot = %d\n", my_rank, ios->comproot); + + /* Get the parameters for this function that the he comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d def_dim_handler ncid = %d namelen %d\n", my_rank, ncid, namelen); + if (!(name = malloc(namelen + 1 * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, 0, + ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d def_dim_handler got parameters namelen = %d " + "name = %s len = %d ncid = %d\n", + my_rank, namelen, name, len, ncid); + + /* Call the create file function. */ + if ((ret = PIOc_def_dim(ncid, name, len, &dimid))) + return ret; + + /* Free resources. */ + free(name); + + printf("%d def_dim_handler succeeded!\n", my_rank); + return PIO_NOERR; +} + +/** This function is run on the IO tasks. It reads or writes an array + * of data to a netCDF variable. + * + * @param ios pointer to the iosystem_desc_t data. + * @param msg the message sent my the comp root task. + * + * @return PIO_NOERR for success, error code otherwise. */ +int vara_handler(iosystem_desc_t *ios, int msg) +{ + int ncid; + int len, namelen; + int iotype; + char *name; + int mode; + int mpierr; + int ret; + int varid; + nc_type xtype; + int ndims; + int *dimids; + void *data; + int size_in_bytes; + PIO_Offset *count, *start; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d def_var_handler comproot = %d\n", my_rank, ios->comproot); + + if (msg == PIO_MSG_PUT_VARA) + { + /* Get the parameters for this function that the he comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((ret = PIOc_inq_varndims(ncid, varid, &ndims))) + return ret; + if (!(start = malloc(ndims * sizeof(int)))) + return PIO_ENOMEM; + if (!(count = malloc(ndims * sizeof(int)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast(start, ndims, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(count, ndims, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + int size = 1; + for (int d = 0; d < ndims; d++) + size *= count[d]; + size_in_bytes = size * sizeof(int); + if (!(data = malloc(size_in_bytes))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast(data, size_in_bytes, MPI_INT, 0, + ios->intercomm))) + return PIO_EIO; + printf("%d def_var_handler got parameters namelen = %d " + "name = %s len = %d ncid = %d\n", + my_rank, namelen, name, len, ncid); + + /* Call the create file function. */ + if ((ret = PIOc_put_vara_int(ncid, varid, start, count, data))) + return ret; + + /* Free resources. */ + free(start); + free(count); + free(data); + } + else + { + + } + + printf("%d vara_handler succeeded!\n", my_rank); + return PIO_NOERR; +} + +/** This function is run on the IO tasks to open a netCDF file. + * + * @param ios pointer to the iosystem_desc_t data. + * + * @return PIO_NOERR for success, error code otherwise. */ +int open_file_handler(iosystem_desc_t *ios) +{ + int ncid; + int len; + int iotype; + char *filename; + int mode; + int mpierr; + int ret; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d open_file_handler comproot = %d\n", my_rank, ios->comproot); + + /* Get the parameters for this function that the he comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d open_file_handler got parameter len = %d\n", my_rank, len); + if (!(filename = malloc(len + 1 * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, 0, + ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d open_file_handler got parameters len = %d " + "filename = %s iotype = %d mode = %d\n", + my_rank, len, filename, iotype, mode); + + /* Call the open file function. */ + if ((ret = PIOc_openfile(ios->iosysid, &ncid, &iotype, filename, mode))) + return ret; + + /* Free resources. */ + free(filename); + + printf("%d open_file_handler succeeded!\n", my_rank); + return PIO_NOERR; +} + +/** This function is run on the IO tasks to delete a netCDF file. + * + * @param ios pointer to the iosystem_desc_t data. + * + * @return PIO_NOERR for success, error code otherwise. */ +int delete_file_handler(iosystem_desc_t *ios) +{ + int ncid; + int len; + char *filename; + int mpierr; + int ret; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d delete_file_handler comproot = %d\n", my_rank, ios->comproot); + + /* Get the parameters for this function that the he comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + printf("%d open_file_handler got parameter len = %d\n", my_rank, len); + if (!(filename = malloc(len + 1 * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, 0, + ios->intercomm))) + return PIO_EIO; + printf("%d delete_file_handler got parameters len = %d filename = %s\n", + my_rank, len, filename); + + /* Call the delete file function. */ + if ((ret = PIOc_deletefile(ios->iosysid, filename))) + return ret; + + /* Free resources. */ + free(filename); + + printf("%d delete_file_handler succeeded!\n", my_rank); + return PIO_NOERR; +} + +int initdecomp_dof_handler(iosystem_desc_t *ios) +{ + return PIO_NOERR; +} + +int writedarray_handler(iosystem_desc_t *ios) +{ + return PIO_NOERR; +} + +int readdarray_handler(iosystem_desc_t *ios) +{ + return PIO_NOERR; +} + +int seterrorhandling_handler(iosystem_desc_t *ios) +{ + return PIO_NOERR; +} + +int var_handler(iosystem_desc_t *ios, int msg) +{ + return PIO_NOERR; +} + +int freedecomp_handler(iosystem_desc_t *ios) +{ + return PIO_NOERR; +} + +int finalize_handler(iosystem_desc_t *ios) +{ + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d finalize_handler called\n", my_rank); + return PIO_NOERR; +} + +int pio_callback_handler(iosystem_desc_t *ios, int msg) +{ + return PIO_NOERR; +} + +/** This function is called by the IO tasks. This function will not + return, unless there is an error. */ +int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) +{ + iosystem_desc_t *my_iosys; + int msg = 0; + MPI_Request req[component_count]; + MPI_Status status; + int index; + int mpierr; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d pio_msg_handler called\n", my_rank); + + /* Have IO comm rank 0 (the ioroot) register to receive + * (non-blocking) for a message from each of the comproots. */ + if (!io_rank) + { + for (int cmp = 0; cmp < component_count; cmp++) + { + my_iosys = &iosys[cmp]; + printf("%d about to call MPI_Irecv\n", my_rank); + mpierr = MPI_Irecv(&msg, 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG, + my_iosys->union_comm, &req[cmp]); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + } + } + + /* If the message is not -1, keep processing messages. */ + while (msg != -1) + { + /* Wait until any one of the requests are complete. */ + if (!io_rank) + { + printf("%d about to call MPI_Waitany req[0] = %d MPI_REQUEST_NULL = %d\n", + my_rank, req[0], MPI_REQUEST_NULL); + mpierr = MPI_Waitany(component_count, req, &index, &status); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + printf("%d Waitany returned index = %d req[%d] = %d\n", my_rank, + index, index, req[index]); + } + + /* Broadcast the index of the computational component that + * originated the request to the rest of the IO tasks. */ + printf("%d about to call index MPI_Bcast index = %d\n", my_rank, index); + mpierr = MPI_Bcast(&index, 1, MPI_INT, 0, iosys->io_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + my_iosys = &iosys[index]; + printf("%d index MPI_Bcast complete index = %d\n", my_rank, index); + + /* Broadcast the msg value to the rest of the IO tasks. */ + printf("%d about to call msg MPI_Bcast\n", my_rank); + mpierr = MPI_Bcast(&msg, 1, MPI_INT, 0, my_iosys->io_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + printf("%d msg MPI_Bcast complete msg = %d\n", my_rank, msg); + + /* Handle the message. This code is run on all IO tasks. */ + switch (msg) + { + case PIO_MSG_CREATE_FILE: + create_file_handler(my_iosys); + break; + case PIO_MSG_SYNC: + sync_file_handler(my_iosys); + break; + case PIO_MSG_ENDDEF: + enddef_file_handler(my_iosys); + break; + case PIO_MSG_OPEN_FILE: + open_file_handler(my_iosys); + break; + case PIO_MSG_CLOSE_FILE: + close_file_handler(my_iosys); + break; + case PIO_MSG_DELETE_FILE: + delete_file_handler(my_iosys); + break; + case PIO_MSG_DEF_DIM: + def_dim_handler(my_iosys); + break; + case PIO_MSG_DEF_VAR: + def_var_handler(my_iosys); + break; + case PIO_MSG_INQ: + case PIO_MSG_INQ_NVARS: + case PIO_MSG_INQ_NDIMS: + case PIO_MSG_INQ_NATTS: + case PIO_MSG_INQ_UNLIMDIM: + inq_handler(my_iosys, msg); + break; + case PIO_MSG_INQ_DIM: + case PIO_MSG_INQ_DIMLEN: + case PIO_MSG_INQ_DIMNAME: + inq_dim_handler(my_iosys, msg); + break; + case PIO_MSG_INQ_DIMID: + inq_dimid_handler(my_iosys); + break; + case PIO_MSG_INQ_VAR: + case PIO_MSG_INQ_VARNATTS: + case PIO_MSG_INQ_VARNAME: + case PIO_MSG_INQ_VARNDIMS: + case PIO_MSG_INQ_VARDIMID: + case PIO_MSG_INQ_VARTYPE: + inq_var_handler(my_iosys, msg); + break; + case PIO_MSG_GET_ATT_INT: + att_handler(my_iosys, msg); + break; + case PIO_MSG_PUT_ATT_INT: + att_handler(my_iosys, msg); + break; + case PIO_MSG_INQ_VARID: + inq_varid_handler(my_iosys); + break; + case PIO_MSG_INITDECOMP_DOF: + initdecomp_dof_handler(my_iosys); + break; + case PIO_MSG_WRITEDARRAY: + writedarray_handler(my_iosys); + break; + case PIO_MSG_READDARRAY: + readdarray_handler(my_iosys); + break; + case PIO_MSG_SETERRORHANDLING: + seterrorhandling_handler(my_iosys); + break; + case PIO_MSG_GET_VAR1: + var_handler(my_iosys, msg); + break; + case PIO_MSG_PUT_VAR1: + var_handler(my_iosys, msg); + break; + case PIO_MSG_PUT_VARA: + vara_handler(my_iosys, msg); + break; + case PIO_MSG_FREEDECOMP: + freedecomp_handler(my_iosys); + break; + case PIO_MSG_EXIT: + finalize_handler(my_iosys); + msg = -1; + break; + default: + pio_callback_handler(my_iosys, msg); + } + + /* Unless finalize was called, listen for another msg from the + * component whose message we just handled. */ + if (!io_rank && msg != -1) + { + my_iosys = &iosys[index]; + mpierr = MPI_Irecv(&msg, 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG, my_iosys->union_comm, + &req[index]); + printf("%d pio_msg_handler called MPI_Irecv req[%d] = %d\n", my_rank, index, req[index]); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + } + + } + + return PIO_NOERR; +} + +int +pio_iosys_print(int my_rank, iosystem_desc_t *iosys) +{ + printf("%d iosysid: %d\n", my_rank, iosys->iosysid); + if (iosys->union_comm == MPI_COMM_NULL) + printf("%d union_comm: MPI_COMM_NULL ", my_rank); + else + printf("%d union_comm: %d ", my_rank, iosys->union_comm); + + if (iosys->comp_comm == MPI_COMM_NULL) + printf("comp_comm: MPI_COMM_NULL "); + else + printf("comp_comm: %d ", iosys->comp_comm); + + if (iosys->io_comm == MPI_COMM_NULL) + printf("io_comm: MPI_COMM_NULL "); + else + printf("io_comm: %d ", iosys->io_comm); + + if (iosys->intercomm == MPI_COMM_NULL) + printf("intercomm: MPI_COMM_NULL\n"); + else + printf("intercomm: %d\n", iosys->intercomm); + + printf("%d num_iotasks=%d num_comptasks=%d union_rank=%d, comp_rank=%d, " + "io_rank=%d async_interface=%d\n", + my_rank, iosys->num_iotasks, iosys->num_comptasks, iosys->union_rank, + iosys->comp_rank, iosys->io_rank, iosys->async_interface); + + printf("%d ioroot=%d comproot=%d iomaster=%d, compmaster=%d\n", + my_rank, iosys->ioroot, iosys->comproot, iosys->iomaster, + iosys->compmaster); + + printf("%d iotasks:", my_rank); + for (int i = 0; i < iosys->num_iotasks; i++) + printf("%d ", iosys->ioranks[i]); + printf("\n"); + return PIO_NOERR; +} + diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index c0b621eb903..dd9693a3abe 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -1022,7 +1022,8 @@ int PIOc_put_var_short (int ncid, int varid, const short *op) /// /// Refer to the netcdf documentation. /// -int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op) +int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], + const int *op) { int ierr; int msg; @@ -1032,6 +1033,9 @@ int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_ var_desc_t *vdesc; PIO_Offset usage; int *request; + int size; + int ret; + int ndims; ierr = PIO_NOERR; @@ -1044,7 +1048,15 @@ int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_ if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if ((ret = PIOc_inq_varndims(ncid, varid, &ndims))) + return ret; + mpierr = MPI_Bcast((void *)start, ndims, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)count, ndims, MPI_INT, ios->compmaster, ios->intercomm); + for (int d = 0, size = 1; d < ndims; d++) + size *= count[d]; + mpierr = MPI_Bcast((void *)op, size, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1122,7 +1134,7 @@ int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const u if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1200,7 +1212,7 @@ int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1278,7 +1290,7 @@ int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1356,7 +1368,7 @@ int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const P if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1434,7 +1446,7 @@ int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1512,7 +1524,7 @@ int PIOc_put_var_int (int ncid, int varid, const int *op) if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1590,7 +1602,7 @@ int PIOc_put_var_longlong (int ncid, int varid, const long long *op) if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1668,7 +1680,7 @@ int PIOc_put_var_schar (int ncid, int varid, const signed char *op) if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1746,7 +1758,7 @@ int PIOc_put_var_uint (int ncid, int varid, const unsigned int *op) if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1824,7 +1836,7 @@ int PIOc_put_var (int ncid, int varid, const void *buf, PIO_Offset bufcount, MPI if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1902,7 +1914,7 @@ int PIOc_put_vara_ushort (int ncid, int varid, const PIO_Offset start[], const P if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1980,7 +1992,7 @@ int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset start[], const PI if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2058,7 +2070,7 @@ int PIOc_put_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2136,7 +2148,7 @@ int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset start[], const PI if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2214,7 +2226,7 @@ int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], cons if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2292,7 +2304,7 @@ int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], const un if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2370,7 +2382,7 @@ int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_ if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2448,7 +2460,7 @@ int PIOc_put_vars_schar (int ncid, int varid, const PIO_Offset start[], const PI if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2526,7 +2538,7 @@ int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *bu if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2604,7 +2616,7 @@ int PIOc_put_vara_float (int ncid, int varid, const PIO_Offset start[], const PI if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2682,7 +2694,7 @@ int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], const fl if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2760,7 +2772,7 @@ int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PI if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2838,7 +2850,7 @@ int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const cha if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2916,7 +2928,7 @@ int PIOc_put_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2994,7 +3006,7 @@ int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -3072,7 +3084,7 @@ int PIOc_put_vars_double (int ncid, int varid, const PIO_Offset start[], const P if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -3150,7 +3162,7 @@ int PIOc_put_vara_longlong (int ncid, int varid, const PIO_Offset start[], const if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -3228,7 +3240,7 @@ int PIOc_put_var_double (int ncid, int varid, const double *op) if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -3306,7 +3318,7 @@ int PIOc_put_var_float (int ncid, int varid, const float *op) if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -3384,7 +3396,7 @@ int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], cons if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -3462,7 +3474,7 @@ int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -3540,7 +3552,7 @@ int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const uns if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -3618,7 +3630,7 @@ int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -3696,7 +3708,7 @@ int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PI if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -3774,7 +3786,7 @@ int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PI if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -3852,7 +3864,7 @@ int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const si if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -3930,7 +3942,7 @@ int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], cons if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4008,7 +4020,7 @@ int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const P if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4086,7 +4098,7 @@ int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offs if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4164,7 +4176,7 @@ int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4242,7 +4254,7 @@ int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const d if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4320,7 +4332,7 @@ int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PI if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4398,7 +4410,7 @@ int PIOc_put_var_text (int ncid, int varid, const char *op) if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4476,7 +4488,7 @@ int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_ if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4554,7 +4566,7 @@ int PIOc_put_var1_short (int ncid, int varid, const PIO_Offset index[], const sh if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4632,7 +4644,7 @@ int PIOc_put_vars_longlong (int ncid, int varid, const PIO_Offset start[], const if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4710,7 +4722,7 @@ int PIOc_put_vara_double (int ncid, int varid, const PIO_Offset start[], const P if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4788,7 +4800,7 @@ int PIOc_put_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offs if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4866,7 +4878,7 @@ int PIOc_put_var_uchar (int ncid, int varid, const unsigned char *op) if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -4944,7 +4956,7 @@ int PIOc_put_var_long (int ncid, int varid, const long *op) if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -5022,7 +5034,7 @@ int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 22d6ea3307f..b9127d47b8b 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -341,6 +341,463 @@ int PIOc_InitDecomp_bc(const int iosysid, const int basetype,const int ndims, co } +/** @ingroup PIO_init + * Library initialization used when IO tasks are distinct from compute + * tasks. + * + * This is a collective call. Input parameters are read on + * comp_rank=0 values on other tasks are ignored. This variation of + * PIO_init sets up a distinct set of tasks to handle IO, these tasks + * do not return from this call. Instead they go to an internal loop + * and wait to receive further instructions from the computational + * tasks. + * + * For 4 tasks, to have 2 of them be computational, and 2 of them + * be IO, I would provide the following: + * + * component_count = 1 + * + * peer_comm = MPI_COMM_WORLD + * + * comp_comms = an array with one element, an MPI (intra) communicator + * that contains the two tasks designated to do computation + * (processors 0, 1). + + * io_comm = an MPI (intra) communicator with the other two tasks (2, + * 3). + * + * iosysidp = pointer that gets the IO system ID. + * + * Fortran function (from PIO1, in piolib_mod.F90) is: + * + * subroutine init_intercom(component_count, peer_comm, comp_comms, + * io_comm, iosystem, rearr_opts) + * + * Some notes from Jim: + * + * Components and Component Count + * ------------------------------ + * + * It's a cesm thing - the cesm model is composed of several component + * models (atm, ocn, ice, lnd, etc) that may or may not be collocated + * on mpi tasks. Since for intercomm the IOCOMM tasks are a subset of + * the compute tasks for a given component we have a separate iocomm + * for each model component. and we call init_inracomm independently + * for each component. + * + * When the IO tasks are independent of any model component then we + * can have all of the components share one set of iotasks and we call + * init_intercomm once with the information for all components. + * + * Inter vs Intra Communicators + * ---------------------------- + * + * ​For an intra you just need to provide the compute comm, pio creates + * an io comm as a subset of that compute comm. + * + * For an inter you need to provide multiple comms - peer comm is the + * communicator that is going to encompass all of the tasks - usually + * this will be mpi_comm_world. Then you need to provide a comm for + * each component model that will share the io server, then an + * io_comm. + * + * Example of Communicators + * ------------------------ + * + * Starting from MPI_COMM_WORLD the calling program will create an + * IO_COMM and one or more COMP_COMMs, I think an example might be best: + * + * Suppose we have 10 tasks and 2 of them will be IO tasks. Then 0:7 + * are in COMP_COMM and 8:9 are in IO_COMM In this case on tasks 0:7 + * COMP_COMM is defined and IO_COMM is MPI_COMM_NULL and on tasks 8:9 + * IO_COMM is defined and COMP_COMM is MPI_COMM_NULL The communicators + * to handle communications between COMP_COMM and IO_COMM are defined + * in init_intercomm and held in a pio internal data structure. + * + * Return or Not + * ------------- + * + * The io_comm tasks do not return from the init_intercomm routine. + * + * Sequence of Events to do Asynch I/O + * ----------------------------------- + * + * Here is the sequence of events that needs to occur when an IO + * operation is called from the collection of compute tasks. I'm + * going to use pio_put_var because write_darray has some special + * characteristics that make it a bit more complicated... + * + * Compute tasks call pio_put_var with an integer argument + * + * The MPI_Send sends a message from comp_rank=0 to io_rank=0 on + * union_comm (a comm defined as the union of io and compute tasks) + * msg is an integer which indicates the function being called, in + * this case the msg is PIO_MSG_PUT_VAR_INT + * + * The iotasks now know what additional arguments they should expect + * to receive from the compute tasks, in this case a file handle, a + * variable id, the length of the array and the array itself. + * + * The iotasks now have the information they need to complete the + * operation and they call the pio_put_var routine. (In pio1 this bit + * of code is in pio_get_put_callbacks.F90.in) + * + * After the netcdf operation is completed (in the case of an inq or + * get operation) the result is communicated back to the compute + * tasks. + * + * + * @param component_count The number of computational (ex. model) + * components to associate with this IO component + * + * @param peer_comm The communicator from which all other communicator + * arguments are derived + * + * @param comp_comms An array containing the computational + * communicator for each of the computational components. The I/O + * tasks pass MPI_COMM_NULL for this parameter. + * +`* @param io_comm The io communicator. Processing tasks pass + * MPI_COMM_NULL for this parameter. + * + * @param iosysidp An array of length component_count. It will get the + * iosysid for each component. + * + * @return PIO_NOERR on success, error code otherwise. + */ +int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, + MPI_Comm *comp_comms, MPI_Comm io_comm, int *iosysidp) +{ + iosystem_desc_t *iosys; + iosystem_desc_t *my_iosys; + int ierr = PIO_NOERR; + int mpierr; + int my_rank; + int iam; + int io_leader, comp_leader; + int root; + + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /* Allocate struct to hold io system info for each component. */ + if (!(iosys = (iosystem_desc_t *) calloc(1, sizeof(iosystem_desc_t) * component_count))) + ierr = PIO_ENOMEM; + + if (!ierr) + for (int cmp = 0; cmp < component_count; cmp++) + { + /* These are used when using the intercomm. */ + int comp_master = MPI_PROC_NULL, io_master = MPI_PROC_NULL; + + /* Get a pointer to the iosys struct */ + my_iosys = &iosys[cmp]; + + /* Create an MPI info object. */ + CheckMPIReturn(MPI_Info_create(&(my_iosys->info)),__FILE__,__LINE__); + + /* This task is part of the computation communicator. */ + if (comp_comms[cmp] != MPI_COMM_NULL) + { + /* Copy the computation communicator. */ + mpierr = MPI_Comm_dup(comp_comms[cmp], &my_iosys->comp_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Create an MPI group with the computation tasks. */ + mpierr = MPI_Comm_group(my_iosys->comp_comm, &my_iosys->compgroup); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find out how many tasks are in this communicator. */ + mpierr = MPI_Comm_size(iosys->comp_comm, &my_iosys->num_comptasks); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Set the rank within the comp_comm. */ + mpierr = MPI_Comm_rank(my_iosys->comp_comm, &my_iosys->comp_rank); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find the rank of the io leader in peer_comm. */ + iam = -1; + mpierr = MPI_Allreduce(&iam, &io_leader, 1, MPI_INT, MPI_MAX, peer_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find the rank of the comp leader in peer_comm. */ + if (!my_iosys->comp_rank) + { + mpierr = MPI_Comm_rank(peer_comm, &iam); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + } + else + iam = -1; + + /* Find the lucky comp_leader task. */ + mpierr = MPI_Allreduce(&iam, &comp_leader, 1, MPI_INT, MPI_MAX, peer_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Is this the compmaster? Only if the comp_rank is zero. */ + if (!my_iosys->comp_rank) + { + my_iosys->compmaster = MPI_ROOT; + comp_master = MPI_ROOT; + } + else + my_iosys->compmaster = MPI_PROC_NULL; + + /* Set up the intercomm from the computation side. */ + mpierr = MPI_Intercomm_create(my_iosys->comp_comm, 0, peer_comm, + io_leader, cmp, &my_iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Create the union communicator. */ + mpierr = MPI_Intercomm_merge(my_iosys->intercomm, 0, &my_iosys->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + } + else + { + my_iosys->comp_comm = MPI_COMM_NULL; + my_iosys->compgroup = MPI_GROUP_NULL; + my_iosys->comp_rank = -1; + } + + /* This task is part of the IO communicator, so set up the + * IO stuff. */ + if (io_comm != MPI_COMM_NULL) + { + /* Copy the IO communicator. */ + mpierr = MPI_Comm_dup(io_comm, &my_iosys->io_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Get an MPI group that includes the io tasks. */ + mpierr = MPI_Comm_group(my_iosys->io_comm, &my_iosys->iogroup); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find out how many tasks are in this communicator. */ + mpierr = MPI_Comm_size(iosys->io_comm, &my_iosys->num_iotasks); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Set the rank within the io_comm. */ + mpierr = MPI_Comm_rank(my_iosys->io_comm, &my_iosys->io_rank); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find the rank of the io leader in peer_comm. */ + if (!my_iosys->io_rank) + { + mpierr = MPI_Comm_rank(peer_comm, &iam); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + } + else + iam = -1; + + /* Find the lucky io_leader task. */ + mpierr = MPI_Allreduce(&iam, &io_leader, 1, MPI_INT, MPI_MAX, peer_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find the rank of the comp leader in peer_comm. */ + iam = -1; + mpierr = MPI_Allreduce(&iam, &comp_leader, 1, MPI_INT, MPI_MAX, peer_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* This is an io task. */ + my_iosys->ioproc = true; + + /* Is this the iomaster? Only if the io_rank is zero. */ + if (!my_iosys->io_rank) + { + my_iosys->iomaster = MPI_ROOT; + io_master = MPI_ROOT; + } + else + my_iosys->iomaster = 0; + + /* Set up the intercomm from the I/O side. */ + mpierr = MPI_Intercomm_create(my_iosys->io_comm, 0, peer_comm, + comp_leader, cmp, &my_iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Create the union communicator. */ + mpierr = MPI_Intercomm_merge(my_iosys->intercomm, 0, &my_iosys->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + } + else + { + my_iosys->io_comm = MPI_COMM_NULL; + my_iosys->iogroup = MPI_GROUP_NULL; + my_iosys->io_rank = -1; + my_iosys->ioproc = false; + my_iosys->iomaster = false; + } + + /* my_comm points to the union communicator for async, and + * the comp_comm for non-async. It should not be freed + * since it is not a proper copy of the commuicator, just + * a copy of the reference to it. */ + my_iosys->my_comm = my_iosys->union_comm; + + /* Find rank in union communicator. */ + mpierr = MPI_Comm_rank(my_iosys->union_comm, &my_iosys->union_rank); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find the rank of the io leader in the union communicator. */ + if (!my_iosys->io_rank) + my_iosys->ioroot = my_iosys->union_rank; + else + my_iosys->ioroot = -1; + + /* Distribute the answer to all tasks. */ + mpierr = MPI_Allreduce(&my_iosys->ioroot, &root, 1, MPI_INT, MPI_MAX, + my_iosys->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + my_iosys->ioroot = root; + + /* Find the rank of the computation leader in the union + * communicator. */ + if (!my_iosys->comp_rank) + my_iosys->comproot = my_iosys->union_rank; + else + my_iosys->comproot = -1; + + /* Distribute the answer to all tasks. */ + mpierr = MPI_Allreduce(&my_iosys->comproot, &root, 1, MPI_INT, MPI_MAX, + my_iosys->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + my_iosys->comproot = root; + + /* Send the number of tasks in the IO and computation + communicators to each other over the intercomm. This is + a one-to-all bcast from the local task that passes + MPI_ROOT as the root (all other local tasks should pass + MPI_PROC_NULL as the root). The bcast is recieved by + all the members of the leaf group which each pass the + rank of the root relative to the root group. */ + if (io_comm != MPI_COMM_NULL) + { + comp_master = 0; + mpierr = MPI_Bcast(&my_iosys->num_comptasks, 1, MPI_INT, comp_master, + my_iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + mpierr = MPI_Bcast(&my_iosys->num_iotasks, 1, MPI_INT, io_master, + my_iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + } + else + { + io_master = 0; + mpierr = MPI_Bcast(&my_iosys->num_comptasks, 1, MPI_INT, comp_master, + my_iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + mpierr = MPI_Bcast(&my_iosys->num_iotasks, 1, MPI_INT, io_master, + my_iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + } + + /* Allocate an array to hold the ranks of the IO tasks + * within the union communicator. */ + printf("%d allocating for %d iotasks\n", my_rank, my_iosys->num_iotasks); + if (!(my_iosys->ioranks = malloc(my_iosys->num_iotasks * sizeof(int)))) + return PIO_ENOMEM; + printf("%d allocated\n", my_rank); + + /* Allocate a temp array to help get the IO ranks. */ + int *tmp_ioranks; + if (!(tmp_ioranks = malloc(my_iosys->num_iotasks * sizeof(int)))) + return PIO_ENOMEM; + + /* Init array, then have IO tasks set their values, then + * use allreduce to distribute results to all tasks. */ + for (int cnt = 0 ; cnt < my_iosys->num_iotasks; cnt++) + tmp_ioranks[cnt] = -1; + if (io_comm != MPI_COMM_NULL) + tmp_ioranks[my_iosys->io_rank] = my_iosys->union_rank; + mpierr = MPI_Allreduce(tmp_ioranks, my_iosys->ioranks, my_iosys->num_iotasks, MPI_INT, MPI_MAX, + my_iosys->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + + /* Free temp array. */ + free(tmp_ioranks); + + /* Set the default error handling. */ + my_iosys->error_handler = PIO_INTERNAL_ERROR; + + /* We do support asynch interface. */ + my_iosys->async_interface = true; + + /* For debug purposes, print the contents of the struct. */ + for (int t = 0; t < my_iosys->num_iotasks + my_iosys->num_comptasks; t++) + { + MPI_Barrier(my_iosys->union_comm); + if (my_rank == t) + pio_iosys_print(my_rank, my_iosys); + } + + /* Add this id to the list of PIO iosystem ids. */ + iosysidp[cmp] = pio_add_to_iosystem_list(my_iosys); + printf("%d added to iosystem_list iosysid = %d\n", my_rank, iosysidp[cmp]); + + /* Now call the function from which the IO tasks will not + * return until the PIO_MSG_EXIT message is sent. */ + if (io_comm != MPI_COMM_NULL) + { + printf("%d about to call pio_msg_handler\n", my_rank); + if ((ierr = pio_msg_handler(my_iosys->io_rank, component_count, iosys))) + return ierr; + } + + } + + /* If there was an error, make sure all tasks see it. */ + if (ierr) + { + /*mpierr = MPI_Bcast(&ierr, 1, MPI_INT, iosys->my_rank, iosys->intercomm);*/ + mpierr = MPI_Bcast(&ierr, 1, MPI_INT, 0, iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + } + + return ierr; +} + /** ** @ingroup PIO_init ** @brief library initialization used when IO tasks are a subset of compute tasks diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 66d8995f7af..3d26a75f18c 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -43,6 +43,11 @@ if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU") PRIVATE -ffree-line-length-none) endif() +if (PIO_ENABLE_ASYNC) + add_executable (test_intercomm EXCLUDE_FROM_ALL test_intercomm.c) + target_link_libraries (test_intercomm pioc) + add_dependencies (tests test_intercomm) +endif () add_executable (test_names EXCLUDE_FROM_ALL test_names.c) target_link_libraries (test_names pioc) add_executable (test_nc4 EXCLUDE_FROM_ALL test_nc4.c) @@ -62,6 +67,10 @@ add_dependencies (tests pio_unit_test) set (DEFAULT_TEST_TIMEOUT 240) if (PIO_USE_MPISERIAL) + if (PIO_ENABLE_ASYNC) + add_test(NAME test_intercomm + COMMAND test_intercomm) + endif () add_test(NAME test_names COMMAND test_names) add_test(NAME test_nc4 @@ -71,6 +80,12 @@ if (PIO_USE_MPISERIAL) set_tests_properties(pio_unit_test PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) else () + if (PIO_ENABLE_ASYNC) + add_mpi_test(test_intercomm + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_intercomm + NUMPROCS 4 + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) + endif () add_mpi_test(test_names EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_names NUMPROCS 4 diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c new file mode 100644 index 00000000000..a78a1d6b71d --- /dev/null +++ b/tests/unit/test_intercomm.c @@ -0,0 +1,399 @@ +/** + * @file Tests for PIOc_Intercomm. This tests the Init_Intercomm() + * function, and basic asynch I/O capability. + * + */ +#include +#ifdef TIMING +#include +#endif + +/** The number of possible output netCDF output flavors available to + * the ParallelIO library. */ +#define NUM_NETCDF_FLAVORS 4 + +/** The number of dimensions in the test data. */ +#define NDIM 1 + +/** The length of our test data. */ +#define DIM_LEN 4 + +/** The length of data on each (of 2) computational task. */ +#define LOCAL_DIM_LEN 2 + +/** The name of the dimension in the netCDF output file. */ +#define DIM_NAME "dim_test_intercomm" + +/** The name of the variable in the netCDF output file. */ +#define VAR_NAME "var_test_intercomm" + +/** The name of the global attribute in the netCDF output file. */ +#define ATT_NAME "gatt_test_intercomm" + +/** Error code for when things go wrong. */ +#define ERR_AWFUL 1111 +#define ERR_WRONG 2222 + +/** Handle MPI errors. This should only be used with MPI library + * function calls. */ +#define MPIERR(e) do { \ + MPI_Error_string(e, err_buffer, &resultlen); \ + fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \ + MPI_Finalize(); \ + return ERR_AWFUL; \ + } while (0) + +/** Handle non-MPI errors by finalizing the MPI library and exiting + * with an exit code. */ +#define ERR(e) do { \ + fprintf(stderr, "Error %d in %s, line %d\n", e, __FILE__, __LINE__); \ + MPI_Finalize(); \ + return e; \ + } while (0) + +/** Global err buffer for MPI. When there is an MPI error, this buffer + * is used to store the error message that is associated with the MPI + * error. */ +char err_buffer[MPI_MAX_ERROR_STRING]; + +/** This is the length of the most recent MPI error message, stored + * int the global error string. */ +int resultlen; + +/** Run Tests for Init_Intercomm + * + * @param argc argument count + * @param argv array of arguments + */ +int +main(int argc, char **argv) +{ + int verbose = 1; + + /** Zero-based rank of processor. */ + int my_rank; + + /** Number of processors involved in current execution. */ + int ntasks; + + /** Specifies the flavor of netCDF output format. */ + int iotype; + + /** Different output flavors. */ + int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, + PIO_IOTYPE_NETCDF, + PIO_IOTYPE_NETCDF4C, + PIO_IOTYPE_NETCDF4P}; + + /** Names for the output files. */ + char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_nc4_pnetcdf.nc", + "test_nc4_classic.nc", + "test_nc4_serial4.nc", + "test_nc4_parallel4.nc"}; + + /** Number of processors that will do IO. In this test we + * will do IO from all processors. */ + int niotasks; + + /** The ID for the parallel I/O system. */ + int iosysid; + + /** The ncid of the netCDF file. */ + int ncid; + + /** The ID of the netCDF varable. */ + int varid; + + /** Return code. */ + int ret; + + /** Index for loops. */ + int fmt, d, d1, i; + +#ifdef TIMING + /* Initialize the GPTL timing library. */ + if ((ret = GPTLinitialize ())) + return ret; +#endif + + /* Initialize MPI. */ + if ((ret = MPI_Init(&argc, &argv))) + MPIERR(ret); + + /* Learn my rank and the total number of processors. */ + if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) + MPIERR(ret); + if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) + MPIERR(ret); + + /* Check that a valid number of processors was specified. */ + if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || + ntasks == 8 || ntasks == 16)) + fprintf(stderr, "test_intercomm Number of processors must be exactly 4!\n"); + if (verbose) + printf("%d: test_intercomm ParallelIO Library test_intercomm running on %d processors.\n", + my_rank, ntasks); + + /* keep things simple - 1 iotask per MPI process */ + niotasks = ntasks; + + /* For example, if I have 4 processors, and I want to have 2 of them be computational, */ + /* and 2 of them be IO: component count is 1 */ + /* peer_comm = MPI_COMM_WORLD */ + /* comp_comms is an array of comms of size 1 with a comm defined just over tasks (0,1) */ + /* io_comm is a comm over tasks (2,3) */ + + /* Initialize the PIO IO system. This specifies how many and which + * processors are involved in I/O. */ +#define COMPONENT_COUNT 1 + MPI_Comm comp_comms; + MPI_Comm io_comm; + MPI_Group io_group; + MPI_Group comp_group; + + /* Tasks 0 and 1 will be computational. Tasks 2 and 3 will be I/O + * tasks. */ + + // Get the group of processes in MPI_COMM_WORLD + MPI_Group world_group; + MPI_Comm_group(MPI_COMM_WORLD, &world_group); + int comp_task; + + if (my_rank == 0 || my_rank == 1) + { + /* We will define comp_comm. The io_comm will get null. */ + io_comm = MPI_COMM_NULL; + int n = 2; + const int ranks[2] = {0, 1}; + + /* Construct a group with ranks 0, 1 in world_group. */ + MPI_Group_incl(world_group, n, ranks, &comp_group); + MPI_Comm_create_group(MPI_COMM_WORLD, comp_group, 0, &comp_comms); + if (verbose) + printf("%d test_intercomm included in comp_group.\n", my_rank); + + comp_task = 1; + } + else + { + /* We will define io_comm. The comp_comms array will get nulls. */ + comp_comms = MPI_COMM_NULL; + int n = 2; + const int ranks[2] = {2, 3}; + + /* Construct a group with ranks 2, 3 in world_group. */ + MPI_Group_incl(world_group, n, ranks, &io_group); + MPI_Comm_create_group(MPI_COMM_WORLD, io_group, 0, &io_comm); + if (verbose) + printf("%d test_intercomm included in io_group.\n", my_rank); + + comp_task = 0; + } + + if ((ret = PIOc_Init_Intercomm(COMPONENT_COUNT, MPI_COMM_WORLD, &comp_comms, + io_comm, &iosysid))) + ERR(ret); + if (verbose) + printf("%d test_intercomm init intercomm returned %d iosysid = %d\n", my_rank, ret, + iosysid); + + /* All the netCDF calls are only executed on the computation + * tasks. The IO tasks have not returned from PIOc_Init_Intercomm, + * and when the do, they should go straight to finalize. */ + if (comp_task) + { + for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) + { + int ncid, varid, dimid; + PIO_Offset start[NDIM], count[NDIM] = {0}; + int data[LOCAL_DIM_LEN]; + + /* Create a netCDF file with one dimension and one variable. */ + if (verbose) + printf("%d test_intercomm creating file %s\n", my_rank, filename[fmt]); + if ((ret = PIOc_createfile(iosysid, &ncid, &format[fmt], filename[fmt], + NC_CLOBBER))) + ERR(ret); + if (verbose) + printf("%d test_intercomm file created ncid = %d\n", my_rank, ncid); + + if (verbose) + printf("%d test_intercomm defining dimension %s\n", my_rank, DIM_NAME); + if ((ret = PIOc_def_dim(ncid, DIM_NAME, DIM_LEN, &dimid))) + ERR(ret); + if (verbose) + printf("%d test_intercomm defining variable %s\n", my_rank, VAR_NAME); + if ((ret = PIOc_def_var(ncid, VAR_NAME, NC_INT, NDIM, &dimid, &varid))) + ERR(ret); + + /* Add a global attribute. */ + /* int att_data = 42; */ + /* if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) */ + /* ERR(ret); */ + + /* Close the file. */ + if (verbose) + printf("%d test_intercomm ending define mode ncid = %d\n", my_rank, ncid); + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Write some data. */ + for (int i = 0; i < LOCAL_DIM_LEN; i++) + data[i] = my_rank; + if (verbose) + printf("%d test_intercomm writing data\n", my_rank); + start[0] = !my_rank ? 0 : 2; + /* if ((ret = PIOc_put_vara_int(ncid, varid, start, count, data))) */ + /* ERR(ret); */ + + /* Close the file. */ + if (verbose) + printf("%d test_intercomm closing file ncid = %d\n", my_rank, ncid); + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Re-open the file to check it. */ + if (verbose) + printf("%d test_intercomm opening file %s\n", my_rank, filename[fmt]); + if ((ret = PIOc_openfile(iosysid, &ncid, &format[fmt], filename[fmt], + NC_NOWRITE))) + ERR(ret); + + /* Find the number of dimensions, variables, and global attributes.*/ + int ndims, nvars, ngatts, unlimdimid; + if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) + ERR(ret); + if (ndims != 1 || nvars != 1 || ngatts != 0 || unlimdimid != -1) + ERR(ERR_WRONG); + int ndims2, nvars2, ngatts2, unlimdimid2; + if ((ret = PIOc_inq_ndims(ncid, &ndims2))) + ERR(ret); + if (ndims2 != 1) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_nvars(ncid, &nvars2))) + ERR(ret); + if (nvars2 != 1) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_natts(ncid, &ngatts2))) + ERR(ret); + if (ngatts2 != 0) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2))) + ERR(ret); + if (unlimdimid != -1) + ERR(ERR_WRONG); + + /* Check out the dimension. */ + char dimname[NC_MAX_NAME + 1]; + PIO_Offset dimlen; + if ((ret = PIOc_inq_dim(ncid, 0, dimname, &dimlen))) + ERR(ret); + printf("%d test_intercomm dim name is %s VAR_NAME is %s\n", my_rank, dimname, VAR_NAME); + if (strcmp(dimname, DIM_NAME) || dimlen != DIM_LEN) + ERR(ERR_WRONG); + char dimname2[NC_MAX_NAME + 1]; + PIO_Offset dimlen2; + if ((ret = PIOc_inq_dimname(ncid, 0, dimname2))) + ERR(ret); + if (strcmp(dimname2, DIM_NAME)) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_dimlen(ncid, 0, &dimlen2))) + ERR(ret); + if (dimlen2 != DIM_LEN) + ERR(ERR_WRONG); + int dimid2; + if ((ret = PIOc_inq_dimid(ncid, DIM_NAME, &dimid2))) + ERR(ret); + if (dimid2 != 0) + ERR(ERR_WRONG); + + /* Check out the variable. */ + char varname[NC_MAX_NAME + 1]; + nc_type vartype; + int varndims, vardimids, varnatts; + if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts))) + ERR(ret); + if (strcmp(varname, VAR_NAME) || vartype != NC_INT || varndims != NDIM || + vardimids != 0 || varnatts != 0) + ERR(ERR_WRONG); + char varname2[NC_MAX_NAME + 1]; + nc_type vartype2; + int varndims2, vardimids2, varnatts2; + if ((ret = PIOc_inq_varname(ncid, 0, varname2))) + ERR(ret); + if (strcmp(varname2, VAR_NAME)) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_vartype(ncid, 0, &vartype2))) + ERR(ret); + if (vartype2 != NC_INT) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_varndims(ncid, 0, &varndims2))) + ERR(ret); + if (varndims2 != NDIM) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_vardimid(ncid, 0, &vardimids2))) + ERR(ret); + if (vardimids2 != 0) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_varnatts(ncid, 0, &varnatts2))) + ERR(ret); + if (varnatts2 != 0) + ERR(ERR_WRONG); + int varid2; + if ((ret = PIOc_inq_varid(ncid, VAR_NAME, &varid2))) + ERR(ret); + if (varid2 != 0) + ERR(ERR_WRONG); + + /* Close the file. */ + if (verbose) + printf("%d test_intercomm closing file (again) ncid = %d\n", my_rank, ncid); + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Now delete the file. */ + if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) + ERR(ret); + /* if ((ret = PIOc_openfile(iosysid, &ncid, &format[fmt], filename[fmt], */ + /* NC_NOWRITE)) != PIO_ENFILE) */ + /* ERR(ERR_AWFUL); */ + + } /* next netcdf format flavor */ + } + + /* Free local MPI resources. */ + if (verbose) + printf("%d test_intercomm Freeing local MPI resources...\n", my_rank); + MPI_Group_free(&world_group); + if (comp_task) + { + MPI_Group_free(&comp_group); + MPI_Comm_free(&comp_comms); + } + else + { + MPI_Group_free(&io_group); + MPI_Comm_free(&io_comm); + } + + /* Finalize the IO system. */ + if (verbose) + printf("%d test_intercomm Freeing PIO resources...\n", my_rank); + if ((ret = PIOc_finalize(iosysid))) + ERR(ret); + + /* Finalize the MPI library. */ + MPI_Finalize(); + +#ifdef TIMING + /* Finalize the GPTL timing library. */ + if ((ret = GPTLfinalize())) + return ret; +#endif + + if (verbose) + printf("%d test_intercomm SUCCESS!!\n", my_rank); + + + return 0; +} From b16142f0337bb356578f2e3a5d2147e5774eaa87 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 5 May 2016 11:02:16 -0400 Subject: [PATCH 014/184] got non-async build working --- src/clib/CMakeLists.txt | 12 +- src/clib/pio_msg.c | 457 +++++++++++++++++++++++++++++++++++++++ src/clib/pioc.c | 458 ---------------------------------------- 3 files changed, 464 insertions(+), 463 deletions(-) diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt index f5448d4d5c2..21d7dcb375e 100644 --- a/src/clib/CMakeLists.txt +++ b/src/clib/CMakeLists.txt @@ -18,14 +18,16 @@ set (PIO_C_SRCS topology.c bget.c) set (PIO_GENNC_SRCS - ${CMAKE_CURRENT_BINARY_DIR}/pio_nc4.c - ${CMAKE_CURRENT_BINARY_DIR}/pio_put_nc.c - ${CMAKE_CURRENT_BINARY_DIR}/pio_get_nc.c) + ${CMAKE_CURRENT_BINARY_DIR}/pio_nc4.c) if (PIO_ENABLE_ASYNC) - set (PIO_ADDL_SRCS pio_nc2.c pio_msg.c) + set (PIO_ADDL_SRCS pio_nc_async.c pio_put_nc_async.c pio_get_nc_async.c pio_msg.c) else () - set (PIO_ADDL_SRCS pio_nc.c) + set (PIO_GENNC_SRCS ${CMAKE_CURRENT_BINARY_DIR}/pio_put_nc.c + ${CMAKE_CURRENT_BINARY_DIR}/pio_get_nc.c + ${CMAKE_CURRENT_BINARY_DIR}/pio_nc4.c + ${CMAKE_CURRENT_BINARY_DIR}/pio_nc.c) + set (PIO_ADDL_SRCS ${PIO_GENNC_SRC}) endif () add_library (pioc ${PIO_C_SRCS} ${PIO_GENNC_SRCS} ${PIO_ADDL_SRCS}) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index a2bc45e955e..ed13ebec41c 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -966,3 +966,460 @@ pio_iosys_print(int my_rank, iosystem_desc_t *iosys) return PIO_NOERR; } +/** @ingroup PIO_init + * Library initialization used when IO tasks are distinct from compute + * tasks. + * + * This is a collective call. Input parameters are read on + * comp_rank=0 values on other tasks are ignored. This variation of + * PIO_init sets up a distinct set of tasks to handle IO, these tasks + * do not return from this call. Instead they go to an internal loop + * and wait to receive further instructions from the computational + * tasks. + * + * For 4 tasks, to have 2 of them be computational, and 2 of them + * be IO, I would provide the following: + * + * component_count = 1 + * + * peer_comm = MPI_COMM_WORLD + * + * comp_comms = an array with one element, an MPI (intra) communicator + * that contains the two tasks designated to do computation + * (processors 0, 1). + + * io_comm = an MPI (intra) communicator with the other two tasks (2, + * 3). + * + * iosysidp = pointer that gets the IO system ID. + * + * Fortran function (from PIO1, in piolib_mod.F90) is: + * + * subroutine init_intercom(component_count, peer_comm, comp_comms, + * io_comm, iosystem, rearr_opts) + * + * Some notes from Jim: + * + * Components and Component Count + * ------------------------------ + * + * It's a cesm thing - the cesm model is composed of several component + * models (atm, ocn, ice, lnd, etc) that may or may not be collocated + * on mpi tasks. Since for intercomm the IOCOMM tasks are a subset of + * the compute tasks for a given component we have a separate iocomm + * for each model component. and we call init_inracomm independently + * for each component. + * + * When the IO tasks are independent of any model component then we + * can have all of the components share one set of iotasks and we call + * init_intercomm once with the information for all components. + * + * Inter vs Intra Communicators + * ---------------------------- + * + * ​For an intra you just need to provide the compute comm, pio creates + * an io comm as a subset of that compute comm. + * + * For an inter you need to provide multiple comms - peer comm is the + * communicator that is going to encompass all of the tasks - usually + * this will be mpi_comm_world. Then you need to provide a comm for + * each component model that will share the io server, then an + * io_comm. + * + * Example of Communicators + * ------------------------ + * + * Starting from MPI_COMM_WORLD the calling program will create an + * IO_COMM and one or more COMP_COMMs, I think an example might be best: + * + * Suppose we have 10 tasks and 2 of them will be IO tasks. Then 0:7 + * are in COMP_COMM and 8:9 are in IO_COMM In this case on tasks 0:7 + * COMP_COMM is defined and IO_COMM is MPI_COMM_NULL and on tasks 8:9 + * IO_COMM is defined and COMP_COMM is MPI_COMM_NULL The communicators + * to handle communications between COMP_COMM and IO_COMM are defined + * in init_intercomm and held in a pio internal data structure. + * + * Return or Not + * ------------- + * + * The io_comm tasks do not return from the init_intercomm routine. + * + * Sequence of Events to do Asynch I/O + * ----------------------------------- + * + * Here is the sequence of events that needs to occur when an IO + * operation is called from the collection of compute tasks. I'm + * going to use pio_put_var because write_darray has some special + * characteristics that make it a bit more complicated... + * + * Compute tasks call pio_put_var with an integer argument + * + * The MPI_Send sends a message from comp_rank=0 to io_rank=0 on + * union_comm (a comm defined as the union of io and compute tasks) + * msg is an integer which indicates the function being called, in + * this case the msg is PIO_MSG_PUT_VAR_INT + * + * The iotasks now know what additional arguments they should expect + * to receive from the compute tasks, in this case a file handle, a + * variable id, the length of the array and the array itself. + * + * The iotasks now have the information they need to complete the + * operation and they call the pio_put_var routine. (In pio1 this bit + * of code is in pio_get_put_callbacks.F90.in) + * + * After the netcdf operation is completed (in the case of an inq or + * get operation) the result is communicated back to the compute + * tasks. + * + * + * @param component_count The number of computational (ex. model) + * components to associate with this IO component + * + * @param peer_comm The communicator from which all other communicator + * arguments are derived + * + * @param comp_comms An array containing the computational + * communicator for each of the computational components. The I/O + * tasks pass MPI_COMM_NULL for this parameter. + * +`* @param io_comm The io communicator. Processing tasks pass + * MPI_COMM_NULL for this parameter. + * + * @param iosysidp An array of length component_count. It will get the + * iosysid for each component. + * + * @return PIO_NOERR on success, error code otherwise. + */ +int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, + MPI_Comm *comp_comms, MPI_Comm io_comm, int *iosysidp) +{ + iosystem_desc_t *iosys; + iosystem_desc_t *my_iosys; + int ierr = PIO_NOERR; + int mpierr; + int my_rank; + int iam; + int io_leader, comp_leader; + int root; + + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /* Allocate struct to hold io system info for each component. */ + if (!(iosys = (iosystem_desc_t *) calloc(1, sizeof(iosystem_desc_t) * component_count))) + ierr = PIO_ENOMEM; + + if (!ierr) + for (int cmp = 0; cmp < component_count; cmp++) + { + /* These are used when using the intercomm. */ + int comp_master = MPI_PROC_NULL, io_master = MPI_PROC_NULL; + + /* Get a pointer to the iosys struct */ + my_iosys = &iosys[cmp]; + + /* Create an MPI info object. */ + CheckMPIReturn(MPI_Info_create(&(my_iosys->info)),__FILE__,__LINE__); + + /* This task is part of the computation communicator. */ + if (comp_comms[cmp] != MPI_COMM_NULL) + { + /* Copy the computation communicator. */ + mpierr = MPI_Comm_dup(comp_comms[cmp], &my_iosys->comp_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Create an MPI group with the computation tasks. */ + mpierr = MPI_Comm_group(my_iosys->comp_comm, &my_iosys->compgroup); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find out how many tasks are in this communicator. */ + mpierr = MPI_Comm_size(iosys->comp_comm, &my_iosys->num_comptasks); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Set the rank within the comp_comm. */ + mpierr = MPI_Comm_rank(my_iosys->comp_comm, &my_iosys->comp_rank); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find the rank of the io leader in peer_comm. */ + iam = -1; + mpierr = MPI_Allreduce(&iam, &io_leader, 1, MPI_INT, MPI_MAX, peer_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find the rank of the comp leader in peer_comm. */ + if (!my_iosys->comp_rank) + { + mpierr = MPI_Comm_rank(peer_comm, &iam); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + } + else + iam = -1; + + /* Find the lucky comp_leader task. */ + mpierr = MPI_Allreduce(&iam, &comp_leader, 1, MPI_INT, MPI_MAX, peer_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Is this the compmaster? Only if the comp_rank is zero. */ + if (!my_iosys->comp_rank) + { + my_iosys->compmaster = MPI_ROOT; + comp_master = MPI_ROOT; + } + else + my_iosys->compmaster = MPI_PROC_NULL; + + /* Set up the intercomm from the computation side. */ + mpierr = MPI_Intercomm_create(my_iosys->comp_comm, 0, peer_comm, + io_leader, cmp, &my_iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Create the union communicator. */ + mpierr = MPI_Intercomm_merge(my_iosys->intercomm, 0, &my_iosys->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + } + else + { + my_iosys->comp_comm = MPI_COMM_NULL; + my_iosys->compgroup = MPI_GROUP_NULL; + my_iosys->comp_rank = -1; + } + + /* This task is part of the IO communicator, so set up the + * IO stuff. */ + if (io_comm != MPI_COMM_NULL) + { + /* Copy the IO communicator. */ + mpierr = MPI_Comm_dup(io_comm, &my_iosys->io_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Get an MPI group that includes the io tasks. */ + mpierr = MPI_Comm_group(my_iosys->io_comm, &my_iosys->iogroup); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find out how many tasks are in this communicator. */ + mpierr = MPI_Comm_size(iosys->io_comm, &my_iosys->num_iotasks); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Set the rank within the io_comm. */ + mpierr = MPI_Comm_rank(my_iosys->io_comm, &my_iosys->io_rank); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find the rank of the io leader in peer_comm. */ + if (!my_iosys->io_rank) + { + mpierr = MPI_Comm_rank(peer_comm, &iam); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + } + else + iam = -1; + + /* Find the lucky io_leader task. */ + mpierr = MPI_Allreduce(&iam, &io_leader, 1, MPI_INT, MPI_MAX, peer_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find the rank of the comp leader in peer_comm. */ + iam = -1; + mpierr = MPI_Allreduce(&iam, &comp_leader, 1, MPI_INT, MPI_MAX, peer_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* This is an io task. */ + my_iosys->ioproc = true; + + /* Is this the iomaster? Only if the io_rank is zero. */ + if (!my_iosys->io_rank) + { + my_iosys->iomaster = MPI_ROOT; + io_master = MPI_ROOT; + } + else + my_iosys->iomaster = 0; + + /* Set up the intercomm from the I/O side. */ + mpierr = MPI_Intercomm_create(my_iosys->io_comm, 0, peer_comm, + comp_leader, cmp, &my_iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Create the union communicator. */ + mpierr = MPI_Intercomm_merge(my_iosys->intercomm, 0, &my_iosys->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + } + else + { + my_iosys->io_comm = MPI_COMM_NULL; + my_iosys->iogroup = MPI_GROUP_NULL; + my_iosys->io_rank = -1; + my_iosys->ioproc = false; + my_iosys->iomaster = false; + } + + /* my_comm points to the union communicator for async, and + * the comp_comm for non-async. It should not be freed + * since it is not a proper copy of the commuicator, just + * a copy of the reference to it. */ + my_iosys->my_comm = my_iosys->union_comm; + + /* Find rank in union communicator. */ + mpierr = MPI_Comm_rank(my_iosys->union_comm, &my_iosys->union_rank); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + + /* Find the rank of the io leader in the union communicator. */ + if (!my_iosys->io_rank) + my_iosys->ioroot = my_iosys->union_rank; + else + my_iosys->ioroot = -1; + + /* Distribute the answer to all tasks. */ + mpierr = MPI_Allreduce(&my_iosys->ioroot, &root, 1, MPI_INT, MPI_MAX, + my_iosys->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + my_iosys->ioroot = root; + + /* Find the rank of the computation leader in the union + * communicator. */ + if (!my_iosys->comp_rank) + my_iosys->comproot = my_iosys->union_rank; + else + my_iosys->comproot = -1; + + /* Distribute the answer to all tasks. */ + mpierr = MPI_Allreduce(&my_iosys->comproot, &root, 1, MPI_INT, MPI_MAX, + my_iosys->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + my_iosys->comproot = root; + + /* Send the number of tasks in the IO and computation + communicators to each other over the intercomm. This is + a one-to-all bcast from the local task that passes + MPI_ROOT as the root (all other local tasks should pass + MPI_PROC_NULL as the root). The bcast is recieved by + all the members of the leaf group which each pass the + rank of the root relative to the root group. */ + if (io_comm != MPI_COMM_NULL) + { + comp_master = 0; + mpierr = MPI_Bcast(&my_iosys->num_comptasks, 1, MPI_INT, comp_master, + my_iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + mpierr = MPI_Bcast(&my_iosys->num_iotasks, 1, MPI_INT, io_master, + my_iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + } + else + { + io_master = 0; + mpierr = MPI_Bcast(&my_iosys->num_comptasks, 1, MPI_INT, comp_master, + my_iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + mpierr = MPI_Bcast(&my_iosys->num_iotasks, 1, MPI_INT, io_master, + my_iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + } + + /* Allocate an array to hold the ranks of the IO tasks + * within the union communicator. */ + printf("%d allocating for %d iotasks\n", my_rank, my_iosys->num_iotasks); + if (!(my_iosys->ioranks = malloc(my_iosys->num_iotasks * sizeof(int)))) + return PIO_ENOMEM; + printf("%d allocated\n", my_rank); + + /* Allocate a temp array to help get the IO ranks. */ + int *tmp_ioranks; + if (!(tmp_ioranks = malloc(my_iosys->num_iotasks * sizeof(int)))) + return PIO_ENOMEM; + + /* Init array, then have IO tasks set their values, then + * use allreduce to distribute results to all tasks. */ + for (int cnt = 0 ; cnt < my_iosys->num_iotasks; cnt++) + tmp_ioranks[cnt] = -1; + if (io_comm != MPI_COMM_NULL) + tmp_ioranks[my_iosys->io_rank] = my_iosys->union_rank; + mpierr = MPI_Allreduce(tmp_ioranks, my_iosys->ioranks, my_iosys->num_iotasks, MPI_INT, MPI_MAX, + my_iosys->union_comm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + + /* Free temp array. */ + free(tmp_ioranks); + + /* Set the default error handling. */ + my_iosys->error_handler = PIO_INTERNAL_ERROR; + + /* We do support asynch interface. */ + my_iosys->async_interface = true; + + /* For debug purposes, print the contents of the struct. */ + /* for (int t = 0; t < my_iosys->num_iotasks + my_iosys->num_comptasks; t++) */ + /* { */ + /* MPI_Barrier(my_iosys->union_comm); */ + /* if (my_rank == t) */ + /* pio_iosys_print(my_rank, my_iosys); */ + /* } */ + + /* Add this id to the list of PIO iosystem ids. */ + iosysidp[cmp] = pio_add_to_iosystem_list(my_iosys); + printf("%d added to iosystem_list iosysid = %d\n", my_rank, iosysidp[cmp]); + + /* Now call the function from which the IO tasks will not + * return until the PIO_MSG_EXIT message is sent. */ + if (io_comm != MPI_COMM_NULL) + { + printf("%d about to call pio_msg_handler\n", my_rank); + if ((ierr = pio_msg_handler(my_iosys->io_rank, component_count, iosys))) + return ierr; + } + + } + + /* If there was an error, make sure all tasks see it. */ + if (ierr) + { + /*mpierr = MPI_Bcast(&ierr, 1, MPI_INT, iosys->my_rank, iosys->intercomm);*/ + mpierr = MPI_Bcast(&ierr, 1, MPI_INT, 0, iosys->intercomm); + CheckMPIReturn(mpierr, __FILE__, __LINE__); + if (mpierr) + ierr = PIO_EIO; + } + + return ierr; +} + diff --git a/src/clib/pioc.c b/src/clib/pioc.c index b9127d47b8b..afe7e85d981 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -340,464 +340,6 @@ int PIOc_InitDecomp_bc(const int iosysid, const int basetype,const int ndims, co return PIO_NOERR; } - -/** @ingroup PIO_init - * Library initialization used when IO tasks are distinct from compute - * tasks. - * - * This is a collective call. Input parameters are read on - * comp_rank=0 values on other tasks are ignored. This variation of - * PIO_init sets up a distinct set of tasks to handle IO, these tasks - * do not return from this call. Instead they go to an internal loop - * and wait to receive further instructions from the computational - * tasks. - * - * For 4 tasks, to have 2 of them be computational, and 2 of them - * be IO, I would provide the following: - * - * component_count = 1 - * - * peer_comm = MPI_COMM_WORLD - * - * comp_comms = an array with one element, an MPI (intra) communicator - * that contains the two tasks designated to do computation - * (processors 0, 1). - - * io_comm = an MPI (intra) communicator with the other two tasks (2, - * 3). - * - * iosysidp = pointer that gets the IO system ID. - * - * Fortran function (from PIO1, in piolib_mod.F90) is: - * - * subroutine init_intercom(component_count, peer_comm, comp_comms, - * io_comm, iosystem, rearr_opts) - * - * Some notes from Jim: - * - * Components and Component Count - * ------------------------------ - * - * It's a cesm thing - the cesm model is composed of several component - * models (atm, ocn, ice, lnd, etc) that may or may not be collocated - * on mpi tasks. Since for intercomm the IOCOMM tasks are a subset of - * the compute tasks for a given component we have a separate iocomm - * for each model component. and we call init_inracomm independently - * for each component. - * - * When the IO tasks are independent of any model component then we - * can have all of the components share one set of iotasks and we call - * init_intercomm once with the information for all components. - * - * Inter vs Intra Communicators - * ---------------------------- - * - * ​For an intra you just need to provide the compute comm, pio creates - * an io comm as a subset of that compute comm. - * - * For an inter you need to provide multiple comms - peer comm is the - * communicator that is going to encompass all of the tasks - usually - * this will be mpi_comm_world. Then you need to provide a comm for - * each component model that will share the io server, then an - * io_comm. - * - * Example of Communicators - * ------------------------ - * - * Starting from MPI_COMM_WORLD the calling program will create an - * IO_COMM and one or more COMP_COMMs, I think an example might be best: - * - * Suppose we have 10 tasks and 2 of them will be IO tasks. Then 0:7 - * are in COMP_COMM and 8:9 are in IO_COMM In this case on tasks 0:7 - * COMP_COMM is defined and IO_COMM is MPI_COMM_NULL and on tasks 8:9 - * IO_COMM is defined and COMP_COMM is MPI_COMM_NULL The communicators - * to handle communications between COMP_COMM and IO_COMM are defined - * in init_intercomm and held in a pio internal data structure. - * - * Return or Not - * ------------- - * - * The io_comm tasks do not return from the init_intercomm routine. - * - * Sequence of Events to do Asynch I/O - * ----------------------------------- - * - * Here is the sequence of events that needs to occur when an IO - * operation is called from the collection of compute tasks. I'm - * going to use pio_put_var because write_darray has some special - * characteristics that make it a bit more complicated... - * - * Compute tasks call pio_put_var with an integer argument - * - * The MPI_Send sends a message from comp_rank=0 to io_rank=0 on - * union_comm (a comm defined as the union of io and compute tasks) - * msg is an integer which indicates the function being called, in - * this case the msg is PIO_MSG_PUT_VAR_INT - * - * The iotasks now know what additional arguments they should expect - * to receive from the compute tasks, in this case a file handle, a - * variable id, the length of the array and the array itself. - * - * The iotasks now have the information they need to complete the - * operation and they call the pio_put_var routine. (In pio1 this bit - * of code is in pio_get_put_callbacks.F90.in) - * - * After the netcdf operation is completed (in the case of an inq or - * get operation) the result is communicated back to the compute - * tasks. - * - * - * @param component_count The number of computational (ex. model) - * components to associate with this IO component - * - * @param peer_comm The communicator from which all other communicator - * arguments are derived - * - * @param comp_comms An array containing the computational - * communicator for each of the computational components. The I/O - * tasks pass MPI_COMM_NULL for this parameter. - * -`* @param io_comm The io communicator. Processing tasks pass - * MPI_COMM_NULL for this parameter. - * - * @param iosysidp An array of length component_count. It will get the - * iosysid for each component. - * - * @return PIO_NOERR on success, error code otherwise. - */ -int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, - MPI_Comm *comp_comms, MPI_Comm io_comm, int *iosysidp) -{ - iosystem_desc_t *iosys; - iosystem_desc_t *my_iosys; - int ierr = PIO_NOERR; - int mpierr; - int my_rank; - int iam; - int io_leader, comp_leader; - int root; - - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - - /* Allocate struct to hold io system info for each component. */ - if (!(iosys = (iosystem_desc_t *) calloc(1, sizeof(iosystem_desc_t) * component_count))) - ierr = PIO_ENOMEM; - - if (!ierr) - for (int cmp = 0; cmp < component_count; cmp++) - { - /* These are used when using the intercomm. */ - int comp_master = MPI_PROC_NULL, io_master = MPI_PROC_NULL; - - /* Get a pointer to the iosys struct */ - my_iosys = &iosys[cmp]; - - /* Create an MPI info object. */ - CheckMPIReturn(MPI_Info_create(&(my_iosys->info)),__FILE__,__LINE__); - - /* This task is part of the computation communicator. */ - if (comp_comms[cmp] != MPI_COMM_NULL) - { - /* Copy the computation communicator. */ - mpierr = MPI_Comm_dup(comp_comms[cmp], &my_iosys->comp_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Create an MPI group with the computation tasks. */ - mpierr = MPI_Comm_group(my_iosys->comp_comm, &my_iosys->compgroup); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Find out how many tasks are in this communicator. */ - mpierr = MPI_Comm_size(iosys->comp_comm, &my_iosys->num_comptasks); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Set the rank within the comp_comm. */ - mpierr = MPI_Comm_rank(my_iosys->comp_comm, &my_iosys->comp_rank); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Find the rank of the io leader in peer_comm. */ - iam = -1; - mpierr = MPI_Allreduce(&iam, &io_leader, 1, MPI_INT, MPI_MAX, peer_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Find the rank of the comp leader in peer_comm. */ - if (!my_iosys->comp_rank) - { - mpierr = MPI_Comm_rank(peer_comm, &iam); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - } - else - iam = -1; - - /* Find the lucky comp_leader task. */ - mpierr = MPI_Allreduce(&iam, &comp_leader, 1, MPI_INT, MPI_MAX, peer_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Is this the compmaster? Only if the comp_rank is zero. */ - if (!my_iosys->comp_rank) - { - my_iosys->compmaster = MPI_ROOT; - comp_master = MPI_ROOT; - } - else - my_iosys->compmaster = MPI_PROC_NULL; - - /* Set up the intercomm from the computation side. */ - mpierr = MPI_Intercomm_create(my_iosys->comp_comm, 0, peer_comm, - io_leader, cmp, &my_iosys->intercomm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Create the union communicator. */ - mpierr = MPI_Intercomm_merge(my_iosys->intercomm, 0, &my_iosys->union_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - } - else - { - my_iosys->comp_comm = MPI_COMM_NULL; - my_iosys->compgroup = MPI_GROUP_NULL; - my_iosys->comp_rank = -1; - } - - /* This task is part of the IO communicator, so set up the - * IO stuff. */ - if (io_comm != MPI_COMM_NULL) - { - /* Copy the IO communicator. */ - mpierr = MPI_Comm_dup(io_comm, &my_iosys->io_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Get an MPI group that includes the io tasks. */ - mpierr = MPI_Comm_group(my_iosys->io_comm, &my_iosys->iogroup); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Find out how many tasks are in this communicator. */ - mpierr = MPI_Comm_size(iosys->io_comm, &my_iosys->num_iotasks); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Set the rank within the io_comm. */ - mpierr = MPI_Comm_rank(my_iosys->io_comm, &my_iosys->io_rank); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Find the rank of the io leader in peer_comm. */ - if (!my_iosys->io_rank) - { - mpierr = MPI_Comm_rank(peer_comm, &iam); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - } - else - iam = -1; - - /* Find the lucky io_leader task. */ - mpierr = MPI_Allreduce(&iam, &io_leader, 1, MPI_INT, MPI_MAX, peer_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Find the rank of the comp leader in peer_comm. */ - iam = -1; - mpierr = MPI_Allreduce(&iam, &comp_leader, 1, MPI_INT, MPI_MAX, peer_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* This is an io task. */ - my_iosys->ioproc = true; - - /* Is this the iomaster? Only if the io_rank is zero. */ - if (!my_iosys->io_rank) - { - my_iosys->iomaster = MPI_ROOT; - io_master = MPI_ROOT; - } - else - my_iosys->iomaster = 0; - - /* Set up the intercomm from the I/O side. */ - mpierr = MPI_Intercomm_create(my_iosys->io_comm, 0, peer_comm, - comp_leader, cmp, &my_iosys->intercomm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Create the union communicator. */ - mpierr = MPI_Intercomm_merge(my_iosys->intercomm, 0, &my_iosys->union_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - } - else - { - my_iosys->io_comm = MPI_COMM_NULL; - my_iosys->iogroup = MPI_GROUP_NULL; - my_iosys->io_rank = -1; - my_iosys->ioproc = false; - my_iosys->iomaster = false; - } - - /* my_comm points to the union communicator for async, and - * the comp_comm for non-async. It should not be freed - * since it is not a proper copy of the commuicator, just - * a copy of the reference to it. */ - my_iosys->my_comm = my_iosys->union_comm; - - /* Find rank in union communicator. */ - mpierr = MPI_Comm_rank(my_iosys->union_comm, &my_iosys->union_rank); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - - /* Find the rank of the io leader in the union communicator. */ - if (!my_iosys->io_rank) - my_iosys->ioroot = my_iosys->union_rank; - else - my_iosys->ioroot = -1; - - /* Distribute the answer to all tasks. */ - mpierr = MPI_Allreduce(&my_iosys->ioroot, &root, 1, MPI_INT, MPI_MAX, - my_iosys->union_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - my_iosys->ioroot = root; - - /* Find the rank of the computation leader in the union - * communicator. */ - if (!my_iosys->comp_rank) - my_iosys->comproot = my_iosys->union_rank; - else - my_iosys->comproot = -1; - - /* Distribute the answer to all tasks. */ - mpierr = MPI_Allreduce(&my_iosys->comproot, &root, 1, MPI_INT, MPI_MAX, - my_iosys->union_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - my_iosys->comproot = root; - - /* Send the number of tasks in the IO and computation - communicators to each other over the intercomm. This is - a one-to-all bcast from the local task that passes - MPI_ROOT as the root (all other local tasks should pass - MPI_PROC_NULL as the root). The bcast is recieved by - all the members of the leaf group which each pass the - rank of the root relative to the root group. */ - if (io_comm != MPI_COMM_NULL) - { - comp_master = 0; - mpierr = MPI_Bcast(&my_iosys->num_comptasks, 1, MPI_INT, comp_master, - my_iosys->intercomm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - mpierr = MPI_Bcast(&my_iosys->num_iotasks, 1, MPI_INT, io_master, - my_iosys->intercomm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - } - else - { - io_master = 0; - mpierr = MPI_Bcast(&my_iosys->num_comptasks, 1, MPI_INT, comp_master, - my_iosys->intercomm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - mpierr = MPI_Bcast(&my_iosys->num_iotasks, 1, MPI_INT, io_master, - my_iosys->intercomm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - } - - /* Allocate an array to hold the ranks of the IO tasks - * within the union communicator. */ - printf("%d allocating for %d iotasks\n", my_rank, my_iosys->num_iotasks); - if (!(my_iosys->ioranks = malloc(my_iosys->num_iotasks * sizeof(int)))) - return PIO_ENOMEM; - printf("%d allocated\n", my_rank); - - /* Allocate a temp array to help get the IO ranks. */ - int *tmp_ioranks; - if (!(tmp_ioranks = malloc(my_iosys->num_iotasks * sizeof(int)))) - return PIO_ENOMEM; - - /* Init array, then have IO tasks set their values, then - * use allreduce to distribute results to all tasks. */ - for (int cnt = 0 ; cnt < my_iosys->num_iotasks; cnt++) - tmp_ioranks[cnt] = -1; - if (io_comm != MPI_COMM_NULL) - tmp_ioranks[my_iosys->io_rank] = my_iosys->union_rank; - mpierr = MPI_Allreduce(tmp_ioranks, my_iosys->ioranks, my_iosys->num_iotasks, MPI_INT, MPI_MAX, - my_iosys->union_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - - /* Free temp array. */ - free(tmp_ioranks); - - /* Set the default error handling. */ - my_iosys->error_handler = PIO_INTERNAL_ERROR; - - /* We do support asynch interface. */ - my_iosys->async_interface = true; - - /* For debug purposes, print the contents of the struct. */ - for (int t = 0; t < my_iosys->num_iotasks + my_iosys->num_comptasks; t++) - { - MPI_Barrier(my_iosys->union_comm); - if (my_rank == t) - pio_iosys_print(my_rank, my_iosys); - } - - /* Add this id to the list of PIO iosystem ids. */ - iosysidp[cmp] = pio_add_to_iosystem_list(my_iosys); - printf("%d added to iosystem_list iosysid = %d\n", my_rank, iosysidp[cmp]); - - /* Now call the function from which the IO tasks will not - * return until the PIO_MSG_EXIT message is sent. */ - if (io_comm != MPI_COMM_NULL) - { - printf("%d about to call pio_msg_handler\n", my_rank); - if ((ierr = pio_msg_handler(my_iosys->io_rank, component_count, iosys))) - return ierr; - } - - } - - /* If there was an error, make sure all tasks see it. */ - if (ierr) - { - /*mpierr = MPI_Bcast(&ierr, 1, MPI_INT, iosys->my_rank, iosys->intercomm);*/ - mpierr = MPI_Bcast(&ierr, 1, MPI_INT, 0, iosys->intercomm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); - if (mpierr) - ierr = PIO_EIO; - } - - return ierr; -} - /** ** @ingroup PIO_init ** @brief library initialization used when IO tasks are a subset of compute tasks From d2664aff6c95c9229ad3979e90ac43aa92c77bda Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 5 May 2016 13:28:00 -0400 Subject: [PATCH 015/184] got async option working --- src/clib/CMakeLists.txt | 13 ++++---- src/clib/pio_msg.c | 60 ++++++++++++++----------------------- src/clib/pio_nc_async.c | 18 +++++++++-- tests/unit/test_intercomm.c | 4 ++- 4 files changed, 47 insertions(+), 48 deletions(-) diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt index 21d7dcb375e..023b7b3dcbe 100644 --- a/src/clib/CMakeLists.txt +++ b/src/clib/CMakeLists.txt @@ -15,22 +15,21 @@ set (PIO_C_SRCS topology.c pio_spmd.c pio_rearrange.c pio_darray.c + pio_nc4.c bget.c) -set (PIO_GENNC_SRCS - ${CMAKE_CURRENT_BINARY_DIR}/pio_nc4.c) +set (PIO_GENNC_SRCS ${CMAKE_CURRENT_BINARY_DIR}/pio_put_nc.c + ${CMAKE_CURRENT_BINARY_DIR}/pio_get_nc.c + ${CMAKE_CURRENT_BINARY_DIR}/pio_nc4.c + ${CMAKE_CURRENT_BINARY_DIR}/pio_nc.c) if (PIO_ENABLE_ASYNC) set (PIO_ADDL_SRCS pio_nc_async.c pio_put_nc_async.c pio_get_nc_async.c pio_msg.c) else () - set (PIO_GENNC_SRCS ${CMAKE_CURRENT_BINARY_DIR}/pio_put_nc.c - ${CMAKE_CURRENT_BINARY_DIR}/pio_get_nc.c - ${CMAKE_CURRENT_BINARY_DIR}/pio_nc4.c - ${CMAKE_CURRENT_BINARY_DIR}/pio_nc.c) set (PIO_ADDL_SRCS ${PIO_GENNC_SRC}) endif () -add_library (pioc ${PIO_C_SRCS} ${PIO_GENNC_SRCS} ${PIO_ADDL_SRCS}) +add_library (pioc ${PIO_C_SRCS} ${PIO_ADDL_SRCS}) # set up include-directories include_directories( diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index ed13ebec41c..480f16ad7a4 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -241,6 +241,11 @@ int att_handler(iosystem_desc_t *ios, int msg) int varid; int mpierr; int ret; + char *name; + size_t namelen; + int len; + nc_type xtype; + int *op; int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); @@ -251,45 +256,25 @@ int att_handler(iosystem_desc_t *ios, int msg) return PIO_EIO; if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d inv_var_handler ncid = %d varid = %d\n", my_rank, ncid, varid); - - /* Call the inq_var function. */ - char name[NC_MAX_NAME + 1], *namep; - nc_type xtype, *xtypep = NULL; - int *ndimsp = NULL, *dimidsp = NULL, *nattsp = NULL; - int ndims, dimids[NC_MAX_DIMS], natts; - switch (msg) - { - case PIO_MSG_INQ_VAR: - namep = name; - xtypep = &xtype; - ndimsp = &ndims; - dimidsp = dimids; - nattsp = &natts; - break; - case PIO_MSG_INQ_VARNATTS: - nattsp = &natts; - break; - case PIO_MSG_INQ_VARNAME: - namep = name; - break; - case PIO_MSG_INQ_VARNDIMS: - ndimsp = &ndims; - break; - case PIO_MSG_INQ_VARDIMID: - dimidsp = dimids; - break; - case PIO_MSG_INQ_VARTYPE: - xtypep = &xtype; - break; - default: - return PIO_EINVAL; - } + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!(name = malloc(namelen * sizeof(char)))) + return PIO_ENOMEM; + mpierr = MPI_Bcast((void *)name, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!(op = malloc(len * sizeof(int)))) + return PIO_ENOMEM; + mpierr = MPI_Bcast(op, len, MPI_INT, ios->compmaster, ios->intercomm); + printf("%d att_handler ncid = %d varid = %d\n", my_rank, ncid, varid); - /* Call the inq function to get the values. */ - if ((ret = PIOc_inq_var(ncid, varid, namep, xtypep, ndimsp, dimidsp, nattsp))) + /* Call the function to write the attribute. */ + if ((ret = PIOc_put_att_int(ncid, varid, name, xtype, len, op))) return ret; - + + /* Free resources. */ + free(name); + free(op); + return PIO_NOERR; } @@ -528,6 +513,7 @@ int def_dim_handler(iosystem_desc_t *ios) * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; + printf("%d def_dim_handler ncid = %d\n", my_rank, ncid); if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; printf("%d def_dim_handler ncid = %d namelen %d\n", my_rank, ncid, namelen); diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 9e9f04a45db..32267c8b27f 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -2874,6 +2874,11 @@ int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_ iosystem_desc_t *ios; file_desc_t *file; char *errstr; + size_t namelen; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_inq_varid ncid = %d name = %s\n", my_rank, ncid, name); errstr = NULL; ierr = PIO_NOERR; @@ -2887,8 +2892,17 @@ int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_ if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + printf("%d PIOc_put_att_int BCast msg = %d\n", my_rank, msg); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + namelen = strlen(name); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)op, len, MPI_INT, ios->compmaster, ios->intercomm); + printf("%d PIOc_put_att_int ncid = %d, varid = %d namelen = %d name = %s xtype = %d len = %d\n", + my_rank, file->fh, varid, namelen, name, xtype, len); } @@ -3965,8 +3979,6 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); namelen = strlen(name); printf("bcasting namelen = %d name = %s len = %d\n", namelen, name, len); - if (!ios->compmaster) - ios->compmaster = MPI_PROC_NULL; mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index a78a1d6b71d..f616b28c1e8 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -227,7 +227,9 @@ main(int argc, char **argv) ERR(ret); /* Add a global attribute. */ - /* int att_data = 42; */ + if (verbose) + printf("%d test_intercomm writing attribute %s\n", my_rank, ATT_NAME); + int att_data = 42; /* if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) */ /* ERR(ret); */ From badb925c40b98326eecb302be283d40f1e9d0817 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 5 May 2016 13:47:50 -0400 Subject: [PATCH 016/184] got non-async build working again --- src/clib/CMakeLists.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt index 023b7b3dcbe..475aa6196a4 100644 --- a/src/clib/CMakeLists.txt +++ b/src/clib/CMakeLists.txt @@ -20,13 +20,12 @@ set (PIO_C_SRCS topology.c set (PIO_GENNC_SRCS ${CMAKE_CURRENT_BINARY_DIR}/pio_put_nc.c ${CMAKE_CURRENT_BINARY_DIR}/pio_get_nc.c - ${CMAKE_CURRENT_BINARY_DIR}/pio_nc4.c ${CMAKE_CURRENT_BINARY_DIR}/pio_nc.c) if (PIO_ENABLE_ASYNC) set (PIO_ADDL_SRCS pio_nc_async.c pio_put_nc_async.c pio_get_nc_async.c pio_msg.c) else () - set (PIO_ADDL_SRCS ${PIO_GENNC_SRC}) + set (PIO_ADDL_SRCS ${PIO_GENNC_SRCS}) endif () add_library (pioc ${PIO_C_SRCS} ${PIO_ADDL_SRCS}) @@ -203,12 +202,9 @@ else () pio_get_nc.c COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc.c pio_nc.c - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc4.c - pio_nc4.c DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/pio_put_nc.c ${CMAKE_CURRENT_SOURCE_DIR}/pio_get_nc.c - ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc.c - ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc4.c) + ${CMAKE_CURRENT_SOURCE_DIR}/pio_nc.c) endif () From 3b0ad2671a782e5a9e52e0ea670b41c6096719a2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 6 May 2016 11:45:24 -0400 Subject: [PATCH 017/184] got attribut put working --- src/clib/pio_file.c | 28 +++++++++++----------- src/clib/pio_msg.c | 34 +++++++++++++++++++------- src/clib/pio_nc_async.c | 10 ++++---- tests/unit/test_intercomm.c | 48 ++++++++++++++++--------------------- 4 files changed, 66 insertions(+), 54 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 461c8d2b4ca..c4ccb4d0fbc 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -113,20 +113,20 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, // If we failed to open a file due to an incompatible type of NetCDF, try it // once with just plain old basic NetCDF -#ifdef _NETCDF - if((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF)) { - if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); - // reset ierr on all tasks - ierr = PIO_NOERR; - // reset file markers for NETCDF on all tasks - file->iotype = PIO_IOTYPE_NETCDF; - - // open netcdf file serially on main task - if(ios->io_rank==0){ - ierr = nc_open(filename, file->mode, &(file->fh)); } - - } -#endif +/* #ifdef _NETCDF */ +/* if((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF)) { */ +/* if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); */ +/* // reset ierr on all tasks */ +/* ierr = PIO_NOERR; */ +/* // reset file markers for NETCDF on all tasks */ +/* file->iotype = PIO_IOTYPE_NETCDF; */ + +/* // open netcdf file serially on main task */ +/* if(ios->io_rank==0){ */ +/* ierr = nc_open(filename, file->mode, &(file->fh)); } */ + +/* } */ +/* #endif */ } ierr = check_netcdf(file, ierr, __FILE__,__LINE__); diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 480f16ad7a4..8dd59e4439d 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -241,14 +241,15 @@ int att_handler(iosystem_desc_t *ios, int msg) int varid; int mpierr; int ret; - char *name; - size_t namelen; + char *name5; + int namelen; int len; nc_type xtype; int *op; int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d att_handler\n", my_rank); /* Get the parameters for this function that the the comp master * task is broadcasting. */ @@ -257,22 +258,28 @@ int att_handler(iosystem_desc_t *ios, int msg) if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!(name = malloc(namelen * sizeof(char)))) + if (!(name5 = malloc((namelen + 1) * sizeof(char)))) return PIO_ENOMEM; - mpierr = MPI_Bcast((void *)name, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name5, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!(op = malloc(len * sizeof(int)))) return PIO_ENOMEM; mpierr = MPI_Bcast(op, len, MPI_INT, ios->compmaster, ios->intercomm); - printf("%d att_handler ncid = %d varid = %d\n", my_rank, ncid, varid); /* Call the function to write the attribute. */ - if ((ret = PIOc_put_att_int(ncid, varid, name, xtype, len, op))) + if ((ret = PIOc_put_att_int(ncid, varid, name5, xtype, len, op))) return ret; + free(op); + if (!(op = malloc(20))) + { + printf("%d att_handler4 ncid = %d varid = %d namelen = %d name = %d\n", + my_rank, ncid, varid, namelen, name5); + } + /* Free resources. */ - free(name); + free(name5); free(op); return PIO_NOERR; @@ -301,7 +308,7 @@ int inq_var_handler(iosystem_desc_t *ios, int msg) return PIO_EIO; if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d inv_var_handler ncid = %d varid = %d\n", my_rank, ncid, varid); + printf("%d inq_var_handler ncid = %d varid = %d\n", my_rank, ncid, varid); /* Call the inq_var function. */ char name[NC_MAX_NAME + 1], *namep; @@ -758,6 +765,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) MPI_Status status; int index; int mpierr; + int ret; int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); @@ -856,7 +864,8 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) inq_var_handler(my_iosys, msg); break; case PIO_MSG_GET_ATT_INT: - att_handler(my_iosys, msg); + ret = att_handler(my_iosys, msg); + printf("att_handler returned %d\n", ret); break; case PIO_MSG_PUT_ATT_INT: att_handler(my_iosys, msg); @@ -896,6 +905,13 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) pio_callback_handler(my_iosys, msg); } + /* If an error was returned by the handler, do something! */ + if (ret) + { + printf("hander returned error code %d\n", ret); + MPI_Finalize(); + } + /* Unless finalize was called, listen for another msg from the * component whose message we just handled. */ if (!io_rank && msg != -1) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 32267c8b27f..b35b2029918 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -562,8 +562,8 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, * the IO root task. */ if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm); namelen = strlen(name); printf("bcasting namelen = %d name = %s\n", namelen, name); if (!ios->compmaster) @@ -2891,13 +2891,15 @@ int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_ if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); printf("%d PIOc_put_att_int BCast msg = %d\n", my_rank, msg); mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); namelen = strlen(name); mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + printf("%d PIOc_put_att_int about to send name = %s\n", my_rank, name); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + printf("%d PIOc_put_att_int sent name = %s\n", my_rank, name); mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast((void *)op, len, MPI_INT, ios->compmaster, ios->intercomm); diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index f616b28c1e8..592b7b5e6ee 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -4,6 +4,7 @@ * */ #include +#include #ifdef TIMING #include #endif @@ -76,9 +77,6 @@ main(int argc, char **argv) /** Number of processors involved in current execution. */ int ntasks; - /** Specifies the flavor of netCDF output format. */ - int iotype; - /** Different output flavors. */ int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, @@ -91,10 +89,6 @@ main(int argc, char **argv) "test_nc4_serial4.nc", "test_nc4_parallel4.nc"}; - /** Number of processors that will do IO. In this test we - * will do IO from all processors. */ - int niotasks; - /** The ID for the parallel I/O system. */ int iosysid; @@ -134,9 +128,6 @@ main(int argc, char **argv) printf("%d: test_intercomm ParallelIO Library test_intercomm running on %d processors.\n", my_rank, ntasks); - /* keep things simple - 1 iotask per MPI process */ - niotasks = ntasks; - /* For example, if I have 4 processors, and I want to have 2 of them be computational, */ /* and 2 of them be IO: component count is 1 */ /* peer_comm = MPI_COMM_WORLD */ @@ -216,11 +207,14 @@ main(int argc, char **argv) ERR(ret); if (verbose) printf("%d test_intercomm file created ncid = %d\n", my_rank, ncid); - + + /* Define a dimension. */ if (verbose) printf("%d test_intercomm defining dimension %s\n", my_rank, DIM_NAME); if ((ret = PIOc_def_dim(ncid, DIM_NAME, DIM_LEN, &dimid))) ERR(ret); + + /* Define a 1-D variable. */ if (verbose) printf("%d test_intercomm defining variable %s\n", my_rank, VAR_NAME); if ((ret = PIOc_def_var(ncid, VAR_NAME, NC_INT, NDIM, &dimid, &varid))) @@ -230,21 +224,21 @@ main(int argc, char **argv) if (verbose) printf("%d test_intercomm writing attribute %s\n", my_rank, ATT_NAME); int att_data = 42; - /* if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) */ - /* ERR(ret); */ + if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) + ERR(ret); - /* Close the file. */ + /* End define mode. */ if (verbose) printf("%d test_intercomm ending define mode ncid = %d\n", my_rank, ncid); if ((ret = PIOc_enddef(ncid))) ERR(ret); /* Write some data. */ - for (int i = 0; i < LOCAL_DIM_LEN; i++) - data[i] = my_rank; - if (verbose) - printf("%d test_intercomm writing data\n", my_rank); - start[0] = !my_rank ? 0 : 2; + /* for (int i = 0; i < LOCAL_DIM_LEN; i++) */ + /* data[i] = my_rank; */ + /* if (verbose) */ + /* printf("%d test_intercomm writing data\n", my_rank); */ + /* start[0] = !my_rank ? 0 : 2; */ /* if ((ret = PIOc_put_vara_int(ncid, varid, start, count, data))) */ /* ERR(ret); */ @@ -258,14 +252,14 @@ main(int argc, char **argv) if (verbose) printf("%d test_intercomm opening file %s\n", my_rank, filename[fmt]); if ((ret = PIOc_openfile(iosysid, &ncid, &format[fmt], filename[fmt], - NC_NOWRITE))) + NC_NOWRITE))) ERR(ret); /* Find the number of dimensions, variables, and global attributes.*/ int ndims, nvars, ngatts, unlimdimid; if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) ERR(ret); - if (ndims != 1 || nvars != 1 || ngatts != 0 || unlimdimid != -1) + if (ndims != 1 || nvars != 1 || ngatts != 1 || unlimdimid != -1) ERR(ERR_WRONG); int ndims2, nvars2, ngatts2, unlimdimid2; if ((ret = PIOc_inq_ndims(ncid, &ndims2))) @@ -278,7 +272,7 @@ main(int argc, char **argv) ERR(ERR_WRONG); if ((ret = PIOc_inq_natts(ncid, &ngatts2))) ERR(ret); - if (ngatts2 != 0) + if (ngatts2 != 1) ERR(ERR_WRONG); if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2))) ERR(ret); @@ -292,13 +286,13 @@ main(int argc, char **argv) ERR(ret); printf("%d test_intercomm dim name is %s VAR_NAME is %s\n", my_rank, dimname, VAR_NAME); if (strcmp(dimname, DIM_NAME) || dimlen != DIM_LEN) - ERR(ERR_WRONG); + ERR(ERR_WRONG); char dimname2[NC_MAX_NAME + 1]; PIO_Offset dimlen2; if ((ret = PIOc_inq_dimname(ncid, 0, dimname2))) ERR(ret); if (strcmp(dimname2, DIM_NAME)) - ERR(ERR_WRONG); + ERR(ERR_WRONG); if ((ret = PIOc_inq_dimlen(ncid, 0, &dimlen2))) ERR(ret); if (dimlen2 != DIM_LEN) @@ -316,7 +310,7 @@ main(int argc, char **argv) if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts))) ERR(ret); if (strcmp(varname, VAR_NAME) || vartype != NC_INT || varndims != NDIM || - vardimids != 0 || varnatts != 0) + vardimids != 0 || varnatts != 0) ERR(ERR_WRONG); char varname2[NC_MAX_NAME + 1]; nc_type vartype2; @@ -354,8 +348,8 @@ main(int argc, char **argv) ERR(ret); /* Now delete the file. */ - if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) - ERR(ret); + /* if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) */ + /* ERR(ret); */ /* if ((ret = PIOc_openfile(iosysid, &ncid, &format[fmt], filename[fmt], */ /* NC_NOWRITE)) != PIO_ENFILE) */ /* ERR(ERR_AWFUL); */ From aee5878a304d2e57de2daa5b3a679991a4d92996 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 9 May 2016 16:41:35 -0400 Subject: [PATCH 018/184] development of async inq functions --- src/clib/pio_internal.h | 8 - src/clib/pio_msg.c | 230 ++++--- src/clib/pio_nc_async.c | 1151 +++++++++++------------------------ tests/unit/test_intercomm.c | 244 +++++--- 4 files changed, 666 insertions(+), 967 deletions(-) diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 740b8fdf1a6..2e9ab413076 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -174,7 +174,6 @@ enum PIO_MSG{ PIO_MSG_PUT_ATT_INT, PIO_MSG_RENAME_ATT, PIO_MSG_DEL_ATT, - PIO_MSG_INQ_NATTS, PIO_MSG_INQ, PIO_MSG_GET_ATT_TEXT, PIO_MSG_GET_ATT_SHORT, @@ -189,21 +188,16 @@ enum PIO_MSG{ PIO_MSG_GET_ATT_ULONGLONG, PIO_MSG_GET_ATT_USHORT, PIO_MSG_PUT_ATT_ULONGLONG, - PIO_MSG_INQ_DIMLEN, PIO_MSG_GET_ATT_UINT, PIO_MSG_GET_ATT_LONGLONG, PIO_MSG_PUT_ATT_SCHAR, PIO_MSG_PUT_ATT_FLOAT, - PIO_MSG_INQ_NVARS, PIO_MSG_RENAME_DIM, PIO_MSG_INQ_VARNDIMS, PIO_MSG_GET_ATT_LONG, PIO_MSG_INQ_DIM, PIO_MSG_INQ_DIMID, - PIO_MSG_INQ_UNLIMDIM, PIO_MSG_INQ_VARDIMID, - PIO_MSG_INQ_ATTLEN, - PIO_MSG_INQ_DIMNAME, PIO_MSG_PUT_ATT_USHORT, PIO_MSG_GET_ATT_FLOAT, PIO_MSG_SYNC, @@ -212,11 +206,9 @@ enum PIO_MSG{ PIO_MSG_GET_ATT_SCHAR, PIO_MSG_INQ_ATTID, PIO_MSG_DEF_DIM, - PIO_MSG_INQ_NDIMS, PIO_MSG_INQ_VARTYPE, PIO_MSG_GET_ATT_INT, PIO_MSG_GET_ATT_DOUBLE, - PIO_MSG_INQ_ATTTYPE, PIO_MSG_PUT_ATT_UCHAR, PIO_MSG_GET_ATT_UCHAR, PIO_MSG_PUT_VARS_UCHAR, diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 8dd59e4439d..732b4a2377d 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -88,50 +88,51 @@ int close_file_handler(iosystem_desc_t *ios) * only ever run on the IO tasks. * * @param ios pointer to the iosystem_desc_t. - * @param msg the message sent my the comp root task. * @return PIO_NOERR for success, error code otherwise. */ -int inq_handler(iosystem_desc_t *ios, int msg) +int inq_handler(iosystem_desc_t *ios) { int ncid; + int ndims, nvars, ngatts, unlimdimid; + int *ndimsp = NULL, *nvarsp = NULL, *ngattsp = NULL, *unlimdimidp = NULL; + char ndims_present, nvars_present, ngatts_present, unlimdimid_present; int mpierr; int ret; int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d inq_handler msg = %d\n", my_rank, msg); + printf("%d inq_handler\n", my_rank); /* Get the parameters for this function that the the comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; + printf("%d inq_handler1\n", my_rank); + if ((mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + printf("%d inq_handler2\n", my_rank); + if ((mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + printf("%d inq_handler3\n", my_rank); + if ((mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + printf("%d inq_handler4\n", my_rank); + if ((mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + printf("%d inq_handler ndims_present = %d nvars_present = %d ngatts_present = %d unlimdimid_present = %d\n", + my_rank, ndims_present, nvars_present, ngatts_present, unlimdimid_present); - /* Call the inq file function. */ - int ndims, nvars, ngatts, unlimdimid; - int *ndimsp = NULL, *nvarsp = NULL, *ngattsp = NULL, *unlimdimidp = NULL; - switch (msg) - { - case PIO_MSG_INQ: + /* NULLs passed in to any of the pointers in the original call + * need to be matched with NULLs here. Assign pointers where + * non-NULL pointers were passed in. */ + if (ndims_present) ndimsp = &ndims; + if (nvars_present) nvarsp = &nvars; + if (ngatts_present) ngattsp = &ngatts; + if (unlimdimid_present) unlimdimidp = &unlimdimid; - break; - case PIO_MSG_INQ_NVARS: - nvarsp = &nvars; - break; - case PIO_MSG_INQ_NDIMS: - ndimsp = &ndims; - break; - case PIO_MSG_INQ_NATTS: - ngattsp = &ngatts; - break; - case PIO_MSG_INQ_UNLIMDIM: - unlimdimidp = &unlimdimid; - break; - default: - return PIO_EINVAL; - } /* Call the inq function to get the values. */ if ((ret = PIOc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp))) @@ -151,12 +152,18 @@ int inq_dim_handler(iosystem_desc_t *ios, int msg) { int ncid; int dimid; + char name_present, len_present; + char *dimnamep = NULL; + PIO_Offset *dimlenp = NULL; + char dimname[NC_MAX_NAME + 1]; + PIO_Offset dimlen; + int mpierr; int ret; int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d inq_handler msg = %d\n", my_rank, msg); + printf("%d inq_dim_handler\n", my_rank); /* Get the parameters for this function that the the comp master * task is broadcasting. */ @@ -164,27 +171,18 @@ int inq_dim_handler(iosystem_desc_t *ios, int msg) return PIO_EIO; if ((mpierr = MPI_Bcast(&dimid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; + if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + printf("%d inq_handler name_present = %d len_present = %d\n", + my_rank, name_present, len_present); - /* Call the inq_dim function. */ - char *dimnamep = NULL; - PIO_Offset *dimlenp = NULL; - char dimname[NC_MAX_NAME + 1]; - PIO_Offset dimlen; - switch (msg) - { - case PIO_MSG_INQ_DIM: + /* Set the non-null pointers. */ + if (name_present) dimnamep = dimname; + if (len_present) dimlenp = &dimlen; - break; - case PIO_MSG_INQ_DIMLEN: - dimlenp = &dimlen; - break; - case PIO_MSG_INQ_DIMNAME: - dimnamep = dimname; - break; - default: - return PIO_EINVAL; - } /* Call the inq function to get the values. */ if ((ret = PIOc_inq_dim(ncid, dimid, dimnamep, dimlenp))) @@ -202,25 +200,42 @@ int inq_dim_handler(iosystem_desc_t *ios, int msg) int inq_dimid_handler(iosystem_desc_t *ios) { int ncid; - int dimid; + int *dimidp = NULL, dimid; int mpierr; + int id_present; int ret; int namelen; char *name; + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d inq_dimid_handler\n", my_rank); + /* Get the parameters for this function that the the comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; + printf("%d inq_dimid_handler ncid = %d\n", my_rank, ncid); if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; + printf("%d inq_dimid_handler ncid = %d namelen = %d\n", my_rank, ncid, namelen); if (!(name = malloc((namelen + 1) * sizeof(char)))) return PIO_ENOMEM; if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; + printf("%d inq_dimid_handler ncid = %d namelen = %d name = %s\n", + my_rank, ncid, namelen, name); + if ((mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + printf("%d inq_dimid_handler ncid = %d namelen = %d name = %s id_present = %d\n", + my_rank, ncid, namelen, name, id_present); + + /* Set non-null pointer. */ + if (id_present) + dimidp = &dimid; /* Call the inq_dimid function. */ - if ((ret = PIOc_inq_dimid(ncid, name, &dimid))) + if ((ret = PIOc_inq_dimid(ncid, name, dimidp))) return ret; /* Free resources. */ @@ -229,13 +244,14 @@ int inq_dimid_handler(iosystem_desc_t *ios) return PIO_NOERR; } -/** Handle attribute operations. This code only runs on IO tasks. +/** Handle attribute inquiry operations. This code only runs on IO + * tasks. * * @param ios pointer to the iosystem_desc_t. * @param msg the message sent my the comp root task. * @return PIO_NOERR for success, error code otherwise. */ -int att_handler(iosystem_desc_t *ios, int msg) +int inq_att_handler(iosystem_desc_t *ios, int msg) { int ncid; int varid; @@ -243,13 +259,13 @@ int att_handler(iosystem_desc_t *ios, int msg) int ret; char *name5; int namelen; - int len; + PIO_Offset attlen; nc_type xtype; - int *op; + int *op, *ip; int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d att_handler\n", my_rank); + printf("%d inq_att_handler\n", my_rank); /* Get the parameters for this function that the the comp master * task is broadcasting. */ @@ -260,27 +276,92 @@ int att_handler(iosystem_desc_t *ios, int msg) mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!(name5 = malloc((namelen + 1) * sizeof(char)))) return PIO_ENOMEM; - mpierr = MPI_Bcast((void *)name5, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!(op = malloc(len * sizeof(int)))) - return PIO_ENOMEM; - mpierr = MPI_Bcast(op, len, MPI_INT, ios->compmaster, ios->intercomm); - - /* Call the function to write the attribute. */ - if ((ret = PIOc_put_att_int(ncid, varid, name5, xtype, len, op))) + mpierr = MPI_Bcast((void *)name5, namelen + 1, MPI_CHAR, ios->compmaster, + ios->intercomm); + + /* Call the function to learn about the attribute. */ + if ((ret = PIOc_inq_att(ncid, varid, name5, &xtype, &attlen))) return ret; - free(op); - if (!(op = malloc(20))) + return PIO_NOERR; +} + +/** Handle attribute operations. This code only runs on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @param msg the message sent my the comp root task. + * @return PIO_NOERR for success, error code otherwise. +*/ +int att_handler(iosystem_desc_t *ios, int msg) +{ + int ncid; + int varid; + int mpierr; + int ret; + char *name5; + int namelen; + PIO_Offset len; + nc_type xtype; + int *op, *ip; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d att_handler\n", my_rank); + + if (msg == PIO_MSG_PUT_ATT_INT) { - printf("%d att_handler4 ncid = %d varid = %d namelen = %d name = %d\n", - my_rank, ncid, varid, namelen, name5); + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!(name5 = malloc((namelen + 1) * sizeof(char)))) + return PIO_ENOMEM; + mpierr = MPI_Bcast((void *)name5, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!(op = malloc(len * sizeof(int)))) + return PIO_ENOMEM; + mpierr = MPI_Bcast(op, len, MPI_INT, ios->compmaster, ios->intercomm); + + /* Call the function to write the attribute. */ + if ((ret = PIOc_put_att_int(ncid, varid, name5, xtype, len, op))) + return ret; + + /* Free resources. */ + free(name5); + free(op); } + else if (msg = PIO_MSG_GET_ATT_INT) + { + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!(name5 = malloc((namelen + 1) * sizeof(char)))) + return PIO_ENOMEM; + mpierr = MPI_Bcast((void *)name5, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - /* Free resources. */ - free(name5); - free(op); + /* Allocate space for the attribute data. */ + if ((ret = PIOc_inq_attlen(ncid, varid, name5, &len))) + return ret; + if (!(ip = malloc(len * sizeof(int)))) + return PIO_ENOMEM; + + /* Call the function to read the attribute. */ + if ((ret = PIOc_get_att_int(ncid, varid, name5, ip))) + return ret; + printf("%d att_handler got att with first element %d\n", my_rank, ip[0]); + + /* Free resources. */ + free(name5); + free(ip); + } return PIO_NOERR; } @@ -841,15 +922,9 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) def_var_handler(my_iosys); break; case PIO_MSG_INQ: - case PIO_MSG_INQ_NVARS: - case PIO_MSG_INQ_NDIMS: - case PIO_MSG_INQ_NATTS: - case PIO_MSG_INQ_UNLIMDIM: - inq_handler(my_iosys, msg); + inq_handler(my_iosys); break; case PIO_MSG_INQ_DIM: - case PIO_MSG_INQ_DIMLEN: - case PIO_MSG_INQ_DIMNAME: inq_dim_handler(my_iosys, msg); break; case PIO_MSG_INQ_DIMID: @@ -864,15 +939,16 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) inq_var_handler(my_iosys, msg); break; case PIO_MSG_GET_ATT_INT: + case PIO_MSG_PUT_ATT_INT: ret = att_handler(my_iosys, msg); printf("att_handler returned %d\n", ret); break; - case PIO_MSG_PUT_ATT_INT: - att_handler(my_iosys, msg); - break; case PIO_MSG_INQ_VARID: inq_varid_handler(my_iosys); break; + case PIO_MSG_INQ_ATT: + inq_att_handler(my_iosys, msg); + break; case PIO_MSG_INITDECOMP_DOF: initdecomp_dof_handler(my_iosys); break; diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index b35b2029918..12a208d7f3d 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -18,136 +18,147 @@ #include #include -/** Internal function to call nc_inq. Call this function only in I/O - * tasks. */ -int -pio_io_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, - int *unlimdimidp) -{ - file_desc_t *file; - char *errstr = NULL; - int ierr = PIO_NOERR; - - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d pio_io_inq ncid = %d\n", my_rank, ncid); - - if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; - - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - printf("%d pio_io_inq file->iosystem->io_rank = %d\n", my_rank, file->iosystem->io_rank); - if (!file->iosystem->io_rank) - { - /* Should not be necessary to do this - nc_inq should - * handle null pointers. This has been reported as a bug - * to netCDF developers. */ - int tmp_ndims, tmp_var, tmp_gatts, tmp_unl; - if (!ndimsp) - ndimsp = &tmp_ndims; - if (!nvarsp) - nvarsp = &tmp_var; - if (!ngattsp) - ngattsp = &tmp_gatts; - if (!unlimdimidp) - unlimdimidp = &tmp_unl; - ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - - if(ierr != PIO_NOERR) - { - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - - return ierr; -} - /** * @ingroup PIOc_inq * The PIO-C interface for the NetCDF function nc_inq. * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. For more information on the underlying + * NetCDF commmand please read about this function in the NetCDF + * documentation at: * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See + * + * @return PIO_NOERR for success, error code otherwise. See * PIOc_Set_File_Error_Handling */ -int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, - int *unlimdimidp) +int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, + int *unlimdimidp) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - + int msg = PIO_MSG_INQ; /** Message for async notification. */ + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + char *errstr = NULL; /** String for error messages. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr; /** Return code from MPI function codes. */ + + /* For debugging purposes. */ int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); printf("%d PIOc_inq ncid = %d\n", my_rank, ncid); - - errstr = NULL; - ierr = PIO_NOERR; - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ; - if(ios->async_interface && ! ios->ioproc){ + /* For async, on non-IO tasks, send the necessary information to + * the IO tasks over the intercomm. Whether or not the pointers + * are NULL are passed as integers, either true or false. */ + if (ios->async_interface && ! ios->ioproc) + { + char ndims_present = ndimsp ? true : false; + char nvars_present = nvarsp ? true : false; + char ngatts_present = ngattsp ? true : false; + char unlimdimid_present = unlimdimidp ? true : false; + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq netcdf Bcast ncid = %d\n", my_rank, file->fh); + mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq netcdf Bcast ndims_present = %d\n", my_rank, ndims_present); + mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq netcdf Bcast nvars_present = %d\n", my_rank, nvars_present); + mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq netcdf Bcast ngatts_present = %d\n", my_rank, ngatts_present); + mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq netcdf Bcast unlimdimid_present = %d\n", my_rank, unlimdimid_present); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + switch (file->iotype) + { +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if (!file->iosystem->io_rank) + { + /* Should not be necessary to do this - nc_inq should + * handle null pointers. This has been reported as a bug + * to netCDF developers. */ + int tmp_ndims, tmp_nvars, tmp_ngatts, tmp_unlimdimid; + ierr = nc_inq(ncid, &tmp_ndims, &tmp_nvars, &tmp_ngatts, &tmp_unlimdimid); + if (ndimsp) + *ndimsp = tmp_ndims; + if (nvarsp) + *nvarsp = tmp_nvars; + if (ngattsp) + *ngattsp = tmp_ngatts; + if (unlimdimidp) + *unlimdimidp = tmp_unlimdimid; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } - if(ios->ioproc){ - ierr = pio_io_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp); - printf("%d PIOc_inq pio_io_inq returned %d\n", my_rank, ierr); + if(ierr != PIO_NOERR) + { + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + printf("%d PIOc_inq netcdf call returned %d\n", my_rank, ierr); ierr = check_netcdf(file, ierr, errstr,__LINE__); } - if(ndimsp != NULL) + + /* Broadcast results to all tasks. If pointers were NULL, then + * ignore that parameter. */ + if (ndimsp) { - printf("%d PIOc_inq Bcast ios->ioroot = %d *ndimsp = %d\n", my_rank, ios->ioroot, *ndimsp); mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + printf("%d PIOc__inq bcast ndims = %d\n", my_rank, *ndimsp); } - if(nvarsp != NULL) + if(nvarsp) + { mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(ngattsp != NULL) + printf("%d PIOc__inq bcast nvars = %d\n", my_rank, *nvarsp); + } + if(ngattsp) + { mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(unlimdimidp != NULL) + printf("%d PIOc__inq bcast ngatts = %d\n", my_rank, *ngattsp); + } + if(unlimdimidp) + { mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); + printf("%d PIOc__inq bcast unlimdimid = %d\n", my_rank, *unlimdimidp); + } + + if(errstr) + free(errstr); + return ierr; } /** - * @ingroup PIOc_inq_dimname - * The PIO-C interface for the NetCDF function nc_inq_dimname. + * @ingroup PIOc_inq_ndims + * The PIO-C interface for the NetCDF function nc_inq_ndims. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -158,72 +169,48 @@ int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, * PIOc_openfile() or PIOc_createfile(). * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_dimname (int ncid, int dimid, char *name) +int PIOc_inq_ndims (int ncid, int *ndimsp) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_DIMNAME; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); - } - + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d calling PIOc_inq_ndims\n", my_rank); + return PIOc_inq(ncid, ndimsp, NULL, NULL, NULL); +} - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dimname(file->fh, dimid, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_dimname(file->fh, dimid, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dimname(file->fh, dimid, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } +/** + * @ingroup PIOc_inq_nvars + * The PIO-C interface for the NetCDF function nc_inq_nvars. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_nvars(int ncid, int *nvarsp) +{ + return PIOc_inq(ncid, NULL, nvarsp, NULL, NULL); +} - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (name) - { - int slen; - if(ios->iomaster) - slen = (int) strlen(name) + 1; - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; +/** + * @ingroup PIOc_inq_natts + * The PIO-C interface for the NetCDF function nc_inq_natts. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_natts(int ncid, int *ngattsp) +{ + return PIOc_inq(ncid, NULL, NULL, ngattsp, NULL); } /** @@ -704,32 +691,41 @@ int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, P * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp) +int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) { - int ierr; - int msg; - int mpierr; iosystem_desc_t *ios; file_desc_t *file; - char *errstr; + char *errstr = NULL; + int ierr = PIO_NOERR; + int msg = PIO_MSG_INQ_DIM; + int mpierr; - errstr = NULL; - ierr = PIO_NOERR; + /* For debugging purposes only... */ + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_inq_dim\n", my_rank); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Get the file info, based on the ncid. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_DIM; - if(ios->async_interface && ! ios->ioproc){ + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface && !ios->ioproc) + { + char name_present = name ? true : false; + char len_present = lenp ? true : false; if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq netcdf Bcast name_present = %d\n", my_rank, name_present); + mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq netcdf Bcast len_present = %d\n", my_rank, len_present); } - + /* Make the call to the netCDF layer. */ if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF @@ -755,18 +751,21 @@ int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp) } } + /* Error handling. */ if(ierr != PIO_NOERR){ errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + + /* BCast the results, if non-null pointers were passed. */ if(name) { int slen; if(ios->iomaster) - slen = (int) strlen(name) + 1; + slen = strlen(name); mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm); } if(lenp != NULL) mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); @@ -774,6 +773,43 @@ int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp) return ierr; } +/** + * @ingroup PIOc_inq_dimname + * The PIO-C interface for the NetCDF function nc_inq_dimname. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_dimname(int ncid, int dimid, char *name) +{ + return PIOc_inq_dim(ncid, dimid, name, NULL); +} + +/** + * @ingroup PIOc_inq_dimlen + * The PIO-C interface for the NetCDF function nc_inq_dimlen. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param lenp a pointer that will get the number of values + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp) +{ + return PIOc_inq_dim(ncid, dimid, NULL, lenp); +} + /** * @ingroup PIOc_get_att_uchar * The PIO-C interface for the NetCDF function nc_get_att_uchar. @@ -1413,63 +1449,9 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) */ int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_ATTLEN; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_attlen(file->fh, varid, name, lenp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} + nc_type dummy; + return PIOc_inq_att(ncid, varid, name, &dummy, lenp); +} /** * @ingroup PIOc_inq_atttype @@ -1486,64 +1468,10 @@ int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) * @param xtypep a pointer that will get the type of the attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep) +int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_ATTTYPE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_atttype(file->fh, varid, name, xtypep);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_atttype(file->fh, varid, name, xtypep);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_atttype(file->fh, varid, name, xtypep);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; + PIO_Offset dummy; + return PIOc_inq_att(ncid, varid, name, xtypep, &dummy); } /** @@ -1619,80 +1547,6 @@ int PIOc_rename_var (int ncid, int varid, const char *name) return ierr; } -/** - * @ingroup PIOc_inq_natts - * The PIO-C interface for the NetCDF function nc_inq_natts. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_natts (int ncid, int *ngattsp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_NATTS; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_natts(file->fh, ngattsp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_natts(file->fh, ngattsp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_natts(file->fh, ngattsp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (ngattsp) - mpierr = MPI_Bcast(ngattsp,1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - /** * @ingroup PIOc_put_att_ulonglong * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. @@ -2026,36 +1880,40 @@ int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, P * @param idp a pointer that will get the id of the variable or attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_dimid (int ncid, const char *name, int *idp) +int PIOc_inq_dimid(int ncid, const char *name, int *idp) { - int ierr; - int msg; - int mpierr; + int msg = PIO_MSG_INQ_DIMID; iosystem_desc_t *ios; file_desc_t *file; - char *errstr; + char *errstr = NULL; + int ierr = PIO_NOERR; + int mpierr; - errstr = NULL; - ierr = PIO_NOERR; + /* For debugging purposes only... */ + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_inq_dimid\n", my_rank); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Get the file info, based on the ncid. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_DIMID; - if(ios->async_interface && ! ios->ioproc){ + /* If using async, and not an IO task, then send parameters. */ + if (ios->async_interface && !ios->ioproc) + { + int namelen; + char id_present = idp ? true : false; if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - int namelen; - if (ios->iomaster) - namelen = strlen(name); + namelen = strlen(name); mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); } - + /* IO tasks call the netCDF functions. */ if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF @@ -2317,245 +2175,9 @@ int PIOc_inq_format (int ncid, int *formatp) return ierr; } -/** - * @ingroup PIOc_get_att_long - * The PIO-C interface for the NetCDF function nc_get_att_long. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_long(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_long(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_long(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_attname - * The PIO-C interface for the NetCDF function nc_inq_attname. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param attnum the attribute ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_ATTNAME; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_attname(file->fh, varid, attnum, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_attname(file->fh, varid, attnum, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name) + 1; - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_att - * The PIO-C interface for the NetCDF function nc_inq_att. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @param lenp a pointer that will get the number of values - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_ATT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(xtypep != NULL) mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(lenp != NULL) mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_put_att_long - * The PIO-C interface for the NetCDF function nc_put_att_long. +/** + * @ingroup PIOc_get_att_long + * The PIO-C interface for the NetCDF function nc_get_att_long. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2567,7 +2189,7 @@ int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Of * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op) +int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) { int ierr; int msg; @@ -2583,7 +2205,7 @@ int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_LONG; + msg = PIO_MSG_GET_ATT_LONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2597,19 +2219,19 @@ int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_get_att_long(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_get_att_long(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_long(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_get_att_long(file->fh, varid, name, ip);; break; #endif default: @@ -2618,28 +2240,35 @@ int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_unlimdim - * The PIO-C interface for the NetCDF function nc_inq_unlimdim. + * @ingroup PIOc_inq_attname + * The PIO-C interface for the NetCDF function nc_inq_attname. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param attnum the attribute ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) +int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) { int ierr; int msg; @@ -2655,7 +2284,7 @@ int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_UNLIMDIM; + msg = PIO_MSG_INQ_ATTNAME; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2669,19 +2298,19 @@ int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_unlimdim(file->fh, unlimdimidp);; + ierr = nc_inq_attname(file->fh, varid, attnum, name);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_unlimdim(file->fh, unlimdimidp);; + ierr = nc_inq_attname(file->fh, varid, attnum, name);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_unlimdim(file->fh, unlimdimidp);; + ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);; break; #endif default: @@ -2694,15 +2323,20 @@ int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (unlimdimidp) - mpierr = MPI_Bcast(unlimdimidp,1, MPI_INT, ios->ioroot, ios->my_comm); + if(name != NULL){ + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_float - * The PIO-C interface for the NetCDF function nc_get_att_float. + * @ingroup PIOc_inq_att + * The PIO-C interface for the NetCDF function nc_inq_att. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2712,9 +2346,12 @@ int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) +int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, + PIO_Offset *lenp) { int ierr; int msg; @@ -2723,40 +2360,47 @@ int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) file_desc_t *file; char *errstr; + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_inq_att ncid = %d varid = %d xtpyep = %d lenp = %d\n", + my_rank, ncid, varid, xtypep, lenp); + errstr = NULL; ierr = PIO_NOERR; - file = pio_get_file_from_id(ncid); - if(file == NULL) + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_FLOAT; + msg = PIO_MSG_INQ_ATT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen = strlen(name); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); } - if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_float(file->fh, varid, name, ip);; + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_float(file->fh, varid, name, ip);; + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_float(file->fh, varid, name, ip);; + ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);; break; #endif default: @@ -2765,33 +2409,33 @@ int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_FLOAT, ios->ioroot, ios->my_comm); - } + if(xtypep) + mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(lenp) + mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_ndims - * The PIO-C interface for the NetCDF function nc_inq_ndims. + * @ingroup PIOc_put_att_long + * The PIO-C interface for the NetCDF function nc_put_att_long. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_ndims (int ncid, int *ndimsp) +int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op) { int ierr; int msg; @@ -2807,7 +2451,7 @@ int PIOc_inq_ndims (int ncid, int *ndimsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_NDIMS; + msg = PIO_MSG_PUT_ATT_LONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2821,19 +2465,19 @@ int PIOc_inq_ndims (int ncid, int *ndimsp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_ndims(file->fh, ndimsp);; + ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_ndims(file->fh, ndimsp);; + ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_ndims(file->fh, ndimsp);; + ierr = ncmpi_put_att_long(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -2846,15 +2490,31 @@ int PIOc_inq_ndims (int ncid, int *ndimsp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (ndimsp) - mpierr = MPI_Bcast(ndimsp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_int - * The PIO-C interface for the NetCDF function nc_put_att_int. + * @ingroup PIOc_inq_unlimdim + * The PIO-C interface for the NetCDF function nc_inq_unlimdim. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_unlimdim(int ncid, int *unlimdimidp) +{ + return PIOc_inq(ncid, NULL, NULL, unlimdimidp, NULL); +} + +/** + * @ingroup PIOc_get_att_float + * The PIO-C interface for the NetCDF function nc_get_att_float. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2866,7 +2526,7 @@ int PIOc_inq_ndims (int ncid, int *ndimsp) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op) +int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) { int ierr; int msg; @@ -2874,11 +2534,6 @@ int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_ iosystem_desc_t *ios; file_desc_t *file; char *errstr; - size_t namelen; - - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d PIOc_inq_varid ncid = %d name = %s\n", my_rank, ncid, name); errstr = NULL; ierr = PIO_NOERR; @@ -2887,24 +2542,12 @@ int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_ if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_INT; + msg = PIO_MSG_GET_ATT_FLOAT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) - mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); - printf("%d PIOc_put_att_int BCast msg = %d\n", my_rank, msg); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - namelen = strlen(name); - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - printf("%d PIOc_put_att_int about to send name = %s\n", my_rank, name); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - printf("%d PIOc_put_att_int sent name = %s\n", my_rank, name); - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)op, len, MPI_INT, ios->compmaster, ios->intercomm); - printf("%d PIOc_put_att_int ncid = %d, varid = %d namelen = %d name = %s xtype = %d len = %d\n", - my_rank, file->fh, varid, namelen, name, xtype, len); + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -2913,19 +2556,19 @@ int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_get_att_float(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_get_att_float(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_int(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_get_att_float(file->fh, varid, name, ip);; break; #endif default: @@ -2934,28 +2577,34 @@ int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_ } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_FLOAT, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_nvars - * The PIO-C interface for the NetCDF function nc_inq_nvars. + * @ingroup PIOc_put_att_int + * The PIO-C interface for the NetCDF function nc_put_att_int. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_nvars (int ncid, int *nvarsp) +int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op) { int ierr; int msg; @@ -2963,6 +2612,11 @@ int PIOc_inq_nvars (int ncid, int *nvarsp) iosystem_desc_t *ios; file_desc_t *file; char *errstr; + size_t namelen; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_inq_varid ncid = %d name = %s\n", my_rank, ncid, name); errstr = NULL; ierr = PIO_NOERR; @@ -2971,33 +2625,39 @@ int PIOc_inq_nvars (int ncid, int *nvarsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_NVARS; + msg = PIO_MSG_PUT_ATT_INT; if(ios->async_interface && ! ios->ioproc){ - if(!ios->comp_rank) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + namelen = strlen(name); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)op, len, MPI_INT, ios->compmaster, ios->intercomm); } - if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_nvars(file->fh, nvarsp);; + ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_nvars(file->fh, nvarsp);; + ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_nvars(file->fh, nvarsp);; + ierr = ncmpi_put_att_int(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -3010,8 +2670,6 @@ int PIOc_inq_nvars (int ncid, int *nvarsp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (nvarsp) - mpierr = MPI_Bcast(nvarsp,1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } @@ -3029,7 +2687,7 @@ int PIOc_inq_nvars (int ncid, int *nvarsp) * PIOc_openfile() or PIOc_createfile(). * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_enddef (int ncid) +int PIOc_enddef(int ncid) { int ierr; int msg; @@ -3541,82 +3199,6 @@ int PIOc_del_att (int ncid, int varid, const char *name) return ierr; } -/** - * @ingroup PIOc_inq_dimlen - * The PIO-C interface for the NetCDF function nc_inq_dimlen. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param lenp a pointer that will get the number of values - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_DIMLEN; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dimlen(file->fh, dimid, lenp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (lenp) - mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - /** * @ingroup PIOc_get_att_schar * The PIO-C interface for the NetCDF function nc_get_att_schar. @@ -3980,7 +3562,6 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); namelen = strlen(name); - printf("bcasting namelen = %d name = %s len = %d\n", namelen, name, len); mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); @@ -4339,6 +3920,7 @@ int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) iosystem_desc_t *ios; file_desc_t *file; char *errstr; + int namelen; errstr = NULL; ierr = PIO_NOERR; @@ -4352,10 +3934,13 @@ int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + namelen = strlen(name); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); } - if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 592b7b5e6ee..5ba73d633b3 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -31,6 +31,9 @@ /** The name of the global attribute in the netCDF output file. */ #define ATT_NAME "gatt_test_intercomm" +/** The value of the global attribute in the netCDF output file. */ +#define ATT_VALUE 42 + /** Error code for when things go wrong. */ #define ERR_AWFUL 1111 #define ERR_WRONG 2222 @@ -61,6 +64,144 @@ char err_buffer[MPI_MAX_ERROR_STRING]; * int the global error string. */ int resultlen; +/* Check the file for correctness. */ +int +check_file(int iosysid, int format, char *filename, int my_rank, int verbose) +{ + int ncid; + int ret; + int ndims, nvars, ngatts, unlimdimid; + int ndims2, nvars2, ngatts2, unlimdimid2; + int dimid2; + char dimname[NC_MAX_NAME + 1]; + PIO_Offset dimlen; + char dimname2[NC_MAX_NAME + 1]; + PIO_Offset dimlen2; + char varname[NC_MAX_NAME + 1]; + nc_type vartype; + int varndims, vardimids, varnatts; + char varname2[NC_MAX_NAME + 1]; + nc_type vartype2; + int varndims2, vardimids2, varnatts2; + int varid2; + int att_data; + + /* Re-open the file to check it. */ + if (verbose) + printf("%d test_intercomm opening file %s\n", my_rank, filename); + if ((ret = PIOc_openfile(iosysid, &ncid, &format, filename, + NC_NOWRITE))) + ERR(ret); + + /* Find the number of dimensions, variables, and global attributes.*/ + if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) + ERR(ret); + if (ndims != 1 || nvars != 1 || ngatts != 1 || unlimdimid != -1) + ERR(ERR_WRONG); + + /* This should return PIO_NOERR. */ + if ((ret = PIOc_inq(ncid, NULL, NULL, NULL, NULL))) + ERR(ret); + + /* Check the other functions that get these values. */ + if ((ret = PIOc_inq_ndims(ncid, &ndims2))) + ERR(ret); + if (ndims2 != 1) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_nvars(ncid, &nvars2))) + ERR(ret); + if (nvars2 != 1) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_natts(ncid, &ngatts2))) + ERR(ret); + if (ngatts2 != 1) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2))) + ERR(ret); + if (unlimdimid != -1) + ERR(ERR_WRONG); + + /* Check out the dimension. */ + if ((ret = PIOc_inq_dim(ncid, 0, dimname, &dimlen))) + ERR(ret); + if (strcmp(dimname, DIM_NAME) || dimlen != DIM_LEN) + ERR(ERR_WRONG); + + /* Check the other functions that get these values. */ + if ((ret = PIOc_inq_dimname(ncid, 0, dimname2))) + ERR(ret); + if (strcmp(dimname2, DIM_NAME)) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_dimlen(ncid, 0, &dimlen2))) + ERR(ret); + if (dimlen2 != DIM_LEN) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_dimid(ncid, DIM_NAME, &dimid2))) + ERR(ret); + if (dimid2 != 0) + ERR(ERR_WRONG); + + /* Check out the variable. */ + if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts))) + ERR(ret); + if (strcmp(varname, VAR_NAME) || vartype != NC_INT || varndims != NDIM || + vardimids != 0 || varnatts != 0) + ERR(ERR_WRONG); + + /* Check the other functions that get these values. */ + if ((ret = PIOc_inq_varname(ncid, 0, varname2))) + ERR(ret); + if (strcmp(varname2, VAR_NAME)) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_vartype(ncid, 0, &vartype2))) + ERR(ret); + if (vartype2 != NC_INT) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_varndims(ncid, 0, &varndims2))) + ERR(ret); + if (varndims2 != NDIM) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_vardimid(ncid, 0, &vardimids2))) + ERR(ret); + if (vardimids2 != 0) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_varnatts(ncid, 0, &varnatts2))) + ERR(ret); + if (varnatts2 != 0) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_varid(ncid, VAR_NAME, &varid2))) + ERR(ret); + if (varid2 != 0) + ERR(ERR_WRONG); + + /* Check out the global attributes. */ + nc_type atttype; + PIO_Offset attlen; + /* if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &atttype, &attlen))) */ + /* ERR(ret); */ + /* if (atttype != NC_INT || attlen != 1) */ + /* ERR(ERR_WRONG); */ + if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &attlen))) + ERR(ret); + if (attlen != 1) + ERR(ERR_WRONG); + /* if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, &att_data))) */ + /* ERR(ret); */ + /* sleep(2); */ + /* if (verbose) */ + /* printf("%d test_intercomm att_data = %d\n", my_rank, att_data); */ + /* if (att_data != ATT_VALUE) */ + /* ERR(ERR_WRONG); */ + + /* Close the file. */ + if (verbose) + printf("%d test_intercomm closing file (again) ncid = %d\n", my_rank, ncid); + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + return 0; +} + /** Run Tests for Init_Intercomm * * @param argc argument count @@ -223,7 +364,7 @@ main(int argc, char **argv) /* Add a global attribute. */ if (verbose) printf("%d test_intercomm writing attribute %s\n", my_rank, ATT_NAME); - int att_data = 42; + int att_data = ATT_VALUE; if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) ERR(ret); @@ -248,104 +389,9 @@ main(int argc, char **argv) if ((ret = PIOc_closefile(ncid))) ERR(ret); - /* Re-open the file to check it. */ - if (verbose) - printf("%d test_intercomm opening file %s\n", my_rank, filename[fmt]); - if ((ret = PIOc_openfile(iosysid, &ncid, &format[fmt], filename[fmt], - NC_NOWRITE))) - ERR(ret); - - /* Find the number of dimensions, variables, and global attributes.*/ - int ndims, nvars, ngatts, unlimdimid; - if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) - ERR(ret); - if (ndims != 1 || nvars != 1 || ngatts != 1 || unlimdimid != -1) - ERR(ERR_WRONG); - int ndims2, nvars2, ngatts2, unlimdimid2; - if ((ret = PIOc_inq_ndims(ncid, &ndims2))) - ERR(ret); - if (ndims2 != 1) - ERR(ERR_WRONG); - if ((ret = PIOc_inq_nvars(ncid, &nvars2))) - ERR(ret); - if (nvars2 != 1) - ERR(ERR_WRONG); - if ((ret = PIOc_inq_natts(ncid, &ngatts2))) - ERR(ret); - if (ngatts2 != 1) - ERR(ERR_WRONG); - if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2))) - ERR(ret); - if (unlimdimid != -1) - ERR(ERR_WRONG); - - /* Check out the dimension. */ - char dimname[NC_MAX_NAME + 1]; - PIO_Offset dimlen; - if ((ret = PIOc_inq_dim(ncid, 0, dimname, &dimlen))) - ERR(ret); - printf("%d test_intercomm dim name is %s VAR_NAME is %s\n", my_rank, dimname, VAR_NAME); - if (strcmp(dimname, DIM_NAME) || dimlen != DIM_LEN) - ERR(ERR_WRONG); - char dimname2[NC_MAX_NAME + 1]; - PIO_Offset dimlen2; - if ((ret = PIOc_inq_dimname(ncid, 0, dimname2))) - ERR(ret); - if (strcmp(dimname2, DIM_NAME)) - ERR(ERR_WRONG); - if ((ret = PIOc_inq_dimlen(ncid, 0, &dimlen2))) - ERR(ret); - if (dimlen2 != DIM_LEN) - ERR(ERR_WRONG); - int dimid2; - if ((ret = PIOc_inq_dimid(ncid, DIM_NAME, &dimid2))) - ERR(ret); - if (dimid2 != 0) - ERR(ERR_WRONG); - - /* Check out the variable. */ - char varname[NC_MAX_NAME + 1]; - nc_type vartype; - int varndims, vardimids, varnatts; - if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts))) - ERR(ret); - if (strcmp(varname, VAR_NAME) || vartype != NC_INT || varndims != NDIM || - vardimids != 0 || varnatts != 0) - ERR(ERR_WRONG); - char varname2[NC_MAX_NAME + 1]; - nc_type vartype2; - int varndims2, vardimids2, varnatts2; - if ((ret = PIOc_inq_varname(ncid, 0, varname2))) - ERR(ret); - if (strcmp(varname2, VAR_NAME)) - ERR(ERR_WRONG); - if ((ret = PIOc_inq_vartype(ncid, 0, &vartype2))) - ERR(ret); - if (vartype2 != NC_INT) - ERR(ERR_WRONG); - if ((ret = PIOc_inq_varndims(ncid, 0, &varndims2))) - ERR(ret); - if (varndims2 != NDIM) - ERR(ERR_WRONG); - if ((ret = PIOc_inq_vardimid(ncid, 0, &vardimids2))) - ERR(ret); - if (vardimids2 != 0) - ERR(ERR_WRONG); - if ((ret = PIOc_inq_varnatts(ncid, 0, &varnatts2))) - ERR(ret); - if (varnatts2 != 0) - ERR(ERR_WRONG); - int varid2; - if ((ret = PIOc_inq_varid(ncid, VAR_NAME, &varid2))) - ERR(ret); - if (varid2 != 0) - ERR(ERR_WRONG); - - /* Close the file. */ - if (verbose) - printf("%d test_intercomm closing file (again) ncid = %d\n", my_rank, ncid); - if ((ret = PIOc_closefile(ncid))) - ERR(ret); + /* Check the file for correctness. */ + if ((ret = check_file(iosysid, format[fmt], filename[fmt], my_rank, verbose))) + ERR(ret); /* Now delete the file. */ /* if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) */ From 90c689facbd2e7eabb8591fae6f4b624deb8eaa3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 9 May 2016 19:29:03 -0400 Subject: [PATCH 019/184] more inq functions working with async --- src/clib/pio_internal.h | 5 - src/clib/pio_msg.c | 64 ++-- src/clib/pio_nc_async.c | 591 ++++++++++-------------------------- tests/unit/test_intercomm.c | 51 ++-- 4 files changed, 219 insertions(+), 492 deletions(-) diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 2e9ab413076..3dbd28b6185 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -166,10 +166,8 @@ enum PIO_MSG{ PIO_MSG_INQ_ATT, PIO_MSG_INQ_FORMAT, PIO_MSG_INQ_VARID, - PIO_MSG_INQ_VARNATTS, PIO_MSG_DEF_VAR, PIO_MSG_INQ_VAR, - PIO_MSG_INQ_VARNAME, PIO_MSG_PUT_ATT_DOUBLE, PIO_MSG_PUT_ATT_INT, PIO_MSG_RENAME_ATT, @@ -193,11 +191,9 @@ enum PIO_MSG{ PIO_MSG_PUT_ATT_SCHAR, PIO_MSG_PUT_ATT_FLOAT, PIO_MSG_RENAME_DIM, - PIO_MSG_INQ_VARNDIMS, PIO_MSG_GET_ATT_LONG, PIO_MSG_INQ_DIM, PIO_MSG_INQ_DIMID, - PIO_MSG_INQ_VARDIMID, PIO_MSG_PUT_ATT_USHORT, PIO_MSG_GET_ATT_FLOAT, PIO_MSG_SYNC, @@ -206,7 +202,6 @@ enum PIO_MSG{ PIO_MSG_GET_ATT_SCHAR, PIO_MSG_INQ_ATTID, PIO_MSG_DEF_DIM, - PIO_MSG_INQ_VARTYPE, PIO_MSG_GET_ATT_INT, PIO_MSG_GET_ATT_DOUBLE, PIO_MSG_PUT_ATT_UCHAR, diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 732b4a2377d..58cfb9ffba8 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -373,15 +373,21 @@ int att_handler(iosystem_desc_t *ios, int msg) * @param msg the message sent my the comp root task. * @return PIO_NOERR for success, error code otherwise. */ -int inq_var_handler(iosystem_desc_t *ios, int msg) +int inq_var_handler(iosystem_desc_t *ios) { int ncid; int varid; int mpierr; + char name_present, xtype_present, ndims_present, dimids_present, natts_present; + char name[NC_MAX_NAME + 1], *namep; + nc_type xtype, *xtypep = NULL; + int *ndimsp = NULL, *dimidsp = NULL, *nattsp = NULL; + int ndims, dimids[NC_MAX_DIMS], natts; int ret; int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d inq_var_handler\n", my_rank); /* Get the parameters for this function that the the comp master * task is broadcasting. */ @@ -389,40 +395,31 @@ int inq_var_handler(iosystem_desc_t *ios, int msg) return PIO_EIO; if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d inq_var_handler ncid = %d varid = %d\n", my_rank, ncid, varid); + if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + printf("%d inq_var_handler ncid = %d varid = %d name_present = %d xtype_present = %d ndims_present = %d " + "dimids_present = %d natts_present = %d\n", + my_rank, ncid, varid, name_present, xtype_present, ndims_present, dimids_present, natts_present); - /* Call the inq_var function. */ - char name[NC_MAX_NAME + 1], *namep; - nc_type xtype, *xtypep = NULL; - int *ndimsp = NULL, *dimidsp = NULL, *nattsp = NULL; - int ndims, dimids[NC_MAX_DIMS], natts; - switch (msg) - { - case PIO_MSG_INQ_VAR: + /* Set the non-NULL pointers. */ + if (name_present) namep = name; + if (xtype_present) xtypep = &xtype; + if (ndims_present) ndimsp = &ndims; + if (dimids_present) dimidsp = dimids; + if (natts_present) nattsp = &natts; - break; - case PIO_MSG_INQ_VARNATTS: - nattsp = &natts; - break; - case PIO_MSG_INQ_VARNAME: - namep = name; - break; - case PIO_MSG_INQ_VARNDIMS: - ndimsp = &ndims; - break; - case PIO_MSG_INQ_VARDIMID: - dimidsp = dimids; - break; - case PIO_MSG_INQ_VARTYPE: - xtypep = &xtype; - break; - default: - return PIO_EINVAL; - } /* Call the inq function to get the values. */ if ((ret = PIOc_inq_var(ncid, varid, namep, xtypep, ndimsp, dimidsp, nattsp))) @@ -931,12 +928,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) inq_dimid_handler(my_iosys); break; case PIO_MSG_INQ_VAR: - case PIO_MSG_INQ_VARNATTS: - case PIO_MSG_INQ_VARNAME: - case PIO_MSG_INQ_VARNDIMS: - case PIO_MSG_INQ_VARDIMID: - case PIO_MSG_INQ_VARTYPE: - inq_var_handler(my_iosys, msg); + inq_var_handler(my_iosys); break; case PIO_MSG_GET_ATT_INT: case PIO_MSG_PUT_ATT_INT: diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 12a208d7f3d..f0830c17a07 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -736,7 +736,7 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ + if (ios->io_rank == 0){ ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; } break; @@ -1037,83 +1037,6 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) return ierr; } -/** - * @ingroup PIOc_inq_vartype - * The PIO-C interface for the NetCDF function nc_inq_vartype. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VARTYPE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh,1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_vartype(file->fh, varid, xtypep);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_vartype(file->fh, varid, xtypep);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_vartype(file->fh, varid, xtypep);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (xtypep) - mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - /** * @ingroup PIOc_put_att_schar * The PIO-C interface for the NetCDF function nc_put_att_schar. @@ -1187,86 +1110,6 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI return ierr; } -/** - * @ingroup PIOc_inq_vardimid - * The PIO-C interface for the NetCDF function nc_inq_vardimid. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VARDIMID; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_vardimid(file->fh, varid, dimidsp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_vardimid(file->fh, varid, dimidsp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_vardimid(file->fh, varid, dimidsp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (!ierr && dimidsp) - { - int ndims; - PIOc_inq_varndims(file->fh, varid, &ndims); - mpierr = MPI_Bcast(dimidsp , ndims, MPI_INT, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - /** * @ingroup PIOc_get_att_ushort * The PIO-C interface for the NetCDF function nc_get_att_ushort. @@ -1636,50 +1479,75 @@ int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype * @param nattsp a pointer that will get the number of attributes * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, - int *dimidsp, int *nattsp) +int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, + int *dimidsp, int *nattsp) { - int ierr; - int msg; - int mpierr; iosystem_desc_t *ios; file_desc_t *file; - char *errstr; + char *errstr = NULL; + int ndims; /** The number of dimensions for this variable. */ + int ierr = PIO_NOERR; + int msg = PIO_MSG_INQ_VAR; + int mpierr; - errstr = NULL; - ierr = PIO_NOERR; + /* For debugging purposes only... */ + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_inq_var\n", my_rank); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Get the file info, based on the ncid. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VAR; + printf("%d PIOc_inq_var got file\n", my_rank); - if(ios->async_interface && ! ios->ioproc){ + /* If using async, and this is not an IO task, send the parameters to the IO task. */ + if (ios->async_interface && !ios->ioproc) + { + char name_present = name ? true : false; + char xtype_present = xtypep ? true : false; + char ndims_present = ndimsp ? true : false; + char dimids_present = dimidsp ? true : false; + char natts_present = nattsp ? true : false; if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq_var ncid = %d\n", my_rank, ncid); mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d dimids_present = %d, natts_present = %d nattsp = %d\n", + my_rank, name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp); } + printf("%d PIOc_inq_var ios->ioproc = %d nattsp = %d\n", my_rank, ios->ioproc, nattsp); - if(ios->ioproc){ + /* Call the netCDF layer. */ + if (ios->ioproc) + { switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + ierr = nc_inq_varndims(file->fh, varid, &ndims); + ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp); break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ + if (ios->io_rank == 0) + { + ierr = nc_inq_varndims(file->fh, varid, &ndims); ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_varndims(file->fh, varid, &ndims); ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; break; #endif @@ -1693,33 +1561,144 @@ int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (xtypep) - mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); - if (ndimsp) - { - mpierr = MPI_Bcast(ndimsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - file->varlist[varid].ndims = (*ndimsp); - } - if (nattsp) - mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + printf("%d called netcdf nc_inq_var ierr = %d\n", my_rank, ierr); + + /* Broadcast the results for non-null pointers. */ if (name) { int slen; if(ios->iomaster) - slen = (int) strlen(name) + 1; + slen = strlen(name); + printf("%d PIOc_inq_var slen = %d\n", my_rank, slen); mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + printf("%d PIOc_inq_var slen = %d\n", my_rank, slen); + mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm); + printf("%d PIOc_inq_var name = %s\n", my_rank, name); + } + if (xtypep) + { + mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); + printf("%d PIOc_inq_var xtype = %d\n", my_rank, *xtypep); + } + if (ndimsp) + { + printf("%d PIOc_inq_var ndims = %d\n", my_rank, *ndimsp); + mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + file->varlist[varid].ndims = (*ndimsp); + printf("%d PIOc_inq_var bcast complete ndims = %d\n", my_rank, *ndimsp); } if (dimidsp) { - int ndims; - PIOc_inq_varndims(file->fh, varid, &ndims); + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm); mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm); } + if (nattsp) + { + printf("%d PIOc_inq_var natts = %d\n", my_rank, *nattsp); + mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + printf("%d PIOc_inq_var natts = %d\n", my_rank, *nattsp); + } if(errstr != NULL) free(errstr); return ierr; } +/** + * @ingroup PIOc_inq_varname + * The PIO-C interface for the NetCDF function nc_inq_varname. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varname (int ncid, int varid, char *name) +{ + return PIOc_inq_var(ncid, varid, name, NULL, NULL, NULL, NULL); +} + +/** + * @ingroup PIOc_inq_vartype + * The PIO-C interface for the NetCDF function nc_inq_vartype. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) +{ + return PIOc_inq_var(ncid, varid, NULL, xtypep, NULL, NULL, NULL); +} + +/** + * @ingroup PIOc_inq_varndims + * The PIO-C interface for the NetCDF function nc_inq_varndims. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) +{ + return PIOc_inq_var(ncid, varid, NULL, NULL, ndimsp, NULL, NULL); +} + +/** + * @ingroup PIOc_inq_vardimid + * The PIO-C interface for the NetCDF function nc_inq_vardimid. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_vardimid(int ncid, int varid, int *dimidsp) +{ + return PIOc_inq_var(ncid, varid, NULL, NULL, NULL, dimidsp, NULL); +} + +/** + * @ingroup PIOc_inq_varnatts + * The PIO-C interface for the NetCDF function nc_inq_varnatts. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param nattsp a pointer that will get the number of attributes + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) +{ + return PIOc_inq_var(ncid, varid, NULL, NULL, NULL, NULL, nattsp); +} + /** * @ingroup PIOc_rename_att * The PIO-C interface for the NetCDF function nc_rename_att. @@ -2893,82 +2872,6 @@ int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, return ierr; } -/** - * @ingroup PIOc_inq_varnatts - * The PIO-C interface for the NetCDF function nc_inq_varnatts. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param nattsp a pointer that will get the number of attributes - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VARNATTS; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varnatts(file->fh, varid, nattsp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_varnatts(file->fh, varid, nattsp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varnatts(file->fh, varid, nattsp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (nattsp) - mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} /** * @ingroup PIOc_get_att_ubyte @@ -3355,170 +3258,6 @@ int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long return ierr; } -/** - * @ingroup PIOc_inq_varndims - * The PIO-C interface for the NetCDF function nc_inq_varndims. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VARNDIMS; - if(file->varlist[varid].ndims > 0){ - (*ndimsp) = file->varlist[varid].ndims; - return PIO_NOERR; - } - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh,1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid,1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varndims(file->fh, varid, ndimsp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_varndims(file->fh, varid, ndimsp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varndims(file->fh, varid, ndimsp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (ndimsp) - { - mpierr = MPI_Bcast(ndimsp,1, MPI_INT, ios->ioroot, ios->my_comm); - file->varlist[varid].ndims = (*ndimsp); - } - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_inq_varname - * The PIO-C interface for the NetCDF function nc_inq_varname. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_varname (int ncid, int varid, char *name) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_VARNAME; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh,1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varname(file->fh, varid, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_varname(file->fh, varid, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varname(file->fh, varid, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name) + 1; - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; -} - /** * @ingroup PIOc_def_dim * The PIO-C interface for the NetCDF function nc_def_dim. diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 5ba73d633b3..dbbc6c39e93 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -88,7 +88,7 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) /* Re-open the file to check it. */ if (verbose) - printf("%d test_intercomm opening file %s\n", my_rank, filename); + printf("%d test_intercomm opening file %s format %d\n", my_rank, filename, format); if ((ret = PIOc_openfile(iosysid, &ncid, &format, filename, NC_NOWRITE))) ERR(ret); @@ -143,36 +143,36 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) /* Check out the variable. */ if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts))) - ERR(ret); + ERR(ret); if (strcmp(varname, VAR_NAME) || vartype != NC_INT || varndims != NDIM || - vardimids != 0 || varnatts != 0) - ERR(ERR_WRONG); + vardimids != 0 || varnatts != 0) + ERR(ERR_WRONG); /* Check the other functions that get these values. */ if ((ret = PIOc_inq_varname(ncid, 0, varname2))) - ERR(ret); + ERR(ret); if (strcmp(varname2, VAR_NAME)) - ERR(ERR_WRONG); + ERR(ERR_WRONG); if ((ret = PIOc_inq_vartype(ncid, 0, &vartype2))) - ERR(ret); + ERR(ret); if (vartype2 != NC_INT) - ERR(ERR_WRONG); + ERR(ERR_WRONG); if ((ret = PIOc_inq_varndims(ncid, 0, &varndims2))) - ERR(ret); + ERR(ret); if (varndims2 != NDIM) - ERR(ERR_WRONG); + ERR(ERR_WRONG); if ((ret = PIOc_inq_vardimid(ncid, 0, &vardimids2))) - ERR(ret); + ERR(ret); if (vardimids2 != 0) - ERR(ERR_WRONG); + ERR(ERR_WRONG); if ((ret = PIOc_inq_varnatts(ncid, 0, &varnatts2))) - ERR(ret); + ERR(ret); if (varnatts2 != 0) - ERR(ERR_WRONG); + ERR(ERR_WRONG); if ((ret = PIOc_inq_varid(ncid, VAR_NAME, &varid2))) - ERR(ret); + ERR(ret); if (varid2 != 0) - ERR(ERR_WRONG); + ERR(ERR_WRONG); /* Check out the global attributes. */ nc_type atttype; @@ -181,10 +181,10 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) /* ERR(ret); */ /* if (atttype != NC_INT || attlen != 1) */ /* ERR(ERR_WRONG); */ - if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &attlen))) - ERR(ret); - if (attlen != 1) - ERR(ERR_WRONG); + /* if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &attlen))) */ + /* ERR(ret); */ + /* if (attlen != 1) */ + /* ERR(ERR_WRONG); */ /* if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, &att_data))) */ /* ERR(ret); */ /* sleep(2); */ @@ -225,10 +225,10 @@ main(int argc, char **argv) PIO_IOTYPE_NETCDF4P}; /** Names for the output files. */ - char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_nc4_pnetcdf.nc", - "test_nc4_classic.nc", - "test_nc4_serial4.nc", - "test_nc4_parallel4.nc"}; + char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_intercomm_pnetcdf.nc", + "test_intercomm_classic.nc", + "test_intercomm_serial4.nc", + "test_intercomm_parallel4.nc"}; /** The ID for the parallel I/O system. */ int iosysid; @@ -334,7 +334,8 @@ main(int argc, char **argv) * and when the do, they should go straight to finalize. */ if (comp_task) { - for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) + /*for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) */ + for (int fmt = 0; fmt < 1; fmt++) { int ncid, varid, dimid; PIO_Offset start[NDIM], count[NDIM] = {0}; From 8600910b76ef546407bb85e920419b065a4bd922 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 10 May 2016 11:40:23 -0400 Subject: [PATCH 020/184] removed some unneeded msg constants --- src/clib/pio_nc.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/clib/pio_nc.c b/src/clib/pio_nc.c index f511475a4f6..52a376f0c26 100644 --- a/src/clib/pio_nc.c +++ b/src/clib/pio_nc.c @@ -122,7 +122,7 @@ int PIOc_inq_dimname (int ncid, int dimid, char *name) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_DIMNAME; + msg = PIO_MSG_INQ_DIM; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -961,7 +961,7 @@ int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARTYPE; + msg = PIO_MSG_INQ_VAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1108,7 +1108,7 @@ int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARDIMID; + msg = PIO_MSG_INQ_VAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1340,7 +1340,7 @@ int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATTLEN; + msg = PIO_MSG_INQ_ATT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1415,7 +1415,7 @@ int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATTTYPE; + msg = PIO_MSG_INQ_ATT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1561,7 +1561,7 @@ int PIOc_inq_natts (int ncid, int *ngattsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_NATTS; + msg = PIO_MSG_INQ; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2322,7 +2322,7 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATTNAME; + msg = PIO_MSG_INQ_ATT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2551,7 +2551,7 @@ int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_UNLIMDIM; + msg = PIO_MSG_INQ; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2702,7 +2702,7 @@ int PIOc_inq_ndims (int ncid, int *ndimsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_NDIMS; + msg = PIO_MSG_INQ; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2848,7 +2848,7 @@ int PIOc_inq_nvars (int ncid, int *nvarsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_NVARS; + msg = PIO_MSG_INQ; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3141,7 +3141,7 @@ int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARNATTS; + msg = PIO_MSG_INQ_VAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3444,7 +3444,7 @@ int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_DIMLEN; + msg = PIO_MSG_INQ_DIM; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3674,7 +3674,7 @@ int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARNDIMS; + msg = PIO_MSG_INQ_VAR; if(file->varlist[varid].ndims > 0){ (*ndimsp) = file->varlist[varid].ndims; return PIO_NOERR; @@ -3753,7 +3753,7 @@ int PIOc_inq_varname (int ncid, int varid, char *name) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARNAME; + msg = PIO_MSG_INQ_VAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) From b312291c23c62b2f8773c7819ea84eeb9411a7c6 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 10 May 2016 13:11:14 -0400 Subject: [PATCH 021/184] added logging --- CMakeLists.txt | 9 +++ src/clib/config.h.in | 3 + src/clib/pio.h | 2 +- src/clib/pio_internal.h | 6 ++ src/clib/pio_msg.c | 6 ++ src/clib/pio_nc_async.c | 126 +++++++++++++++++++++++++----------- tests/unit/test_intercomm.c | 7 +- 7 files changed, 118 insertions(+), 41 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b284128d2da..b13a09dbebb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,18 +15,27 @@ mark_as_advanced(VERSION_MAJOR VERSION_MINOR VERSION_PATCH) option (PIO_ENABLE_FORTRAN "Enable the Fortran library builds" ON) option (PIO_ENABLE_TIMING "Enable the use of the GPTL timing library" ON) option (PIO_ENABLE_ASYNC "Enable the use of asychronus IO operations" OFF) +option (PIO_ENABLE_LOGGING "Enable debug logging (large output possible)" OFF) option (PIO_TEST_BIG_ENDIAN "Enable test to see if machine is big endian" ON) option (PIO_USE_MPIIO "Enable support for MPI-IO auto detect" ON) option (PIO_USE_MPISERIAL "Enable mpi-serial support (instead of MPI)" OFF) option (PIO_USE_MALLOC "Use native malloc (instead of bget package)" OFF) option (WITH_PNETCDF "Require the use of PnetCDF" ON) +# Set a variable that appears in the config.h.in file. if(PIO_USE_MALLOC) set(USE_MALLOC 1) else() set(USE_MALLOC 0) endif() +# Set a variable that appears in the config.h.in file. +if(PIO_ENABLE_LOGGING) + set(ENABLE_LOGGING 1) +else() + set(ENABLE_LOGGING 0) +endif() + #===== Library Variables ===== set (PIO_FILESYSTEM_HINTS IGNORE CACHE STRING "Filesystem hints (lustre or gpfs)") diff --git a/src/clib/config.h.in b/src/clib/config.h.in index e4be61d9e0c..1722872c305 100644 --- a/src/clib/config.h.in +++ b/src/clib/config.h.in @@ -19,4 +19,7 @@ * will use the included bget() package for memory management. */ #define PIO_USE_MALLOC @USE_MALLOC@ +/** Set to non-zero to turn on logging. Output may be large. */ +#define PIO_ENABLE_LOGGING @ENABLE_LOGGING@ + #endif /* _PIO_CONFIG_ */ diff --git a/src/clib/pio.h b/src/clib/pio.h index e96f754e57e..44507edcdc7 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -647,7 +647,7 @@ int PIOc_set_blocksize(const int newblocksize); int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf); int PIOc_get_var_schar (int ncid, int varid, signed char *buf); int PIOc_iotype_available(const int iotype); - + int PIOc_set_log_level(int level); #if defined(__cplusplus) } #endif diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 3dbd28b6185..78297f54f9e 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -17,6 +17,12 @@ #include #endif +#ifdef PIO_ENABLE_LOGGING +void nc_log(int severity, const char *fmt, ...); +#define LOG(e) pio_log e +#else +#define LOG(e) +#endif /* PIO_ENABLE_LOGGING */ #define max(a,b) \ ({ __typeof__ (a) _a = (a); \ diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 58cfb9ffba8..e29c6b769af 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -7,9 +7,15 @@ * @see http://code.google.com/p/parallelio/ */ +#include #include #include +#ifdef PIO_ENABLE_LOGGING +extern int my_rank; +extern int pio_log_level; +#endif /* PIO_ENABLE_LOGGING */ + /** This function is run on the IO tasks to create a netCDF file. */ int create_file_handler(iosystem_desc_t *ios) { diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index f0830c17a07..144a5a2095f 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -15,9 +15,80 @@ * @date Feburary 2014, April 2016 */ +#include +#ifdef PIO_ENABLE_LOGGING +#include +#endif /* PIO_ENABLE_LOGGING */ #include #include +#ifdef PIO_ENABLE_LOGGING +int pio_log_level = 0; +int my_rank; +#endif /* PIO_ENABLE_LOGGING */ + +/** Set the logging level. Set to -1 for nothing, 0 for errors only, 1 + * for important logging, and so on. Log levels below 1 are only + * printed on the io/component root. If the library is not built with + * logging, this function does nothing. */ +int PIOc_set_log_level(int level) +{ +#ifdef PIO_ENABLE_LOGGING + printf("setting log level to %d\n", level); + pio_log_level = level; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + return PIO_NOERR; +#endif /* PIO_ENABLE_LOGGING */ +} + +#ifdef PIO_ENABLE_LOGGING +/** This function prints out a message, if the severity of the message + is lower than the global pio_log_level. To use it, do something + like this: + + pio_log(0, "this computer will explode in %d seconds", i); + + After the first arg (the severity), use the rest like a normal + printf statement. Output will appear on stdout. + This function is heavily based on the function in section 15.5 of + the C FAQ. +*/ +void +pio_log(int severity, const char *fmt, ...) +{ + va_list argp; + int t; + + /* If the severity is greater than the log level, we don't print + this message. */ + if (severity > pio_log_level) + return; + + /* If the severity is 1 or less, only print on rank 0. */ + if (severity < 2 && my_rank != 0) + return; + + /* If the severity is zero, this is an error. Otherwise insert that + many tabs before the message. */ + if (!severity) + fprintf(stdout, "ERROR: "); + for (t = 0; t < severity; t++) + fprintf(stdout, "\t"); + + /* Show the rank. */ + fprintf(stdout, "%d ", my_rank); + + /* Print out the variable list of args with vprintf. */ + va_start(argp, fmt); + vfprintf(stdout, fmt, argp); + va_end(argp); + + /* Put on a final linefeed. */ + fprintf(stdout, "\n"); + fflush(stdout); +} +#endif /* PIO_ENABLE_LOGGING */ + /** * @ingroup PIOc_inq * The PIO-C interface for the NetCDF function nc_inq. @@ -44,10 +115,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr; /** Return code from MPI function codes. */ - /* For debugging purposes. */ - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d PIOc_inq ncid = %d\n", my_rank, ncid); + LOG((1, "PIOc_inq ncid = %d", ncid)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) @@ -67,15 +135,11 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - printf("%d PIOc_inq netcdf Bcast ncid = %d\n", my_rank, file->fh); mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - printf("%d PIOc_inq netcdf Bcast ndims_present = %d\n", my_rank, ndims_present); mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - printf("%d PIOc_inq netcdf Bcast nvars_present = %d\n", my_rank, nvars_present); mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - printf("%d PIOc_inq netcdf Bcast ngatts_present = %d\n", my_rank, ngatts_present); mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - printf("%d PIOc_inq netcdf Bcast unlimdimid_present = %d\n", my_rank, unlimdimid_present); + LOG((2, "PIOc_inq netcdf Bcast unlimdimid_present = %d", unlimdimid_present)); } /* If this is an IO task, then call the netCDF function. */ @@ -132,22 +196,22 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, if (ndimsp) { mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - printf("%d PIOc__inq bcast ndims = %d\n", my_rank, *ndimsp); + LOG((2, "PIOc__inq bcast ndims = %d\n", *ndimsp)); } if(nvarsp) { mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - printf("%d PIOc__inq bcast nvars = %d\n", my_rank, *nvarsp); + LOG((2, "%d PIOc__inq bcast nvars = %d\n", my_rank, *nvarsp)); } if(ngattsp) { mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - printf("%d PIOc__inq bcast ngatts = %d\n", my_rank, *ngattsp); + LOG((2, "%d PIOc__inq bcast ngatts = %d\n", my_rank, *ngattsp)); } if(unlimdimidp) { mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm); - printf("%d PIOc__inq bcast unlimdimid = %d\n", my_rank, *unlimdimidp); + LOG((2, "%d PIOc__inq bcast unlimdimid = %d\n", my_rank, *unlimdimidp)); } if(errstr) @@ -171,9 +235,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, */ int PIOc_inq_ndims (int ncid, int *ndimsp) { - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d calling PIOc_inq_ndims\n", my_rank); + LOG((1, "PIOc_inq_ndims")); return PIOc_inq(ncid, ndimsp, NULL, NULL, NULL); } @@ -552,7 +614,7 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm); namelen = strlen(name); - printf("bcasting namelen = %d name = %s\n", namelen, name); + LOG((2, "bcasting namelen = %d name = %s\n", namelen, name)); if (!ios->compmaster) ios->compmaster = MPI_PROC_NULL; mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); @@ -700,10 +762,7 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) int msg = PIO_MSG_INQ_DIM; int mpierr; - /* For debugging purposes only... */ - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d PIOc_inq_dim\n", my_rank); + LOG((1, "PIOc_inq_dim")); /* Get the file info, based on the ncid. */ if (!(file = pio_get_file_from_id(ncid))) @@ -720,9 +779,9 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - printf("%d PIOc_inq netcdf Bcast name_present = %d\n", my_rank, name_present); + LOG((2, "PIOc_inq netcdf Bcast name_present = %d", name_present)); mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - printf("%d PIOc_inq netcdf Bcast len_present = %d\n", my_rank, len_present); + LOG((2, "PIOc_inq netcdf Bcast len_present = %d", len_present)); } /* Make the call to the netCDF layer. */ @@ -1221,9 +1280,7 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) ios = file->iosystem; msg = PIO_MSG_INQ_VARID; - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d PIOc_inq_varid ncid = %d name = %s\n", my_rank, ncid, name); + LOG((1, "PIOc_inq_varid ncid = %d name = %s", ncid, name)); if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1232,11 +1289,8 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); int namelen; namelen = strlen(name); - printf("%d PIOc_inq_varid BCast namelen = %d\n", my_rank, namelen); mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - printf("%d PIOc_inq_varid BCast name = %s\n", my_rank, name); mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - printf("%d PIOc_inq_varid BCast done\n", my_rank); } if(ios->ioproc){ @@ -1490,16 +1544,12 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int msg = PIO_MSG_INQ_VAR; int mpierr; - /* For debugging purposes only... */ - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d PIOc_inq_var\n", my_rank); + LOG((1, "PIOc_inq_var\n")); /* Get the file info, based on the ncid. */ if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - printf("%d PIOc_inq_var got file\n", my_rank); /* If using async, and this is not an IO task, send the parameters to the IO task. */ if (ios->async_interface && !ios->ioproc) @@ -1512,19 +1562,17 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - printf("%d PIOc_inq_var ncid = %d\n", my_rank, ncid); mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - printf("%d PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d dimids_present = %d, natts_present = %d nattsp = %d\n", - my_rank, name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp); + LOG((2, "PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d " + "dimids_present = %d, natts_present = %d nattsp = %d", + name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp)); } - printf("%d PIOc_inq_var ios->ioproc = %d nattsp = %d\n", my_rank, ios->ioproc, nattsp); - /* Call the netCDF layer. */ if (ios->ioproc) { diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index dbbc6c39e93..6ab8cb03186 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -321,7 +321,12 @@ main(int argc, char **argv) comp_task = 0; } - + + /* Turn on logging. */ + if ((ret = PIOc_set_log_level(2))) + ERR(ret); + + /* Initialize the async setup. */ if ((ret = PIOc_Init_Intercomm(COMPONENT_COUNT, MPI_COMM_WORLD, &comp_comms, io_comm, &iosysid))) ERR(ret); From bb17ac4d96745de164f72c1f80421d400065b398 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 10 May 2016 15:00:47 -0400 Subject: [PATCH 022/184] rearranged order of functions, started to use LOG in pio_msg --- src/clib/pio_internal.h | 2 +- src/clib/pio_msg.c | 101 ++- src/clib/pio_nc_async.c | 1334 ++++++++++++++++++++------------------- 3 files changed, 713 insertions(+), 724 deletions(-) diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 78297f54f9e..9efa132843d 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -18,7 +18,7 @@ #endif #ifdef PIO_ENABLE_LOGGING -void nc_log(int severity, const char *fmt, ...); +void pio_log(int severity, const char *fmt, ...); #define LOG(e) pio_log e #else #define LOG(e) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index e29c6b769af..76a1f69ab6e 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -27,15 +27,13 @@ int create_file_handler(iosystem_desc_t *ios) int mpierr; int ret; - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d create_file_handler comproot = %d\n", my_rank, ios->comproot); + LOG((1, "create_file_handler comproot = %d\n", ios->comproot)); /* Get the parameters for this function that the he comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d create_file_handler got parameter len = %d\n", my_rank, len); + LOG((1, "create_file_handler got parameter len = %d\n", len)); if (!(filename = malloc(len + 1 * sizeof(char)))) return PIO_ENOMEM; if ((mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, 0, @@ -45,9 +43,9 @@ int create_file_handler(iosystem_desc_t *ios) return PIO_EIO; if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d create_file_handler got parameters len = %d " + LOG((1, "create_file_handler got parameters len = %d " "filename = %s iotype = %d mode = %d\n", - my_rank, len, filename, iotype, mode); + len, filename, iotype, mode)); /* Call the create file function. */ if ((ret = PIOc_createfile(ios->iosysid, &ncid, &iotype, filename, mode))) @@ -56,7 +54,7 @@ int create_file_handler(iosystem_desc_t *ios) /* Free resources. */ free(filename); - printf("%d create_file_handler succeeded!\n", my_rank); + LOG((1, "create_file_handler succeeded!\n", my_rank)); return PIO_NOERR; } @@ -74,19 +72,19 @@ int close_file_handler(iosystem_desc_t *ios) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d close_file_handler\n", my_rank); + LOG((1, "%d close_file_handler\n", my_rank)); /* Get the parameters for this function that the the comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d create_file_handler got parameter ncid = %d\n", my_rank, ncid); + LOG((1, "%d create_file_handler got parameter ncid = %d\n", ncid)); /* Call the close file function. */ if ((ret = PIOc_closefile(ncid))) return ret; - printf("%d close_file_handler succeeded!\n", my_rank); + LOG((1, "close_file_handler succeeded!\n", my_rank)); return PIO_NOERR; } @@ -107,26 +105,22 @@ int inq_handler(iosystem_desc_t *ios) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d inq_handler\n", my_rank); + LOG((1, "%d inq_handler\n", my_rank)); /* Get the parameters for this function that the the comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d inq_handler1\n", my_rank); if ((mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - printf("%d inq_handler2\n", my_rank); if ((mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - printf("%d inq_handler3\n", my_rank); if ((mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - printf("%d inq_handler4\n", my_rank); if ((mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - printf("%d inq_handler ndims_present = %d nvars_present = %d ngatts_present = %d unlimdimid_present = %d\n", - my_rank, ndims_present, nvars_present, ngatts_present, unlimdimid_present); + LOG((1, "%d inq_handler ndims_present = %d nvars_present = %d ngatts_present = %d unlimdimid_present = %d\n", + ndims_present, nvars_present, ngatts_present, unlimdimid_present)); /* NULLs passed in to any of the pointers in the original call * need to be matched with NULLs here. Assign pointers where @@ -169,7 +163,7 @@ int inq_dim_handler(iosystem_desc_t *ios, int msg) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d inq_dim_handler\n", my_rank); + LOG((1, "inq_dim_handler\n", my_rank)); /* Get the parameters for this function that the the comp master * task is broadcasting. */ @@ -182,7 +176,7 @@ int inq_dim_handler(iosystem_desc_t *ios, int msg) if ((mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; printf("%d inq_handler name_present = %d len_present = %d\n", - my_rank, name_present, len_present); + name_present, len_present); /* Set the non-null pointers. */ if (name_present) @@ -215,26 +209,22 @@ int inq_dimid_handler(iosystem_desc_t *ios) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d inq_dimid_handler\n", my_rank); + LOG((1, "inq_dimid_handler\n", my_rank)); /* Get the parameters for this function that the the comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d inq_dimid_handler ncid = %d\n", my_rank, ncid); if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d inq_dimid_handler ncid = %d namelen = %d\n", my_rank, ncid, namelen); if (!(name = malloc((namelen + 1) * sizeof(char)))) return PIO_ENOMEM; if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - printf("%d inq_dimid_handler ncid = %d namelen = %d name = %s\n", - my_rank, ncid, namelen, name); if ((mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - printf("%d inq_dimid_handler ncid = %d namelen = %d name = %s id_present = %d\n", - my_rank, ncid, namelen, name, id_present); + LOG((1, "%d inq_dimid_handler ncid = %d namelen = %d name = %s id_present = %d\n", + ncid, namelen, name, id_present)); /* Set non-null pointer. */ if (id_present) @@ -271,7 +261,7 @@ int inq_att_handler(iosystem_desc_t *ios, int msg) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d inq_att_handler\n", my_rank); + LOG((1, "inq_att_handler\n", my_rank)); /* Get the parameters for this function that the the comp master * task is broadcasting. */ @@ -312,7 +302,7 @@ int att_handler(iosystem_desc_t *ios, int msg) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d att_handler\n", my_rank); + LOG((1, "%d att_handler\n", my_rank)); if (msg == PIO_MSG_PUT_ATT_INT) { @@ -362,7 +352,6 @@ int att_handler(iosystem_desc_t *ios, int msg) /* Call the function to read the attribute. */ if ((ret = PIOc_get_att_int(ncid, varid, name5, ip))) return ret; - printf("%d att_handler got att with first element %d\n", my_rank, ip[0]); /* Free resources. */ free(name5); @@ -393,7 +382,7 @@ int inq_var_handler(iosystem_desc_t *ios) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d inq_var_handler\n", my_rank); + LOG((1, "%d inq_var_handler\n", my_rank)); /* Get the parameters for this function that the the comp master * task is broadcasting. */ @@ -483,13 +472,13 @@ int sync_file_handler(iosystem_desc_t *ios) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d sync_file_handler\n", my_rank); + LOG((1, "%d sync_file_handler\n", my_rank)); /* Get the parameters for this function that the comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d sync_file_handler got parameter ncid = %d\n", my_rank, ncid); + LOG((1, "%d sync_file_handler got parameter ncid = %d\n", my_rank, ncid)); /* Call the sync file function. */ if ((ret = PIOc_sync(ncid))) @@ -512,19 +501,18 @@ int enddef_file_handler(iosystem_desc_t *ios) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d enddef_file_handler\n", my_rank); + LOG((1, "%d enddef_file_handler\n", my_rank)); /* Get the parameters for this function that the comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d enddef_file_handler got parameter ncid = %d\n", my_rank, ncid); /* Call the sync file function. */ if ((ret = PIOc_enddef(ncid))) return ret; - printf("%d enddef_file_handler succeeded!\n", my_rank); + LOG((1, "%d enddef_file_handler succeeded!\n", my_rank)); return PIO_NOERR; } @@ -546,7 +534,7 @@ int def_var_handler(iosystem_desc_t *ios) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d def_var_handler comproot = %d\n", my_rank, ios->comproot); + LOG((1, "%d def_var_handler comproot = %d\n", my_rank, ios->comproot)); /* Get the parameters for this function that the he comp master * task is broadcasting. */ @@ -567,9 +555,9 @@ int def_var_handler(iosystem_desc_t *ios) return PIO_ENOMEM; if ((mpierr = MPI_Bcast(dimids, ndims, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d def_var_handler got parameters namelen = %d " + LOG((1, "%d def_var_handler got parameters namelen = %d " "name = %s len = %d ncid = %d\n", - my_rank, namelen, name, len, ncid); + my_rank, namelen, name, len, ncid)); /* Call the create file function. */ if ((ret = PIOc_def_var(ncid, name, xtype, ndims, dimids, &varid))) @@ -579,7 +567,7 @@ int def_var_handler(iosystem_desc_t *ios) free(name); free(dimids); - printf("%d def_var_handler succeeded!\n", my_rank); + LOG((1, "%d def_var_handler succeeded!\n", my_rank)); return PIO_NOERR; } @@ -598,7 +586,7 @@ int def_dim_handler(iosystem_desc_t *ios) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d def_dim_handler comproot = %d\n", my_rank, ios->comproot); + LOG((1, "%d def_dim_handler comproot = %d\n", my_rank, ios->comproot)); /* Get the parameters for this function that the he comp master * task is broadcasting. */ @@ -626,7 +614,7 @@ int def_dim_handler(iosystem_desc_t *ios) /* Free resources. */ free(name); - printf("%d def_dim_handler succeeded!\n", my_rank); + LOG((1, "%d def_dim_handler succeeded!\n", my_rank)); return PIO_NOERR; } @@ -656,7 +644,7 @@ int vara_handler(iosystem_desc_t *ios, int msg) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d def_var_handler comproot = %d\n", my_rank, ios->comproot); + LOG((1, "%d def_var_handler comproot = %d\n", my_rank, ios->comproot)); if (msg == PIO_MSG_PUT_VARA) { @@ -703,7 +691,7 @@ int vara_handler(iosystem_desc_t *ios, int msg) } - printf("%d vara_handler succeeded!\n", my_rank); + LOG((1, "%d vara_handler succeeded!\n", my_rank)); return PIO_NOERR; } @@ -724,7 +712,7 @@ int open_file_handler(iosystem_desc_t *ios) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d open_file_handler comproot = %d\n", my_rank, ios->comproot); + LOG((1, "%d open_file_handler comproot = %d\n", my_rank, ios->comproot)); /* Get the parameters for this function that the he comp master * task is broadcasting. */ @@ -740,9 +728,9 @@ int open_file_handler(iosystem_desc_t *ios) return PIO_EIO; if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d open_file_handler got parameters len = %d " + LOG((1, "%d open_file_handler got parameters len = %d " "filename = %s iotype = %d mode = %d\n", - my_rank, len, filename, iotype, mode); + my_rank, len, filename, iotype, mode)); /* Call the open file function. */ if ((ret = PIOc_openfile(ios->iosysid, &ncid, &iotype, filename, mode))) @@ -751,7 +739,7 @@ int open_file_handler(iosystem_desc_t *ios) /* Free resources. */ free(filename); - printf("%d open_file_handler succeeded!\n", my_rank); + LOG((1, "%d open_file_handler succeeded!\n", my_rank)); return PIO_NOERR; } @@ -770,20 +758,19 @@ int delete_file_handler(iosystem_desc_t *ios) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d delete_file_handler comproot = %d\n", my_rank, ios->comproot); + LOG((1, "%d delete_file_handler comproot = %d\n", my_rank, ios->comproot)); /* Get the parameters for this function that the he comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d open_file_handler got parameter len = %d\n", my_rank, len); if (!(filename = malloc(len + 1 * sizeof(char)))) return PIO_ENOMEM; if ((mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - printf("%d delete_file_handler got parameters len = %d filename = %s\n", - my_rank, len, filename); + LOG((1, "%d delete_file_handler got parameters len = %d filename = %s\n", + my_rank, len, filename)); /* Call the delete file function. */ if ((ret = PIOc_deletefile(ios->iosysid, filename))) @@ -792,7 +779,7 @@ int delete_file_handler(iosystem_desc_t *ios) /* Free resources. */ free(filename); - printf("%d delete_file_handler succeeded!\n", my_rank); + LOG((1, "%d delete_file_handler succeeded!\n", my_rank)); return PIO_NOERR; } @@ -830,7 +817,7 @@ int finalize_handler(iosystem_desc_t *ios) { int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d finalize_handler called\n", my_rank); + LOG((1, "%d finalize_handler called\n", my_rank)); return PIO_NOERR; } @@ -853,7 +840,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d pio_msg_handler called\n", my_rank); + LOG((1, "%d pio_msg_handler called\n", my_rank)); /* Have IO comm rank 0 (the ioroot) register to receive * (non-blocking) for a message from each of the comproots. */ @@ -862,7 +849,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) for (int cmp = 0; cmp < component_count; cmp++) { my_iosys = &iosys[cmp]; - printf("%d about to call MPI_Irecv\n", my_rank); + LOG((1, "%d about to call MPI_Irecv\n", my_rank)); mpierr = MPI_Irecv(&msg, 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG, my_iosys->union_comm, &req[cmp]); CheckMPIReturn(mpierr, __FILE__, __LINE__); @@ -875,8 +862,8 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) /* Wait until any one of the requests are complete. */ if (!io_rank) { - printf("%d about to call MPI_Waitany req[0] = %d MPI_REQUEST_NULL = %d\n", - my_rank, req[0], MPI_REQUEST_NULL); + LOG((1, "%d about to call MPI_Waitany req[0] = %d MPI_REQUEST_NULL = %d\n", + my_rank, req[0], MPI_REQUEST_NULL)); mpierr = MPI_Waitany(component_count, req, &index, &status); CheckMPIReturn(mpierr, __FILE__, __LINE__); printf("%d Waitany returned index = %d req[%d] = %d\n", my_rank, diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 144a5a2095f..45dd6c4ea38 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -276,62 +276,69 @@ int PIOc_inq_natts(int ncid, int *ngattsp) } /** - * @ingroup PIOc_put_att_short - * The PIO-C interface for the NetCDF function nc_put_att_short. + * @ingroup PIOc_inq_dim + * The PIO-C interface for the NetCDF function nc_inq_dim. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. + * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op) +int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) { - int ierr; - int msg; - int mpierr; iosystem_desc_t *ios; file_desc_t *file; - char *errstr; + char *errstr = NULL; + int ierr = PIO_NOERR; + int msg = PIO_MSG_INQ_DIM; + int mpierr; - errstr = NULL; - ierr = PIO_NOERR; + LOG((1, "PIOc_inq_dim")); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Get the file info, based on the ncid. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_SHORT; - if(ios->async_interface && ! ios->ioproc){ + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface && !ios->ioproc) + { + char name_present = name ? true : false; + char len_present = lenp ? true : false; if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_inq netcdf Bcast name_present = %d", name_present)); + mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_inq netcdf Bcast len_present = %d", len_present)); } - + /* Make the call to the netCDF layer. */ if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; + if (ios->io_rank == 0){ + ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_short(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);; break; #endif default: @@ -339,18 +346,31 @@ int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PI } } + /* Error handling. */ if(ierr != PIO_NOERR){ errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + + /* BCast the results, if non-null pointers were passed. */ + if(name) + { + int slen; + if(ios->iomaster) + slen = strlen(name); + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(lenp != NULL) + mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_rename_dim - * The PIO-C interface for the NetCDF function nc_rename_dim. + * @ingroup PIOc_inq_dimname + * The PIO-C interface for the NetCDF function nc_inq_dimname. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -361,49 +381,96 @@ int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PI * PIOc_openfile() or PIOc_createfile(). * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_rename_dim (int ncid, int dimid, const char *name) +int PIOc_inq_dimname(int ncid, int dimid, char *name) { - int ierr; - int msg; - int mpierr; + return PIOc_inq_dim(ncid, dimid, name, NULL); +} + +/** + * @ingroup PIOc_inq_dimlen + * The PIO-C interface for the NetCDF function nc_inq_dimlen. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param lenp a pointer that will get the number of values + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp) +{ + return PIOc_inq_dim(ncid, dimid, NULL, lenp); +} + +/** + * @ingroup PIOc_inq_dimid + * The PIO-C interface for the NetCDF function nc_inq_dimid. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param idp a pointer that will get the id of the variable or attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_dimid(int ncid, const char *name, int *idp) +{ + int msg = PIO_MSG_INQ_DIMID; iosystem_desc_t *ios; file_desc_t *file; - char *errstr; + char *errstr = NULL; + int ierr = PIO_NOERR; + int mpierr; - errstr = NULL; - ierr = PIO_NOERR; + /* For debugging purposes only... */ + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_inq_dimid\n", my_rank); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Get the file info, based on the ncid. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_RENAME_DIM; - if(ios->async_interface && ! ios->ioproc){ + /* If using async, and not an IO task, then send parameters. */ + if (ios->async_interface && !ios->ioproc) + { + int namelen; + char id_present = idp ? true : false; if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + namelen = strlen(name); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); } - + /* IO tasks call the netCDF functions. */ if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_dim(file->fh, dimid, name);; + ierr = nc_inq_dimid(file->fh, name, idp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_rename_dim(file->fh, dimid, name);; + ierr = nc_inq_dimid(file->fh, name, idp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_dim(file->fh, dimid, name);; + ierr = ncmpi_inq_dimid(file->fh, name, idp);; break; #endif default: @@ -416,67 +483,92 @@ int PIOc_rename_dim (int ncid, int dimid, const char *name) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (idp) + mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_double - * The PIO-C interface for the NetCDF function nc_get_att_double. + * @ingroup PIOc_inq_var + * The PIO-C interface for the NetCDF function nc_inq_var. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @param nattsp a pointer that will get the number of attributes * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) +int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, + int *dimidsp, int *nattsp) { - int ierr; - int msg; - int mpierr; iosystem_desc_t *ios; file_desc_t *file; - char *errstr; + char *errstr = NULL; + int ndims; /** The number of dimensions for this variable. */ + int ierr = PIO_NOERR; + int msg = PIO_MSG_INQ_VAR; + int mpierr; - errstr = NULL; - ierr = PIO_NOERR; + LOG((1, "PIOc_inq_var\n")); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Get the file info, based on the ncid. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_DOUBLE; - if(ios->async_interface && ! ios->ioproc){ + /* If using async, and this is not an IO task, send the parameters to the IO task. */ + if (ios->async_interface && !ios->ioproc) + { + char name_present = name ? true : false; + char xtype_present = xtypep ? true : false; + char ndims_present = ndimsp ? true : false; + char dimids_present = dimidsp ? true : false; + char natts_present = nattsp ? true : false; if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d " + "dimids_present = %d, natts_present = %d nattsp = %d", + name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp)); } - - if(ios->ioproc){ + /* Call the netCDF layer. */ + if (ios->ioproc) + { switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_double(file->fh, varid, name, ip);; + ierr = nc_inq_varndims(file->fh, varid, &ndims); + ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp); break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_double(file->fh, varid, name, ip);; + if (ios->io_rank == 0) + { + ierr = nc_inq_varndims(file->fh, varid, &ndims); + ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_double(file->fh, varid, name, ip);; + ierr = ncmpi_inq_varndims(file->fh, varid, &ndims); + ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; break; #endif default: @@ -485,94 +577,73 @@ int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_DOUBLE, ios->ioroot, ios->my_comm); + printf("%d called netcdf nc_inq_var ierr = %d\n", my_rank, ierr); + + /* Broadcast the results for non-null pointers. */ + if (name) + { + int slen; + if(ios->iomaster) + slen = strlen(name); + printf("%d PIOc_inq_var slen = %d\n", my_rank, slen); + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + printf("%d PIOc_inq_var slen = %d\n", my_rank, slen); + mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm); + printf("%d PIOc_inq_var name = %s\n", my_rank, name); + } + if (xtypep) + { + mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); + printf("%d PIOc_inq_var xtype = %d\n", my_rank, *xtypep); + } + if (ndimsp) + { + printf("%d PIOc_inq_var ndims = %d\n", my_rank, *ndimsp); + mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + file->varlist[varid].ndims = (*ndimsp); + printf("%d PIOc_inq_var bcast complete ndims = %d\n", my_rank, *ndimsp); + } + if (dimidsp) + { + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm); + } + if (nattsp) + { + printf("%d PIOc_inq_var natts = %d\n", my_rank, *nattsp); + mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + printf("%d PIOc_inq_var natts = %d\n", my_rank, *nattsp); } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_set_fill - * The PIO-C interface for the NetCDF function nc_set_fill. + * @ingroup PIOc_inq_varname + * The PIO-C interface for the NetCDF function nc_inq_varname. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_set_fill (int ncid, int fillmode, int *old_modep) +int PIOc_inq_varname (int ncid, int varid, char *name) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_SET_FILL; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_set_fill(file->fh, fillmode, old_modep);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_set_fill(file->fh, fillmode, old_modep);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_set_fill(file->fh, fillmode, old_modep);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_inq_var(ncid, varid, name, NULL, NULL, NULL, NULL); } /** - * @ingroup PIOc_def_var - * The PIO-C interface for the NetCDF function nc_def_var. + * @ingroup PIOc_inq_vartype + * The PIO-C interface for the NetCDF function nc_inq_vartype. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -582,105 +653,88 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param varidp a pointer that will get the variable id - * @return PIO_NOERR for success, error code otherwise. See - * PIOc_Set_File_Error_Handling + * @param xtypep a pointer that will get the type of the attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, - const int *dimidsp, int *varidp) +int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - int namelen; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_DEF_VAR; + return PIOc_inq_var(ncid, varid, NULL, xtypep, NULL, NULL, NULL); +} - /* If async is in use, and this is not an IO tasks, then bcast the - * function parameters to the intercomm, where it will be read by - * the IO root task. */ - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm); - namelen = strlen(name); - LOG((2, "bcasting namelen = %d name = %s\n", namelen, name)); - if (!ios->compmaster) - ios->compmaster = MPI_PROC_NULL; - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmaster, ios->intercomm); - } +/** + * @ingroup PIOc_inq_varndims + * The PIO-C interface for the NetCDF function nc_inq_varndims. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) +{ + return PIOc_inq_var(ncid, varid, NULL, NULL, ndimsp, NULL, NULL); +} - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; - break; - case PIO_IOTYPE_NETCDF4C: - if(ios->io_rank==0){ - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); - if(ierr == PIO_NOERR){ - ierr = nc_def_var_deflate(file->fh, *varidp, 0,1,1); - } - } - break; -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } +/** + * @ingroup PIOc_inq_vardimid + * The PIO-C interface for the NetCDF function nc_inq_vardimid. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_vardimid(int ncid, int varid, int *dimidsp) +{ + return PIOc_inq_var(ncid, varid, NULL, NULL, NULL, dimidsp, NULL); +} - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; +/** + * @ingroup PIOc_inq_varnatts + * The PIO-C interface for the NetCDF function nc_inq_varnatts. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param nattsp a pointer that will get the number of attributes + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) +{ + return PIOc_inq_var(ncid, varid, NULL, NULL, NULL, NULL, nattsp); } /** - * @ingroup PIOc_put_att_double - * The PIO-C interface for the NetCDF function nc_put_att_double. + * @ingroup PIOc_inq_varid + * The PIO-C interface for the NetCDF function nc_inq_varid. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. + * @param varidp a pointer that will get the variable id * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op) +int PIOc_inq_varid (int ncid, const char *name, int *varidp) { int ierr; int msg; @@ -696,33 +750,39 @@ int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, P if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_DOUBLE; + msg = PIO_MSG_INQ_VARID; + + LOG((1, "PIOc_inq_varid ncid = %d name = %s", ncid, name)); if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + printf("%d PIOc_inq_varid BCast ncid = %d\n", my_rank, ncid); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen; + namelen = strlen(name); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); } - if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_varid(file->fh, name, varidp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_varid(file->fh, name, varidp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_double(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_inq_varid(file->fh, name, varidp);; break; #endif default: @@ -735,74 +795,79 @@ int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, P sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if (varidp) + mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_dim - * The PIO-C interface for the NetCDF function nc_inq_dim. + * @ingroup PIOc_inq_att + * The PIO-C interface for the NetCDF function nc_inq_att. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) +int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, + PIO_Offset *lenp) { + int ierr; + int msg; + int mpierr; iosystem_desc_t *ios; file_desc_t *file; - char *errstr = NULL; - int ierr = PIO_NOERR; - int msg = PIO_MSG_INQ_DIM; - int mpierr; + char *errstr; - LOG((1, "PIOc_inq_dim")); + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_inq_att ncid = %d varid = %d xtpyep = %d lenp = %d\n", + my_rank, ncid, varid, xtypep, lenp); + + errstr = NULL; + ierr = PIO_NOERR; - /* Get the file info, based on the ncid. */ if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; + msg = PIO_MSG_INQ_ATT; - /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface && !ios->ioproc) - { - char name_present = name ? true : false; - char len_present = lenp ? true : false; + if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq netcdf Bcast name_present = %d", name_present)); - mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq netcdf Bcast len_present = %d", len_present)); + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen = strlen(name); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); } - /* Make the call to the netCDF layer. */ if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: - if (ios->io_rank == 0){ - ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; + if(ios->io_rank==0){ + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);; + ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);; break; #endif default: @@ -810,68 +875,64 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) } } - /* Error handling. */ if(ierr != PIO_NOERR){ errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - - /* BCast the results, if non-null pointers were passed. */ - if(name) - { - int slen; - if(ios->iomaster) - slen = strlen(name); - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(lenp != NULL) - mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + if(xtypep) + mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(lenp) + mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_dimname - * The PIO-C interface for the NetCDF function nc_inq_dimname. + * @ingroup PIOc_inq_attlen + * The PIO-C interface for the NetCDF function nc_inq_attlen. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_dimname(int ncid, int dimid, char *name) +int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) { - return PIOc_inq_dim(ncid, dimid, name, NULL); + nc_type dummy; + return PIOc_inq_att(ncid, varid, name, &dummy, lenp); } /** - * @ingroup PIOc_inq_dimlen - * The PIO-C interface for the NetCDF function nc_inq_dimlen. + * @ingroup PIOc_inq_atttype + * The PIO-C interface for the NetCDF function nc_inq_atttype. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param lenp a pointer that will get the number of values + * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp) +int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep) { - return PIOc_inq_dim(ncid, dimid, NULL, lenp); + PIO_Offset dummy; + return PIOc_inq_att(ncid, varid, name, xtypep, &dummy); } /** - * @ingroup PIOc_get_att_uchar - * The PIO-C interface for the NetCDF function nc_get_att_uchar. + * @ingroup PIOc_inq_attname + * The PIO-C interface for the NetCDF function nc_inq_attname. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -881,9 +942,10 @@ int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp) * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. + * @param attnum the attribute ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip) +int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) { int ierr; int msg; @@ -899,7 +961,7 @@ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_UCHAR; + msg = PIO_MSG_INQ_ATTNAME; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -913,19 +975,19 @@ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_uchar(file->fh, varid, name, ip);; + ierr = nc_inq_attname(file->fh, varid, attnum, name);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_uchar(file->fh, varid, name, ip);; + ierr = nc_inq_attname(file->fh, varid, attnum, name);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_uchar(file->fh, varid, name, ip);; + ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);; break; #endif default: @@ -934,34 +996,37 @@ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_CHAR, ios->ioroot, ios->my_comm); + if(name != NULL){ + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_var_fill - * The PIO-C interface for the NetCDF function nc_inq_var_fill. + * @ingroup PIOc_inq_attid + * The PIO-C interface for the NetCDF function nc_inq_attid. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. + * @param idp a pointer that will get the id of the variable or attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) +int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) { int ierr; int msg; @@ -977,7 +1042,7 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VAR_FILL; + msg = PIO_MSG_INQ_ATTID; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -991,19 +1056,19 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; + ierr = nc_inq_attid(file->fh, varid, name, idp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; + ierr = nc_inq_attid(file->fh, varid, name, idp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_value);; + ierr = ncmpi_inq_attid(file->fh, varid, name, idp);; break; #endif default: @@ -1016,14 +1081,14 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(fill_value, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_attid - * The PIO-C interface for the NetCDF function nc_inq_attid. + * @ingroup PIOc_put_att_short + * The PIO-C interface for the NetCDF function nc_put_att_short. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1033,10 +1098,9 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param idp a pointer that will get the id of the variable or attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) +int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op) { int ierr; int msg; @@ -1052,7 +1116,7 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATTID; + msg = PIO_MSG_PUT_ATT_SHORT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1066,19 +1130,19 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_attid(file->fh, varid, name, idp);; + ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_attid(file->fh, varid, name, idp);; + ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_attid(file->fh, varid, name, idp);; + ierr = ncmpi_put_att_short(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -1091,26 +1155,24 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_schar - * The PIO-C interface for the NetCDF function nc_put_att_schar. + * @ingroup PIOc_rename_dim + * The PIO-C interface for the NetCDF function nc_rename_dim. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op) +int PIOc_rename_dim (int ncid, int dimid, const char *name) { int ierr; int msg; @@ -1126,7 +1188,7 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_SCHAR; + msg = PIO_MSG_RENAME_DIM; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1140,19 +1202,19 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_rename_dim(file->fh, dimid, name);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_rename_dim(file->fh, dimid, name);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_schar(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_rename_dim(file->fh, dimid, name);; break; #endif default: @@ -1170,8 +1232,8 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI } /** - * @ingroup PIOc_get_att_ushort - * The PIO-C interface for the NetCDF function nc_get_att_ushort. + * @ingroup PIOc_get_att_double + * The PIO-C interface for the NetCDF function nc_get_att_double. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1183,7 +1245,7 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip) +int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) { int ierr; int msg; @@ -1199,7 +1261,7 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_USHORT; + msg = PIO_MSG_GET_ATT_DOUBLE; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1213,19 +1275,19 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_ushort(file->fh, varid, name, ip);; + ierr = nc_get_att_double(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_ushort(file->fh, varid, name, ip);; + ierr = nc_get_att_double(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_ushort(file->fh, varid, name, ip);; + ierr = ncmpi_get_att_double(file->fh, varid, name, ip);; break; #endif default: @@ -1241,28 +1303,26 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * if(ierr == PIO_NOERR){ PIO_Offset attlen; PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_SHORT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_DOUBLE, ios->ioroot, ios->my_comm); } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_varid - * The PIO-C interface for the NetCDF function nc_inq_varid. + * @ingroup PIOc_set_fill + * The PIO-C interface for the NetCDF function nc_set_fill. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param varidp a pointer that will get the variable id * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_varid (int ncid, const char *name, int *varidp) +int PIOc_set_fill (int ncid, int fillmode, int *old_modep) { int ierr; int msg; @@ -1278,39 +1338,33 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARID; - - LOG((1, "PIOc_inq_varid ncid = %d name = %s", ncid, name)); + msg = PIO_MSG_SET_FILL; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - printf("%d PIOc_inq_varid BCast ncid = %d\n", my_rank, ncid); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - int namelen; - namelen = strlen(name); - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } + if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varid(file->fh, name, varidp);; + ierr = nc_set_fill(file->fh, fillmode, old_modep);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_varid(file->fh, name, varidp);; + ierr = nc_set_fill(file->fh, fillmode, old_modep);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varid(file->fh, name, varidp);; + ierr = ncmpi_set_fill(file->fh, fillmode, old_modep);; break; #endif default: @@ -1323,69 +1377,123 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (varidp) - mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_attlen - * The PIO-C interface for the NetCDF function nc_inq_attlen. + * @ingroup PIOc_def_var + * The PIO-C interface for the NetCDF function nc_def_var. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param lenp a pointer that will get the number of values - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + * @param varidp a pointer that will get the variable id + * @return PIO_NOERR for success, error code otherwise. See + * PIOc_Set_File_Error_Handling */ -int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) +int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, + const int *dimidsp, int *varidp) { - nc_type dummy; - return PIOc_inq_att(ncid, varid, name, &dummy, lenp); -} + int msg = PIO_MSG_DEF_VAR; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + int ierr = PIO_NOERR; + char *errstr = NULL; + int namelen; -/** - * @ingroup PIOc_inq_atttype - * The PIO-C interface for the NetCDF function nc_inq_atttype. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep) -{ - PIO_Offset dummy; - return PIOc_inq_att(ncid, varid, name, xtypep, &dummy); + /* Get the file information. */ + if (!(file = pio_get_file_from_id(ncid))) + return PIO_EBADID; + ios = file->iosystem; + + /* If async is in use, and this is not an IO tasks, then bcast the + * function parameters to the intercomm, where it will be read by + * the IO root task. */ + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm); + namelen = strlen(name); + LOG((2, "bcasting namelen = %d name = %s\n", namelen, name)); + if (!ios->compmaster) + ios->compmaster = MPI_PROC_NULL; + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmaster, ios->intercomm); + } + + if(ios->ioproc){ + switch(file->iotype) + { +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; + break; + case PIO_IOTYPE_NETCDF4C: + if (ios->io_rank == 0) + { + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); + if (!ierr) + ierr = nc_def_var_deflate(file->fh, *varidp, 0,1,1); + } + break; +#endif + case PIO_IOTYPE_NETCDF: + if (ios->io_rank == 0) + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + /* Handle errors. */ + if (ierr) + { + errstr = (char *)malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr, "in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm); + + /* Free error string if allocated. */ + if(errstr) + free(errstr); + + return ierr; } /** - * @ingroup PIOc_rename_var - * The PIO-C interface for the NetCDF function nc_rename_var. + * @ingroup PIOc_put_att_double + * The PIO-C interface for the NetCDF function nc_put_att_double. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_rename_var (int ncid, int varid, const char *name) +int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op) { int ierr; int msg; @@ -1401,7 +1509,7 @@ int PIOc_rename_var (int ncid, int varid, const char *name) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_RENAME_VAR; + msg = PIO_MSG_PUT_ATT_DOUBLE; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1415,19 +1523,19 @@ int PIOc_rename_var (int ncid, int varid, const char *name) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_var(file->fh, varid, name);; + ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_rename_var(file->fh, varid, name);; + ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_var(file->fh, varid, name);; + ierr = ncmpi_put_att_double(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -1445,8 +1553,8 @@ int PIOc_rename_var (int ncid, int varid, const char *name) } /** - * @ingroup PIOc_put_att_ulonglong - * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. + * @ingroup PIOc_get_att_uchar + * The PIO-C interface for the NetCDF function nc_get_att_uchar. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1458,7 +1566,7 @@ int PIOc_rename_var (int ncid, int varid, const char *name) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op) +int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip) { int ierr; int msg; @@ -1474,7 +1582,7 @@ int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_ULONGLONG; + msg = PIO_MSG_GET_ATT_UCHAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1488,19 +1596,19 @@ int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_get_att_uchar(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_get_att_uchar(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_ulonglong(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_get_att_uchar(file->fh, varid, name, ip);; break; #endif default: @@ -1509,17 +1617,22 @@ int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_CHAR, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_var - * The PIO-C interface for the NetCDF function nc_inq_var. + * @ingroup PIOc_inq_var_fill + * The PIO-C interface for the NetCDF function nc_inq_var_fill. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1529,74 +1642,51 @@ int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @param nattsp a pointer that will get the number of attributes * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, - int *dimidsp, int *nattsp) +int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) { + int ierr; + int msg; + int mpierr; iosystem_desc_t *ios; file_desc_t *file; - char *errstr = NULL; - int ndims; /** The number of dimensions for this variable. */ - int ierr = PIO_NOERR; - int msg = PIO_MSG_INQ_VAR; - int mpierr; + char *errstr; - LOG((1, "PIOc_inq_var\n")); + errstr = NULL; + ierr = PIO_NOERR; - /* Get the file info, based on the ncid. */ - if (!(file = pio_get_file_from_id(ncid))) + file = pio_get_file_from_id(ncid); + if(file == NULL) return PIO_EBADID; ios = file->iosystem; + msg = PIO_MSG_INQ_VAR_FILL; - /* If using async, and this is not an IO task, send the parameters to the IO task. */ - if (ios->async_interface && !ios->ioproc) - { - char name_present = name ? true : false; - char xtype_present = xtypep ? true : false; - char ndims_present = ndimsp ? true : false; - char dimids_present = dimidsp ? true : false; - char natts_present = nattsp ? true : false; + if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d " - "dimids_present = %d, natts_present = %d nattsp = %d", - name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp)); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } - /* Call the netCDF layer. */ - if (ios->ioproc) - { + + if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varndims(file->fh, varid, &ndims); - ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp); + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: - if (ios->io_rank == 0) - { - ierr = nc_inq_varndims(file->fh, varid, &ndims); - ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + if(ios->io_rank==0){ + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varndims(file->fh, varid, &ndims); - ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_value);; break; #endif default: @@ -1609,147 +1699,87 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - printf("%d called netcdf nc_inq_var ierr = %d\n", my_rank, ierr); - - /* Broadcast the results for non-null pointers. */ - if (name) - { - int slen; - if(ios->iomaster) - slen = strlen(name); - printf("%d PIOc_inq_var slen = %d\n", my_rank, slen); - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - printf("%d PIOc_inq_var slen = %d\n", my_rank, slen); - mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm); - printf("%d PIOc_inq_var name = %s\n", my_rank, name); - } - if (xtypep) - { - mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); - printf("%d PIOc_inq_var xtype = %d\n", my_rank, *xtypep); - } - if (ndimsp) - { - printf("%d PIOc_inq_var ndims = %d\n", my_rank, *ndimsp); - mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - file->varlist[varid].ndims = (*ndimsp); - printf("%d PIOc_inq_var bcast complete ndims = %d\n", my_rank, *ndimsp); - } - if (dimidsp) - { - mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm); - } - if (nattsp) - { - printf("%d PIOc_inq_var natts = %d\n", my_rank, *nattsp); - mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - printf("%d PIOc_inq_var natts = %d\n", my_rank, *nattsp); - } + mpierr = MPI_Bcast(fill_value, 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_varname - * The PIO-C interface for the NetCDF function nc_inq_varname. + * @ingroup PIOc_put_att_schar + * The PIO-C interface for the NetCDF function nc_put_att_schar. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_varname (int ncid, int varid, char *name) +int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op) { - return PIOc_inq_var(ncid, varid, name, NULL, NULL, NULL, NULL); -} + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; -/** - * @ingroup PIOc_inq_vartype - * The PIO-C interface for the NetCDF function nc_inq_vartype. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) -{ - return PIOc_inq_var(ncid, varid, NULL, xtypep, NULL, NULL, NULL); -} + errstr = NULL; + ierr = PIO_NOERR; -/** - * @ingroup PIOc_inq_varndims - * The PIO-C interface for the NetCDF function nc_inq_varndims. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) -{ - return PIOc_inq_var(ncid, varid, NULL, NULL, ndimsp, NULL, NULL); -} + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_SCHAR; -/** - * @ingroup PIOc_inq_vardimid - * The PIO-C interface for the NetCDF function nc_inq_vardimid. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_vardimid(int ncid, int varid, int *dimidsp) -{ - return PIOc_inq_var(ncid, varid, NULL, NULL, NULL, dimidsp, NULL); -} + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } -/** - * @ingroup PIOc_inq_varnatts - * The PIO-C interface for the NetCDF function nc_inq_varnatts. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param nattsp a pointer that will get the number of attributes - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) -{ - return PIOc_inq_var(ncid, varid, NULL, NULL, NULL, NULL, nattsp); + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att_schar(file->fh, varid, name, xtype, len, op);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; } /** - * @ingroup PIOc_rename_att - * The PIO-C interface for the NetCDF function nc_rename_att. + * @ingroup PIOc_get_att_ushort + * The PIO-C interface for the NetCDF function nc_get_att_ushort. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1761,7 +1791,7 @@ int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) +int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip) { int ierr; int msg; @@ -1777,7 +1807,7 @@ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_RENAME_ATT; + msg = PIO_MSG_GET_ATT_USHORT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1791,19 +1821,19 @@ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_att(file->fh, varid, name, newname);; + ierr = nc_get_att_ushort(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_rename_att(file->fh, varid, name, newname);; + ierr = nc_get_att_ushort(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_att(file->fh, varid, name, newname);; + ierr = ncmpi_get_att_ushort(file->fh, varid, name, ip);; break; #endif default: @@ -1812,29 +1842,34 @@ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_SHORT, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_ushort - * The PIO-C interface for the NetCDF function nc_put_att_ushort. + * @ingroup PIOc_rename_var + * The PIO-C interface for the NetCDF function nc_rename_var. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op) +int PIOc_rename_var (int ncid, int varid, const char *name) { int ierr; int msg; @@ -1850,7 +1885,7 @@ int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, P if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_USHORT; + msg = PIO_MSG_RENAME_VAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1864,19 +1899,19 @@ int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, P #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_rename_var(file->fh, varid, name);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_rename_var(file->fh, varid, name);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_ushort(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_rename_var(file->fh, varid, name);; break; #endif default: @@ -1894,71 +1929,62 @@ int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, P } /** - * @ingroup PIOc_inq_dimid - * The PIO-C interface for the NetCDF function nc_inq_dimid. + * @ingroup PIOc_put_att_ulonglong + * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param idp a pointer that will get the id of the variable or attribute. + * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_dimid(int ncid, const char *name, int *idp) +int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op) { - int msg = PIO_MSG_INQ_DIMID; + int ierr; + int msg; + int mpierr; iosystem_desc_t *ios; file_desc_t *file; - char *errstr = NULL; - int ierr = PIO_NOERR; - int mpierr; + char *errstr; - /* For debugging purposes only... */ - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d PIOc_inq_dimid\n", my_rank); + errstr = NULL; + ierr = PIO_NOERR; - /* Get the file info, based on the ncid. */ - if (!(file = pio_get_file_from_id(ncid))) + file = pio_get_file_from_id(ncid); + if(file == NULL) return PIO_EBADID; ios = file->iosystem; + msg = PIO_MSG_PUT_ATT_ULONGLONG; - /* If using async, and not an IO task, then send parameters. */ - if (ios->async_interface && !ios->ioproc) - { - int namelen; - char id_present = idp ? true : false; + if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - namelen = strlen(name); - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } - /* IO tasks call the netCDF functions. */ + if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dimid(file->fh, name, idp);; + ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_dimid(file->fh, name, idp);; + ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dimid(file->fh, name, idp);; + ierr = ncmpi_put_att_ulonglong(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -1971,15 +1997,13 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if (idp) - mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_text - * The PIO-C interface for the NetCDF function nc_put_att_text. + * @ingroup PIOc_rename_att + * The PIO-C interface for the NetCDF function nc_rename_att. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1991,7 +2015,7 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, const char *op) +int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) { int ierr; int msg; @@ -2007,7 +2031,7 @@ int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, co if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_TEXT; + msg = PIO_MSG_RENAME_ATT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2021,19 +2045,19 @@ int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, co #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; + ierr = nc_rename_att(file->fh, varid, name, newname);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; + ierr = nc_rename_att(file->fh, varid, name, newname);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_text(file->fh, varid, name, len, op);; + ierr = ncmpi_rename_att(file->fh, varid, name, newname);; break; #endif default: @@ -2051,8 +2075,8 @@ int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, co } /** - * @ingroup PIOc_get_att_uint - * The PIO-C interface for the NetCDF function nc_get_att_uint. + * @ingroup PIOc_put_att_ushort + * The PIO-C interface for the NetCDF function nc_put_att_ushort. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2064,7 +2088,7 @@ int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, co * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) +int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op) { int ierr; int msg; @@ -2080,7 +2104,7 @@ int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_UINT; + msg = PIO_MSG_PUT_ATT_USHORT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2094,19 +2118,19 @@ int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_uint(file->fh, varid, name, ip);; + ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_uint(file->fh, varid, name, ip);; + ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_uint(file->fh, varid, name, ip);; + ierr = ncmpi_put_att_ushort(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -2115,34 +2139,29 @@ int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED, ios->ioroot, ios->my_comm); - } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_format - * The PIO-C interface for the NetCDF function nc_inq_format. + * @ingroup PIOc_put_att_text + * The PIO-C interface for the NetCDF function nc_put_att_text. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param formatp a pointer that will get the file format + * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_format (int ncid, int *formatp) +int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, const char *op) { int ierr; int msg; @@ -2158,7 +2177,7 @@ int PIOc_inq_format (int ncid, int *formatp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_FORMAT; + msg = PIO_MSG_PUT_ATT_TEXT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2172,19 +2191,19 @@ int PIOc_inq_format (int ncid, int *formatp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_format(file->fh, formatp);; + ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_format(file->fh, formatp);; + ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_format(file->fh, formatp);; + ierr = ncmpi_put_att_text(file->fh, varid, name, len, op);; break; #endif default: @@ -2197,14 +2216,13 @@ int PIOc_inq_format (int ncid, int *formatp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_long - * The PIO-C interface for the NetCDF function nc_get_att_long. + * @ingroup PIOc_get_att_uint + * The PIO-C interface for the NetCDF function nc_get_att_uint. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2216,7 +2234,7 @@ int PIOc_inq_format (int ncid, int *formatp) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) +int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) { int ierr; int msg; @@ -2232,7 +2250,7 @@ int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_LONG; + msg = PIO_MSG_GET_ATT_UINT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2246,19 +2264,19 @@ int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_long(file->fh, varid, name, ip);; + ierr = nc_get_att_uint(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_long(file->fh, varid, name, ip);; + ierr = nc_get_att_uint(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_long(file->fh, varid, name, ip);; + ierr = ncmpi_get_att_uint(file->fh, varid, name, ip);; break; #endif default: @@ -2274,28 +2292,27 @@ int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) if(ierr == PIO_NOERR){ PIO_Offset attlen; PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED, ios->ioroot, ios->my_comm); } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_attname - * The PIO-C interface for the NetCDF function nc_inq_attname. + * @ingroup PIOc_inq_format + * The PIO-C interface for the NetCDF function nc_inq_format. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param attnum the attribute ID. + * @param formatp a pointer that will get the file format * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) +int PIOc_inq_format (int ncid, int *formatp) { int ierr; int msg; @@ -2311,7 +2328,7 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATTNAME; + msg = PIO_MSG_INQ_FORMAT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2325,19 +2342,19 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_attname(file->fh, varid, attnum, name);; + ierr = nc_inq_format(file->fh, formatp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_attname(file->fh, varid, attnum, name);; + ierr = nc_inq_format(file->fh, formatp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);; + ierr = ncmpi_inq_format(file->fh, formatp);; break; #endif default: @@ -2350,20 +2367,14 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name) + 1; - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } + mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_att - * The PIO-C interface for the NetCDF function nc_inq_att. + * @ingroup PIOc_get_att_long + * The PIO-C interface for the NetCDF function nc_get_att_long. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2373,12 +2384,9 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, - PIO_Offset *lenp) +int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) { int ierr; int msg; @@ -2387,47 +2395,40 @@ int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, file_desc_t *file; char *errstr; - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d PIOc_inq_att ncid = %d varid = %d xtpyep = %d lenp = %d\n", - my_rank, ncid, varid, xtypep, lenp); - errstr = NULL; ierr = PIO_NOERR; - if (!(file = pio_get_file_from_id(ncid))) + file = pio_get_file_from_id(ncid); + if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATT; + msg = PIO_MSG_GET_ATT_LONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - int namelen = strlen(name); - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } + if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; + ierr = nc_get_att_long(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; + ierr = nc_get_att_long(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);; + ierr = ncmpi_get_att_long(file->fh, varid, name, ip);; break; #endif default: @@ -2436,14 +2437,15 @@ int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(xtypep) - mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(lenp) - mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } From b16d251b1798f3607c9a2ab19e38bde22aa781e6 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 10 May 2016 16:55:44 -0400 Subject: [PATCH 023/184] cleaning up error handling --- src/clib/pio_msg.c | 323 ++++++++++++++--------------- src/clib/pio_nc_async.c | 444 +++++++++++++++++++++------------------- 2 files changed, 401 insertions(+), 366 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 76a1f69ab6e..e30dc8dce2b 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -1,5 +1,5 @@ /** - * @file + * @file * @author Ed Hartnett * @date 2016 * @brief PIO async msg handling @@ -26,7 +26,7 @@ int create_file_handler(iosystem_desc_t *ios) int mode; int mpierr; int ret; - + LOG((1, "create_file_handler comproot = %d\n", ios->comproot)); /* Get the parameters for this function that the he comp master @@ -50,28 +50,28 @@ int create_file_handler(iosystem_desc_t *ios) /* Call the create file function. */ if ((ret = PIOc_createfile(ios->iosysid, &ncid, &iotype, filename, mode))) return ret; - + /* Free resources. */ free(filename); - + LOG((1, "create_file_handler succeeded!\n", my_rank)); return PIO_NOERR; } /** This function is run on the IO tasks to close a netCDF file. It is - * only ever run on the IO tasks. + * only ever run on the IO tasks. * - * @param ios pointer to the iosystem_desc_t. - * @return PIO_NOERR for success, error code otherwise. + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. */ int close_file_handler(iosystem_desc_t *ios) { int ncid; int mpierr; int ret; - + int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "%d close_file_handler\n", my_rank)); /* Get the parameters for this function that the the comp master @@ -83,16 +83,16 @@ int close_file_handler(iosystem_desc_t *ios) /* Call the close file function. */ if ((ret = PIOc_closefile(ncid))) return ret; - + LOG((1, "close_file_handler succeeded!\n", my_rank)); return PIO_NOERR; } /** This function is run on the IO tasks to inq a netCDF file. It is - * only ever run on the IO tasks. + * only ever run on the IO tasks. * - * @param ios pointer to the iosystem_desc_t. - * @return PIO_NOERR for success, error code otherwise. + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. */ int inq_handler(iosystem_desc_t *ios) { @@ -102,9 +102,9 @@ int inq_handler(iosystem_desc_t *ios) char ndims_present, nvars_present, ngatts_present, unlimdimid_present; int mpierr; int ret; - + int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "%d inq_handler\n", my_rank)); /* Get the parameters for this function that the the comp master @@ -137,22 +137,22 @@ int inq_handler(iosystem_desc_t *ios) /* Call the inq function to get the values. */ if ((ret = PIOc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp))) return ret; - + return PIO_NOERR; } /** Do an inq_dim on a netCDF dimension. This function is only run on * IO tasks. * - * @param ios pointer to the iosystem_desc_t. - * @param msg the message sent my the comp root task. - * @return PIO_NOERR for success, error code otherwise. + * @param ios pointer to the iosystem_desc_t. + * @param msg the message sent my the comp root task. + * @return PIO_NOERR for success, error code otherwise. */ int inq_dim_handler(iosystem_desc_t *ios, int msg) { int ncid; int dimid; - char name_present, len_present; + char name_present, len_present; char *dimnamep = NULL; PIO_Offset *dimlenp = NULL; char dimname[NC_MAX_NAME + 1]; @@ -160,9 +160,9 @@ int inq_dim_handler(iosystem_desc_t *ios, int msg) int mpierr; int ret; - + int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "inq_dim_handler\n", my_rank)); /* Get the parameters for this function that the the comp master @@ -187,15 +187,15 @@ int inq_dim_handler(iosystem_desc_t *ios, int msg) /* Call the inq function to get the values. */ if ((ret = PIOc_inq_dim(ncid, dimid, dimnamep, dimlenp))) return ret; - + return PIO_NOERR; } /** Do an inq_dimid on a netCDF dimension name. This function is only * run on IO tasks. * - * @param ios pointer to the iosystem_desc_t. - * @return PIO_NOERR for success, error code otherwise. + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. */ int inq_dimid_handler(iosystem_desc_t *ios) { @@ -206,9 +206,9 @@ int inq_dimid_handler(iosystem_desc_t *ios) int ret; int namelen; char *name; - + int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "inq_dimid_handler\n", my_rank)); /* Get the parameters for this function that the the comp master @@ -224,7 +224,7 @@ int inq_dimid_handler(iosystem_desc_t *ios) if ((mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; LOG((1, "%d inq_dimid_handler ncid = %d namelen = %d name = %s id_present = %d\n", - ncid, namelen, name, id_present)); + ncid, namelen, name, id_present)); /* Set non-null pointer. */ if (id_present) @@ -236,18 +236,18 @@ int inq_dimid_handler(iosystem_desc_t *ios) /* Free resources. */ free(name); - + return PIO_NOERR; } /** Handle attribute inquiry operations. This code only runs on IO * tasks. * - * @param ios pointer to the iosystem_desc_t. - * @param msg the message sent my the comp root task. - * @return PIO_NOERR for success, error code otherwise. + * @param ios pointer to the iosystem_desc_t. + * @param msg the message sent my the comp root task. + * @return PIO_NOERR for success, error code otherwise. */ -int inq_att_handler(iosystem_desc_t *ios, int msg) +int inq_att_handler(iosystem_desc_t *ios) { int ncid; int varid; @@ -256,12 +256,12 @@ int inq_att_handler(iosystem_desc_t *ios, int msg) char *name5; int namelen; PIO_Offset attlen; - nc_type xtype; int *op, *ip; - - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - LOG((1, "inq_att_handler\n", my_rank)); + nc_type xtype, *xtypep = NULL; + PIO_Offset len, *lenp = NULL; + char xtype_present, len_present; + + LOG((1, "inq_att_handler")); /* Get the parameters for this function that the the comp master * task is broadcasting. */ @@ -269,14 +269,26 @@ int inq_att_handler(iosystem_desc_t *ios, int msg) return PIO_EIO; if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm))) + return PIO_EIO; if (!(name5 = malloc((namelen + 1) * sizeof(char)))) return PIO_ENOMEM; - mpierr = MPI_Bcast((void *)name5, namelen + 1, MPI_CHAR, ios->compmaster, - ios->intercomm); - + if ((mpierr = MPI_Bcast((void *)name5, namelen + 1, MPI_CHAR, ios->compmaster, + ios->intercomm))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + + /* Match NULLs in collective function call. */ + if (xtype_present) + xtypep = &xtype; + if (len_present) + lenp = &len; + /* Call the function to learn about the attribute. */ - if ((ret = PIOc_inq_att(ncid, varid, name5, &xtype, &attlen))) + if ((ret = PIOc_inq_att(ncid, varid, name5, xtypep, lenp))) return ret; return PIO_NOERR; @@ -284,9 +296,9 @@ int inq_att_handler(iosystem_desc_t *ios, int msg) /** Handle attribute operations. This code only runs on IO tasks. * - * @param ios pointer to the iosystem_desc_t. - * @param msg the message sent my the comp root task. - * @return PIO_NOERR for success, error code otherwise. + * @param ios pointer to the iosystem_desc_t. + * @param msg the message sent my the comp root task. + * @return PIO_NOERR for success, error code otherwise. */ int att_handler(iosystem_desc_t *ios, int msg) { @@ -299,7 +311,7 @@ int att_handler(iosystem_desc_t *ios, int msg) PIO_Offset len; nc_type xtype; int *op, *ip; - + int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "%d att_handler\n", my_rank)); @@ -321,7 +333,7 @@ int att_handler(iosystem_desc_t *ios, int msg) if (!(op = malloc(len * sizeof(int)))) return PIO_ENOMEM; mpierr = MPI_Bcast(op, len, MPI_INT, ios->compmaster, ios->intercomm); - + /* Call the function to write the attribute. */ if ((ret = PIOc_put_att_int(ncid, varid, name5, xtype, len, op))) return ret; @@ -348,7 +360,7 @@ int att_handler(iosystem_desc_t *ios, int msg) return ret; if (!(ip = malloc(len * sizeof(int)))) return PIO_ENOMEM; - + /* Call the function to read the attribute. */ if ((ret = PIOc_get_att_int(ncid, varid, name5, ip))) return ret; @@ -364,9 +376,9 @@ int att_handler(iosystem_desc_t *ios, int msg) /** Do an inq_var on a netCDF variable. This function is only run on * IO tasks. * - * @param ios pointer to the iosystem_desc_t. - * @param msg the message sent my the comp root task. - * @return PIO_NOERR for success, error code otherwise. + * @param ios pointer to the iosystem_desc_t. + * @param msg the message sent my the comp root task. + * @return PIO_NOERR for success, error code otherwise. */ int inq_var_handler(iosystem_desc_t *ios) { @@ -376,10 +388,10 @@ int inq_var_handler(iosystem_desc_t *ios) char name_present, xtype_present, ndims_present, dimids_present, natts_present; char name[NC_MAX_NAME + 1], *namep; nc_type xtype, *xtypep = NULL; - int *ndimsp = NULL, *dimidsp = NULL, *nattsp = NULL; - int ndims, dimids[NC_MAX_DIMS], natts; + int *ndimsp = NULL, *dimidsp = NULL, *nattsp = NULL; + int ndims, dimids[NC_MAX_DIMS], natts; int ret; - + int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "%d inq_var_handler\n", my_rank)); @@ -419,15 +431,15 @@ int inq_var_handler(iosystem_desc_t *ios) /* Call the inq function to get the values. */ if ((ret = PIOc_inq_var(ncid, varid, namep, xtypep, ndimsp, dimidsp, nattsp))) return ret; - + return PIO_NOERR; } /** Do an inq_varid on a netCDF variable name. This function is only * run on IO tasks. * - * @param ios pointer to the iosystem_desc_t. - * @return PIO_NOERR for success, error code otherwise. + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. */ int inq_varid_handler(iosystem_desc_t *ios) { @@ -437,7 +449,7 @@ int inq_varid_handler(iosystem_desc_t *ios) int ret; int namelen; char *name; - + /* Get the parameters for this function that the the comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) @@ -455,23 +467,23 @@ int inq_varid_handler(iosystem_desc_t *ios) /* Free resources. */ free(name); - + return PIO_NOERR; } -/** This function is run on the IO tasks to sync a netCDF file. +/** This function is run on the IO tasks to sync a netCDF file. * - * @param ios pointer to the iosystem_desc_t. - * @return PIO_NOERR for success, error code otherwise. + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. */ int sync_file_handler(iosystem_desc_t *ios) { int ncid; int mpierr; int ret; - + int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "%d sync_file_handler\n", my_rank)); /* Get the parameters for this function that the comp master @@ -483,24 +495,24 @@ int sync_file_handler(iosystem_desc_t *ios) /* Call the sync file function. */ if ((ret = PIOc_sync(ncid))) return ret; - + printf("%d sync_file_handler succeeded!\n", my_rank); return PIO_NOERR; } -/** This function is run on the IO tasks to enddef a netCDF file. +/** This function is run on the IO tasks to enddef a netCDF file. * - * @param ios pointer to the iosystem_desc_t. - * @return PIO_NOERR for success, error code otherwise. + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. */ int enddef_file_handler(iosystem_desc_t *ios) { int ncid; int mpierr; int ret; - + int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "%d enddef_file_handler\n", my_rank)); /* Get the parameters for this function that the comp master @@ -511,7 +523,7 @@ int enddef_file_handler(iosystem_desc_t *ios) /* Call the sync file function. */ if ((ret = PIOc_enddef(ncid))) return ret; - + LOG((1, "%d enddef_file_handler succeeded!\n", my_rank)); return PIO_NOERR; } @@ -531,9 +543,9 @@ int def_var_handler(iosystem_desc_t *ios) nc_type xtype; int ndims; int *dimids; - + int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "%d def_var_handler comproot = %d\n", my_rank, ios->comproot)); /* Get the parameters for this function that the he comp master @@ -562,11 +574,11 @@ int def_var_handler(iosystem_desc_t *ios) /* Call the create file function. */ if ((ret = PIOc_def_var(ncid, name, xtype, ndims, dimids, &varid))) return ret; - + /* Free resources. */ free(name); free(dimids); - + LOG((1, "%d def_var_handler succeeded!\n", my_rank)); return PIO_NOERR; } @@ -583,19 +595,19 @@ int def_dim_handler(iosystem_desc_t *ios) int mpierr; int ret; int dimid; - + int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "%d def_dim_handler comproot = %d\n", my_rank, ios->comproot)); /* Get the parameters for this function that the he comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d def_dim_handler ncid = %d\n", my_rank, ncid); + printf("%d def_dim_handler ncid = %d\n", my_rank, ncid); if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d def_dim_handler ncid = %d namelen %d\n", my_rank, ncid, namelen); + printf("%d def_dim_handler ncid = %d namelen %d\n", my_rank, ncid, namelen); if (!(name = malloc(namelen + 1 * sizeof(char)))) return PIO_ENOMEM; if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, 0, @@ -610,19 +622,19 @@ int def_dim_handler(iosystem_desc_t *ios) /* Call the create file function. */ if ((ret = PIOc_def_dim(ncid, name, len, &dimid))) return ret; - + /* Free resources. */ free(name); - + LOG((1, "%d def_dim_handler succeeded!\n", my_rank)); return PIO_NOERR; } /** This function is run on the IO tasks. It reads or writes an array - * of data to a netCDF variable. + * of data to a netCDF variable. * * @param ios pointer to the iosystem_desc_t data. - * @param msg the message sent my the comp root task. + * @param msg the message sent my the comp root task. * * @return PIO_NOERR for success, error code otherwise. */ int vara_handler(iosystem_desc_t *ios, int msg) @@ -641,9 +653,9 @@ int vara_handler(iosystem_desc_t *ios, int msg) void *data; int size_in_bytes; PIO_Offset *count, *start; - + int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "%d def_var_handler comproot = %d\n", my_rank, ios->comproot)); if (msg == PIO_MSG_PUT_VARA) @@ -669,7 +681,7 @@ int vara_handler(iosystem_desc_t *ios, int msg) size *= count[d]; size_in_bytes = size * sizeof(int); if (!(data = malloc(size_in_bytes))) - return PIO_ENOMEM; + return PIO_ENOMEM; if ((mpierr = MPI_Bcast(data, size_in_bytes, MPI_INT, 0, ios->intercomm))) return PIO_EIO; @@ -680,7 +692,7 @@ int vara_handler(iosystem_desc_t *ios, int msg) /* Call the create file function. */ if ((ret = PIOc_put_vara_int(ncid, varid, start, count, data))) return ret; - + /* Free resources. */ free(start); free(count); @@ -688,14 +700,14 @@ int vara_handler(iosystem_desc_t *ios, int msg) } else { - + } LOG((1, "%d vara_handler succeeded!\n", my_rank)); return PIO_NOERR; } -/** This function is run on the IO tasks to open a netCDF file. +/** This function is run on the IO tasks to open a netCDF file. * * @param ios pointer to the iosystem_desc_t data. * @@ -709,16 +721,16 @@ int open_file_handler(iosystem_desc_t *ios) int mode; int mpierr; int ret; - + int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "%d open_file_handler comproot = %d\n", my_rank, ios->comproot)); /* Get the parameters for this function that the he comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d open_file_handler got parameter len = %d\n", my_rank, len); + LOG((2, "open_file_handler got parameter len = %d", len)); if (!(filename = malloc(len + 1 * sizeof(char)))) return PIO_ENOMEM; if ((mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, 0, @@ -735,15 +747,15 @@ int open_file_handler(iosystem_desc_t *ios) /* Call the open file function. */ if ((ret = PIOc_openfile(ios->iosysid, &ncid, &iotype, filename, mode))) return ret; - + /* Free resources. */ free(filename); - + LOG((1, "%d open_file_handler succeeded!\n", my_rank)); return PIO_NOERR; } -/** This function is run on the IO tasks to delete a netCDF file. +/** This function is run on the IO tasks to delete a netCDF file. * * @param ios pointer to the iosystem_desc_t data. * @@ -755,9 +767,9 @@ int delete_file_handler(iosystem_desc_t *ios) char *filename; int mpierr; int ret; - + int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "%d delete_file_handler comproot = %d\n", my_rank, ios->comproot)); /* Get the parameters for this function that the he comp master @@ -775,10 +787,10 @@ int delete_file_handler(iosystem_desc_t *ios) /* Call the delete file function. */ if ((ret = PIOc_deletefile(ios->iosysid, filename))) return ret; - + /* Free resources. */ free(filename); - + LOG((1, "%d delete_file_handler succeeded!\n", my_rank)); return PIO_NOERR; } @@ -830,7 +842,7 @@ int pio_callback_handler(iosystem_desc_t *ios, int msg) return, unless there is an error. */ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) { - iosystem_desc_t *my_iosys; + iosystem_desc_t *my_iosys; int msg = 0; MPI_Request req[component_count]; MPI_Status status; @@ -841,7 +853,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); LOG((1, "%d pio_msg_handler called\n", my_rank)); - + /* Have IO comm rank 0 (the ioroot) register to receive * (non-blocking) for a message from each of the comproots. */ if (!io_rank) @@ -849,7 +861,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) for (int cmp = 0; cmp < component_count; cmp++) { my_iosys = &iosys[cmp]; - LOG((1, "%d about to call MPI_Irecv\n", my_rank)); + LOG((1, "%d about to call MPI_Irecv\n", my_rank)); mpierr = MPI_Irecv(&msg, 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG, my_iosys->union_comm, &req[cmp]); CheckMPIReturn(mpierr, __FILE__, __LINE__); @@ -863,26 +875,25 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) if (!io_rank) { LOG((1, "%d about to call MPI_Waitany req[0] = %d MPI_REQUEST_NULL = %d\n", - my_rank, req[0], MPI_REQUEST_NULL)); + my_rank, req[0], MPI_REQUEST_NULL)); mpierr = MPI_Waitany(component_count, req, &index, &status); CheckMPIReturn(mpierr, __FILE__, __LINE__); - printf("%d Waitany returned index = %d req[%d] = %d\n", my_rank, - index, index, req[index]); + LOG((3, "Waitany returned index = %d req[%d] = %d", + index, index, req[index])); } /* Broadcast the index of the computational component that * originated the request to the rest of the IO tasks. */ - printf("%d about to call index MPI_Bcast index = %d\n", my_rank, index); mpierr = MPI_Bcast(&index, 1, MPI_INT, 0, iosys->io_comm); CheckMPIReturn(mpierr, __FILE__, __LINE__); my_iosys = &iosys[index]; - printf("%d index MPI_Bcast complete index = %d\n", my_rank, index); + LOG((3, "index MPI_Bcast complete index = %d", index)); /* Broadcast the msg value to the rest of the IO tasks. */ - printf("%d about to call msg MPI_Bcast\n", my_rank); + LOG((3, "about to call msg MPI_Bcast")); mpierr = MPI_Bcast(&msg, 1, MPI_INT, 0, my_iosys->io_comm); CheckMPIReturn(mpierr, __FILE__, __LINE__); - printf("%d msg MPI_Bcast complete msg = %d\n", my_rank, msg); + LOG((3, "msg MPI_Bcast complete msg = %d", msg)); /* Handle the message. This code is run on all IO tasks. */ switch (msg) @@ -926,13 +937,12 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) case PIO_MSG_GET_ATT_INT: case PIO_MSG_PUT_ATT_INT: ret = att_handler(my_iosys, msg); - printf("att_handler returned %d\n", ret); break; case PIO_MSG_INQ_VARID: inq_varid_handler(my_iosys); break; case PIO_MSG_INQ_ATT: - inq_att_handler(my_iosys, msg); + inq_att_handler(my_iosys); break; case PIO_MSG_INITDECOMP_DOF: initdecomp_dof_handler(my_iosys); @@ -969,7 +979,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) /* If an error was returned by the handler, do something! */ if (ret) { - printf("hander returned error code %d\n", ret); + LOG((0, "hander returned error code %d", ret)); MPI_Finalize(); } @@ -980,10 +990,10 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) my_iosys = &iosys[index]; mpierr = MPI_Irecv(&msg, 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG, my_iosys->union_comm, &req[index]); - printf("%d pio_msg_handler called MPI_Irecv req[%d] = %d\n", my_rank, index, req[index]); + LOG((3, "pio_msg_handler called MPI_Irecv req[%d] = %d\n", index, req[index])); CheckMPIReturn(mpierr, __FILE__, __LINE__); } - + } return PIO_NOERR; @@ -997,22 +1007,22 @@ pio_iosys_print(int my_rank, iosystem_desc_t *iosys) printf("%d union_comm: MPI_COMM_NULL ", my_rank); else printf("%d union_comm: %d ", my_rank, iosys->union_comm); - + if (iosys->comp_comm == MPI_COMM_NULL) printf("comp_comm: MPI_COMM_NULL "); else printf("comp_comm: %d ", iosys->comp_comm); - + if (iosys->io_comm == MPI_COMM_NULL) printf("io_comm: MPI_COMM_NULL "); else printf("io_comm: %d ", iosys->io_comm); - + if (iosys->intercomm == MPI_COMM_NULL) printf("intercomm: MPI_COMM_NULL\n"); else printf("intercomm: %d\n", iosys->intercomm); - + printf("%d num_iotasks=%d num_comptasks=%d union_rank=%d, comp_rank=%d, " "io_rank=%d async_interface=%d\n", my_rank, iosys->num_iotasks, iosys->num_comptasks, iosys->union_rank, @@ -1029,7 +1039,7 @@ pio_iosys_print(int my_rank, iosystem_desc_t *iosys) return PIO_NOERR; } -/** @ingroup PIO_init +/** @ingroup PIO_init * Library initialization used when IO tasks are distinct from compute * tasks. * @@ -1105,7 +1115,7 @@ pio_iosys_print(int my_rank, iosystem_desc_t *iosys) * Return or Not * ------------- * - * The io_comm tasks do not return from the init_intercomm routine. + * The io_comm tasks do not return from the init_intercomm routine. * * Sequence of Events to do Asynch I/O * ----------------------------------- @@ -1179,7 +1189,7 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, /* Get a pointer to the iosys struct */ my_iosys = &iosys[cmp]; - + /* Create an MPI info object. */ CheckMPIReturn(MPI_Info_create(&(my_iosys->info)),__FILE__,__LINE__); @@ -1188,32 +1198,32 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, { /* Copy the computation communicator. */ mpierr = MPI_Comm_dup(comp_comms[cmp], &my_iosys->comp_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; /* Create an MPI group with the computation tasks. */ mpierr = MPI_Comm_group(my_iosys->comp_comm, &my_iosys->compgroup); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; /* Find out how many tasks are in this communicator. */ mpierr = MPI_Comm_size(iosys->comp_comm, &my_iosys->num_comptasks); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; /* Set the rank within the comp_comm. */ mpierr = MPI_Comm_rank(my_iosys->comp_comm, &my_iosys->comp_rank); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; /* Find the rank of the io leader in peer_comm. */ iam = -1; mpierr = MPI_Allreduce(&iam, &io_leader, 1, MPI_INT, MPI_MAX, peer_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; @@ -1221,7 +1231,7 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, if (!my_iosys->comp_rank) { mpierr = MPI_Comm_rank(peer_comm, &iam); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; } @@ -1230,7 +1240,7 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, /* Find the lucky comp_leader task. */ mpierr = MPI_Allreduce(&iam, &comp_leader, 1, MPI_INT, MPI_MAX, peer_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; @@ -1242,17 +1252,17 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, } else my_iosys->compmaster = MPI_PROC_NULL; - + /* Set up the intercomm from the computation side. */ mpierr = MPI_Intercomm_create(my_iosys->comp_comm, 0, peer_comm, io_leader, cmp, &my_iosys->intercomm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; /* Create the union communicator. */ mpierr = MPI_Intercomm_merge(my_iosys->intercomm, 0, &my_iosys->union_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; } @@ -1269,25 +1279,25 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, { /* Copy the IO communicator. */ mpierr = MPI_Comm_dup(io_comm, &my_iosys->io_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; /* Get an MPI group that includes the io tasks. */ mpierr = MPI_Comm_group(my_iosys->io_comm, &my_iosys->iogroup); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; /* Find out how many tasks are in this communicator. */ mpierr = MPI_Comm_size(iosys->io_comm, &my_iosys->num_iotasks); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; /* Set the rank within the io_comm. */ mpierr = MPI_Comm_rank(my_iosys->io_comm, &my_iosys->io_rank); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; @@ -1295,7 +1305,7 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, if (!my_iosys->io_rank) { mpierr = MPI_Comm_rank(peer_comm, &iam); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; } @@ -1304,17 +1314,17 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, /* Find the lucky io_leader task. */ mpierr = MPI_Allreduce(&iam, &io_leader, 1, MPI_INT, MPI_MAX, peer_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; /* Find the rank of the comp leader in peer_comm. */ iam = -1; mpierr = MPI_Allreduce(&iam, &comp_leader, 1, MPI_INT, MPI_MAX, peer_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; - + /* This is an io task. */ my_iosys->ioproc = true; @@ -1325,18 +1335,18 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, io_master = MPI_ROOT; } else - my_iosys->iomaster = 0; + my_iosys->iomaster = 0; /* Set up the intercomm from the I/O side. */ mpierr = MPI_Intercomm_create(my_iosys->io_comm, 0, peer_comm, comp_leader, cmp, &my_iosys->intercomm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; /* Create the union communicator. */ mpierr = MPI_Intercomm_merge(my_iosys->intercomm, 0, &my_iosys->union_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; @@ -1358,7 +1368,7 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, /* Find rank in union communicator. */ mpierr = MPI_Comm_rank(my_iosys->union_comm, &my_iosys->union_rank); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; @@ -1371,11 +1381,11 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, /* Distribute the answer to all tasks. */ mpierr = MPI_Allreduce(&my_iosys->ioroot, &root, 1, MPI_INT, MPI_MAX, my_iosys->union_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; my_iosys->ioroot = root; - + /* Find the rank of the computation leader in the union * communicator. */ if (!my_iosys->comp_rank) @@ -1386,7 +1396,7 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, /* Distribute the answer to all tasks. */ mpierr = MPI_Allreduce(&my_iosys->comproot, &root, 1, MPI_INT, MPI_MAX, my_iosys->union_comm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) ierr = PIO_EIO; my_iosys->comproot = root; @@ -1403,7 +1413,7 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, comp_master = 0; mpierr = MPI_Bcast(&my_iosys->num_comptasks, 1, MPI_INT, comp_master, my_iosys->intercomm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); mpierr = MPI_Bcast(&my_iosys->num_iotasks, 1, MPI_INT, io_master, my_iosys->intercomm); CheckMPIReturn(mpierr, __FILE__, __LINE__); @@ -1413,7 +1423,7 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, io_master = 0; mpierr = MPI_Bcast(&my_iosys->num_comptasks, 1, MPI_INT, comp_master, my_iosys->intercomm); - CheckMPIReturn(mpierr, __FILE__, __LINE__); + CheckMPIReturn(mpierr, __FILE__, __LINE__); mpierr = MPI_Bcast(&my_iosys->num_iotasks, 1, MPI_INT, io_master, my_iosys->intercomm); CheckMPIReturn(mpierr, __FILE__, __LINE__); @@ -1421,11 +1431,9 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, /* Allocate an array to hold the ranks of the IO tasks * within the union communicator. */ - printf("%d allocating for %d iotasks\n", my_rank, my_iosys->num_iotasks); if (!(my_iosys->ioranks = malloc(my_iosys->num_iotasks * sizeof(int)))) return PIO_ENOMEM; - printf("%d allocated\n", my_rank); - + /* Allocate a temp array to help get the IO ranks. */ int *tmp_ioranks; if (!(tmp_ioranks = malloc(my_iosys->num_iotasks * sizeof(int)))) @@ -1434,7 +1442,7 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, /* Init array, then have IO tasks set their values, then * use allreduce to distribute results to all tasks. */ for (int cnt = 0 ; cnt < my_iosys->num_iotasks; cnt++) - tmp_ioranks[cnt] = -1; + tmp_ioranks[cnt] = -1; if (io_comm != MPI_COMM_NULL) tmp_ioranks[my_iosys->io_rank] = my_iosys->union_rank; mpierr = MPI_Allreduce(tmp_ioranks, my_iosys->ioranks, my_iosys->num_iotasks, MPI_INT, MPI_MAX, @@ -1460,17 +1468,13 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, /* Add this id to the list of PIO iosystem ids. */ iosysidp[cmp] = pio_add_to_iosystem_list(my_iosys); - printf("%d added to iosystem_list iosysid = %d\n", my_rank, iosysidp[cmp]); + LOG((2, "added to iosystem_list iosysid = %d", iosysidp[cmp])); /* Now call the function from which the IO tasks will not * return until the PIO_MSG_EXIT message is sent. */ if (io_comm != MPI_COMM_NULL) - { - printf("%d about to call pio_msg_handler\n", my_rank); if ((ierr = pio_msg_handler(my_iosys->io_rank, component_count, iosys))) return ierr; - } - } /* If there was an error, make sure all tasks see it. */ @@ -1482,7 +1486,6 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, if (mpierr) ierr = PIO_EIO; } - + return ierr; } - diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 45dd6c4ea38..b2e6bc31a7a 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -89,6 +89,24 @@ pio_log(int severity, const char *fmt, ...) } #endif /* PIO_ENABLE_LOGGING */ +/** + ** @brief Wrapper for MPI calls to print the Error string on error + */ +void check_mpi(file_desc_t *file, const int ierr, const char *filename, const int line) +{ + if (ierr != MPI_SUCCESS) + { + char errstring[MPI_MAX_ERROR_STRING]; + int errstrlen; + int mpierr = MPI_Error_string(ierr, errstring, &errstrlen); + + fprintf(stderr, "MPI ERROR: %s in file %s at line %d\n", + errstring, filename, line); + + check_netcdf(file, PIO_EIO, filename, line); + } +} + /** * @ingroup PIOc_inq * The PIO-C interface for the NetCDF function nc_inq. @@ -111,7 +129,6 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int msg = PIO_MSG_INQ; /** Message for async notification. */ iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ - char *errstr = NULL; /** String for error messages. */ int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr; /** Return code from MPI function codes. */ @@ -125,23 +142,36 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, /* For async, on non-IO tasks, send the necessary information to * the IO tasks over the intercomm. Whether or not the pointers * are NULL are passed as integers, either true or false. */ - if (ios->async_interface && ! ios->ioproc) + if (ios->async_interface) { - char ndims_present = ndimsp ? true : false; - char nvars_present = nvarsp ? true : false; - char ngatts_present = ngattsp ? true : false; - char unlimdimid_present = unlimdimidp ? true : false; - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq netcdf Bcast unlimdimid_present = %d", unlimdimid_present)); - } + if (!ios->ioproc) + { + char ndims_present = ndimsp ? true : false; + char nvars_present = nvarsp ? true : false; + char ngatts_present = ngattsp ? true : false; + char unlimdimid_present = unlimdimidp ? true : false; + + if (ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_inq netcdf Bcast unlimdimid_present = %d", unlimdimid_present)); + } + /* Handle MPI errors. */ + mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + check_mpi(file, mpierr, __FILE__, __LINE__); + } + /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { @@ -182,41 +212,31 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, ierr = iotype_error(file->iotype,__FILE__,__LINE__); } - if(ierr != PIO_NOERR) - { - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - printf("%d PIOc_inq netcdf call returned %d\n", my_rank, ierr); - ierr = check_netcdf(file, ierr, errstr,__LINE__); + LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } - /* Broadcast results to all tasks. If pointers were NULL, then - * ignore that parameter. */ + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. Ignore NULL parameters. */ if (ndimsp) - { - mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - LOG((2, "PIOc__inq bcast ndims = %d\n", *ndimsp)); - } + if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + if(nvarsp) - { - mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - LOG((2, "%d PIOc__inq bcast nvars = %d\n", my_rank, *nvarsp)); - } + if ((mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + if(ngattsp) - { - mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - LOG((2, "%d PIOc__inq bcast ngatts = %d\n", my_rank, *ngattsp)); - } + if ((mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + if(unlimdimidp) - { - mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm); - LOG((2, "%d PIOc__inq bcast unlimdimid = %d\n", my_rank, *unlimdimidp)); - } + if ((mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; - if(errstr) - free(errstr); - return ierr; } @@ -293,9 +313,8 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) { iosystem_desc_t *ios; file_desc_t *file; - char *errstr = NULL; - int ierr = PIO_NOERR; int msg = PIO_MSG_INQ_DIM; + int ierr = PIO_NOERR; int mpierr; LOG((1, "PIOc_inq_dim")); @@ -306,18 +325,29 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface && !ios->ioproc) + if (ios->async_interface) { - char name_present = name ? true : false; - char len_present = lenp ? true : false; - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq netcdf Bcast name_present = %d", name_present)); - mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq netcdf Bcast len_present = %d", len_present)); + if (!ios->ioproc) + { + char name_present = name ? true : false; + char len_present = lenp ? true : false; + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_inq netcdf Bcast name_present = %d", name_present)); + if (!mpierr) + mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_inq netcdf Bcast len_present = %d", len_present)); + } + + /* Handle MPI errors. */ + mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* Make the call to the netCDF layer. */ @@ -346,25 +376,27 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) } } - /* Error handling. */ - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); - /* BCast the results, if non-null pointers were passed. */ - if(name) + /* Broadcast results to all tasks. Ignore NULL parameters. */ + if (name) { int slen; - if(ios->iomaster) + if (ios->iomaster) slen = strlen(name); - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm); + if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) + return PIO_EIO; } - if(lenp != NULL) - mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); + + if (lenp) + if ((mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) + return PIO_EIO; + return ierr; } @@ -424,14 +456,14 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) int msg = PIO_MSG_INQ_DIMID; iosystem_desc_t *ios; file_desc_t *file; - char *errstr = NULL; int ierr = PIO_NOERR; int mpierr; - /* For debugging purposes only... */ - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d PIOc_inq_dimid\n", my_rank); + /* Name must be provided. */ + if (!name) + return PIO_EINVAL; + + LOG((1, "PIOc_inq_dimid name = %s", name)); /* Get the file info, based on the ncid. */ if (!(file = pio_get_file_from_id(ncid))) @@ -478,14 +510,16 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) } } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results. */ if (idp) - mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); + if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + return ierr; } @@ -510,13 +544,12 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, { iosystem_desc_t *ios; file_desc_t *file; - char *errstr = NULL; int ndims; /** The number of dimensions for this variable. */ int ierr = PIO_NOERR; int msg = PIO_MSG_INQ_VAR; int mpierr; - LOG((1, "PIOc_inq_var\n")); + LOG((1, "PIOc_inq_var ncid = %d varid = %d", ncid, varid)); /* Get the file info, based on the ncid. */ if (!(file = pio_get_file_from_id(ncid))) @@ -576,12 +609,10 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, } } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - printf("%d called netcdf nc_inq_var ierr = %d\n", my_rank, ierr); + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast the results for non-null pointers. */ if (name) @@ -589,36 +620,32 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int slen; if(ios->iomaster) slen = strlen(name); - printf("%d PIOc_inq_var slen = %d\n", my_rank, slen); - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - printf("%d PIOc_inq_var slen = %d\n", my_rank, slen); - mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm); - printf("%d PIOc_inq_var name = %s\n", my_rank, name); + if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) + return PIO_EIO; } if (xtypep) - { - mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); - printf("%d PIOc_inq_var xtype = %d\n", my_rank, *xtypep); - } + if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + if (ndimsp) { - printf("%d PIOc_inq_var ndims = %d\n", my_rank, *ndimsp); - mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; file->varlist[varid].ndims = (*ndimsp); - printf("%d PIOc_inq_var bcast complete ndims = %d\n", my_rank, *ndimsp); } if (dimidsp) { - mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm); + if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; } if (nattsp) - { - printf("%d PIOc_inq_var natts = %d\n", my_rank, *nattsp); - mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - printf("%d PIOc_inq_var natts = %d\n", my_rank, *nattsp); - } - if(errstr != NULL) free(errstr); + if ((mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + return ierr; } @@ -736,36 +763,48 @@ int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) */ int PIOc_inq_varid (int ncid, const char *name, int *varidp) { - int ierr; - int msg; - int mpierr; + int ierr = PIO_NOERR; + int msg = PIO_MSG_INQ_VARID; + int mpierr = 0; iosystem_desc_t *ios; file_desc_t *file; - char *errstr; - errstr = NULL; - ierr = PIO_NOERR; + /* Caller must provide name. */ + if (!name) + return PIO_EINVAL; - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Get file info based on ncid. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARID; LOG((1, "PIOc_inq_varid ncid = %d name = %s", ncid, name)); - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - printf("%d PIOc_inq_varid BCast ncid = %d\n", my_rank, ncid); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - int namelen; - namelen = strlen(name); - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (ios->async_interface) + { + if (!ios->ioproc) + { + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen; + namelen = strlen(name); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + check_mpi(file, mpierr, __FILE__, __LINE__); } - if(ios->ioproc){ + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 @@ -790,14 +829,16 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) } } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. Ignore NULL parameters. */ if (varidp) - mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); + if ((mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + return ierr; } @@ -817,30 +858,34 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, - PIO_Offset *lenp) +int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, + PIO_Offset *lenp) { - int ierr; - int msg; - int mpierr; + int msg = PIO_MSG_INQ_ATT; iosystem_desc_t *ios; file_desc_t *file; - char *errstr; + int mpierr; + int ierr = PIO_NOERR; - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d PIOc_inq_att ncid = %d varid = %d xtpyep = %d lenp = %d\n", - my_rank, ncid, varid, xtypep, lenp); + /* Caller must provide a name. */ + if (!name) + return PIO_EINVAL; - errstr = NULL; - ierr = PIO_NOERR; + LOG((1, "PIOc_inq_att ncid = %d varid = %d xtpyep = %d lenp = %d", + ncid, varid, xtypep, lenp)); + /* Find file based on ncid. */ if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATT; - if(ios->async_interface && ! ios->ioproc){ + /* If using async and this is not an IO task, send parameter data + * over the intercomm. */ + if(ios->async_interface && !ios->ioproc) + { + char xtype_present = xtypep ? true : false; + char len_present = lenp ? true : false; + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); @@ -848,26 +893,29 @@ int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, int namelen = strlen(name); mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); } - if(ios->ioproc){ - switch(file->iotype){ + if (ios->ioproc) + { + switch (file->iotype) + { #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp); break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; - } + if (ios->io_rank == 0) + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp); break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);; + ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp); break; #endif default: @@ -875,16 +923,19 @@ int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, } } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); + /* Handle errors. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results. */ if(xtypep) - mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm); + if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; if(lenp) - mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); + if ((mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) + return PIO_EIO; + return ierr; } @@ -905,8 +956,7 @@ int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, */ int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) { - nc_type dummy; - return PIOc_inq_att(ncid, varid, name, &dummy, lenp); + return PIOc_inq_att(ncid, varid, name, NULL, lenp); } /** @@ -926,8 +976,7 @@ int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) */ int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep) { - PIO_Offset dummy; - return PIOc_inq_att(ncid, varid, name, xtypep, &dummy); + return PIOc_inq_att(ncid, varid, name, xtypep, NULL); } /** @@ -947,21 +996,15 @@ int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep) */ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) { - int ierr; - int msg; + int ierr = PIO_NOERR; + int msg = PIO_MSG_INQ_ATTNAME; int mpierr; iosystem_desc_t *ios; file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - file = pio_get_file_from_id(ncid); - if(file == NULL) + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATTNAME; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -969,7 +1012,6 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } - if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF @@ -995,19 +1037,16 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) } } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(name != NULL){ + check_netcdf(file, ierr, __FILE__, __LINE__); + if (name) + { int slen; if(ios->iomaster) - slen = (int) strlen(name) + 1; + slen = (int) strlen(name); mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast((void *)name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm); } - if(errstr != NULL) free(errstr); + return ierr; } @@ -1028,21 +1067,15 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) */ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) { - int ierr; - int msg; + int ierr = PIO_NOERR; + int msg = PIO_MSG_INQ_ATTID; int mpierr; iosystem_desc_t *ios; file_desc_t *file; - char *errstr; - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATTID; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1050,7 +1083,6 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } - if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF @@ -1076,13 +1108,13 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) } } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); + /* Handle errors. */ + mpierr = MPI_Bcast(&ierr , 1, MPI_INT, ios->ioroot, ios->my_comm); + check_netcdf(file, ierr, __FILE__, __LINE__); + + if (idp) + mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); + return ierr; } From af76675fa258385227c30b0ea9fdedb9cf4db0be Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 11 May 2016 09:25:27 -0400 Subject: [PATCH 024/184] further async development --- src/clib/pio_file.c | 115 +++++++++++++++++++++++------------- src/clib/pio_internal.h | 2 + src/clib/pio_nc_async.c | 101 ++++++++++++++++++------------- tests/unit/test_intercomm.c | 16 ++--- 4 files changed, 145 insertions(+), 89 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index c4ccb4d0fbc..fac23a4146a 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -13,28 +13,33 @@ */ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, - const char filename[], const int mode) + const char *filename, const int mode) { - int ierr; - int msg; - int mpierr; + int msg = PIO_MSG_OPEN_FILE; size_t len; iosystem_desc_t *ios; file_desc_t *file; + int ierr = PIO_NOERR; + int mpierr = MPI_SUCCESS; + + /* Get the IO system info from the iosysid. */ + if (!(ios = pio_get_iosystem_from_id(iosysid))) + { + printf("bad iosysid %d\n",iosysid); + return PIO_EBADID; + } - ierr = PIO_NOERR; + /* User must provide valid input for these parameters. */ + if (!ncidp || !iotype || !filename) + return PIO_EINVAL; + if (*iotype < PIO_IOTYPE_PNETCDF || *iotype > PIO_IOTYPE_NETCDF4P) + return PIO_ENOMEM; - msg = PIO_MSG_OPEN_FILE; - ios = pio_get_iosystem_from_id(iosysid); - if(ios==NULL){ - printf("bad iosysid %d\n",iosysid); - return PIO_EBADID; - } + /* Allocate space for the file info. */ + if (!(file = (file_desc_t *) malloc(sizeof(*file)))) + return PIO_ENOMEM; - file = (file_desc_t *) malloc(sizeof(*file)); - if(file==NULL){ - return PIO_ENOMEM; - } + /* Fill in some file values. */ file->iotype = *iotype; file->next = NULL; file->iosystem = ios; @@ -57,19 +62,35 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, file->buffer.frame=NULL; file->buffer.fillvalue=NULL; - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - len = strlen(filename); - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + if(ios->comp_rank==0) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + len = strlen(filename); + if (!mpierr) + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); + } - if(ios->ioproc){ + /* Handle MPI errors. */ + mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + check_mpi(file, mpierr, __FILE__, __LINE__); + } - switch(file->iotype){ + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + switch (file->iotype) + { #ifdef _NETCDF #ifdef _NETCDF4 @@ -78,7 +99,7 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, ierr = nc_open(filename, file->mode, &(file->fh)); #else file->mode = file->mode | NC_MPIIO; - ierr = nc_open_par(filename, file->mode, ios->io_comm,ios->info, &(file->fh)); + ierr = nc_open_par(filename, file->mode, ios->io_comm, ios->info, &file->fh); #endif break; @@ -89,18 +110,20 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_open(filename, file->mode, &(file->fh)); + ierr = nc_open(filename, file->mode, &file->fh); } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &(file->fh)); + ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &file->fh); // This should only be done with a file opened to append - if(ierr == PIO_NOERR && (file->mode & PIO_WRITE)){ - if(ios->iomaster) printf("%d Setting IO buffer %ld\n",__LINE__,PIO_BUFFER_SIZE_LIMIT); + if (ierr == PIO_NOERR && (file->mode & PIO_WRITE)) + { + if(ios->iomaster) + printf("%d Setting IO buffer %ld\n",__LINE__,PIO_BUFFER_SIZE_LIMIT); ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT ); } break; @@ -129,19 +152,29 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, /* #endif */ } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if (!ierr) { - mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm); - *ncidp = file->fh; - pio_add_to_file_list(file); - *ncidp = file->fh; + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. Ignore NULL parameters. */ + if (!ierr) + { + if ((mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm))) + return PIO_EIO; + + if ((mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm))) + return PIO_EIO; + + *ncidp = file->fh; + pio_add_to_file_list(file); } - if(ios->io_rank==0){ - printf("Open file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr); + + if (ios->io_rank==0){ + printf("Open file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr); // if(file->fh==5) print_trace(stdout); } + return ierr; } diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 9efa132843d..16e5120ead1 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -152,6 +152,8 @@ typedef struct pio_swapm_defaults void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk); void piomemerror(iosystem_desc_t ios, size_t req, char *fname, const int line); void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc); + void check_mpi(file_desc_t *file, const int mpierr, const char *filename, + const int line); #ifdef BGQ void identity(MPI_Comm comm, int *iotask); diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index b2e6bc31a7a..ea5d3ec1c8f 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -89,20 +89,28 @@ pio_log(int severity, const char *fmt, ...) } #endif /* PIO_ENABLE_LOGGING */ -/** - ** @brief Wrapper for MPI calls to print the Error string on error +/** Handle MPI errors. An error message is sent to stderr, then the + check_netcdf() function is called with PIO_EIO. + + @param file pointer to the file_desc_t info + @param mpierr the MPI return code to handle + @param filename the name of the code file where error occured. + @param line the line of code where error occured. */ -void check_mpi(file_desc_t *file, const int ierr, const char *filename, const int line) +void check_mpi(file_desc_t *file, const int mpierr, const char *filename, + const int line) { - if (ierr != MPI_SUCCESS) + if (mpierr) { char errstring[MPI_MAX_ERROR_STRING]; int errstrlen; - int mpierr = MPI_Error_string(ierr, errstring, &errstrlen); - fprintf(stderr, "MPI ERROR: %s in file %s at line %d\n", - errstring, filename, line); + /* If we can get an error string from MPI, print it to stderr. */ + if (!MPI_Error_string(mpierr, errstring, &errstrlen)) + fprintf(stderr, "MPI ERROR: %s in file %s at line %d\n", + errstring, filename, line); + /* Handle all MPI errors as PIO_EIO. */ check_netcdf(file, PIO_EIO, filename, line); } } @@ -130,7 +138,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr; /** Return code from MPI function codes. */ + int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ LOG((1, "PIOc_inq ncid = %d", ncid)); @@ -139,9 +147,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, return PIO_EBADID; ios = file->iosystem; - /* For async, on non-IO tasks, send the necessary information to - * the IO tasks over the intercomm. Whether or not the pointers - * are NULL are passed as integers, either true or false. */ + /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { if (!ios->ioproc) @@ -221,22 +227,25 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ - if (ndimsp) - if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; - - if(nvarsp) - if ((mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; - - if(ngattsp) - if ((mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; - - if(unlimdimidp) - if ((mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + if (!ierr) + { + if (ndimsp) + if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if (nvarsp) + if ((mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + + if (ngattsp) + if ((mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + + if (unlimdimidp) + if ((mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + } + return ierr; } @@ -471,22 +480,35 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) ios = file->iosystem; /* If using async, and not an IO task, then send parameters. */ - if (ios->async_interface && !ios->ioproc) + if (ios->async_interface) { - int namelen; - char id_present = idp ? true : false; - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - namelen = strlen(name); - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!ios->ioproc) + { + char id_present = idp ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen = strlen(name); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* IO tasks call the netCDF functions. */ - if(ios->ioproc){ - switch(file->iotype){ + if (ios->ioproc) + { + switch(file->iotype) + { #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: @@ -495,9 +517,8 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ + if (ios->io_rank == 0) ierr = nc_inq_dimid(file->fh, name, idp);; - } break; #endif #ifdef _PNETCDF diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 6ab8cb03186..cfddf1e600a 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -177,14 +177,14 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) /* Check out the global attributes. */ nc_type atttype; PIO_Offset attlen; - /* if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &atttype, &attlen))) */ - /* ERR(ret); */ - /* if (atttype != NC_INT || attlen != 1) */ - /* ERR(ERR_WRONG); */ - /* if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &attlen))) */ - /* ERR(ret); */ - /* if (attlen != 1) */ - /* ERR(ERR_WRONG); */ + if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &atttype, &attlen))) + ERR(ret); + if (atttype != NC_INT || attlen != 1) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &attlen))) + ERR(ret); + if (attlen != 1) + ERR(ERR_WRONG); /* if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, &att_data))) */ /* ERR(ret); */ /* sleep(2); */ From fc48722e78d2f59d24a7c88dfac4dd5c25eb7856 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 11 May 2016 09:32:57 -0400 Subject: [PATCH 025/184] better error handling --- src/clib/pio_nc_async.c | 84 +++++++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index ea5d3ec1c8f..6006905181b 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -391,21 +391,24 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ - if (name) - { - int slen; - if (ios->iomaster) - slen = strlen(name); - if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; - if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) - return PIO_EIO; + if (!ierr) + { + if (name) + { + int slen; + if (ios->iomaster) + slen = strlen(name); + if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + } + + if (lenp) + if ((mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); } - - if (lenp) - if ((mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) - return PIO_EIO; - + return ierr; } @@ -533,13 +536,13 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results. */ if (idp) if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); return ierr; } @@ -632,7 +635,7 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast the results for non-null pointers. */ @@ -642,30 +645,30 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, if(ios->iomaster) slen = strlen(name); if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); } if (xtypep) if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); if (ndimsp) { if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); file->varlist[varid].ndims = (*ndimsp); } if (dimidsp) { if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); } if (nattsp) if ((mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); return ierr; } @@ -852,13 +855,13 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ if (varidp) if ((mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); return ierr; } @@ -946,16 +949,19 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, /* Handle errors. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results. */ - if(xtypep) - if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; - if(lenp) - if ((mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) - return PIO_EIO; + if (!ierr) + { + if(xtypep) + if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if(lenp) + if ((mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + } return ierr; } @@ -1130,11 +1136,17 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) } /* Handle errors. */ - mpierr = MPI_Bcast(&ierr , 1, MPI_INT, ios->ioroot, ios->my_comm); + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); - if (idp) - mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); + /* Broadcast results. */ + if (!ierr) + { + if (idp) + if ((mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + } return ierr; } From 90f714e087eff15da7e3f413cf175dee8e629d0a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 11 May 2016 10:28:00 -0400 Subject: [PATCH 026/184] continued development of async --- src/clib/pio.h | 2 + src/clib/pio_nc_async.c | 411 +++++++++++++++++++++++----------------- src/clib/pioc_support.c | 26 +++ 3 files changed, 269 insertions(+), 170 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index 44507edcdc7..2be57ca845c 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -648,6 +648,8 @@ int PIOc_set_blocksize(const int newblocksize); int PIOc_get_var_schar (int ncid, int varid, signed char *buf); int PIOc_iotype_available(const int iotype); int PIOc_set_log_level(int level); + int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, size_t len, const void *op); + int PIOc_get_att(int ncid, int varid, const char *name, void *ip); #if defined(__cplusplus) } #endif diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 6006905181b..dcaf033ed3e 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -89,32 +89,6 @@ pio_log(int severity, const char *fmt, ...) } #endif /* PIO_ENABLE_LOGGING */ -/** Handle MPI errors. An error message is sent to stderr, then the - check_netcdf() function is called with PIO_EIO. - - @param file pointer to the file_desc_t info - @param mpierr the MPI return code to handle - @param filename the name of the code file where error occured. - @param line the line of code where error occured. - */ -void check_mpi(file_desc_t *file, const int mpierr, const char *filename, - const int line) -{ - if (mpierr) - { - char errstring[MPI_MAX_ERROR_STRING]; - int errstrlen; - - /* If we can get an error string from MPI, print it to stderr. */ - if (!MPI_Error_string(mpierr, errstring, &errstrlen)) - fprintf(stderr, "MPI ERROR: %s in file %s at line %d\n", - errstring, filename, line); - - /* Handle all MPI errors as PIO_EIO. */ - check_netcdf(file, PIO_EIO, filename, line); - } -} - /** * @ingroup PIOc_inq * The PIO-C interface for the NetCDF function nc_inq. @@ -1151,79 +1125,6 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) return ierr; } -/** - * @ingroup PIOc_put_att_short - * The PIO-C interface for the NetCDF function nc_put_att_short. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_short(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - /** * @ingroup PIOc_rename_dim * The PIO-C interface for the NetCDF function nc_rename_dim. @@ -1297,20 +1198,20 @@ int PIOc_rename_dim (int ncid, int dimid, const char *name) } /** - * @ingroup PIOc_get_att_double - * The PIO-C interface for the NetCDF function nc_get_att_double. + * @ingroup PIOc_rename_var + * The PIO-C interface for the NetCDF function nc_rename_var. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) +int PIOc_rename_var (int ncid, int varid, const char *name) { int ierr; int msg; @@ -1326,7 +1227,7 @@ int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_DOUBLE; + msg = PIO_MSG_RENAME_VAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1340,19 +1241,19 @@ int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_double(file->fh, varid, name, ip);; + ierr = nc_rename_var(file->fh, varid, name);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_double(file->fh, varid, name, ip);; + ierr = nc_rename_var(file->fh, varid, name);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_double(file->fh, varid, name, ip);; + ierr = ncmpi_rename_var(file->fh, varid, name);; break; #endif default: @@ -1361,15 +1262,10 @@ int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_DOUBLE, ios->ioroot, ios->my_comm); - } if(errstr != NULL) free(errstr); return ierr; } @@ -1545,20 +1441,20 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, } /** - * @ingroup PIOc_put_att_double - * The PIO-C interface for the NetCDF function nc_put_att_double. + * @ingroup PIOc_inq_var_fill + * The PIO-C interface for the NetCDF function nc_inq_var_fill. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op) +int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) { int ierr; int msg; @@ -1574,7 +1470,7 @@ int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, P if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_DOUBLE; + msg = PIO_MSG_INQ_VAR_FILL; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1588,19 +1484,19 @@ int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, P #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_double(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_value);; break; #endif default: @@ -1613,13 +1509,111 @@ int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, P sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(fill_value, 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_uchar - * The PIO-C interface for the NetCDF function nc_get_att_uchar. + * @ingroup PIOc_get_att + * The PIO-C interface for the NetCDF function nc_get_att. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See + * PIOc_Set_File_Error_Handling + */ +int PIOc_get_att(int ncid, int varid, const char *name, void *ip) +{ + int msg = PIO_MSG_GET_ATT_INT; + iosystem_desc_t *ios; + file_desc_t *file; + int ierr = PIO_NOERR; + int mpierr = MPI_SUCCESS; + + /* User must provide a name and destination pointer. */ + if (!name || !ip) + return PIO_EINVAL; + + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) + return PIO_EBADID; + ios = file->iosystem; + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen = strlen(name); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + check_mpi(file, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + switch (file->iotype) + { +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att(file->fh, varid, name, ip); + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if (ios->io_rank == 0) + ierr = nc_get_att(file->fh, varid, name, ip); + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att(file->fh, varid, name, ip); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. */ + if (!ierr) + { + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_INT, ios->ioroot, ios->my_comm); + } + return ierr; +} + +/** + * @ingroup PIOc_put_att_short + * The PIO-C interface for the NetCDF function nc_put_att_short. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1631,7 +1625,7 @@ int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, P * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip) +int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op) { int ierr; int msg; @@ -1647,7 +1641,7 @@ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_UCHAR; + msg = PIO_MSG_PUT_ATT_SHORT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1661,19 +1655,19 @@ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_uchar(file->fh, varid, name, ip);; + ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_uchar(file->fh, varid, name, ip);; + ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_uchar(file->fh, varid, name, ip);; + ierr = ncmpi_put_att_short(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -1682,34 +1676,29 @@ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_CHAR, ios->ioroot, ios->my_comm); - } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_var_fill - * The PIO-C interface for the NetCDF function nc_inq_var_fill. + * @ingroup PIOc_get_att_double + * The PIO-C interface for the NetCDF function nc_get_att_double. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) +int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) { int ierr; int msg; @@ -1725,7 +1714,7 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VAR_FILL; + msg = PIO_MSG_GET_ATT_DOUBLE; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1739,19 +1728,19 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; + ierr = nc_get_att_double(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; + ierr = nc_get_att_double(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_value);; + ierr = ncmpi_get_att_double(file->fh, varid, name, ip);; break; #endif default: @@ -1760,18 +1749,22 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(fill_value, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_DOUBLE, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_schar - * The PIO-C interface for the NetCDF function nc_put_att_schar. + * @ingroup PIOc_put_att_double + * The PIO-C interface for the NetCDF function nc_put_att_double. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1783,7 +1776,7 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op) +int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op) { int ierr; int msg; @@ -1799,7 +1792,7 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_SCHAR; + msg = PIO_MSG_PUT_ATT_DOUBLE; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1813,19 +1806,19 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_schar(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_put_att_double(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -1843,8 +1836,8 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI } /** - * @ingroup PIOc_get_att_ushort - * The PIO-C interface for the NetCDF function nc_get_att_ushort. + * @ingroup PIOc_get_att_uchar + * The PIO-C interface for the NetCDF function nc_get_att_uchar. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1856,7 +1849,7 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip) +int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip) { int ierr; int msg; @@ -1872,7 +1865,7 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_USHORT; + msg = PIO_MSG_GET_ATT_UCHAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1886,19 +1879,19 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_ushort(file->fh, varid, name, ip);; + ierr = nc_get_att_uchar(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_ushort(file->fh, varid, name, ip);; + ierr = nc_get_att_uchar(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_ushort(file->fh, varid, name, ip);; + ierr = ncmpi_get_att_uchar(file->fh, varid, name, ip);; break; #endif default: @@ -1914,27 +1907,27 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * if(ierr == PIO_NOERR){ PIO_Offset attlen; PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_SHORT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_CHAR, ios->ioroot, ios->my_comm); } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_rename_var - * The PIO-C interface for the NetCDF function nc_rename_var. + * @ingroup PIOc_put_att_schar + * The PIO-C interface for the NetCDF function nc_put_att_schar. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_rename_var (int ncid, int varid, const char *name) +int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op) { int ierr; int msg; @@ -1950,7 +1943,7 @@ int PIOc_rename_var (int ncid, int varid, const char *name) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_RENAME_VAR; + msg = PIO_MSG_PUT_ATT_SCHAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1964,19 +1957,19 @@ int PIOc_rename_var (int ncid, int varid, const char *name) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_var(file->fh, varid, name);; + ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_rename_var(file->fh, varid, name);; + ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_var(file->fh, varid, name);; + ierr = ncmpi_put_att_schar(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -1993,6 +1986,84 @@ int PIOc_rename_var (int ncid, int varid, const char *name) return ierr; } +/** + * @ingroup PIOc_get_att_ushort + * The PIO-C interface for the NetCDF function nc_get_att_ushort. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_ATT_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_att_ushort(file->fh, varid, name, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_get_att_ushort(file->fh, varid, name, ip);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_get_att_ushort(file->fh, varid, name, ip);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_SHORT, ios->ioroot, ios->my_comm); + } + if(errstr != NULL) free(errstr); + return ierr; +} + /** * @ingroup PIOc_put_att_ulonglong * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 0a5cc4baa46..bd16fa0aff5 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -114,6 +114,32 @@ void pioassert(_Bool expression, const char *msg, const char *fname, const int l } +/** Handle MPI errors. An error message is sent to stderr, then the + check_netcdf() function is called with PIO_EIO. + + @param file pointer to the file_desc_t info + @param mpierr the MPI return code to handle + @param filename the name of the code file where error occured. + @param line the line of code where error occured. + */ +void check_mpi(file_desc_t *file, const int mpierr, const char *filename, + const int line) +{ + if (mpierr) + { + char errstring[MPI_MAX_ERROR_STRING]; + int errstrlen; + + /* If we can get an error string from MPI, print it to stderr. */ + if (!MPI_Error_string(mpierr, errstring, &errstrlen)) + fprintf(stderr, "MPI ERROR: %s in file %s at line %d\n", + errstring, filename, line); + + /* Handle all MPI errors as PIO_EIO. */ + check_netcdf(file, PIO_EIO, filename, line); + } +} + /** Check the result of a netCDF API call. * * @param file pointer to the PIO structure describing this file. From fb378ca2acc95d5ef4de45309d345f17a546a14b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 11 May 2016 14:58:49 -0400 Subject: [PATCH 027/184] added log message --- src/clib/pio_file.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index fac23a4146a..b68ce023768 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -1,3 +1,4 @@ +#include #include #include /** @@ -22,6 +23,8 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, int ierr = PIO_NOERR; int mpierr = MPI_SUCCESS; + LOG((1, "PIOc_openfile iosysid = %d", iosysid)); + /* Get the IO system info from the iosysid. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) { From 07830738067c7de55c2687329ba2738acb814971 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 11 May 2016 15:08:38 -0400 Subject: [PATCH 028/184] further development --- src/clib/pio_msg.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index e30dc8dce2b..3b26f1f598b 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -308,7 +308,7 @@ int att_handler(iosystem_desc_t *ios, int msg) int ret; char *name5; int namelen; - PIO_Offset len; + PIO_Offset attlen; nc_type xtype; int *op, *ip; @@ -329,13 +329,13 @@ int att_handler(iosystem_desc_t *ios, int msg) return PIO_ENOMEM; mpierr = MPI_Bcast((void *)name5, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); - if (!(op = malloc(len * sizeof(int)))) + mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!(op = malloc(attlen * sizeof(int)))) return PIO_ENOMEM; - mpierr = MPI_Bcast(op, len, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(op, attlen, MPI_INT, ios->compmaster, ios->intercomm); /* Call the function to write the attribute. */ - if ((ret = PIOc_put_att_int(ncid, varid, name5, xtype, len, op))) + if ((ret = PIOc_put_att_int(ncid, varid, name5, xtype, attlen, op))) return ret; /* Free resources. */ @@ -356,9 +356,9 @@ int att_handler(iosystem_desc_t *ios, int msg) mpierr = MPI_Bcast((void *)name5, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); /* Allocate space for the attribute data. */ - if ((ret = PIOc_inq_attlen(ncid, varid, name5, &len))) + if ((ret = PIOc_inq_attlen(ncid, varid, name5, &attlen))) return ret; - if (!(ip = malloc(len * sizeof(int)))) + if (!(ip = malloc(attlen * sizeof(int)))) return PIO_ENOMEM; /* Call the function to read the attribute. */ @@ -496,7 +496,7 @@ int sync_file_handler(iosystem_desc_t *ios) if ((ret = PIOc_sync(ncid))) return ret; - printf("%d sync_file_handler succeeded!\n", my_rank); + LOG((2, "sync_file_handler succeeded!")); return PIO_NOERR; } @@ -598,16 +598,14 @@ int def_dim_handler(iosystem_desc_t *ios) int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - LOG((1, "%d def_dim_handler comproot = %d\n", my_rank, ios->comproot)); + LOG((1, "def_dim_handler comproot = %d", ios->comproot)); /* Get the parameters for this function that the he comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d def_dim_handler ncid = %d\n", my_rank, ncid); if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d def_dim_handler ncid = %d namelen %d\n", my_rank, ncid, namelen); if (!(name = malloc(namelen + 1 * sizeof(char)))) return PIO_ENOMEM; if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, 0, @@ -615,9 +613,8 @@ int def_dim_handler(iosystem_desc_t *ios) return PIO_EIO; if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d def_dim_handler got parameters namelen = %d " - "name = %s len = %d ncid = %d\n", - my_rank, namelen, name, len, ncid); + LOG((2, "def_dim_handler got parameters namelen = %d " + "name = %s len = %d ncid = %d", namelen, name, len, ncid)); /* Call the create file function. */ if ((ret = PIOc_def_dim(ncid, name, len, &dimid))) @@ -685,9 +682,8 @@ int vara_handler(iosystem_desc_t *ios, int msg) if ((mpierr = MPI_Bcast(data, size_in_bytes, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - printf("%d def_var_handler got parameters namelen = %d " - "name = %s len = %d ncid = %d\n", - my_rank, namelen, name, len, ncid); + LOG((2, " def_var_handler got parameters namelen = %d " + "name = %s len = %d ncid = %d", namelen, name, len, ncid)); /* Call the create file function. */ if ((ret = PIOc_put_vara_int(ncid, varid, start, count, data))) @@ -893,7 +889,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) LOG((3, "about to call msg MPI_Bcast")); mpierr = MPI_Bcast(&msg, 1, MPI_INT, 0, my_iosys->io_comm); CheckMPIReturn(mpierr, __FILE__, __LINE__); - LOG((3, "msg MPI_Bcast complete msg = %d", msg)); + LOG((1, "msg MPI_Bcast complete msg = %d", msg)); /* Handle the message. This code is run on all IO tasks. */ switch (msg) From e2504bb0be58ef296156f3ad8454f1fd38d8d693 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 11 May 2016 15:27:00 -0400 Subject: [PATCH 029/184] further development of async --- src/clib/pio_internal.h | 4 +- src/clib/pio_msg.c | 106 ++++++++++++++++++++++++++++------------ src/clib/pio_nc_async.c | 2 + 3 files changed, 81 insertions(+), 31 deletions(-) diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 16e5120ead1..a288a15de1d 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -370,7 +370,9 @@ enum PIO_MSG{ PIO_MSG_FREEDECOMP, PIO_MSG_CLOSE_FILE, PIO_MSG_DELETE_FILE, - PIO_MSG_EXIT + PIO_MSG_EXIT, + PIO_MSG_GET_ATT, + PIO_MSG_PUT_ATT }; #endif diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 3b26f1f598b..c8fb89e9d2d 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -255,7 +255,6 @@ int inq_att_handler(iosystem_desc_t *ios) int ret; char *name5; int namelen; - PIO_Offset attlen; int *op, *ip; nc_type xtype, *xtypep = NULL; PIO_Offset len, *lenp = NULL; @@ -309,12 +308,10 @@ int att_handler(iosystem_desc_t *ios, int msg) char *name5; int namelen; PIO_Offset attlen; - nc_type xtype; + nc_type atttype; int *op, *ip; - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - LOG((1, "%d att_handler\n", my_rank)); + LOG((1, "att_handler msg = %d", msg)); if (msg == PIO_MSG_PUT_ATT_INT) { @@ -328,47 +325,94 @@ int att_handler(iosystem_desc_t *ios, int msg) if (!(name5 = malloc((namelen + 1) * sizeof(char)))) return PIO_ENOMEM; mpierr = MPI_Bcast((void *)name5, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&atttype, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!(op = malloc(attlen * sizeof(int)))) return PIO_ENOMEM; mpierr = MPI_Bcast(op, attlen, MPI_INT, ios->compmaster, ios->intercomm); /* Call the function to write the attribute. */ - if ((ret = PIOc_put_att_int(ncid, varid, name5, xtype, attlen, op))) + if ((ret = PIOc_put_att_int(ncid, varid, name5, atttype, attlen, op))) return ret; /* Free resources. */ free(name5); free(op); } - else if (msg = PIO_MSG_GET_ATT_INT) - { - /* Get the parameters for this function that the the comp master - * task is broadcasting. */ - if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) - return PIO_EIO; - if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) - return PIO_EIO; - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!(name5 = malloc((namelen + 1) * sizeof(char)))) - return PIO_ENOMEM; - mpierr = MPI_Bcast((void *)name5, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - /* Allocate space for the attribute data. */ - if ((ret = PIOc_inq_attlen(ncid, varid, name5, &attlen))) - return ret; - if (!(ip = malloc(attlen * sizeof(int)))) - return PIO_ENOMEM; + return PIO_NOERR; +} - /* Call the function to read the attribute. */ - if ((ret = PIOc_get_att_int(ncid, varid, name5, ip))) - return ret; +/** Handle attribute operations. This code only runs on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @param msg the message sent my the comp root task. + * @return PIO_NOERR for success, error code otherwise. +*/ +int att_get_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int mpierr; + int ierr; + char *name; + int namelen; + PIO_Offset attlen; + nc_type atttype; + int *op, *ip; + int iotype; - /* Free resources. */ - free(name5); - free(ip); + LOG((1, "att_get_handler")); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!(name = malloc((namelen + 1) * sizeof(char)))) + return PIO_ENOMEM; + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, + ios->intercomm); + if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + + /* Get the length and the type of the attribute. */ + switch (iotype) + { +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_att(ncid, varid, name, &atttype, (size_t *)&attlen); + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if (ios->io_rank == 0) + ierr = nc_inq_att(ncid, varid, name, &atttype, (size_t *)&attlen); + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_att(ncid, varid, name, &atttype, &attlen); + break; +#endif + default: + ierr = iotype_error(iotype,__FILE__,__LINE__); } + + /* Allocate space for the attribute data. */ + if (!(ip = malloc(attlen * sizeof(int)))) + return PIO_ENOMEM; + + /* Call the function to read the attribute. */ + if ((ierr = PIOc_get_att_int(ncid, varid, name, ip))) + return ierr; + + /* Free resources. */ + free(name); + free(ip); return PIO_NOERR; } @@ -931,6 +975,8 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) inq_var_handler(my_iosys); break; case PIO_MSG_GET_ATT_INT: + ret = att_get_handler(my_iosys); + break; case PIO_MSG_PUT_ATT_INT: ret = att_handler(my_iosys, msg); break; diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index dcaf033ed3e..13e0c002376 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1562,6 +1562,8 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); } /* Handle MPI errors. */ From 2a77fe14cfb275e7ccf6522c59576a8d918eab9e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 11 May 2016 16:18:15 -0400 Subject: [PATCH 030/184] further development of async --- src/clib/pio_msg.c | 8 +- src/clib/pio_nc_async.c | 827 ++---------------------------------- tests/unit/test_intercomm.c | 44 +- 3 files changed, 65 insertions(+), 814 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index c8fb89e9d2d..f04eda2ecd0 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -377,8 +377,11 @@ int att_get_handler(iosystem_desc_t *ios) ios->intercomm); if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; + LOG((1, "att_get_handler ncid = %d varid = %d namelen = %d name = %s iotype = %d", + ncid, varid, namelen, name, iotype)); /* Get the length and the type of the attribute. */ + LOG((1, "att_get_handler magic iotype = %d", iotype)); switch (iotype) { #ifdef _NETCDF @@ -780,9 +783,8 @@ int open_file_handler(iosystem_desc_t *ios) return PIO_EIO; if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - LOG((1, "%d open_file_handler got parameters len = %d " - "filename = %s iotype = %d mode = %d\n", - my_rank, len, filename, iotype, mode)); + LOG((2, "open_file_handler got parameters len = %d filename = %s iotype = %d mode = %d\n", + len, filename, iotype, mode)); /* Call the open file function. */ if ((ret = PIOc_openfile(ios->iosysid, &ncid, &iotype, filename, mode))) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 13e0c002376..fbd0313206b 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -18,6 +18,7 @@ #include #ifdef PIO_ENABLE_LOGGING #include +#include #endif /* PIO_ENABLE_LOGGING */ #include #include @@ -64,8 +65,8 @@ pio_log(int severity, const char *fmt, ...) if (severity > pio_log_level) return; - /* If the severity is 1 or less, only print on rank 0. */ - if (severity < 2 && my_rank != 0) + /* If the severity is 0, only print on rank 0. */ + if (severity < 1 && my_rank != 0) return; /* If the severity is zero, this is an error. Otherwise insert that @@ -1534,6 +1535,8 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) int msg = PIO_MSG_GET_ATT_INT; iosystem_desc_t *ios; file_desc_t *file; + PIO_Offset attlen; + nc_type atttype; int ierr = PIO_NOERR; int mpierr = MPI_SUCCESS; @@ -1541,6 +1544,8 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) if (!name || !ip) return PIO_EINVAL; + LOG((1, "PIOc_get_att ncid %d varid %d name %s", ncid, varid, name)); + /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; @@ -1564,6 +1569,7 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + LOG((2, "broadcast iotype = %d", file->iotype)); } /* Handle MPI errors. */ @@ -1580,23 +1586,32 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_get_att(file->fh, varid, name, ip); + if (!ierr) + ierr = nc_inq_att(file->fh, varid, name, &atttype, (size_t *)&attlen); break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if (ios->io_rank == 0) + { ierr = nc_get_att(file->fh, varid, name, ip); + if (!ierr) + ierr = nc_inq_att(file->fh, varid, name, &atttype, (size_t *)&attlen); + } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: ierr = ncmpi_get_att(file->fh, varid, name, ip); + if (!ierr) + ierr = ncmpi_inq_att(file->fh, varid, name, &atttype, &attlen); break; #endif default: ierr = iotype_error(file->iotype,__FILE__,__LINE__); } } + LOG((2, "PIOc_get_att called netcdf layer ierr = %d", ierr)); /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) @@ -1606,9 +1621,12 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) /* Broadcast results to all tasks. */ if (!ierr) { - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_INT, ios->ioroot, ios->my_comm); + LOG((2, "PIOc_get_att broadcasting attlen = %d", attlen)); + mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + LOG((2, "PIOc_get_att done broadcasting attlen = %d", attlen)); + LOG((2, "PIOc_get_att broadcasting att data")); + mpierr = MPI_Bcast(ip, (int)attlen, MPI_INT, ios->ioroot, ios->my_comm); + LOG((2, "PIOc_get_att done broadcasting att data")); } return ierr; } @@ -1700,68 +1718,9 @@ int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PI * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) +int PIOc_get_att_double(int ncid, int varid, const char *name, double *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_double(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_double(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_double(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_DOUBLE, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** @@ -1853,66 +1812,7 @@ int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, P */ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_uchar(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_uchar(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_uchar(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_CHAR, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** @@ -2004,66 +1904,7 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI */ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_USHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_ushort(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_ushort(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_ushort(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_SHORT, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** @@ -2374,66 +2215,7 @@ int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, co */ int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_uint(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_uint(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_uint(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** @@ -2526,66 +2308,7 @@ int PIOc_inq_format (int ncid, int *formatp) */ int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_long(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_long(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_long(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** @@ -2695,66 +2418,7 @@ int PIOc_inq_unlimdim(int ncid, int *unlimdimidp) */ int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_float(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_float(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_float(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_FLOAT, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** @@ -3077,66 +2741,7 @@ int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, */ int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_UBYTE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_ubyte(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_ubyte(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_ubyte(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_BYTE, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** @@ -3155,66 +2760,7 @@ int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip */ int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_text(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_text(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_text(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** @@ -3306,66 +2852,7 @@ int PIOc_del_att (int ncid, int varid, const char *name) */ int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_schar(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_schar(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_schar(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** @@ -3384,66 +2871,7 @@ int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) */ int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_ulonglong(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_LONG_LONG, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** @@ -3618,66 +3046,7 @@ int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO */ int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_short(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_short(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_short(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_SHORT, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** @@ -3841,70 +3210,7 @@ int PIOc_put_att_ubyte (int ncid, int varid, const char *name, nc_type xtype, PI */ int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - int namelen; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - namelen = strlen(name); - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_int(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_int(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_int(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_INT, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** @@ -3923,66 +3229,7 @@ int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) */ int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_ATT_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_longlong(file->fh, varid, name, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_get_att_longlong(file->fh, varid, name, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_longlong(file->fh, varid, name, ip);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG_LONG, ios->ioroot, ios->my_comm); - } - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index cfddf1e600a..cf99c672a05 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -92,6 +92,7 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) if ((ret = PIOc_openfile(iosysid, &ncid, &format, filename, NC_NOWRITE))) ERR(ret); + sleep(2); /* Find the number of dimensions, variables, and global attributes.*/ if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) @@ -136,10 +137,10 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) ERR(ret); if (dimlen2 != DIM_LEN) ERR(ERR_WRONG); - if ((ret = PIOc_inq_dimid(ncid, DIM_NAME, &dimid2))) - ERR(ret); - if (dimid2 != 0) - ERR(ERR_WRONG); + /* if ((ret = PIOc_inq_dimid(ncid, DIM_NAME, &dimid2))) */ + /* ERR(ret); */ + /* if (dimid2 != 0) */ + /* ERR(ERR_WRONG); */ /* Check out the variable. */ if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts))) @@ -169,29 +170,30 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) ERR(ret); if (varnatts2 != 0) ERR(ERR_WRONG); - if ((ret = PIOc_inq_varid(ncid, VAR_NAME, &varid2))) - ERR(ret); - if (varid2 != 0) - ERR(ERR_WRONG); + /* if ((ret = PIOc_inq_varid(ncid, VAR_NAME, &varid2))) */ + /* ERR(ret); */ + /* if (varid2 != 0) */ + /* ERR(ERR_WRONG); */ /* Check out the global attributes. */ nc_type atttype; PIO_Offset attlen; - if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &atttype, &attlen))) - ERR(ret); - if (atttype != NC_INT || attlen != 1) - ERR(ERR_WRONG); - if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &attlen))) - ERR(ret); - if (attlen != 1) - ERR(ERR_WRONG); - /* if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, &att_data))) */ + /* if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &atttype, &attlen))) */ + /* ERR(ret); */ + /* if (atttype != NC_INT || attlen != 1) */ + /* ERR(ERR_WRONG); */ + /* if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &attlen))) */ /* ERR(ret); */ - /* sleep(2); */ - /* if (verbose) */ - /* printf("%d test_intercomm att_data = %d\n", my_rank, att_data); */ - /* if (att_data != ATT_VALUE) */ + /* if (attlen != 1) */ /* ERR(ERR_WRONG); */ + sleep(2); + if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, &att_data))) + ERR(ret); + sleep(2); + if (verbose) + printf("%d test_intercomm att_data = %d\n", my_rank, att_data); + if (att_data != ATT_VALUE) + ERR(ERR_WRONG); /* Close the file. */ if (verbose) From f028866a1d06ccede498ac4a785cca97e254aa21 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 11 May 2016 16:18:32 -0400 Subject: [PATCH 031/184] further development of async --- tests/unit/test_intercomm.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index cf99c672a05..e36a508f6ec 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -178,18 +178,16 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) /* Check out the global attributes. */ nc_type atttype; PIO_Offset attlen; - /* if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &atttype, &attlen))) */ - /* ERR(ret); */ - /* if (atttype != NC_INT || attlen != 1) */ - /* ERR(ERR_WRONG); */ - /* if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &attlen))) */ - /* ERR(ret); */ - /* if (attlen != 1) */ - /* ERR(ERR_WRONG); */ - sleep(2); + if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &atttype, &attlen))) + ERR(ret); + if (atttype != NC_INT || attlen != 1) + ERR(ERR_WRONG); + if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &attlen))) + ERR(ret); + if (attlen != 1) + ERR(ERR_WRONG); if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, &att_data))) ERR(ret); - sleep(2); if (verbose) printf("%d test_intercomm att_data = %d\n", my_rank, att_data); if (att_data != ATT_VALUE) From 891b29c28370d1f2c41e68531407c635872e7155 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 11 May 2016 16:19:25 -0400 Subject: [PATCH 032/184] further development of async --- tests/unit/test_intercomm.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index e36a508f6ec..7e46d59d093 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -170,10 +170,10 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) ERR(ret); if (varnatts2 != 0) ERR(ERR_WRONG); - /* if ((ret = PIOc_inq_varid(ncid, VAR_NAME, &varid2))) */ - /* ERR(ret); */ - /* if (varid2 != 0) */ - /* ERR(ERR_WRONG); */ + if ((ret = PIOc_inq_varid(ncid, VAR_NAME, &varid2))) + ERR(ret); + if (varid2 != 0) + ERR(ERR_WRONG); /* Check out the global attributes. */ nc_type atttype; From 8b460808550218894b1f719a55641ad350103284 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 11 May 2016 16:28:09 -0400 Subject: [PATCH 033/184] further development of async --- src/clib/pio_msg.c | 6 ++---- src/clib/pio_nc_async.c | 5 +++-- tests/unit/test_intercomm.c | 10 ++++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index f04eda2ecd0..530ea5c2693 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -207,9 +207,7 @@ int inq_dimid_handler(iosystem_desc_t *ios) int namelen; char *name; - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - LOG((1, "inq_dimid_handler\n", my_rank)); + LOG((1, "inq_dimid_handler")); /* Get the parameters for this function that the the comp master * task is broadcasting. */ @@ -223,7 +221,7 @@ int inq_dimid_handler(iosystem_desc_t *ios) return PIO_EIO; if ((mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - LOG((1, "%d inq_dimid_handler ncid = %d namelen = %d name = %s id_present = %d\n", + LOG((1, "inq_dimid_handler ncid = %d namelen = %d name = %s id_present = %d", ncid, namelen, name, id_present)); /* Set non-null pointer. */ diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index fbd0313206b..f92c81924ef 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -440,11 +440,10 @@ int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp) */ int PIOc_inq_dimid(int ncid, const char *name, int *idp) { - int msg = PIO_MSG_INQ_DIMID; iosystem_desc_t *ios; file_desc_t *file; int ierr = PIO_NOERR; - int mpierr; + int mpierr = MPI_SUCCESS; /* Name must be provided. */ if (!name) @@ -462,10 +461,12 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) { if (!ios->ioproc) { + int msg = PIO_MSG_INQ_DIMID; char id_present = idp ? true : false; if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if (!mpierr) mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); int namelen = strlen(name); diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 7e46d59d093..b427ab7b466 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -137,10 +137,12 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) ERR(ret); if (dimlen2 != DIM_LEN) ERR(ERR_WRONG); - /* if ((ret = PIOc_inq_dimid(ncid, DIM_NAME, &dimid2))) */ - /* ERR(ret); */ - /* if (dimid2 != 0) */ - /* ERR(ERR_WRONG); */ + sleep(2); + if ((ret = PIOc_inq_dimid(ncid, DIM_NAME, &dimid2))) + ERR(ret); + if (dimid2 != 0) + ERR(ERR_WRONG); + sleep(2); /* Check out the variable. */ if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts))) From 80a361f60384d27934a1a6f7b7f40716ea1e991b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 May 2016 12:11:10 -0400 Subject: [PATCH 034/184] further async development --- src/clib/pio.h | 2 +- src/clib/pio_nc_async.c | 381 +++++++++++++++++++++++++----------- tests/unit/test_intercomm.c | 2 +- 3 files changed, 268 insertions(+), 117 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index 2be57ca845c..5dc08cedee2 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -648,7 +648,7 @@ int PIOc_set_blocksize(const int newblocksize); int PIOc_get_var_schar (int ncid, int varid, signed char *buf); int PIOc_iotype_available(const int iotype); int PIOc_set_log_level(int level); - int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, size_t len, const void *op); + int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const void *op); int PIOc_get_att(int ncid, int varid, const char *name, void *ip); #if defined(__cplusplus) } diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index f92c81924ef..e5bd7fc693c 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -113,7 +113,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ LOG((1, "PIOc_inq ncid = %d", ncid)); @@ -149,7 +149,8 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, } /* Handle MPI errors. */ - mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -297,9 +298,8 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) { iosystem_desc_t *ios; file_desc_t *file; - int msg = PIO_MSG_INQ_DIM; int ierr = PIO_NOERR; - int mpierr; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ LOG((1, "PIOc_inq_dim")); @@ -313,10 +313,13 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) { if (!ios->ioproc) { + int msg = PIO_MSG_INQ_DIM; char name_present = name ? true : false; - char len_present = lenp ? true : false; + char len_present = lenp ? true : false; + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if (!mpierr) mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -330,7 +333,8 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) } /* Handle MPI errors. */ - mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -443,7 +447,7 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) iosystem_desc_t *ios; file_desc_t *file; int ierr = PIO_NOERR; - int mpierr = MPI_SUCCESS; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ /* Name must be provided. */ if (!name) @@ -479,7 +483,8 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) } /* Handle MPI errors. */ - mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -516,9 +521,10 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results. */ - if (idp) - if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + if (!ierr) + if (idp) + if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); return ierr; } @@ -546,8 +552,7 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, file_desc_t *file; int ndims; /** The number of dimensions for this variable. */ int ierr = PIO_NOERR; - int msg = PIO_MSG_INQ_VAR; - int mpierr; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ LOG((1, "PIOc_inq_var ncid = %d varid = %d", ncid, varid)); @@ -556,32 +561,51 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, return PIO_EBADID; ios = file->iosystem; - /* If using async, and this is not an IO task, send the parameters to the IO task. */ - if (ios->async_interface && !ios->ioproc) + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) { - char name_present = name ? true : false; - char xtype_present = xtypep ? true : false; - char ndims_present = ndimsp ? true : false; - char dimids_present = dimidsp ? true : false; - char natts_present = nattsp ? true : false; - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_VAR; + char name_present = name ? true : false; + char xtype_present = xtypep ? true : false; + char ndims_present = ndimsp ? true : false; + char dimids_present = dimidsp ? true : false; + char natts_present = nattsp ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); LOG((2, "PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d " "dimids_present = %d, natts_present = %d nattsp = %d", name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp)); - } + } + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); + } + /* Call the netCDF layer. */ if (ios->ioproc) { - switch(file->iotype){ + switch(file->iotype) + { #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: @@ -615,36 +639,39 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast the results for non-null pointers. */ - if (name) - { - int slen; - if(ios->iomaster) - slen = strlen(name); - if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - } - if (xtypep) - if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - - if (ndimsp) - { - if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - file->varlist[varid].ndims = (*ndimsp); - } - if (dimidsp) + if (!ierr) { - if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + if (name) + { + int slen; + if(ios->iomaster) + slen = strlen(name); + if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + } + if (xtypep) + if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + + if (ndimsp) + { + if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + file->varlist[varid].ndims = (*ndimsp); + } + if (dimidsp) + { + if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + } + if (nattsp) + if ((mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); } - if (nattsp) - if ((mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); return ierr; } @@ -763,11 +790,10 @@ int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) */ int PIOc_inq_varid (int ncid, const char *name, int *varidp) { - int ierr = PIO_NOERR; - int msg = PIO_MSG_INQ_VARID; - int mpierr = 0; iosystem_desc_t *ios; file_desc_t *file; + int ierr = PIO_NOERR; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ /* Caller must provide name. */ if (!name) @@ -784,6 +810,8 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) { if (!ios->ioproc) { + int msg = PIO_MSG_INQ_VARID; + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); @@ -798,7 +826,8 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) } /* Handle MPI errors. */ - mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -864,7 +893,7 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, int msg = PIO_MSG_INQ_ATT; iosystem_desc_t *ios; file_desc_t *file; - int mpierr; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ int ierr = PIO_NOERR; /* Caller must provide a name. */ @@ -923,10 +952,10 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, } } - /* Handle errors. */ - if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - check_netcdf(file, ierr, __FILE__, __LINE__); + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); /* Broadcast results. */ if (!ierr) @@ -1001,7 +1030,7 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) { int ierr = PIO_NOERR; int msg = PIO_MSG_INQ_ATTNAME; - int mpierr; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ iosystem_desc_t *ios; file_desc_t *file; @@ -1072,7 +1101,7 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) { int ierr = PIO_NOERR; int msg = PIO_MSG_INQ_ATTID; - int mpierr; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ iosystem_desc_t *ios; file_desc_t *file; @@ -1111,10 +1140,10 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) } } - /* Handle errors. */ - if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - check_netcdf(file, ierr, __FILE__, __LINE__); + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); /* Broadcast results. */ if (!ierr) @@ -1144,7 +1173,7 @@ int PIOc_rename_dim (int ncid, int dimid, const char *name) { int ierr; int msg; - int mpierr; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ iosystem_desc_t *ios; file_desc_t *file; char *errstr; @@ -1217,7 +1246,7 @@ int PIOc_rename_var (int ncid, int varid, const char *name) { int ierr; int msg; - int mpierr; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ iosystem_desc_t *ios; file_desc_t *file; char *errstr; @@ -1289,7 +1318,7 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) { int ierr; int msg; - int mpierr; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ iosystem_desc_t *ios; file_desc_t *file; char *errstr; @@ -1363,39 +1392,63 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp) { - int msg = PIO_MSG_DEF_VAR; - int mpierr; iosystem_desc_t *ios; file_desc_t *file; int ierr = PIO_NOERR; - char *errstr = NULL; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ int namelen; + /* User must provide name and storage for varid. */ + if (!name || !varidp) + { + check_netcdf(file, PIO_EINVAL, __FILE__, __LINE__); + return PIO_EINVAL; + } + /* Get the file information. */ if (!(file = pio_get_file_from_id(ncid))) + { + check_netcdf(file, PIO_EBADID, __FILE__, __LINE__); return PIO_EBADID; + } ios = file->iosystem; - /* If async is in use, and this is not an IO tasks, then bcast the - * function parameters to the intercomm, where it will be read by - * the IO root task. */ - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm); - namelen = strlen(name); - LOG((2, "bcasting namelen = %d name = %s\n", namelen, name)); - if (!ios->compmaster) - ios->compmaster = MPI_PROC_NULL; - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmaster, ios->intercomm); + /* If using async, and not an IO task, then send parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_DEF_VAR; + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + if (!mpierr) + mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm); + namelen = strlen(name); + LOG((2, "bcasting namelen = %d name = %s\n", namelen, name)); + if (!ios->compmaster) + ios->compmaster = MPI_PROC_NULL; + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } - if(ios->ioproc){ - switch(file->iotype) + /* IO tasks call the netCDF functions. */ + if (ios->ioproc) + { + switch (file->iotype) { #ifdef _NETCDF #ifdef _NETCDF4 @@ -1426,19 +1479,16 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, } } - /* Handle errors. */ - if (ierr) - { - errstr = (char *)malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr, "in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm); + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results. */ + if (!ierr) + if (varidp) + mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm); - /* Free error string if allocated. */ - if(errstr) - free(errstr); - return ierr; } @@ -1460,7 +1510,7 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) { int ierr; int msg; - int mpierr; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ iosystem_desc_t *ios; file_desc_t *file; char *errstr; @@ -1533,13 +1583,12 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) */ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) { - int msg = PIO_MSG_GET_ATT_INT; iosystem_desc_t *ios; file_desc_t *file; PIO_Offset attlen; nc_type atttype; int ierr = PIO_NOERR; - int mpierr = MPI_SUCCESS; + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ /* User must provide a name and destination pointer. */ if (!name || !ip) @@ -1557,8 +1606,11 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) { if (!ios->ioproc) { + int msg = PIO_MSG_GET_ATT_INT; + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if (!mpierr) mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -1574,7 +1626,8 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) } /* Handle MPI errors. */ - mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -1632,6 +1685,104 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) return ierr; } +/** + * @ingroup PIOc_put_att + * The PIO-C interface for the NetCDF function nc_put_att. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const void *op) +{ + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + + LOG((1, "PIOc_inq_varid ncid = %d name = %s", ncid, name)); + + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) + return PIO_EBADID; + ios = file->iosystem; + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_PUT_ATT; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen = strlen(name); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)op, len, MPI_INT, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + switch (file->iotype) + { +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_put_att(file->fh, varid, name, xtype, (size_t)len, op); + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_att(file->fh, varid, name, xtype, (size_t)len, op); + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_put_att(file->fh, varid, name, xtype, len, op); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + + return ierr; +} + /** * @ingroup PIOc_put_att_short * The PIO-C interface for the NetCDF function nc_put_att_short. @@ -1650,7 +1801,7 @@ int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PI { int ierr; int msg; - int mpierr; + int mpierr = MPI_SUCCESS; iosystem_desc_t *ios; file_desc_t *file; char *errstr; diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index b427ab7b466..f6dbfc966c5 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -371,7 +371,7 @@ main(int argc, char **argv) /* Add a global attribute. */ if (verbose) - printf("%d test_intercomm writing attribute %s\n", my_rank, ATT_NAME); + printf("%d test_intercomm writing attributes %s\n", my_rank, ATT_NAME); int att_data = ATT_VALUE; if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) ERR(ret); From 10b178ac88a72cb16190b4943398bfdc42d7c840 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 May 2016 12:31:51 -0400 Subject: [PATCH 035/184] code cleanup --- src/clib/pio_nc_async.c | 335 ++++++---------------------------------- 1 file changed, 44 insertions(+), 291 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index e5bd7fc693c..278b19f759f 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -280,6 +280,24 @@ int PIOc_inq_natts(int ncid, int *ngattsp) return PIOc_inq(ncid, NULL, NULL, ngattsp, NULL); } +/** + * @ingroup PIOc_inq_unlimdim + * The PIO-C interface for the NetCDF function nc_inq_unlimdim. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_unlimdim(int ncid, int *unlimdimidp) +{ + return PIOc_inq(ncid, NULL, NULL, unlimdimidp, NULL); +} + /** * @ingroup PIOc_inq_dim * The PIO-C interface for the NetCDF function nc_inq_dim. @@ -1786,18 +1804,9 @@ int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, /** * @ingroup PIOc_put_att_short * The PIO-C interface for the NetCDF function nc_put_att_short. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op) +int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const short *op) { int ierr; int msg; @@ -1859,16 +1868,6 @@ int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PI /** * @ingroup PIOc_get_att_double * The PIO-C interface for the NetCDF function nc_get_att_double. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_double(int ncid, int varid, const char *name, double *ip) { @@ -1878,18 +1877,9 @@ int PIOc_get_att_double(int ncid, int varid, const char *name, double *ip) /** * @ingroup PIOc_put_att_double * The PIO-C interface for the NetCDF function nc_put_att_double. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op) +int PIOc_put_att_double(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const double *op) { int ierr; int msg; @@ -1951,16 +1941,6 @@ int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, P /** * @ingroup PIOc_get_att_uchar * The PIO-C interface for the NetCDF function nc_get_att_uchar. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip) { @@ -1970,18 +1950,9 @@ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip /** * @ingroup PIOc_put_att_schar * The PIO-C interface for the NetCDF function nc_put_att_schar. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op) +int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const signed char *op) { int ierr; int msg; @@ -2043,16 +2014,6 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI /** * @ingroup PIOc_get_att_ushort * The PIO-C interface for the NetCDF function nc_get_att_ushort. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip) { @@ -2062,18 +2023,9 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * /** * @ingroup PIOc_put_att_ulonglong * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op) +int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const unsigned long long *op) { int ierr; int msg; @@ -2208,18 +2160,9 @@ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) /** * @ingroup PIOc_put_att_ushort * The PIO-C interface for the NetCDF function nc_put_att_ushort. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op) +int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const unsigned short *op) { int ierr; int msg; @@ -2281,18 +2224,9 @@ int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, P /** * @ingroup PIOc_put_att_text * The PIO-C interface for the NetCDF function nc_put_att_text. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, const char *op) +int PIOc_put_att_text(int ncid, int varid, const char *name, + PIO_Offset len, const char *op) { int ierr; int msg; @@ -2354,16 +2288,6 @@ int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, co /** * @ingroup PIOc_get_att_uint * The PIO-C interface for the NetCDF function nc_get_att_uint. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) { @@ -2447,16 +2371,6 @@ int PIOc_inq_format (int ncid, int *formatp) /** * @ingroup PIOc_get_att_long * The PIO-C interface for the NetCDF function nc_get_att_long. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) { @@ -2466,18 +2380,9 @@ int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) /** * @ingroup PIOc_put_att_long * The PIO-C interface for the NetCDF function nc_put_att_long. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op) +int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const long *op) { int ierr; int msg; @@ -2536,37 +2441,9 @@ int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO return ierr; } -/** - * @ingroup PIOc_inq_unlimdim - * The PIO-C interface for the NetCDF function nc_inq_unlimdim. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_inq_unlimdim(int ncid, int *unlimdimidp) -{ - return PIOc_inq(ncid, NULL, NULL, unlimdimidp, NULL); -} - /** * @ingroup PIOc_get_att_float * The PIO-C interface for the NetCDF function nc_get_att_float. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) { @@ -2576,18 +2453,9 @@ int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) /** * @ingroup PIOc_put_att_int * The PIO-C interface for the NetCDF function nc_put_att_int. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op) +int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const int *op) { int ierr; int msg; @@ -2733,18 +2601,9 @@ int PIOc_enddef(int ncid) /** * @ingroup PIOc_put_att_uchar * The PIO-C interface for the NetCDF function nc_put_att_uchar. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) +int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const unsigned char *op) { int ierr; int msg; @@ -2806,18 +2665,9 @@ int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PI /** * @ingroup PIOc_put_att_longlong * The PIO-C interface for the NetCDF function nc_put_att_longlong. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long long *op) +int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const long long *op) { int ierr; int msg; @@ -2880,16 +2730,6 @@ int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, /** * @ingroup PIOc_get_att_ubyte * The PIO-C interface for the NetCDF function nc_get_att_ubyte. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip) { @@ -2899,16 +2739,6 @@ int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip /** * @ingroup PIOc_get_att_text * The PIO-C interface for the NetCDF function nc_get_att_text. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) { @@ -2991,16 +2821,6 @@ int PIOc_del_att (int ncid, int varid, const char *name) /** * @ingroup PIOc_get_att_schar * The PIO-C interface for the NetCDF function nc_get_att_schar. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) { @@ -3010,16 +2830,6 @@ int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) /** * @ingroup PIOc_get_att_ulonglong * The PIO-C interface for the NetCDF function nc_get_att_ulonglong. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip) { @@ -3112,18 +2922,9 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) /** * @ingroup PIOc_put_att_uint * The PIO-C interface for the NetCDF function nc_put_att_uint. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned int *op) +int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const unsigned int *op) { int ierr; int msg; @@ -3185,16 +2986,6 @@ int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO /** * @ingroup PIOc_get_att_short * The PIO-C interface for the NetCDF function nc_get_att_short. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) { @@ -3276,18 +3067,9 @@ int PIOc_redef (int ncid) /** * @ingroup PIOc_put_att_ubyte * The PIO-C interface for the NetCDF function nc_put_att_ubyte. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_ubyte (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) +int PIOc_put_att_ubyte(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const unsigned char *op) { int ierr; int msg; @@ -3349,16 +3131,6 @@ int PIOc_put_att_ubyte (int ncid, int varid, const char *name, nc_type xtype, PI /** * @ingroup PIOc_get_att_int * The PIO-C interface for the NetCDF function nc_get_att_int. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) { @@ -3368,16 +3140,6 @@ int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) /** * @ingroup PIOc_get_att_longlong * The PIO-C interface for the NetCDF function nc_get_att_longlong. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) { @@ -3387,18 +3149,9 @@ int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) /** * @ingroup PIOc_put_att_float * The PIO-C interface for the NetCDF function nc_put_att_float. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const float *op) +int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const float *op) { int ierr; int msg; From a0aaea9fe56ef3d092235e9b06d2acef8efce899 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 May 2016 12:42:54 -0400 Subject: [PATCH 036/184] further cleanup of async code --- src/clib/pio_nc_async.c | 56 +---------------------------------------- 1 file changed, 1 insertion(+), 55 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 278b19f759f..b8737990ef0 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1808,61 +1808,7 @@ int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op) { - int ierr; - int msg; - int mpierr = MPI_SUCCESS; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_short(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** From 96e74980a53a53a7a34a34a8c667f7f61b3a946a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 May 2016 12:46:55 -0400 Subject: [PATCH 037/184] further cleanup of async code --- src/clib/pio_nc_async.c | 448 +--------------------------------------- 1 file changed, 8 insertions(+), 440 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index b8737990ef0..ad646e26573 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1827,61 +1827,7 @@ int PIOc_get_att_double(int ncid, int varid, const char *name, double *ip) int PIOc_put_att_double(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_double(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_double(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** @@ -1900,61 +1846,7 @@ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_schar(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** @@ -1973,61 +1865,7 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_ulonglong(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** @@ -2110,61 +1948,7 @@ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_USHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_ushort(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** @@ -2174,61 +1958,7 @@ int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype, int PIOc_put_att_text(int ncid, int varid, const char *name, PIO_Offset len, const char *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_text(file->fh, varid, name, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } /** @@ -2330,61 +2060,7 @@ int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_long(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } /** @@ -2551,61 +2227,7 @@ int PIOc_enddef(int ncid) int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_uchar(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } /** @@ -2615,61 +2237,7 @@ int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype, int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long long *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_longlong(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } From f788f6b923e9c502556c640953d8eade88af28a2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 May 2016 12:48:19 -0400 Subject: [PATCH 038/184] further cleanup of async code --- src/clib/pio_nc_async.c | 117 ++-------------------------------------- 1 file changed, 4 insertions(+), 113 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index ad646e26573..ed62343c874 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -2240,7 +2240,6 @@ int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype, return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } - /** * @ingroup PIOc_get_att_ubyte * The PIO-C interface for the NetCDF function nc_get_att_ubyte. @@ -2440,61 +2439,7 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned int *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_uint(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } /** @@ -2585,68 +2530,14 @@ int PIOc_redef (int ncid) int PIOc_put_att_ubyte(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_UBYTE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_ubyte(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_ubyte(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_ubyte(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } /** * @ingroup PIOc_get_att_int * The PIO-C interface for the NetCDF function nc_get_att_int. */ -int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) +int PIOc_get_att_int(int ncid, int varid, const char *name, int *ip) { return PIOc_get_att(ncid, varid, name, (void *)ip); } @@ -2655,7 +2546,7 @@ int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) * @ingroup PIOc_get_att_longlong * The PIO-C interface for the NetCDF function nc_get_att_longlong. */ -int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) +int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip) { return PIOc_get_att(ncid, varid, name, (void *)ip); } From 0a6fb4f902cc3a76bf62e01f5c29eb398687d939 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 May 2016 12:50:30 -0400 Subject: [PATCH 039/184] further cleanup of async code --- src/clib/pio_nc_async.c | 110 ++++++++++------------------------------ 1 file changed, 28 insertions(+), 82 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index ed62343c874..e1075701e48 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -2349,6 +2349,33 @@ int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long return PIOc_get_att(ncid, varid, name, (void *)ip); } +/** + * @ingroup PIOc_get_att_short + * The PIO-C interface for the NetCDF function nc_get_att_short. + */ +int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) +{ + return PIOc_get_att(ncid, varid, name, (void *)ip); +} + +/** + * @ingroup PIOc_get_att_int + * The PIO-C interface for the NetCDF function nc_get_att_int. + */ +int PIOc_get_att_int(int ncid, int varid, const char *name, int *ip) +{ + return PIOc_get_att(ncid, varid, name, (void *)ip); +} + +/** + * @ingroup PIOc_get_att_longlong + * The PIO-C interface for the NetCDF function nc_get_att_longlong. + */ +int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip) +{ + return PIOc_get_att(ncid, varid, name, (void *)ip); +} + /** * @ingroup PIOc_def_dim * The PIO-C interface for the NetCDF function nc_def_dim. @@ -2442,15 +2469,6 @@ int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } -/** - * @ingroup PIOc_get_att_short - * The PIO-C interface for the NetCDF function nc_get_att_short. - */ -int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) -{ - return PIOc_get_att(ncid, varid, name, (void *)ip); -} - /** * @ingroup PIOc_redef * The PIO-C interface for the NetCDF function nc_redef. @@ -2533,24 +2551,6 @@ int PIOc_put_att_ubyte(int ncid, int varid, const char *name, nc_type xtype, return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } -/** - * @ingroup PIOc_get_att_int - * The PIO-C interface for the NetCDF function nc_get_att_int. - */ -int PIOc_get_att_int(int ncid, int varid, const char *name, int *ip) -{ - return PIOc_get_att(ncid, varid, name, (void *)ip); -} - -/** - * @ingroup PIOc_get_att_longlong - * The PIO-C interface for the NetCDF function nc_get_att_longlong. - */ -int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip) -{ - return PIOc_get_att(ncid, varid, name, (void *)ip); -} - /** * @ingroup PIOc_put_att_float * The PIO-C interface for the NetCDF function nc_put_att_float. @@ -2558,60 +2558,6 @@ int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip) int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const float *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_float(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } From 54c8287f32cb3378cf78dded89367656316e69f1 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 May 2016 12:53:36 -0400 Subject: [PATCH 040/184] further cleanup of async code --- src/clib/pio_nc_async.c | 456 ++++++++++++++++++++-------------------- 1 file changed, 228 insertions(+), 228 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index e1075701e48..e3e1de1b9b1 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1391,6 +1391,234 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) return ierr; } +/** + * @ingroup PIOc_enddef + * The PIO-C interface for the NetCDF function nc_enddef. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_enddef(int ncid) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_ENDDEF; + + if(ios->async_interface && ! ios->ioproc){ + if(!ios->comp_rank) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + printf("PIOc_enddef file->fh = %d\n", file->fh); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_enddef(file->fh);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_enddef(file->fh);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_enddef(file->fh);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_redef + * The PIO-C interface for the NetCDF function nc_redef. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_redef (int ncid) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_REDEF; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_redef(file->fh);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_redef(file->fh);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_redef(file->fh);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_def_dim + * The PIO-C interface for the NetCDF function nc_def_dim. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param idp a pointer that will get the id of the variable or attribute. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + int namelen; + + errstr = NULL; + ierr = PIO_NOERR; + + int my_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + printf("%d PIOc_def_dim ncid = %d name = %s len = %d\n", my_rank, + ncid, name, len); + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_DEF_DIM; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + namelen = strlen(name); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_def_dim(file->fh, name, len, idp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + /** * @ingroup PIOc_def_var * The PIO-C interface for the NetCDF function nc_def_var. @@ -2147,79 +2375,6 @@ int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype, return ierr; } -/** - * @ingroup PIOc_enddef - * The PIO-C interface for the NetCDF function nc_enddef. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_enddef(int ncid) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_ENDDEF; - - if(ios->async_interface && ! ios->ioproc){ - if(!ios->comp_rank) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - printf("PIOc_enddef file->fh = %d\n", file->fh); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_enddef(file->fh);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_enddef(file->fh);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_enddef(file->fh);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - /** * @ingroup PIOc_put_att_uchar * The PIO-C interface for the NetCDF function nc_put_att_uchar. @@ -2376,89 +2531,6 @@ int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip) return PIOc_get_att(ncid, varid, name, (void *)ip); } -/** - * @ingroup PIOc_def_dim - * The PIO-C interface for the NetCDF function nc_def_dim. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param idp a pointer that will get the id of the variable or attribute. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - int namelen; - - errstr = NULL; - ierr = PIO_NOERR; - - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d PIOc_def_dim ncid = %d name = %s len = %d\n", my_rank, - ncid, name, len); - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_DEF_DIM; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - namelen = strlen(name); - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); - } - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_def_dim(file->fh, name, len, idp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; -} - /** * @ingroup PIOc_put_att_uint * The PIO-C interface for the NetCDF function nc_put_att_uint. @@ -2469,78 +2541,6 @@ int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } -/** - * @ingroup PIOc_redef - * The PIO-C interface for the NetCDF function nc_redef. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_redef (int ncid) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_REDEF; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_redef(file->fh);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_redef(file->fh);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_redef(file->fh);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - /** * @ingroup PIOc_put_att_ubyte * The PIO-C interface for the NetCDF function nc_put_att_ubyte. From 19ea32285ecf07e4033f86cd18f7f05015c71afa Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 May 2016 14:08:21 -0400 Subject: [PATCH 041/184] further development of async code --- src/clib/pio_nc_async.c | 582 ++++++++++++++++++++-------------------- 1 file changed, 291 insertions(+), 291 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index e3e1de1b9b1..51bb7f7813a 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -298,6 +298,80 @@ int PIOc_inq_unlimdim(int ncid, int *unlimdimidp) return PIOc_inq(ncid, NULL, NULL, unlimdimidp, NULL); } +/** + * @ingroup PIOc_inq_format + * The PIO-C interface for the NetCDF function nc_inq_format. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param formatp a pointer that will get the file format + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_inq_format (int ncid, int *formatp) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_INQ_FORMAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_format(file->fh, formatp);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_inq_format(file->fh, formatp);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_inq_format(file->fh, formatp);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); + return ierr; +} + /** * @ingroup PIOc_inq_dim * The PIO-C interface for the NetCDF function nc_inq_dim. @@ -1319,6 +1393,152 @@ int PIOc_rename_var (int ncid, int varid, const char *name) return ierr; } +/** + * @ingroup PIOc_rename_att + * The PIO-C interface for the NetCDF function nc_rename_att. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_RENAME_ATT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_rename_att(file->fh, varid, name, newname);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_rename_att(file->fh, varid, name, newname);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_rename_att(file->fh, varid, name, newname);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + +/** + * @ingroup PIOc_del_att + * The PIO-C interface for the NetCDF function nc_del_att. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_del_att (int ncid, int varid, const char *name) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + char *errstr; + + errstr = NULL; + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_DEL_ATT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_del_att(file->fh, varid, name);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_del_att(file->fh, varid, name);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_del_att(file->fh, varid, name);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + if(ierr != PIO_NOERR){ + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); + } + ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(errstr != NULL) free(errstr); + return ierr; +} + /** * @ingroup PIOc_set_fill * The PIO-C interface for the NetCDF function nc_set_fill. @@ -2087,196 +2307,82 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * } /** - * @ingroup PIOc_put_att_ulonglong - * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. + * @ingroup PIOc_get_att_uint + * The PIO-C interface for the NetCDF function nc_get_att_uint. */ -int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const unsigned long long *op) +int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) { - return PIOc_put_att(ncid, varid, name, xtype, len, op); + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** - * @ingroup PIOc_rename_att - * The PIO-C interface for the NetCDF function nc_rename_att. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + * @ingroup PIOc_get_att_long + * The PIO-C interface for the NetCDF function nc_get_att_long. */ -int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) +int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; + return PIOc_get_att(ncid, varid, name, (void *)ip); +} - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_RENAME_ATT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_att(file->fh, varid, name, newname);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_rename_att(file->fh, varid, name, newname);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_att(file->fh, varid, name, newname);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; +/** + * @ingroup PIOc_get_att_ubyte + * The PIO-C interface for the NetCDF function nc_get_att_ubyte. + */ +int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip) +{ + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** - * @ingroup PIOc_put_att_ushort - * The PIO-C interface for the NetCDF function nc_put_att_ushort. + * @ingroup PIOc_get_att_text + * The PIO-C interface for the NetCDF function nc_get_att_text. */ -int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const unsigned short *op) +int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) { - return PIOc_put_att(ncid, varid, name, xtype, len, op); + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** - * @ingroup PIOc_put_att_text - * The PIO-C interface for the NetCDF function nc_put_att_text. + * @ingroup PIOc_get_att_schar + * The PIO-C interface for the NetCDF function nc_get_att_schar. */ -int PIOc_put_att_text(int ncid, int varid, const char *name, - PIO_Offset len, const char *op) +int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) { - return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** - * @ingroup PIOc_get_att_uint - * The PIO-C interface for the NetCDF function nc_get_att_uint. + * @ingroup PIOc_get_att_ulonglong + * The PIO-C interface for the NetCDF function nc_get_att_ulonglong. */ -int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) +int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip) { return PIOc_get_att(ncid, varid, name, (void *)ip); } /** - * @ingroup PIOc_inq_format - * The PIO-C interface for the NetCDF function nc_inq_format. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param formatp a pointer that will get the file format - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + * @ingroup PIOc_get_att_short + * The PIO-C interface for the NetCDF function nc_get_att_short. */ -int PIOc_inq_format (int ncid, int *formatp) +int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_INQ_FORMAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_format(file->fh, formatp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_format(file->fh, formatp);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_format(file->fh, formatp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + return PIOc_get_att(ncid, varid, name, (void *)ip); +} - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); - return ierr; +/** + * @ingroup PIOc_get_att_int + * The PIO-C interface for the NetCDF function nc_get_att_int. + */ +int PIOc_get_att_int(int ncid, int varid, const char *name, int *ip) +{ + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** - * @ingroup PIOc_get_att_long - * The PIO-C interface for the NetCDF function nc_get_att_long. + * @ingroup PIOc_get_att_longlong + * The PIO-C interface for the NetCDF function nc_get_att_longlong. */ -int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) +int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip) { return PIOc_get_att(ncid, varid, name, (void *)ip); } @@ -2396,167 +2502,61 @@ int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype, } /** - * @ingroup PIOc_get_att_ubyte - * The PIO-C interface for the NetCDF function nc_get_att_ubyte. - */ -int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip) -{ - return PIOc_get_att(ncid, varid, name, (void *)ip); -} - -/** - * @ingroup PIOc_get_att_text - * The PIO-C interface for the NetCDF function nc_get_att_text. - */ -int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) -{ - return PIOc_get_att(ncid, varid, name, (void *)ip); -} - -/** - * @ingroup PIOc_del_att - * The PIO-C interface for the NetCDF function nc_del_att. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_del_att (int ncid, int varid, const char *name) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_DEL_ATT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_del_att(file->fh, varid, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_del_att(file->fh, varid, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_del_att(file->fh, varid, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; -} - -/** - * @ingroup PIOc_get_att_schar - * The PIO-C interface for the NetCDF function nc_get_att_schar. - */ -int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) -{ - return PIOc_get_att(ncid, varid, name, (void *)ip); -} - -/** - * @ingroup PIOc_get_att_ulonglong - * The PIO-C interface for the NetCDF function nc_get_att_ulonglong. - */ -int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip) -{ - return PIOc_get_att(ncid, varid, name, (void *)ip); -} - -/** - * @ingroup PIOc_get_att_short - * The PIO-C interface for the NetCDF function nc_get_att_short. + * @ingroup PIOc_put_att_uint + * The PIO-C interface for the NetCDF function nc_put_att_uint. */ -int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) +int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const unsigned int *op) { - return PIOc_get_att(ncid, varid, name, (void *)ip); + return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } /** - * @ingroup PIOc_get_att_int - * The PIO-C interface for the NetCDF function nc_get_att_int. + * @ingroup PIOc_put_att_ubyte + * The PIO-C interface for the NetCDF function nc_put_att_ubyte. */ -int PIOc_get_att_int(int ncid, int varid, const char *name, int *ip) +int PIOc_put_att_ubyte(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const unsigned char *op) { - return PIOc_get_att(ncid, varid, name, (void *)ip); + return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } /** - * @ingroup PIOc_get_att_longlong - * The PIO-C interface for the NetCDF function nc_get_att_longlong. + * @ingroup PIOc_put_att_float + * The PIO-C interface for the NetCDF function nc_put_att_float. */ -int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip) +int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const float *op) { - return PIOc_get_att(ncid, varid, name, (void *)ip); + return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } /** - * @ingroup PIOc_put_att_uint - * The PIO-C interface for the NetCDF function nc_put_att_uint. + * @ingroup PIOc_put_att_ulonglong + * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. */ -int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const unsigned int *op) +int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const unsigned long long *op) { - return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** - * @ingroup PIOc_put_att_ubyte - * The PIO-C interface for the NetCDF function nc_put_att_ubyte. + * @ingroup PIOc_put_att_ushort + * The PIO-C interface for the NetCDF function nc_put_att_ushort. */ -int PIOc_put_att_ubyte(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const unsigned char *op) +int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const unsigned short *op) { - return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** - * @ingroup PIOc_put_att_float - * The PIO-C interface for the NetCDF function nc_put_att_float. + * @ingroup PIOc_put_att_text + * The PIO-C interface for the NetCDF function nc_put_att_text. */ -int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const float *op) +int PIOc_put_att_text(int ncid, int varid, const char *name, + PIO_Offset len, const char *op) { return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } From 8938469e6b5749b3e8621cf6f4537b532a2be119 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 May 2016 14:25:34 -0400 Subject: [PATCH 042/184] further async development --- src/clib/pio_nc_async.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 51bb7f7813a..e4c4cb21c0a 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -2074,6 +2074,13 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) { int msg = PIO_MSG_GET_ATT_INT; + /* Get the type and length of the attribute. */ + if ((ierr = PIOc_inq_att(file->fh, varid, name, &atttype, &attlen))) + { + check_netcdf(file, ierr, __FILE__, __LINE__); + return ierr; + } + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); From 6e7e780a23b5739cd6481ecbafc59e51cddf19c8 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 May 2016 14:26:39 -0400 Subject: [PATCH 043/184] further async development --- src/clib/pio_msg.c | 4 ++++ src/clib/pio_nc_async.c | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 530ea5c2693..25d106e9c3c 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -375,6 +375,10 @@ int att_get_handler(iosystem_desc_t *ios) ios->intercomm); if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; + if ((mpierr = MPI_Bcast(&atttype, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, 0, ios->intercomm))) + return PIO_EIO; LOG((1, "att_get_handler ncid = %d varid = %d namelen = %d name = %s iotype = %d", ncid, varid, namelen, name, iotype)); diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index e4c4cb21c0a..dbaeb017af3 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -2095,7 +2095,10 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); - LOG((2, "broadcast iotype = %d", file->iotype)); + if (!mpierr) + mpierr = MPI_Bcast(&atttype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); } /* Handle MPI errors. */ From cd4421f3920293fe59af2dcfc283674ff639970b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 May 2016 14:27:46 -0400 Subject: [PATCH 044/184] further async development --- src/clib/pio_msg.c | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 25d106e9c3c..0553cd94ce9 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -382,31 +382,6 @@ int att_get_handler(iosystem_desc_t *ios) LOG((1, "att_get_handler ncid = %d varid = %d namelen = %d name = %s iotype = %d", ncid, varid, namelen, name, iotype)); - /* Get the length and the type of the attribute. */ - LOG((1, "att_get_handler magic iotype = %d", iotype)); - switch (iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_att(ncid, varid, name, &atttype, (size_t *)&attlen); - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if (ios->io_rank == 0) - ierr = nc_inq_att(ncid, varid, name, &atttype, (size_t *)&attlen); - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_att(ncid, varid, name, &atttype, &attlen); - break; -#endif - default: - ierr = iotype_error(iotype,__FILE__,__LINE__); - } - /* Allocate space for the attribute data. */ if (!(ip = malloc(attlen * sizeof(int)))) return PIO_ENOMEM; From ae43f4771a108f31804a1cb69cf9e2b371597f94 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 13 May 2016 12:43:46 -0400 Subject: [PATCH 045/184] added PIOc_inq_type --- src/clib/pio.h | 1 + src/clib/pio_internal.h | 3 +- src/clib/pio_msg.c | 46 ++++++++++++- src/clib/pio_nc_async.c | 128 ++++++++++++++++++++++++++++++++++++ tests/unit/test_intercomm.c | 22 +++++-- 5 files changed, 192 insertions(+), 8 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index 5dc08cedee2..3b20458f294 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -650,6 +650,7 @@ int PIOc_set_blocksize(const int newblocksize); int PIOc_set_log_level(int level); int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const void *op); int PIOc_get_att(int ncid, int varid, const char *name, void *ip); + int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep); #if defined(__cplusplus) } #endif diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index a288a15de1d..3425517818a 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -372,7 +372,8 @@ enum PIO_MSG{ PIO_MSG_DELETE_FILE, PIO_MSG_EXIT, PIO_MSG_GET_ATT, - PIO_MSG_PUT_ATT + PIO_MSG_PUT_ATT, + PIO_MSG_INQ_TYPE }; #endif diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 0553cd94ce9..42444ce13e2 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -16,6 +16,45 @@ extern int my_rank; extern int pio_log_level; #endif /* PIO_ENABLE_LOGGING */ +/** This function is run on the IO tasks to find netCDF type + * length. */ +int inq_type_handler(iosystem_desc_t *ios) +{ + int ncid; + int xtype; + char name_present, size_present; + char *namep = NULL, name[NC_MAX_NAME + 1]; + PIO_Offset *sizep = NULL, size; + int mpierr; + int ret; + + LOG((1, "typelen_handler")); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + LOG((1, "inq_type_handler got parameters ncid = %d datatype = %d", ncid, xtype)); + + if (name_present) + namep = name; + if (size_present) + sizep = &size; + + /* Call the function. */ + if ((ret = PIOc_inq_type(ncid, xtype, namep, sizep))) + return ret; + + LOG((1, "inq_type_handler succeeded!")); + return PIO_NOERR; +} + /** This function is run on the IO tasks to create a netCDF file. */ int create_file_handler(iosystem_desc_t *ios) { @@ -54,7 +93,7 @@ int create_file_handler(iosystem_desc_t *ios) /* Free resources. */ free(filename); - LOG((1, "create_file_handler succeeded!\n", my_rank)); + LOG((1, "create_file_handler succeeded!")); return PIO_NOERR; } @@ -387,7 +426,7 @@ int att_get_handler(iosystem_desc_t *ios) return PIO_ENOMEM; /* Call the function to read the attribute. */ - if ((ierr = PIOc_get_att_int(ncid, varid, name, ip))) + if ((ierr = PIOc_get_att(ncid, varid, name, ip))) return ierr; /* Free resources. */ @@ -917,6 +956,9 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) /* Handle the message. This code is run on all IO tasks. */ switch (msg) { + case PIO_MSG_INQ_TYPE: + inq_type_handler(my_iosys); + break; case PIO_MSG_CREATE_FILE: create_file_handler(my_iosys); break; diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index dbaeb017af3..b5ba8022bdc 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -298,6 +298,134 @@ int PIOc_inq_unlimdim(int ncid, int *unlimdimidp) return PIOc_inq(ncid, NULL, NULL, unlimdimidp, NULL); } +/** + * @ingroup PIOc_typelen + * The PIO-C interface for the NetCDF function nctypelen. + */ +int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) +{ + int msg = PIO_MSG_INQ_TYPE; /** Message for async notification. */ + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + int typelen; + + LOG((1, "PIOc_inq_type ncid = %d xtype = %d", ncid, xtype)); + + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) + return PIO_EBADID; + ios = file->iosystem; + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + char name_present = name ? true : false; + char size_present = sizep ? true : false; + + if (ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + switch (file->iotype) + { +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_inq_type(ncid, xtype, name, (size_t *)sizep); + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if (!file->iosystem->io_rank) + ierr = nc_inq_type(ncid, xtype, name, (size_t *)sizep); + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + switch (xtype) + { + case NC_UBYTE: + case NC_BYTE: + case NC_CHAR: + typelen = 1; + break; + case NC_SHORT: + case NC_USHORT: + typelen = 2; + break; + case NC_UINT: + case NC_INT: + case NC_FLOAT: + typelen = 4; + break; + case NC_UINT64: + case NC_INT64: + case NC_DOUBLE: + typelen = 8; + break; + } + + if (sizep) + *sizep = typelen; + if (name) + strcpy(name, "some type"); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + + LOG((2, "PIOc_inq_type netcdf call returned %d", ierr)); + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. Ignore NULL parameters. */ + if (!ierr) + { + if (name) + { + int slen; + if (ios->iomaster) + slen = strlen(name); + if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + } + if (sizep) + if ((mpierr = MPI_Bcast(sizep , 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + } + + return ierr; +} + /** * @ingroup PIOc_inq_format * The PIO-C interface for the NetCDF function nc_inq_format. diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index f6dbfc966c5..a538e3bdd45 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -92,7 +92,6 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) if ((ret = PIOc_openfile(iosysid, &ncid, &format, filename, NC_NOWRITE))) ERR(ret); - sleep(2); /* Find the number of dimensions, variables, and global attributes.*/ if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) @@ -137,12 +136,10 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) ERR(ret); if (dimlen2 != DIM_LEN) ERR(ERR_WRONG); - sleep(2); if ((ret = PIOc_inq_dimid(ncid, DIM_NAME, &dimid2))) ERR(ret); if (dimid2 != 0) ERR(ERR_WRONG); - sleep(2); /* Check out the variable. */ if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts))) @@ -341,8 +338,7 @@ main(int argc, char **argv) * and when the do, they should go straight to finalize. */ if (comp_task) { - /*for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) */ - for (int fmt = 0; fmt < 1; fmt++) + for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) { int ncid, varid, dimid; PIO_Offset start[NDIM], count[NDIM] = {0}; @@ -357,6 +353,22 @@ main(int argc, char **argv) if (verbose) printf("%d test_intercomm file created ncid = %d\n", my_rank, ncid); + /* Test the inq_type function for atomic types. */ + char type_name[NC_MAX_NAME + 1]; + PIO_Offset type_size; + #define NUM_TYPES 11 + nc_type xtype[NUM_TYPES] = {NC_CHAR, NC_BYTE, NC_SHORT, NC_INT, NC_FLOAT, NC_DOUBLE, + NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, NC_UINT64}; + int type_len[NUM_TYPES] = {1, 1, 2, 4, 4, 8, 1, 2, 4, 8, 8}; + int max_type = format[fmt] == PIO_IOTYPE_NETCDF ? NC_DOUBLE : NC_UINT64; + for (int i = 0; i < max_type; i++) + { + if ((ret = PIOc_inq_type(ncid, xtype[i], type_name, &type_size))) + ERR(ret); + if (type_size != type_len[i]) + ERR(ERR_AWFUL); + } + /* Define a dimension. */ if (verbose) printf("%d test_intercomm defining dimension %s\n", my_rank, DIM_NAME); From 5912c2a6e39ce23f8d76ba615e3d2155acf89300 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 13 May 2016 12:48:01 -0400 Subject: [PATCH 046/184] added PIOc_inq_type --- src/clib/pio_msg.c | 9 ++++++--- src/clib/pio_nc_async.c | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 42444ce13e2..b14b834215a 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -394,7 +394,7 @@ int att_get_handler(iosystem_desc_t *ios) int ierr; char *name; int namelen; - PIO_Offset attlen; + PIO_Offset attlen, typelen; nc_type atttype; int *op, *ip; int iotype; @@ -418,8 +418,11 @@ int att_get_handler(iosystem_desc_t *ios) return PIO_EIO; if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, 0, ios->intercomm))) return PIO_EIO; - LOG((1, "att_get_handler ncid = %d varid = %d namelen = %d name = %s iotype = %d", - ncid, varid, namelen, name, iotype)); + if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm))) + return PIO_EIO; + LOG((1, "att_get_handler ncid = %d varid = %d namelen = %d name = %s iotype = %d" + "atttype = %d attlen = %d typelen = %d", + ncid, varid, namelen, name, iotype, atttype, attlen, typelen)); /* Allocate space for the attribute data. */ if (!(ip = malloc(attlen * sizeof(int)))) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index b5ba8022bdc..7dbce2660c9 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -2179,7 +2179,7 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) { iosystem_desc_t *ios; file_desc_t *file; - PIO_Offset attlen; + PIO_Offset attlen, typelen; nc_type atttype; int ierr = PIO_NOERR; int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ @@ -2209,9 +2209,18 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) return ierr; } + /* Get the length (in bytes) of the type. */ + if ((ierr = PIOc_inq_type(file->fh, atttype, NULL, &typelen))) + { + check_netcdf(file, ierr, __FILE__, __LINE__); + return ierr; + } + + /* Send the message to IO master. */ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - + + /* Send the function parameters. */ if (!mpierr) mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -2227,6 +2236,8 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) mpierr = MPI_Bcast(&atttype, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); } /* Handle MPI errors. */ From fed2d0f2fa7e478f4bc60fde23628708613c59b7 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 13 May 2016 12:58:16 -0400 Subject: [PATCH 047/184] further development of async get_att --- src/clib/pio_msg.c | 2 +- src/clib/pio_nc_async.c | 43 +++++++++++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index b14b834215a..df51c1149e7 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -425,7 +425,7 @@ int att_get_handler(iosystem_desc_t *ios) ncid, varid, namelen, name, iotype, atttype, attlen, typelen)); /* Allocate space for the attribute data. */ - if (!(ip = malloc(attlen * sizeof(int)))) + if (!(ip = malloc(attlen * typelen))) return PIO_ENOMEM; /* Call the function to read the attribute. */ diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 7dbce2660c9..4dfab6de19d 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -2195,27 +2195,34 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) return PIO_EBADID; ios = file->iosystem; - /* If async is in use, and this is not an IO task, bcast the parameters. */ + /* Run these on all tasks if async is not in use, but only on + * non-IO tasks if async is in use. */ + if (!ios->async_interface || !ios->ioproc) + { + /* Get the type and length of the attribute. */ + if ((ierr = PIOc_inq_att(file->fh, varid, name, &atttype, &attlen))) + { + check_netcdf(file, ierr, __FILE__, __LINE__); + return ierr; + } + + /* Get the length (in bytes) of the type. */ + if ((ierr = PIOc_inq_type(file->fh, atttype, NULL, &typelen))) + { + check_netcdf(file, ierr, __FILE__, __LINE__); + return ierr; + } + } + + + /* If async is in use, and this is not an IO task, bcast the + * parameters and the attribute and type information we fetched. */ if (ios->async_interface) { if (!ios->ioproc) { int msg = PIO_MSG_GET_ATT_INT; - /* Get the type and length of the attribute. */ - if ((ierr = PIOc_inq_att(file->fh, varid, name, &atttype, &attlen))) - { - check_netcdf(file, ierr, __FILE__, __LINE__); - return ierr; - } - - /* Get the length (in bytes) of the type. */ - if ((ierr = PIOc_inq_type(file->fh, atttype, NULL, &typelen))) - { - check_netcdf(file, ierr, __FILE__, __LINE__); - return ierr; - } - /* Send the message to IO master. */ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); @@ -2244,6 +2251,12 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); + + /* Broadcast values. */ + if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ From f5b581a6abb54c00121079e708fd8c7fcd793c83 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 13 May 2016 13:13:26 -0400 Subject: [PATCH 048/184] now get_att generalized by type --- src/clib/pio_nc_async.c | 30 +++++++++++++----------------- tests/unit/test_intercomm.c | 3 ++- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 4dfab6de19d..51e7bb25c2c 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -2252,11 +2252,13 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); - /* Broadcast values. */ - if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + /* Broadcast values currently only known on computation tasks to IO tasks. */ + LOG((2, "PIOc_get_att bcast from comproot = %d attlen = %d typelen = %d", ios->comproot, attlen, typelen)); + if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + LOG((2, "PIOc_get_att bcast complete attlen = %d typelen = %d", attlen, typelen)); } /* If this is an IO task, then call the netCDF function. */ @@ -2268,25 +2270,17 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_get_att(file->fh, varid, name, ip); - if (!ierr) - ierr = nc_inq_att(file->fh, varid, name, &atttype, (size_t *)&attlen); break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if (ios->io_rank == 0) - { ierr = nc_get_att(file->fh, varid, name, ip); - if (!ierr) - ierr = nc_inq_att(file->fh, varid, name, &atttype, (size_t *)&attlen); - } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: ierr = ncmpi_get_att(file->fh, varid, name, ip); - if (!ierr) - ierr = ncmpi_inq_att(file->fh, varid, name, &atttype, &attlen); break; #endif default: @@ -2303,11 +2297,13 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) /* Broadcast results to all tasks. */ if (!ierr) { - LOG((2, "PIOc_get_att broadcasting attlen = %d", attlen)); - mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - LOG((2, "PIOc_get_att done broadcasting attlen = %d", attlen)); LOG((2, "PIOc_get_att broadcasting att data")); - mpierr = MPI_Bcast(ip, (int)attlen, MPI_INT, ios->ioroot, ios->my_comm); + if ((mpierr = MPI_Bcast(ip, (int)attlen * typelen, MPI_BYTE, ios->ioroot, + ios->my_comm))) + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } LOG((2, "PIOc_get_att done broadcasting att data")); } return ierr; diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index a538e3bdd45..fdf85df0094 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -338,7 +338,8 @@ main(int argc, char **argv) * and when the do, they should go straight to finalize. */ if (comp_task) { - for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) +/* for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) */ + for (int fmt = 0; fmt < 1; fmt++) { int ncid, varid, dimid; PIO_Offset start[NDIM], count[NDIM] = {0}; From 1c351820a6314a90ec957c210e48e7de0d066d24 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 13 May 2016 13:21:33 -0400 Subject: [PATCH 049/184] now get_att generalized by type --- tests/unit/test_intercomm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index fdf85df0094..6d57d58bb5d 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -338,8 +338,8 @@ main(int argc, char **argv) * and when the do, they should go straight to finalize. */ if (comp_task) { -/* for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) */ - for (int fmt = 0; fmt < 1; fmt++) + for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) +/* for (int fmt = 0; fmt < 1; fmt++) */ { int ncid, varid, dimid; PIO_Offset start[NDIM], count[NDIM] = {0}; From e6675619f1513b178bba86fa75378d0f2923f938 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 13 May 2016 13:22:38 -0400 Subject: [PATCH 050/184] now get_att generalized by type --- tests/unit/test_intercomm.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 6d57d58bb5d..bf50759815a 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -338,8 +338,8 @@ main(int argc, char **argv) * and when the do, they should go straight to finalize. */ if (comp_task) { - for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) -/* for (int fmt = 0; fmt < 1; fmt++) */ +/* for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) */ + for (int fmt = 0; fmt < 1; fmt++) { int ncid, varid, dimid; PIO_Offset start[NDIM], count[NDIM] = {0}; @@ -386,6 +386,9 @@ main(int argc, char **argv) if (verbose) printf("%d test_intercomm writing attributes %s\n", my_rank, ATT_NAME); int att_data = ATT_VALUE; + short short_att_data = ATT_VALUE; + if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &short_att_data))) + ERR(ret); if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) ERR(ret); From 06b279e09da4daa12a29079cfc258b7648151d96 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 13 May 2016 18:02:12 -0400 Subject: [PATCH 051/184] got put_att working with async --- src/clib/pio_msg.c | 73 ++++++++++++++++++++----------------- src/clib/pio_nc_async.c | 27 ++++++++++++-- tests/unit/test_intercomm.c | 8 ++-- 3 files changed, 69 insertions(+), 39 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index df51c1149e7..b73a6da02b3 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -336,46 +336,53 @@ int inq_att_handler(iosystem_desc_t *ios) * @param msg the message sent my the comp root task. * @return PIO_NOERR for success, error code otherwise. */ -int att_handler(iosystem_desc_t *ios, int msg) +int att_put_handler(iosystem_desc_t *ios) { int ncid; int varid; int mpierr; - int ret; - char *name5; + int ierr; + char *name; int namelen; - PIO_Offset attlen; + PIO_Offset attlen, typelen; nc_type atttype; int *op, *ip; + int iotype; - LOG((1, "att_handler msg = %d", msg)); - - if (msg == PIO_MSG_PUT_ATT_INT) - { - /* Get the parameters for this function that the the comp master - * task is broadcasting. */ - if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) - return PIO_EIO; - if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) - return PIO_EIO; - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!(name5 = malloc((namelen + 1) * sizeof(char)))) - return PIO_ENOMEM; - mpierr = MPI_Bcast((void *)name5, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&atttype, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); - if (!(op = malloc(attlen * sizeof(int)))) - return PIO_ENOMEM; - mpierr = MPI_Bcast(op, attlen, MPI_INT, ios->compmaster, ios->intercomm); + LOG((1, "att_put_handler")); - /* Call the function to write the attribute. */ - if ((ret = PIOc_put_att_int(ncid, varid, name5, atttype, attlen, op))) - return ret; + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!(name = malloc((namelen + 1) * sizeof(char)))) + return PIO_ENOMEM; + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, + ios->intercomm); + if ((mpierr = MPI_Bcast(&atttype, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm))) + return PIO_EIO; + if (!(op = malloc(attlen * typelen))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast((void *)op, attlen * typelen, MPI_BYTE, 0, ios->intercomm))) + return PIO_EIO; + LOG((1, "att_put_handler ncid = %d varid = %d namelen = %d name = %s iotype = %d" + "atttype = %d attlen = %d typelen = %d", + ncid, varid, namelen, name, iotype, atttype, attlen, typelen)); - /* Free resources. */ - free(name5); - free(op); - } + /* Call the function to read the attribute. */ + if ((ierr = PIOc_put_att(ncid, varid, name, atttype, attlen, op))) + return ierr; + + /* Free resources. */ + free(name); + free(op); return PIO_NOERR; } @@ -998,11 +1005,11 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) case PIO_MSG_INQ_VAR: inq_var_handler(my_iosys); break; - case PIO_MSG_GET_ATT_INT: + case PIO_MSG_GET_ATT: ret = att_get_handler(my_iosys); break; - case PIO_MSG_PUT_ATT_INT: - ret = att_handler(my_iosys, msg); + case PIO_MSG_PUT_ATT: + ret = att_put_handler(my_iosys); break; case PIO_MSG_INQ_VARID: inq_varid_handler(my_iosys); diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 51e7bb25c2c..b755e331fc0 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -2221,7 +2221,7 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) { if (!ios->ioproc) { - int msg = PIO_MSG_GET_ATT_INT; + int msg = PIO_MSG_GET_ATT; /* Send the message to IO master. */ if(ios->compmaster) @@ -2328,16 +2328,29 @@ int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, { iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ + PIO_Offset typelen; /** Length (in bytes) of the type. */ int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - LOG((1, "PIOc_inq_varid ncid = %d name = %s", ncid, name)); + LOG((1, "PIOc_put_att ncid = %d varid = %d name = %s", ncid, varid, name)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; + /* Run these on all tasks if async is not in use, but only on + * non-IO tasks if async is in use. */ + if (!ios->async_interface || !ios->ioproc) + { + /* Get the length (in bytes) of the type. */ + if ((ierr = PIOc_inq_type(file->fh, xtype, NULL, &typelen))) + { + check_netcdf(file, ierr, __FILE__, __LINE__); + return ierr; + } + } + /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { @@ -2362,13 +2375,21 @@ int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, if (!mpierr) mpierr = MPI_Bcast(&len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast((void *)op, len, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)op, len * typelen, MPI_BYTE, ios->compmaster, + ios->intercomm); } /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); + + /* Broadcast values currently only known on computation tasks to IO tasks. */ + LOG((2, "PIOc_put_att bcast from comproot = %d typelen = %d", ios->comproot, typelen)); + if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index bf50759815a..c0d7c93c6be 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -387,8 +387,10 @@ main(int argc, char **argv) printf("%d test_intercomm writing attributes %s\n", my_rank, ATT_NAME); int att_data = ATT_VALUE; short short_att_data = ATT_VALUE; + sleep(2); if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &short_att_data))) ERR(ret); + sleep(2); if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) ERR(ret); @@ -413,9 +415,9 @@ main(int argc, char **argv) if ((ret = PIOc_closefile(ncid))) ERR(ret); - /* Check the file for correctness. */ - if ((ret = check_file(iosysid, format[fmt], filename[fmt], my_rank, verbose))) - ERR(ret); + /* /\* Check the file for correctness. *\/ */ + /* if ((ret = check_file(iosysid, format[fmt], filename[fmt], my_rank, verbose))) */ + /* ERR(ret); */ /* Now delete the file. */ /* if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) */ From 99632fd170033056eb337db3ea3003dd0cfacdfb Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 13 May 2016 18:37:11 -0400 Subject: [PATCH 052/184] further development of async code --- src/clib/pio_msg.c | 6 ++- src/clib/pio_nc_async.c | 74 +++---------------------------------- tests/unit/test_intercomm.c | 27 +++++++++----- 3 files changed, 28 insertions(+), 79 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index b73a6da02b3..ab4232f68d3 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -28,7 +28,7 @@ int inq_type_handler(iosystem_desc_t *ios) int mpierr; int ret; - LOG((1, "typelen_handler")); + LOG((1, "inq_type_handler")); /* Get the parameters for this function that the the comp master * task is broadcasting. */ @@ -40,7 +40,7 @@ int inq_type_handler(iosystem_desc_t *ios) return PIO_EIO; if ((mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - LOG((1, "inq_type_handler got parameters ncid = %d datatype = %d", ncid, xtype)); + LOG((2, "inq_type_handler got parameters ncid = %d datatype = %d", ncid, xtype)); if (name_present) namep = name; @@ -51,6 +51,8 @@ int inq_type_handler(iosystem_desc_t *ios) if ((ret = PIOc_inq_type(ncid, xtype, namep, sizep))) return ret; + if (sizep) + LOG((2, "inq_type_handler size = %d", *sizep)); LOG((1, "inq_type_handler succeeded!")); return PIO_NOERR; } diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index b755e331fc0..7977726a09b 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -402,7 +402,7 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -2344,11 +2344,12 @@ int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, if (!ios->async_interface || !ios->ioproc) { /* Get the length (in bytes) of the type. */ - if ((ierr = PIOc_inq_type(file->fh, xtype, NULL, &typelen))) + if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &typelen))) { check_netcdf(file, ierr, __FILE__, __LINE__); return ierr; } + LOG((2, "PIOc_put_att typelen = %d", ncid, typelen)); } /* If async is in use, and this is not an IO task, bcast the parameters. */ @@ -2379,6 +2380,8 @@ int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, if (!mpierr) mpierr = MPI_Bcast((void *)op, len * typelen, MPI_BYTE, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_put_att finished bcast ncid = %d varid = %d namelen = %d name = %s " + "len = %d typelen = %d", file->fh, varid, namelen, name, len, typelen)); } /* Handle MPI errors. */ @@ -2592,72 +2595,7 @@ int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - size_t namelen; - - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d PIOc_inq_varid ncid = %d name = %s\n", my_rank, ncid, name); - - errstr = NULL; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - namelen = strlen(name); - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)op, len, MPI_INT, ios->compmaster, ios->intercomm); - } - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_int(file->fh, varid, name, xtype, len, op);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); - return ierr; + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index c0d7c93c6be..3e4f937fcba 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -30,6 +30,7 @@ /** The name of the global attribute in the netCDF output file. */ #define ATT_NAME "gatt_test_intercomm" +#define SHORT_ATT_NAME "short_gatt_test_intercomm" /** The value of the global attribute in the netCDF output file. */ #define ATT_VALUE 42 @@ -85,6 +86,7 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) int varndims2, vardimids2, varnatts2; int varid2; int att_data; + short short_att_data; /* Re-open the file to check it. */ if (verbose) @@ -96,7 +98,7 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) /* Find the number of dimensions, variables, and global attributes.*/ if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) ERR(ret); - if (ndims != 1 || nvars != 1 || ngatts != 1 || unlimdimid != -1) + if (ndims != 1 || nvars != 1 || ngatts != 2 || unlimdimid != -1) ERR(ERR_WRONG); /* This should return PIO_NOERR. */ @@ -114,7 +116,7 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) ERR(ERR_WRONG); if ((ret = PIOc_inq_natts(ncid, &ngatts2))) ERR(ret); - if (ngatts2 != 1) + if (ngatts2 != 2) ERR(ERR_WRONG); if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2))) ERR(ret); @@ -191,6 +193,15 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) printf("%d test_intercomm att_data = %d\n", my_rank, att_data); if (att_data != ATT_VALUE) ERR(ERR_WRONG); + if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, SHORT_ATT_NAME, &atttype, &attlen))) + ERR(ret); + if (atttype != NC_SHORT || attlen != 1) + ERR(ERR_WRONG); + if ((ret = PIOc_get_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME, &short_att_data))) + ERR(ret); + if (short_att_data != ATT_VALUE) + ERR(ERR_WRONG); + /* Close the file. */ if (verbose) @@ -387,12 +398,10 @@ main(int argc, char **argv) printf("%d test_intercomm writing attributes %s\n", my_rank, ATT_NAME); int att_data = ATT_VALUE; short short_att_data = ATT_VALUE; - sleep(2); - if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &short_att_data))) - ERR(ret); - sleep(2); if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) ERR(ret); + if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME, NC_SHORT, 1, &short_att_data))) + ERR(ret); /* End define mode. */ if (verbose) @@ -415,9 +424,9 @@ main(int argc, char **argv) if ((ret = PIOc_closefile(ncid))) ERR(ret); - /* /\* Check the file for correctness. *\/ */ - /* if ((ret = check_file(iosysid, format[fmt], filename[fmt], my_rank, verbose))) */ - /* ERR(ret); */ + /* Check the file for correctness. */ + if ((ret = check_file(iosysid, format[fmt], filename[fmt], my_rank, verbose))) + ERR(ret); /* Now delete the file. */ /* if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) */ From 4866ec14ec1780a996b6f47ed724a4b640d3023b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 13 May 2016 18:50:25 -0400 Subject: [PATCH 053/184] further development of async code --- src/clib/pio_nc_async.c | 10 +++++----- tests/unit/test_intercomm.c | 22 ++++++++++++++++++++-- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 7977726a09b..25e642a29c6 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -2605,7 +2605,7 @@ int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype, int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) { - return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** @@ -2615,7 +2615,7 @@ int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype, int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long long *op) { - return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** @@ -2625,7 +2625,7 @@ int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype, int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned int *op) { - return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** @@ -2635,7 +2635,7 @@ int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, int PIOc_put_att_ubyte(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) { - return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** @@ -2645,7 +2645,7 @@ int PIOc_put_att_ubyte(int ncid, int varid, const char *name, nc_type xtype, int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const float *op) { - return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); + return PIOc_put_att(ncid, varid, name, xtype, len, op); } /** diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 3e4f937fcba..108aa4929a5 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -31,6 +31,8 @@ /** The name of the global attribute in the netCDF output file. */ #define ATT_NAME "gatt_test_intercomm" #define SHORT_ATT_NAME "short_gatt_test_intercomm" +#define FLOAT_ATT_NAME "float_gatt_test_intercomm" +#define DOUBLE_ATT_NAME "double_gatt_test_intercomm" /** The value of the global attribute in the netCDF output file. */ #define ATT_VALUE 42 @@ -87,6 +89,8 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) int varid2; int att_data; short short_att_data; + float float_att_data; + double double_att_data; /* Re-open the file to check it. */ if (verbose) @@ -98,7 +102,7 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) /* Find the number of dimensions, variables, and global attributes.*/ if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) ERR(ret); - if (ndims != 1 || nvars != 1 || ngatts != 2 || unlimdimid != -1) + if (ndims != 1 || nvars != 1 || ngatts != 4 || unlimdimid != -1) ERR(ERR_WRONG); /* This should return PIO_NOERR. */ @@ -116,7 +120,7 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) ERR(ERR_WRONG); if ((ret = PIOc_inq_natts(ncid, &ngatts2))) ERR(ret); - if (ngatts2 != 2) + if (ngatts2 != 4) ERR(ERR_WRONG); if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2))) ERR(ret); @@ -201,6 +205,14 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) ERR(ret); if (short_att_data != ATT_VALUE) ERR(ERR_WRONG); + if ((ret = PIOc_get_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME, &float_att_data))) + ERR(ret); + if (float_att_data != ATT_VALUE) + ERR(ERR_WRONG); + if ((ret = PIOc_get_att_double(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, &double_att_data))) + ERR(ret); + if (double_att_data != ATT_VALUE) + ERR(ERR_WRONG); /* Close the file. */ @@ -398,10 +410,16 @@ main(int argc, char **argv) printf("%d test_intercomm writing attributes %s\n", my_rank, ATT_NAME); int att_data = ATT_VALUE; short short_att_data = ATT_VALUE; + float float_att_data = ATT_VALUE; + double double_att_data = ATT_VALUE; if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) ERR(ret); if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME, NC_SHORT, 1, &short_att_data))) ERR(ret); + if ((ret = PIOc_put_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME, NC_FLOAT, 1, &float_att_data))) + ERR(ret); + if ((ret = PIOc_put_att_double(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, NC_DOUBLE, 1, &double_att_data))) + ERR(ret); /* End define mode. */ if (verbose) From 9020e2d167b178b6bfc88a077ed5cb6adf81bb14 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 13 May 2016 19:12:23 -0400 Subject: [PATCH 054/184] moved logging code to pioc_support --- src/clib/pio_nc_async.c | 71 ---------------------------------------- src/clib/pioc_support.c | 72 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 71 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 25e642a29c6..41f36a5742c 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -16,80 +16,9 @@ */ #include -#ifdef PIO_ENABLE_LOGGING -#include -#include -#endif /* PIO_ENABLE_LOGGING */ #include #include -#ifdef PIO_ENABLE_LOGGING -int pio_log_level = 0; -int my_rank; -#endif /* PIO_ENABLE_LOGGING */ - -/** Set the logging level. Set to -1 for nothing, 0 for errors only, 1 - * for important logging, and so on. Log levels below 1 are only - * printed on the io/component root. If the library is not built with - * logging, this function does nothing. */ -int PIOc_set_log_level(int level) -{ -#ifdef PIO_ENABLE_LOGGING - printf("setting log level to %d\n", level); - pio_log_level = level; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - return PIO_NOERR; -#endif /* PIO_ENABLE_LOGGING */ -} - -#ifdef PIO_ENABLE_LOGGING -/** This function prints out a message, if the severity of the message - is lower than the global pio_log_level. To use it, do something - like this: - - pio_log(0, "this computer will explode in %d seconds", i); - - After the first arg (the severity), use the rest like a normal - printf statement. Output will appear on stdout. - This function is heavily based on the function in section 15.5 of - the C FAQ. -*/ -void -pio_log(int severity, const char *fmt, ...) -{ - va_list argp; - int t; - - /* If the severity is greater than the log level, we don't print - this message. */ - if (severity > pio_log_level) - return; - - /* If the severity is 0, only print on rank 0. */ - if (severity < 1 && my_rank != 0) - return; - - /* If the severity is zero, this is an error. Otherwise insert that - many tabs before the message. */ - if (!severity) - fprintf(stdout, "ERROR: "); - for (t = 0; t < severity; t++) - fprintf(stdout, "\t"); - - /* Show the rank. */ - fprintf(stdout, "%d ", my_rank); - - /* Print out the variable list of args with vprintf. */ - va_start(argp, fmt); - vfprintf(stdout, fmt, argp); - va_end(argp); - - /* Put on a final linefeed. */ - fprintf(stdout, "\n"); - fflush(stdout); -} -#endif /* PIO_ENABLE_LOGGING */ - /** * @ingroup PIOc_inq * The PIO-C interface for the NetCDF function nc_inq. diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index bd16fa0aff5..579a9e64fc6 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -1,12 +1,84 @@ /** @file * Support functions. */ +#include +#ifdef PIO_ENABLE_LOGGING +#include +#include +#endif /* PIO_ENABLE_LOGGING */ #include #include #include #define versno 2001 +#ifdef PIO_ENABLE_LOGGING +int pio_log_level = 0; +int my_rank; +#endif /* PIO_ENABLE_LOGGING */ + +/** Set the logging level. Set to -1 for nothing, 0 for errors only, 1 + * for important logging, and so on. Log levels below 1 are only + * printed on the io/component root. If the library is not built with + * logging, this function does nothing. */ +int PIOc_set_log_level(int level) +{ +#ifdef PIO_ENABLE_LOGGING + printf("setting log level to %d\n", level); + pio_log_level = level; + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + return PIO_NOERR; +#endif /* PIO_ENABLE_LOGGING */ +} + +#ifdef PIO_ENABLE_LOGGING +/** This function prints out a message, if the severity of the message + is lower than the global pio_log_level. To use it, do something + like this: + + pio_log(0, "this computer will explode in %d seconds", i); + + After the first arg (the severity), use the rest like a normal + printf statement. Output will appear on stdout. + This function is heavily based on the function in section 15.5 of + the C FAQ. +*/ +void +pio_log(int severity, const char *fmt, ...) +{ + va_list argp; + int t; + + /* If the severity is greater than the log level, we don't print + this message. */ + if (severity > pio_log_level) + return; + + /* If the severity is 0, only print on rank 0. */ + if (severity < 1 && my_rank != 0) + return; + + /* If the severity is zero, this is an error. Otherwise insert that + many tabs before the message. */ + if (!severity) + fprintf(stdout, "ERROR: "); + for (t = 0; t < severity; t++) + fprintf(stdout, "\t"); + + /* Show the rank. */ + fprintf(stdout, "%d ", my_rank); + + /* Print out the variable list of args with vprintf. */ + va_start(argp, fmt); + vfprintf(stdout, fmt, argp); + va_end(argp); + + /* Put on a final linefeed. */ + fprintf(stdout, "\n"); + fflush(stdout); +} +#endif /* PIO_ENABLE_LOGGING */ + static pio_swapm_defaults swapm_defaults; bool PIO_Save_Decomps=false; /** From 0337dbec63b11eb89732872a0d9e087a46340daf Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sat, 14 May 2016 06:18:20 -0400 Subject: [PATCH 055/184] cleaning up call to netcdf layer in PIOc_nc_async --- src/clib/pio.h | 4 ++ src/clib/pio_file.c | 61 ++++++++++++----- src/clib/pio_nc_async.c | 129 +++++++++++++----------------------- src/clib/pioc.c | 12 ++-- tests/unit/test_intercomm.c | 4 +- 5 files changed, 101 insertions(+), 109 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index 3b20458f294..596e10c0e29 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -259,6 +259,10 @@ typedef struct file_desc_t int mode; struct wmulti_buffer buffer; struct file_desc_t *next; + + /** True if this task should participate in IO (only true for one + * task with netcdf serial files. */ + int do_io; } file_desc_t; /** diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index b68ce023768..03cce3e9c25 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -65,6 +65,14 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, file->buffer.frame=NULL; file->buffer.fillvalue=NULL; + /** Set to true if this task should participate in IO (only true for + * one task with netcdf serial files. */ + if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF || + ios->io_rank == 0) + file->do_io = 1; + else + file->do_io = 0; + /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { @@ -75,13 +83,13 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, len = strlen(filename); if (!mpierr) - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); } /* Handle MPI errors. */ @@ -193,7 +201,7 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, ** @param mode : The netcdf mode for the open operation */ -int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, +int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, const char filename[], const int mode) { int ierr; @@ -235,18 +243,37 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, msg = PIO_MSG_CREATE_FILE; file->mode = mode; + /** Set to true if this task should participate in IO (only true for + * one task with netcdf serial files. */ + if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF || + ios->io_rank == 0) + file->do_io = 1; + else + file->do_io = 0; - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); - len = strlen(filename); - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); - } - + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + if(ios->comp_rank==0) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + len = strlen(filename); + if (!mpierr) + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + /* Handle MPI errors. */ + mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + check_mpi(file, mpierr, __FILE__, __LINE__); + } + if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF @@ -288,9 +315,9 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, ierr = check_netcdf(file, ierr, __FILE__,__LINE__); if(ierr == PIO_NOERR){ - mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm); + mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm); file->mode = file->mode | PIO_WRITE; // This flag is implied by netcdf create functions but we need to know if its set - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm); *ncidp = file->fh; pio_add_to_file_list(file); *ncidp = file->fh; @@ -403,7 +430,7 @@ int PIOc_deletefile(const int iosysid, const char filename[]) if(ios->comp_rank==0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); len = strlen(filename); - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); } // The barriers are needed to assure that no task is trying to operate on the file while it is being deleted. diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 41f36a5742c..338cfef612f 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -38,7 +38,6 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp) { - int msg = PIO_MSG_INQ; /** Message for async notification. */ iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ int ierr = PIO_NOERR; /** Return code from function calls. */ @@ -56,6 +55,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, { if (!ios->ioproc) { + int msg = PIO_MSG_INQ; /** Message for async notification. */ char ndims_present = ndimsp ? true : false; char nvars_present = nvarsp ? true : false; char ngatts_present = ngattsp ? true : false; @@ -86,43 +86,29 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if (!file->iosystem->io_rank) - { - /* Should not be necessary to do this - nc_inq should - * handle null pointers. This has been reported as a bug - * to netCDF developers. */ - int tmp_ndims, tmp_nvars, tmp_ngatts, tmp_unlimdimid; - ierr = nc_inq(ncid, &tmp_ndims, &tmp_nvars, &tmp_ngatts, &tmp_unlimdimid); - if (ndimsp) - *ndimsp = tmp_ndims; - if (nvarsp) - *nvarsp = tmp_nvars; - if (ngattsp) - *ngattsp = tmp_ngatts; - if (unlimdimidp) - *unlimdimidp = tmp_unlimdimid; - } - break; -#endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype == PIO_IOTYPE_NETCDF && file->do_io) + { + /* Should not be necessary to do this - nc_inq should + * handle null pointers. This has been reported as a bug + * to netCDF developers. */ + int tmp_ndims, tmp_nvars, tmp_ngatts, tmp_unlimdimid; + ierr = nc_inq(ncid, &tmp_ndims, &tmp_nvars, &tmp_ngatts, &tmp_unlimdimid); + if (ndimsp) + *ndimsp = tmp_ndims; + if (nvarsp) + *nvarsp = tmp_nvars; + if (ngattsp) + *ngattsp = tmp_ngatts; + if (unlimdimidp) + *unlimdimidp = tmp_unlimdimid; + } else if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); +#endif /* _NETCDF */ LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } @@ -274,25 +260,13 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) check_mpi(file, mpierr, __FILE__, __LINE__); } + /* If this is an IO task, then call the netCDF function. */ /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_type(ncid, xtype, name, (size_t *)sizep); - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if (!file->iosystem->io_rank) - ierr = nc_inq_type(ncid, xtype, name, (size_t *)sizep); - break; -#endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + if (file->iotype == PIO_IOTYPE_PNETCDF) + { switch (xtype) { case NC_UBYTE: @@ -320,12 +294,12 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) *sizep = typelen; if (name) strcpy(name, "some type"); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - + } +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_type(ncid, xtype, name, (size_t *)sizep); +#endif /* _NETCDF */ LOG((2, "PIOc_inq_type netcdf call returned %d", ierr)); } @@ -445,9 +419,9 @@ int PIOc_inq_format (int ncid, int *formatp) */ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) { - iosystem_desc_t *ios; - file_desc_t *file; - int ierr = PIO_NOERR; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ LOG((1, "PIOc_inq_dim")); @@ -487,30 +461,17 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) check_mpi(file, mpierr, __FILE__, __LINE__); } - /* Make the call to the netCDF layer. */ - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if (ios->io_rank == 0){ - ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; - } - break; -#endif + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);; +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; +#endif /* _NETCDF */ } /* Broadcast and check the return code. */ diff --git a/src/clib/pioc.c b/src/clib/pioc.c index afe7e85d981..612d99bde7c 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -350,10 +350,9 @@ int PIOc_InitDecomp_bc(const int iosysid, const int basetype,const int ndims, co ** @param rearr the rearranger to use by default, this may be overriden in the @ref PIO_initdecomp ** @param iosysidp index of the defined system descriptor */ - -int PIOc_Init_Intracomm(const MPI_Comm comp_comm, - const int num_iotasks, const int stride, - const int base,const int rearr, int *iosysidp) +int PIOc_Init_Intracomm(const MPI_Comm comp_comm, const int num_iotasks, + const int stride, const int base, const int rearr, + int *iosysidp) { iosystem_desc_t *iosys; int ierr = PIO_NOERR; @@ -437,10 +436,11 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, &(iosys->iogroup)),__FILE__,__LINE__); /* Create an MPI communicator for the IO tasks. */ - CheckMPIReturn(MPI_Comm_create(iosys->comp_comm, iosys->iogroup, &(iosys->io_comm)),__FILE__,__LINE__); + CheckMPIReturn(MPI_Comm_create(iosys->comp_comm, iosys->iogroup, &(iosys->io_comm)) + ,__FILE__,__LINE__); /* For the tasks that are doing IO, get their rank. */ - if(iosys->ioproc) + if (iosys->ioproc) CheckMPIReturn(MPI_Comm_rank(iosys->io_comm, &(iosys->io_rank)),__FILE__,__LINE__); else iosys->io_rank = -1; diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 108aa4929a5..18f3f3238ab 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -361,8 +361,8 @@ main(int argc, char **argv) * and when the do, they should go straight to finalize. */ if (comp_task) { -/* for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) */ - for (int fmt = 0; fmt < 1; fmt++) + for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) +/* for (int fmt = 0; fmt < 1; fmt++) */ { int ncid, varid, dimid; PIO_Offset start[NDIM], count[NDIM] = {0}; From ddaa30a8049a3e89bb643b8a2023e2eb5d0d80ab Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sat, 14 May 2016 06:30:01 -0400 Subject: [PATCH 056/184] cleaning up call to netcdf layer in PIOc_nc_async --- src/clib/pio_nc_async.c | 305 +++++++++------------------------------- 1 file changed, 63 insertions(+), 242 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 338cfef612f..da6bd4a872c 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -143,15 +143,6 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, /** * @ingroup PIOc_inq_ndims * The PIO-C interface for the NetCDF function nc_inq_ndims. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_ndims (int ncid, int *ndimsp) { @@ -162,15 +153,6 @@ int PIOc_inq_ndims (int ncid, int *ndimsp) /** * @ingroup PIOc_inq_nvars * The PIO-C interface for the NetCDF function nc_inq_nvars. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_nvars(int ncid, int *nvarsp) { @@ -180,15 +162,6 @@ int PIOc_inq_nvars(int ncid, int *nvarsp) /** * @ingroup PIOc_inq_natts * The PIO-C interface for the NetCDF function nc_inq_natts. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_natts(int ncid, int *ngattsp) { @@ -198,15 +171,6 @@ int PIOc_inq_natts(int ncid, int *ngattsp) /** * @ingroup PIOc_inq_unlimdim * The PIO-C interface for the NetCDF function nc_inq_unlimdim. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_unlimdim(int ncid, int *unlimdimidp) { @@ -332,16 +296,6 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) /** * @ingroup PIOc_inq_format * The PIO-C interface for the NetCDF function nc_inq_format. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param formatp a pointer that will get the file format - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_format (int ncid, int *formatp) { @@ -504,15 +458,6 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) /** * @ingroup PIOc_inq_dimname * The PIO-C interface for the NetCDF function nc_inq_dimname. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_dimname(int ncid, int dimid, char *name) { @@ -522,16 +467,6 @@ int PIOc_inq_dimname(int ncid, int dimid, char *name) /** * @ingroup PIOc_inq_dimlen * The PIO-C interface for the NetCDF function nc_inq_dimlen. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param lenp a pointer that will get the number of values - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp) { @@ -601,28 +536,14 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) /* IO tasks call the netCDF functions. */ if (ios->ioproc) { - switch(file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dimid(file->fh, name, idp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if (ios->io_rank == 0) - ierr = nc_inq_dimid(file->fh, name, idp);; - break; -#endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + if (file->iotype == PIO_IOTYPE_PNETCDF) ierr = ncmpi_inq_dimid(file->fh, name, idp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_dimid(file->fh, name, idp);; +#endif /* _NETCDF */ } /* Broadcast and check the return code. */ @@ -714,33 +635,20 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, /* Call the netCDF layer. */ if (ios->ioproc) { - switch(file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varndims(file->fh, varid, &ndims); - ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp); - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if (ios->io_rank == 0) - { - ierr = nc_inq_varndims(file->fh, varid, &ndims); - ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; - } - break; -#endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + if (file->iotype == PIO_IOTYPE_PNETCDF) + { ierr = ncmpi_inq_varndims(file->fh, varid, &ndims); ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); } +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + { + ierr = nc_inq_varndims(file->fh, varid, &ndims); + ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp); + } +#endif /* _NETCDF */ } /* Broadcast and check the return code. */ @@ -789,16 +697,6 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, /** * @ingroup PIOc_inq_varname * The PIO-C interface for the NetCDF function nc_inq_varname. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_varname (int ncid, int varid, char *name) { @@ -808,17 +706,6 @@ int PIOc_inq_varname (int ncid, int varid, char *name) /** * @ingroup PIOc_inq_vartype * The PIO-C interface for the NetCDF function nc_inq_vartype. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) { @@ -828,16 +715,6 @@ int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) /** * @ingroup PIOc_inq_varndims * The PIO-C interface for the NetCDF function nc_inq_varndims. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) { @@ -847,16 +724,6 @@ int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) /** * @ingroup PIOc_inq_vardimid * The PIO-C interface for the NetCDF function nc_inq_vardimid. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_vardimid(int ncid, int varid, int *dimidsp) { @@ -866,17 +733,6 @@ int PIOc_inq_vardimid(int ncid, int varid, int *dimidsp) /** * @ingroup PIOc_inq_varnatts * The PIO-C interface for the NetCDF function nc_inq_varnatts. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param nattsp a pointer that will get the number of attributes - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) { @@ -944,28 +800,14 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varid(file->fh, name, varidp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_varid(file->fh, name, varidp);; - } - break; -#endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varid(file->fh, name, varidp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq_varid(file->fh, name, varidp);; +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_varid(file->fh, name, varidp); +#endif /* _NETCDF */ } /* Broadcast and check the return code. */ @@ -1084,17 +926,6 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, /** * @ingroup PIOc_inq_attlen * The PIO-C interface for the NetCDF function nc_inq_attlen. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param lenp a pointer that will get the number of values - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) { @@ -1104,17 +935,6 @@ int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) /** * @ingroup PIOc_inq_atttype * The PIO-C interface for the NetCDF function nc_inq_atttype. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep) { @@ -2321,16 +2141,6 @@ int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, return ierr; } -/** - * @ingroup PIOc_put_att_short - * The PIO-C interface for the NetCDF function nc_put_att_short. - */ -int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const short *op) -{ - return PIOc_put_att(ncid, varid, name, xtype, len, op); -} - /** * @ingroup PIOc_get_att_double * The PIO-C interface for the NetCDF function nc_get_att_double. @@ -2340,16 +2150,6 @@ int PIOc_get_att_double(int ncid, int varid, const char *name, double *ip) return PIOc_get_att(ncid, varid, name, (void *)ip); } -/** - * @ingroup PIOc_put_att_double - * The PIO-C interface for the NetCDF function nc_put_att_double. - */ -int PIOc_put_att_double(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const double *op) -{ - return PIOc_put_att(ncid, varid, name, xtype, len, op); -} - /** * @ingroup PIOc_get_att_uchar * The PIO-C interface for the NetCDF function nc_get_att_uchar. @@ -2359,16 +2159,6 @@ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip return PIOc_get_att(ncid, varid, name, (void *)ip); } -/** - * @ingroup PIOc_put_att_schar - * The PIO-C interface for the NetCDF function nc_put_att_schar. - */ -int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const signed char *op) -{ - return PIOc_put_att(ncid, varid, name, xtype, len, op); -} - /** * @ingroup PIOc_get_att_ushort * The PIO-C interface for the NetCDF function nc_get_att_ushort. @@ -2460,22 +2250,32 @@ int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip) } /** - * @ingroup PIOc_put_att_long - * The PIO-C interface for the NetCDF function nc_put_att_long. + * @ingroup PIOc_get_att_float + * The PIO-C interface for the NetCDF function nc_get_att_float. */ -int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const long *op) +int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) { - return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); + return PIOc_get_att(ncid, varid, name, (void *)ip); } /** - * @ingroup PIOc_get_att_float - * The PIO-C interface for the NetCDF function nc_get_att_float. + * @ingroup PIOc_put_att_schar + * The PIO-C interface for the NetCDF function nc_put_att_schar. */ -int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) +int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const signed char *op) { - return PIOc_get_att(ncid, varid, name, (void *)ip); + return PIOc_put_att(ncid, varid, name, xtype, len, op); +} + +/** + * @ingroup PIOc_put_att_long + * The PIO-C interface for the NetCDF function nc_put_att_long. + */ +int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const long *op) +{ + return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } /** @@ -2568,3 +2368,24 @@ int PIOc_put_att_text(int ncid, int varid, const char *name, return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } +/** + * @ingroup PIOc_put_att_short + * The PIO-C interface for the NetCDF function nc_put_att_short. + */ +int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const short *op) +{ + return PIOc_put_att(ncid, varid, name, xtype, len, op); +} + +/** + * @ingroup PIOc_put_att_double + * The PIO-C interface for the NetCDF function nc_put_att_double. + */ +int PIOc_put_att_double(int ncid, int varid, const char *name, nc_type xtype, + PIO_Offset len, const double *op) +{ + return PIOc_put_att(ncid, varid, name, xtype, len, op); +} + + From 16e37990aa5f6d40d53bdd0f9bae045db1c1b039 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sat, 14 May 2016 07:02:03 -0400 Subject: [PATCH 057/184] got inq_format working with async --- src/clib/pio_msg.c | 39 +++++++++++++++ src/clib/pio_nc_async.c | 95 ++++++++++++++++++++----------------- tests/unit/test_intercomm.c | 11 +++++ 3 files changed, 101 insertions(+), 44 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index ab4232f68d3..fc49680c74b 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -57,6 +57,42 @@ int inq_type_handler(iosystem_desc_t *ios) return PIO_NOERR; } +/** This function is run on the IO tasks to find netCDF file + * format. */ +int inq_format_handler(iosystem_desc_t *ios) +{ + int ncid; + int *formatp = NULL, format; + char format_present; + int mpierr; + int ret; + + LOG((1, "inq_format_handler")); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&format_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + LOG((2, "inq_format_handler got parameters ncid = %d format_present = %d", + ncid, format_present)); + + /* Manage NULL pointers. */ + if (format_present) + formatp = &format; + + /* Call the function. */ + if ((ret = PIOc_inq_format(ncid, formatp))) + return ret; + + if (formatp) + LOG((2, "inq_format_handler format = %d", *formatp)); + LOG((1, "inq_format_handler succeeded!")); + + return PIO_NOERR; +} + /** This function is run on the IO tasks to create a netCDF file. */ int create_file_handler(iosystem_desc_t *ios) { @@ -971,6 +1007,9 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) case PIO_MSG_INQ_TYPE: inq_type_handler(my_iosys); break; + case PIO_MSG_INQ_FORMAT: + inq_format_handler(my_iosys); + break; case PIO_MSG_CREATE_FILE: create_file_handler(my_iosys); break; diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index da6bd4a872c..04f48d91f07 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -299,61 +299,68 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) */ int PIOc_inq_format (int ncid, int *formatp) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - errstr = NULL; - ierr = PIO_NOERR; + LOG((1, "PIOc_inq ncid = %d", ncid)); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_FORMAT; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_FORMAT; + char format_present = formatp ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if (!mpierr) + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&format_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_format(file->fh, formatp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_format(file->fh, formatp);; - } - break; -#endif + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_format(file->fh, formatp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq_format(file->fh, formatp); +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_format(file->fh, formatp); +#endif /* _NETCDF */ + LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. Ignore NULL parameters. */ + if (!ierr) + { + if (formatp) + if ((mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); + return ierr; } diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 18f3f3238ab..686ad181c09 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -377,6 +377,17 @@ main(int argc, char **argv) if (verbose) printf("%d test_intercomm file created ncid = %d\n", my_rank, ncid); + /* Test the inq_format function. */ + int myformat; + if ((ret = PIOc_inq_format(ncid, &myformat))) + ERR(ret); + if ((format[fmt] == PIO_IOTYPE_PNETCDF || format[fmt] == PIO_IOTYPE_NETCDF) && + myformat != 1) + ERR(ERR_AWFUL); + else if ((format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P) && + myformat != 3) + ERR(ERR_AWFUL); + /* Test the inq_type function for atomic types. */ char type_name[NC_MAX_NAME + 1]; PIO_Offset type_size; From 2a250cfe947a264e9c57a4ecf4aaec6e6d08d208 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sat, 14 May 2016 07:32:41 -0400 Subject: [PATCH 058/184] got inq_attname working with async --- src/clib/pio_msg.c | 46 +++++++++++++ src/clib/pio_nc_async.c | 124 +++++++++++++++++++----------------- tests/unit/test_intercomm.c | 5 ++ 3 files changed, 117 insertions(+), 58 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index fc49680c74b..3640d201c0c 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -368,6 +368,49 @@ int inq_att_handler(iosystem_desc_t *ios) return PIO_NOERR; } +/** Handle attribute inquiry operations. This code only runs on IO + * tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @param msg the message sent my the comp root task. + * @return PIO_NOERR for success, error code otherwise. +*/ +int inq_attname_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int attnum; + char name[NC_MAX_NAME + 1], *namep = NULL; + char name_present; + int mpierr; + int ret; + + LOG((1, "inq_att_name_handler")); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmaster, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + LOG((2, "inq_attname_handler got ncid = %d varid = %d attnum = %d name_present = %d", + ncid, varid, attnum, name_present)); + + /* Match NULLs in collective function call. */ + if (name_present) + namep = name; + + /* Call the function to learn about the attribute. */ + if ((ret = PIOc_inq_attname(ncid, varid, attnum, namep))) + return ret; + + return PIO_NOERR; +} + /** Handle attribute operations. This code only runs on IO tasks. * * @param ios pointer to the iosystem_desc_t. @@ -1058,6 +1101,9 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) case PIO_MSG_INQ_ATT: inq_att_handler(my_iosys); break; + case PIO_MSG_INQ_ATTNAME: + inq_attname_handler(my_iosys); + break; case PIO_MSG_INITDECOMP_DOF: initdecomp_dof_handler(my_iosys); break; diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 04f48d91f07..34f0141ef8d 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -885,30 +885,18 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); } + /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp); - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if (ios->io_rank == 0) - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp); - break; -#endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + if (file->iotype == PIO_IOTYPE_PNETCDF) ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp); +#endif /* _NETCDF */ + LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } /* Handle MPI errors. */ @@ -963,58 +951,78 @@ int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep) * @param attnum the attribute ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) +int PIOc_inq_attname(int ncid, int varid, int attnum, char *name) { - int ierr = PIO_NOERR; - int msg = PIO_MSG_INQ_ATTNAME; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - iosystem_desc_t *ios; - file_desc_t *file; + LOG((1, "PIOc_inq_attname ncid = %d varid = %d attnum = %d", ncid, varid, + attnum)); + + /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_ATTNAME; + char name_present = name ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_attname(file->fh, varid, attnum, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_attname(file->fh, varid, attnum, name);; - } - break; -#endif + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + if (file->iotype == PIO_IOTYPE_PNETCDF) ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_attname(file->fh, varid, attnum, name);; +#endif /* _NETCDF */ + LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr)); } + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; check_netcdf(file, ierr, __FILE__, __LINE__); - if (name) - { - int slen; - if(ios->iomaster) - slen = (int) strlen(name); - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm); - } + + /* Broadcast results to all tasks. Ignore NULL parameters. */ + if (!ierr) + if (name) + { + int namelen = strlen(name); + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->ioroot, + ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + } return ierr; } diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 686ad181c09..3587a5e03da 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -183,6 +183,7 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) /* Check out the global attributes. */ nc_type atttype; PIO_Offset attlen; + char myattname[NC_MAX_NAME + 1]; if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &atttype, &attlen))) ERR(ret); if (atttype != NC_INT || attlen != 1) @@ -191,6 +192,10 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) ERR(ret); if (attlen != 1) ERR(ERR_WRONG); + if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, myattname))) + ERR(ret); + if (strcmp(ATT_NAME, myattname)) + ERR(ERR_WRONG); if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, &att_data))) ERR(ret); if (verbose) From 5b140219be69862622ad1246fcba4bfe7ff308dd Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sat, 14 May 2016 07:49:01 -0400 Subject: [PATCH 059/184] got inq_attid working for async --- src/clib/pio_msg.c | 55 +++++++++++++++++++++++++ src/clib/pio_nc_async.c | 81 +++++++++++++++++++++++-------------- tests/unit/test_intercomm.c | 5 +++ 3 files changed, 110 insertions(+), 31 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 3640d201c0c..51575f697bc 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -411,6 +411,58 @@ int inq_attname_handler(iosystem_desc_t *ios) return PIO_NOERR; } +/** Handle attribute inquiry operations. This code only runs on IO + * tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @param msg the message sent my the comp root task. + * @return PIO_NOERR for success, error code otherwise. +*/ +int inq_attid_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int attnum; + char *name; + int namelen; + int id, *idp = NULL; + char id_present; + int mpierr; + int ret; + + LOG((1, "inq_attid_handler")); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm))) + return PIO_EIO; + if (!(name = malloc((namelen + 1) * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + LOG((2, "inq_attid_handler got ncid = %d varid = %d attnum = %d id_present = %d", + ncid, varid, attnum, id_present)); + + /* Match NULLs in collective function call. */ + if (id_present) + idp = &id; + + /* Call the function to learn about the attribute. */ + if ((ret = PIOc_inq_attid(ncid, varid, name, idp))) + return ret; + + /* Free resources. */ + free(name); + + return PIO_NOERR; +} + /** Handle attribute operations. This code only runs on IO tasks. * * @param ios pointer to the iosystem_desc_t. @@ -1104,6 +1156,9 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) case PIO_MSG_INQ_ATTNAME: inq_attname_handler(my_iosys); break; + case PIO_MSG_INQ_ATTID: + inq_attid_handler(my_iosys); + break; case PIO_MSG_INITDECOMP_DOF: initdecomp_dof_handler(my_iosys); break; diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 34f0141ef8d..0a9fe0d8d20 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1042,47 +1042,66 @@ int PIOc_inq_attname(int ncid, int varid, int attnum, char *name) * @param idp a pointer that will get the id of the variable or attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) +int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp) { - int ierr = PIO_NOERR; - int msg = PIO_MSG_INQ_ATTID; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - iosystem_desc_t *ios; - file_desc_t *file; + /* User must provide name shorter than NC_MAX_NAME +1. */ + if (!name || strlen(name) > NC_MAX_NAME) + return PIO_EINVAL; + + LOG((1, "PIOc_inq_attid ncid = %d varid = %d name = %s", ncid, varid, name)); + + /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_ATTID; + int namelen = strlen(name); + char id_present = idp ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_attid(file->fh, varid, name, idp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_attid(file->fh, varid, name, idp);; - } - break; -#endif + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + if (file->iotype == PIO_IOTYPE_PNETCDF) ierr = ncmpi_inq_attid(file->fh, varid, name, idp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_attid(file->fh, varid, name, idp);; +#endif /* _NETCDF */ + LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr)); } /* Handle MPI errors. */ @@ -1094,7 +1113,7 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) if (!ierr) { if (idp) - if ((mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm))) check_mpi(file, mpierr, __FILE__, __LINE__); } diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 3587a5e03da..30d06d9e392 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -184,6 +184,7 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) nc_type atttype; PIO_Offset attlen; char myattname[NC_MAX_NAME + 1]; + int myid; if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &atttype, &attlen))) ERR(ret); if (atttype != NC_INT || attlen != 1) @@ -196,6 +197,10 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) ERR(ret); if (strcmp(ATT_NAME, myattname)) ERR(ERR_WRONG); + if ((ret = PIOc_inq_attid(ncid, NC_GLOBAL, ATT_NAME, &myid))) + ERR(ret); + if (myid != 0) + ERR(ERR_WRONG); if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, &att_data))) ERR(ret); if (verbose) From 10ff1436da7f206c88b4b69261fe041b5e323694 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sat, 14 May 2016 07:58:53 -0400 Subject: [PATCH 060/184] got inq_attid working for async --- src/clib/pio_msg.c | 45 +++++++++++++++++++ src/clib/pio_nc_async.c | 95 ++++++++++++++++++++++------------------- 2 files changed, 97 insertions(+), 43 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 51575f697bc..1a65d67f5ea 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -833,6 +833,48 @@ int def_dim_handler(iosystem_desc_t *ios) return PIO_NOERR; } +/** This function is run on the IO tasks to rename a netCDF + * dimension. */ +int rename_dim_handler(iosystem_desc_t *ios) +{ + int ncid; + int len, namelen; + int iotype; + char *name; + int mode; + int mpierr; + int ret; + int dimid; + + LOG((1, "rename_dim_handler")); + + /* Get the parameters for this function that the he comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&dimid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if (!(name = malloc(namelen + 1 * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, 0, + ios->intercomm))) + return PIO_EIO; + LOG((2, "rename_dim_handler got parameters namelen = %d " + "name = %s ncid = %d varid = %d", namelen, name, ncid, varid)); + + /* Call the create file function. */ + if ((ret = PIOc_rename_dim(ncid, dimid, name))) + return ret; + + /* Free resources. */ + free(name); + + LOG((1, "%d rename_dim_handler succeeded!\n", my_rank)); + return PIO_NOERR; +} + /** This function is run on the IO tasks. It reads or writes an array * of data to a netCDF variable. * @@ -1123,6 +1165,9 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) case PIO_MSG_DELETE_FILE: delete_file_handler(my_iosys); break; + case PIO_MSG_RENAME_DIM: + rename_dim_handler(my_iosys); + break; case PIO_MSG_DEF_DIM: def_dim_handler(my_iosys); break; diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 0a9fe0d8d20..13f5b310bd9 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1133,62 +1133,71 @@ int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp) * PIOc_openfile() or PIOc_createfile(). * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_rename_dim (int ncid, int dimid, const char *name) +int PIOc_rename_dim(int ncid, int dimid, const char *name) { - int ierr; - int msg; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - errstr = NULL; - ierr = PIO_NOERR; + LOG((1, "PIOc_inq ncid = %d", ncid)); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* User must provide name of correct length. */ + if (!name || strlen(name) > NC_MAX_NAME) + return PIO_EINVAL; + + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_RENAME_DIM; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_RENAME_DIM; /** Message for async notification. */ + int namelen = strlen(name); + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_dim(file->fh, dimid, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_rename_dim(file->fh, dimid, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_dim(file->fh, dimid, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { +#ifdef _PNETCDF + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_rename_dim(file->fh, dimid, name); +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_rename_dim(file->fh, dimid, name);; +#endif /* _NETCDF */ + LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + return ierr; } From 6c0ee7b3783040a22246790562c8c2998e6e916c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sat, 14 May 2016 08:17:31 -0400 Subject: [PATCH 061/184] got rename_dim working --- src/clib/pio_msg.c | 12 ++++++------ src/clib/pio_nc_async.c | 6 ++++-- tests/unit/test_intercomm.c | 12 +++++++++++- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 1a65d67f5ea..c1b8aa42ab2 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -845,6 +845,7 @@ int rename_dim_handler(iosystem_desc_t *ios) int mpierr; int ret; int dimid; + char name1[NC_MAX_NAME + 1]; LOG((1, "rename_dim_handler")); @@ -856,13 +857,12 @@ int rename_dim_handler(iosystem_desc_t *ios) return PIO_EIO; if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - if (!(name = malloc(namelen + 1 * sizeof(char)))) - return PIO_ENOMEM; - if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, 0, - ios->intercomm))) - return PIO_EIO; + if (!(name = malloc((namelen + 1) * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; LOG((2, "rename_dim_handler got parameters namelen = %d " - "name = %s ncid = %d varid = %d", namelen, name, ncid, varid)); + "name = %s ncid = %d dimid = %d", namelen, name, ncid, dimid)); /* Call the create file function. */ if ((ret = PIOc_rename_dim(ncid, dimid, name))) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 13f5b310bd9..aa67d8b42e3 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1167,9 +1167,11 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name) if (!mpierr) mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) - mpierr = MPI_Bcast(&name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_rename_dim Bcast file->fh = %d dimid = %d namelen = %d name = %s", + file->fh, dimid, namelen, name)); } /* Handle MPI errors. */ diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 30d06d9e392..c4d2bc37547 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -23,12 +23,15 @@ #define LOCAL_DIM_LEN 2 /** The name of the dimension in the netCDF output file. */ +#define FIRST_DIM_NAME "jojo" #define DIM_NAME "dim_test_intercomm" /** The name of the variable in the netCDF output file. */ +#define FIRST_VAR_NAME "bill" #define VAR_NAME "var_test_intercomm" /** The name of the global attribute in the netCDF output file. */ +#define FIRST_ATT_NAME "willy" #define ATT_NAME "gatt_test_intercomm" #define SHORT_ATT_NAME "short_gatt_test_intercomm" #define FLOAT_ATT_NAME "float_gatt_test_intercomm" @@ -415,9 +418,16 @@ main(int argc, char **argv) } /* Define a dimension. */ + char dimname2[NC_MAX_NAME + 1]; if (verbose) printf("%d test_intercomm defining dimension %s\n", my_rank, DIM_NAME); - if ((ret = PIOc_def_dim(ncid, DIM_NAME, DIM_LEN, &dimid))) + if ((ret = PIOc_def_dim(ncid, FIRST_DIM_NAME, DIM_LEN, &dimid))) + ERR(ret); + if ((ret = PIOc_inq_dimname(ncid, 0, dimname2))) + ERR(ret); + if (strcmp(dimname2, FIRST_DIM_NAME)) + ERR(ERR_WRONG); + if ((ret = PIOc_rename_dim(ncid, 0, DIM_NAME))) ERR(ret); /* Define a 1-D variable. */ From f44671845e06f2f512cd4b011f82938f25ccbb3a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sat, 14 May 2016 08:24:58 -0400 Subject: [PATCH 062/184] got rename_var working with async --- src/clib/pio_msg.c | 45 ++++++++++++++++ src/clib/pio_nc_async.c | 101 ++++++++++++++++++++---------------- tests/unit/test_intercomm.c | 9 +++- 3 files changed, 109 insertions(+), 46 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index c1b8aa42ab2..a4b663fe136 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -875,6 +875,48 @@ int rename_dim_handler(iosystem_desc_t *ios) return PIO_NOERR; } +/** This function is run on the IO tasks to rename a netCDF + * dimension. */ +int rename_var_handler(iosystem_desc_t *ios) +{ + int ncid; + int len, namelen; + int iotype; + char *name; + int mode; + int mpierr; + int ret; + int varid; + char name1[NC_MAX_NAME + 1]; + + LOG((1, "rename_var_handler")); + + /* Get the parameters for this function that the he comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if (!(name = malloc((namelen + 1) * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + LOG((2, "rename_var_handler got parameters namelen = %d " + "name = %s ncid = %d varid = %d", namelen, name, ncid, varid)); + + /* Call the create file function. */ + if ((ret = PIOc_rename_var(ncid, varid, name))) + return ret; + + /* Free resources. */ + free(name); + + LOG((1, "%d rename_var_handler succeeded!\n", my_rank)); + return PIO_NOERR; +} + /** This function is run on the IO tasks. It reads or writes an array * of data to a netCDF variable. * @@ -1168,6 +1210,9 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) case PIO_MSG_RENAME_DIM: rename_dim_handler(my_iosys); break; + case PIO_MSG_RENAME_VAR: + rename_var_handler(my_iosys); + break; case PIO_MSG_DEF_DIM: def_dim_handler(my_iosys); break; diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index aa67d8b42e3..6c15bf74758 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1140,12 +1140,12 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name) int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - LOG((1, "PIOc_inq ncid = %d", ncid)); - /* User must provide name of correct length. */ if (!name || strlen(name) > NC_MAX_NAME) return PIO_EINVAL; + LOG((1, "PIOc_rename_dim ncid = %d dimid = %d name = %s", ncid, dimid, name)); + /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; @@ -1217,62 +1217,73 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_rename_var (int ncid, int varid, const char *name) +int PIOc_rename_var(int ncid, int varid, const char *name) { - int ierr; - int msg; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - errstr = NULL; - ierr = PIO_NOERR; + /* User must provide name of correct length. */ + if (!name || strlen(name) > NC_MAX_NAME) + return PIO_EINVAL; - file = pio_get_file_from_id(ncid); - if(file == NULL) + LOG((1, "PIOc_rename_var ncid = %d varid = %d name = %s", ncid, varid, name)); + + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_RENAME_VAR; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_RENAME_VAR; /** Message for async notification. */ + int namelen = strlen(name); + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_var(file->fh, varid, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_rename_var(file->fh, varid, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_var(file->fh, varid, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_rename_var Bcast file->fh = %d varid = %d namelen = %d name = %s", + file->fh, varid, namelen, name)); } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { +#ifdef _PNETCDF + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_rename_var(file->fh, varid, name); +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_rename_var(file->fh, varid, name);; +#endif /* _NETCDF */ + LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + return ierr; } diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index c4d2bc37547..dfce3827c1f 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -431,9 +431,16 @@ main(int argc, char **argv) ERR(ret); /* Define a 1-D variable. */ + char varname2[NC_MAX_NAME + 1]; if (verbose) printf("%d test_intercomm defining variable %s\n", my_rank, VAR_NAME); - if ((ret = PIOc_def_var(ncid, VAR_NAME, NC_INT, NDIM, &dimid, &varid))) + if ((ret = PIOc_def_var(ncid, FIRST_VAR_NAME, NC_INT, NDIM, &dimid, &varid))) + ERR(ret); + if ((ret = PIOc_inq_varname(ncid, 0, varname2))) + ERR(ret); + if (strcmp(varname2, FIRST_VAR_NAME)) + ERR(ERR_WRONG); + if ((ret = PIOc_rename_var(ncid, 0, VAR_NAME))) ERR(ret); /* Add a global attribute. */ From 0ce232fd523cb6ec39018a2d5dbc0e29f9f68775 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sat, 14 May 2016 09:01:59 -0400 Subject: [PATCH 063/184] got rename_var working with async --- src/clib/pio_msg.c | 49 +++++++++++++++++++ src/clib/pio_nc_async.c | 105 ++++++++++++++++++++++++---------------- 2 files changed, 111 insertions(+), 43 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index a4b663fe136..cdd7dd84e6c 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -917,6 +917,52 @@ int rename_var_handler(iosystem_desc_t *ios) return PIO_NOERR; } +/** This function is run on the IO tasks to rename a netCDF + * attribute. */ +int rename_att_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int namelen, newnamelen; + char *name, *newname; + int mpierr; + int ret; + + LOG((1, "rename_att_handler")); + + /* Get the parameters for this function that the he comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if (!(name = malloc((namelen + 1) * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if (!(newname = malloc((newnamelen + 1) * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast(newname, newnamelen + 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + LOG((2, "rename_var_handler got parameters namelen = %d name = %s ncid = %d varid = %d" + "newnamelen = %d newname = %s", namelen, name, ncid, varid, newnamelen, newname)); + + /* Call the create file function. */ + if ((ret = PIOc_rename_att(ncid, varid, name, newname))) + return ret; + + /* Free resources. */ + free(name); + free(newname); + + LOG((1, "%d rename_att_handler succeeded!\n", my_rank)); + return PIO_NOERR; +} + /** This function is run on the IO tasks. It reads or writes an array * of data to a netCDF variable. * @@ -1213,6 +1259,9 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) case PIO_MSG_RENAME_VAR: rename_var_handler(my_iosys); break; + case PIO_MSG_RENAME_ATT: + rename_att_handler(my_iosys); + break; case PIO_MSG_DEF_DIM: def_dim_handler(my_iosys); break; diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 6c15bf74758..87927295a5f 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1303,60 +1303,79 @@ int PIOc_rename_var(int ncid, int varid, const char *name) */ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; +{ + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - errstr = NULL; - ierr = PIO_NOERR; + /* User must provide names of correct length. */ + if (!name || strlen(name) > NC_MAX_NAME || + !newname || strlen(newname) > NC_MAX_NAME) + return PIO_EINVAL; - file = pio_get_file_from_id(ncid); - if(file == NULL) + LOG((1, "PIOc_rename_att ncid = %d varid = %d name = %s newname = %s", + ncid, varid, name, newname)); + + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_RENAME_ATT; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_RENAME_ATT; /** Message for async notification. */ + int namelen = strlen(name); + int newnamelen = strlen(newname); + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_att(file->fh, varid, name, newname);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_rename_att(file->fh, varid, name, newname);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_att(file->fh, varid, name, newname);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((char *)newname, newnamelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_rename_att Bcast file->fh = %d varid = %d namelen = %d name = %s" + "newnamelen = %s newname = %s", file->fh, varid, namelen, name, newnamelen, newname)); } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { +#ifdef _PNETCDF + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_rename_att(file->fh, varid, name, newname); +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_rename_att(file->fh, varid, name, newname); +#endif /* _NETCDF */ + LOG((2, "PIOc_rename_att netcdf call returned %d", ierr)); } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + return ierr; } From 36c065ce62f738454488c653f861d9c8b499257e Mon Sep 17 00:00:00 2001 From: katetc Date: Mon, 16 May 2016 15:04:34 -0600 Subject: [PATCH 064/184] Changed Pnetcdf required version to 1.6.1 in documentation. --- doc/source/Installing.txt | 2 +- doc/source/mach_walkthrough.txt | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/source/Installing.txt b/doc/source/Installing.txt index 0c0020d39ac..6f281152248 100644 --- a/doc/source/Installing.txt +++ b/doc/source/Installing.txt @@ -20,7 +20,7 @@ The PIO code is currently stored on github at module load git/2.3.0
%> module load cmake/3.0.2
%> module load netcdf-mpi/4.3.3.1
- %> module load pnetcdf/1.6.0
+ %> module load pnetcdf/1.6.1
+ GNU @@ -54,7 +54,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load git/2.3.0
%> module load cmake/3.0.2
%> module load netcdf-mpi/4.3.3.1
- %> module load pnetcdf/1.6.0
+ %> module load pnetcdf/1.6.1
+ PGI @@ -64,7 +64,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load git/2.3.0
%> module load cmake/3.0.2
%> module load netcdf-mpi/4.3.3.1
- %> module load pnetcdf/1.6.0
+ %> module load pnetcdf/1.6.1
  • Environment Variables @@ -103,7 +103,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake/3.0.0
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
    + GNU @@ -117,7 +117,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake/3.0.0
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
    + Cray @@ -132,7 +132,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake/3.0.0
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
  • Environment Variables @@ -178,7 +178,7 @@ And then set the following environment variables to add in the rest of the libra %> setenv LIBZ /soft/libraries/alcf/current/xl/ZLIB
    %> setenv HDF5 /soft/libraries/hdf5/1.8.14/cnk-xl/V1R2M2-20150213
    %> setenv NETCDF /soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/V1R2M2-20150213
    - %> setenv PNETCDF /soft/libraries/pnetcdf/1.6.0/cnk-xl/V1R2M2-20150213
    + %> setenv PNETCDF /soft/libraries/pnetcdf/1.6.1/cnk-xl/V1R2M2-20150213
    %> setenv CC /soft/compilers/wrappers/xl/mpixlc_r
    %> setenv FC /soft/compilers/wrappers/xl/mpixlf90_r
    @@ -209,7 +209,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
    + PGI @@ -219,7 +219,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
  • Environment Variables @@ -477,5 +477,5 @@ variables to these tests will increase the time significantly. -_Last updated: 11-18-2015_ +_Last updated: 05-16-2016_ */ From a055a395115b65458fbd74538a6b8dbf1ba40967 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 07:09:51 -0400 Subject: [PATCH 065/184] fixed rename_att function for async --- src/clib/pio_msg.c | 15 +++++++++++++-- src/clib/pio_nc_async.c | 30 +++++++++++++++++++++--------- tests/unit/test_intercomm.c | 21 ++++++++++++++++++--- 3 files changed, 52 insertions(+), 14 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index cdd7dd84e6c..86fd5d04f53 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -929,31 +929,41 @@ int rename_att_handler(iosystem_desc_t *ios) int ret; LOG((1, "rename_att_handler")); + LOG((1, "rename_att_handler2")); /* Get the parameters for this function that the he comp master * task is broadcasting. */ + LOG((1, "rename_att_handler about to get ncid")); if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; + LOG((2, "rename_att_handler ncid = %d", ncid)); if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; + LOG((2, "rename_att_handler varid = %d", varid)); if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; + LOG((2, "rename_att_handler namelen = %d", namelen)); if (!(name = malloc((namelen + 1) * sizeof(char)))) return PIO_ENOMEM; if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; + LOG((2, "rename_att_handler name = %s", name)); if ((mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; + LOG((2, "rename_att_handler newnamelen = %d", newnamelen)); if (!(newname = malloc((newnamelen + 1) * sizeof(char)))) return PIO_ENOMEM; if ((mpierr = MPI_Bcast(newname, newnamelen + 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - LOG((2, "rename_var_handler got parameters namelen = %d name = %s ncid = %d varid = %d" + LOG((2, "rename_att_handler got parameters namelen = %d name = %s ncid = %d varid = %d " "newnamelen = %d newname = %s", namelen, name, ncid, varid, newnamelen, newname)); /* Call the create file function. */ if ((ret = PIOc_rename_att(ncid, varid, name, newname))) + { + LOG((2, "rename_att_handler rename_att returned %d", ret)); return ret; + } /* Free resources. */ free(name); @@ -1224,7 +1234,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) LOG((3, "about to call msg MPI_Bcast")); mpierr = MPI_Bcast(&msg, 1, MPI_INT, 0, my_iosys->io_comm); CheckMPIReturn(mpierr, __FILE__, __LINE__); - LOG((1, "msg MPI_Bcast complete msg = %d", msg)); + LOG((1, "pio_msg_handler msg MPI_Bcast complete msg = %d", msg)); /* Handle the message. This code is run on all IO tasks. */ switch (msg) @@ -1261,6 +1271,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) break; case PIO_MSG_RENAME_ATT: rename_att_handler(my_iosys); + LOG((1, "rename_att_handler returned")); break; case PIO_MSG_DEF_DIM: def_dim_handler(my_iosys); diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 87927295a5f..e80f80bc3db 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1299,15 +1299,16 @@ int PIOc_rename_var(int ncid, int varid, const char *name) * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + * @return PIO_NOERR for success, error code otherwise. See + * PIOc_Set_File_Error_Handling */ -int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) -{ +int PIOc_rename_att (int ncid, int varid, const char *name, + const char *newname) { iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ /* User must provide names of correct length. */ if (!name || strlen(name) > NC_MAX_NAME || @@ -1322,6 +1323,8 @@ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) return PIO_EBADID; ios = file->iosystem; + LOG((2, "PIOc_rename_att2 ncid = %d varid = %d name = %s newname = %s", + ncid, varid, name, newname)); /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { @@ -1331,9 +1334,12 @@ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) int namelen = strlen(name); int newnamelen = strlen(newname); - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + LOG((2, "PIOc_rename_att sending msg")); + if (ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + LOG((2, "PIOc_rename_att done sending msg")); if (!mpierr) mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -1346,18 +1352,19 @@ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast((char *)newname, newnamelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_rename_att Bcast file->fh = %d varid = %d namelen = %d name = %s" - "newnamelen = %s newname = %s", file->fh, varid, namelen, name, newnamelen, newname)); } + /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } + LOG((2, "PIOc_rename_att calling netcdf function")); /* If this is an IO task, then call the netCDF function. */ + LOG((2, "PIOc_rename_att calling netcdf function")); if (ios->ioproc) { #ifdef _PNETCDF @@ -1372,10 +1379,15 @@ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) } /* Broadcast and check the return code. */ + LOG((2, "PIOc_rename_att Bcasting %d", ierr)); if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + { + LOG((2, "PIOc_rename_att ERROR")); return PIO_EIO; + } + LOG((2, "PIOc_rename_att Bcast complete")); check_netcdf(file, ierr, __FILE__, __LINE__); - + LOG((2, "PIOc_rename_att succeeded")); return ierr; } diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index dfce3827c1f..d2253fc6f30 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -31,7 +31,7 @@ #define VAR_NAME "var_test_intercomm" /** The name of the global attribute in the netCDF output file. */ -#define FIRST_ATT_NAME "willy" +#define FIRST_ATT_NAME "willy_gatt_test_intercomm" #define ATT_NAME "gatt_test_intercomm" #define SHORT_ATT_NAME "short_gatt_test_intercomm" #define FLOAT_ATT_NAME "float_gatt_test_intercomm" @@ -450,8 +450,23 @@ main(int argc, char **argv) short short_att_data = ATT_VALUE; float float_att_data = ATT_VALUE; double double_att_data = ATT_VALUE; - if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) - ERR(ret); + char attname2[NC_MAX_NAME + 1]; + if (fmt == 0) + { + if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) + ERR(ret); + } + else + { + if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data))) + ERR(ret); + if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, attname2))) + ERR(ret); + if (strcmp(attname2, FIRST_ATT_NAME)) + ERR(ERR_WRONG); + if ((ret = PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, ATT_NAME))) + ERR(ret); + } if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME, NC_SHORT, 1, &short_att_data))) ERR(ret); if ((ret = PIOc_put_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME, NC_FLOAT, 1, &float_att_data))) From 01679e5be991eb12c35e1fc7839d6c0d161d8a20 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 08:19:42 -0400 Subject: [PATCH 066/184] more cleanup --- src/clib/pio_nc_async.c | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index e80f80bc3db..4e9544a17dc 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1323,8 +1323,6 @@ int PIOc_rename_att (int ncid, int varid, const char *name, return PIO_EBADID; ios = file->iosystem; - LOG((2, "PIOc_rename_att2 ncid = %d varid = %d name = %s newname = %s", - ncid, varid, name, newname)); /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { @@ -1334,8 +1332,6 @@ int PIOc_rename_att (int ncid, int varid, const char *name, int namelen = strlen(name); int newnamelen = strlen(newname); - LOG((2, "PIOc_rename_att sending msg")); - if (ios->compmaster) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); @@ -1354,17 +1350,13 @@ int PIOc_rename_att (int ncid, int varid, const char *name, mpierr = MPI_Bcast((char *)newname, newnamelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); } - /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } - LOG((2, "PIOc_rename_att calling netcdf function")); - /* If this is an IO task, then call the netCDF function. */ - LOG((2, "PIOc_rename_att calling netcdf function")); if (ios->ioproc) { #ifdef _PNETCDF @@ -1375,18 +1367,13 @@ int PIOc_rename_att (int ncid, int varid, const char *name, if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) ierr = nc_rename_att(file->fh, varid, name, newname); #endif /* _NETCDF */ - LOG((2, "PIOc_rename_att netcdf call returned %d", ierr)); } /* Broadcast and check the return code. */ - LOG((2, "PIOc_rename_att Bcasting %d", ierr)); if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - { - LOG((2, "PIOc_rename_att ERROR")); - return PIO_EIO; - } - LOG((2, "PIOc_rename_att Bcast complete")); + check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); + LOG((2, "PIOc_rename_att succeeded")); return ierr; } From 1728b3e9e3270f3ba069cc84d024c5675209938c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 08:33:10 -0400 Subject: [PATCH 067/184] continued async development --- src/clib/pio_nc_async.c | 341 ++++++++++++++++++++-------------------- 1 file changed, 167 insertions(+), 174 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 4e9544a17dc..0bc417c5f0a 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1392,62 +1392,63 @@ int PIOc_rename_att (int ncid, int varid, const char *name, * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_del_att (int ncid, int varid, const char *name) +int PIOc_del_att(int ncid, int varid, const char *name) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ - errstr = NULL; - ierr = PIO_NOERR; + /* User must provide name of correct length. */ + if (!name || strlen(name) > NC_MAX_NAME) + return PIO_EINVAL; - file = pio_get_file_from_id(ncid); - if(file == NULL) + LOG((1, "PIOc_del_att ncid = %d varid = %d name = %s", ncid, varid, name)); + + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_DEL_ATT; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_DEL_ATT; + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_del_att(file->fh, varid, name);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_del_att(file->fh, varid, name);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_del_att(file->fh, varid, name);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + if (!mpierr) + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { +#ifdef _PNETCDF + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_del_att(file->fh, varid, name); +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_del_att(file->fh, varid, name); +#endif /* _NETCDF */ } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + check_netcdf(file, ierr, __FILE__, __LINE__); + + LOG((2, "PIOc_del_att succeeded")); return ierr; } @@ -1466,60 +1467,59 @@ int PIOc_del_att (int ncid, int varid, const char *name) */ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) { - int ierr; - int msg; - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ - errstr = NULL; - ierr = PIO_NOERR; + LOG((1, "PIOc_set_fill ncid = %d fillmode = %d old_modep = %d", ncid, fillmode, + old_modep)); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_SET_FILL; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_SET_FILL; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_set_fill(file->fh, fillmode, old_modep);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_set_fill(file->fh, fillmode, old_modep);; - } - break; -#endif + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_set_fill(file->fh, fillmode, old_modep);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_set_fill(file->fh, fillmode, old_modep); +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_set_fill(file->fh, fillmode, old_modep); +#endif /* _NETCDF */ } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + check_netcdf(file, ierr, __FILE__, __LINE__); + + LOG((2, "PIOc_set_fill succeeded")); return ierr; } @@ -1538,61 +1538,57 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) */ int PIOc_enddef(int ncid) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ - errstr = NULL; - ierr = PIO_NOERR; + LOG((1, "PIOc_enddef ncid = %d", ncid)); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_ENDDEF; - if(ios->async_interface && ! ios->ioproc){ - if(!ios->comp_rank) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - printf("PIOc_enddef file->fh = %d\n", file->fh); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_ENDDEF; + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_enddef(file->fh);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_enddef(file->fh);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_enddef(file->fh);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { +#ifdef _PNETCDF + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_enddef(file->fh); +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_enddef(file->fh); +#endif /* _NETCDF */ } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + check_netcdf(file, ierr, __FILE__, __LINE__); + + LOG((2, "PIOc_enddef succeeded")); return ierr; } @@ -1611,60 +1607,57 @@ int PIOc_enddef(int ncid) */ int PIOc_redef (int ncid) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ - errstr = NULL; - ierr = PIO_NOERR; + LOG((1, "PIOc_redef ncid = %d", ncid)); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_REDEF; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_ENDDEF; + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_redef(file->fh);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_redef(file->fh);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_redef(file->fh);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { +#ifdef _PNETCDF + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_redef(file->fh); +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_redef(file->fh); +#endif /* _NETCDF */ } - ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(errstr != NULL) free(errstr); + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + check_netcdf(file, ierr, __FILE__, __LINE__); + + LOG((2, "PIOc_redef succeeded")); return ierr; } From 2e1746024e2ebde549e8e1fe23e0ef6c210f7f28 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 10:05:05 -0400 Subject: [PATCH 068/184] continued async development --- tests/unit/test_intercomm.c | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index d2253fc6f30..57968c48aad 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -451,22 +451,14 @@ main(int argc, char **argv) float float_att_data = ATT_VALUE; double double_att_data = ATT_VALUE; char attname2[NC_MAX_NAME + 1]; - if (fmt == 0) - { - if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_data))) - ERR(ret); - } - else - { - if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data))) - ERR(ret); - if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, attname2))) - ERR(ret); - if (strcmp(attname2, FIRST_ATT_NAME)) - ERR(ERR_WRONG); - if ((ret = PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, ATT_NAME))) - ERR(ret); - } + if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data))) + ERR(ret); + if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, attname2))) + ERR(ret); + if (strcmp(attname2, FIRST_ATT_NAME)) + ERR(ERR_WRONG); + if ((ret = PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, ATT_NAME))) + ERR(ret); if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME, NC_SHORT, 1, &short_att_data))) ERR(ret); if ((ret = PIOc_put_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME, NC_FLOAT, 1, &float_att_data))) From f60fe9656657acc133759bc5a1909e739488acd3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 10:11:50 -0400 Subject: [PATCH 069/184] continued async development --- tests/unit/test_intercomm.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 57968c48aad..b2c2d2d3d70 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -451,6 +451,7 @@ main(int argc, char **argv) float float_att_data = ATT_VALUE; double double_att_data = ATT_VALUE; char attname2[NC_MAX_NAME + 1]; + /* Write an att and rename it. */ if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data))) ERR(ret); if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, attname2))) @@ -459,6 +460,16 @@ main(int argc, char **argv) ERR(ERR_WRONG); if ((ret = PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, ATT_NAME))) ERR(ret); + + /* Write an att and delete it. */ + /* if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data))) */ + /* ERR(ret); */ + /* if ((ret = PIOc_del_att(ncid, NC_GLOBAL, FIRST_ATT_NAME))) */ + /* ERR(ret); */ + /* if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, NULL, NULL)) != PIO_ENOTATT) */ + /* ERR(ERR_AWFUL); */ + + /* Write some atts of different types. */ if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME, NC_SHORT, 1, &short_att_data))) ERR(ret); if ((ret = PIOc_put_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME, NC_FLOAT, 1, &float_att_data))) From 149d94e58e1742310394bb0ceb7d7e4d499b6a76 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 10:29:27 -0400 Subject: [PATCH 070/184] continued async development --- src/clib/pio_msg.c | 53 +++++++++++++++++++++++++++++-------- src/clib/pio_nc_async.c | 52 ++++++++++++++++++++---------------- tests/unit/test_intercomm.c | 12 ++++++--- 3 files changed, 79 insertions(+), 38 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 86fd5d04f53..5c2f511da50 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -929,28 +929,21 @@ int rename_att_handler(iosystem_desc_t *ios) int ret; LOG((1, "rename_att_handler")); - LOG((1, "rename_att_handler2")); /* Get the parameters for this function that the he comp master * task is broadcasting. */ - LOG((1, "rename_att_handler about to get ncid")); if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - LOG((2, "rename_att_handler ncid = %d", ncid)); if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - LOG((2, "rename_att_handler varid = %d", varid)); if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - LOG((2, "rename_att_handler namelen = %d", namelen)); if (!(name = malloc((namelen + 1) * sizeof(char)))) return PIO_ENOMEM; if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - LOG((2, "rename_att_handler name = %s", name)); if ((mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - LOG((2, "rename_att_handler newnamelen = %d", newnamelen)); if (!(newname = malloc((newnamelen + 1) * sizeof(char)))) return PIO_ENOMEM; if ((mpierr = MPI_Bcast(newname, newnamelen + 1, MPI_CHAR, 0, ios->intercomm))) @@ -960,10 +953,7 @@ int rename_att_handler(iosystem_desc_t *ios) /* Call the create file function. */ if ((ret = PIOc_rename_att(ncid, varid, name, newname))) - { - LOG((2, "rename_att_handler rename_att returned %d", ret)); return ret; - } /* Free resources. */ free(name); @@ -973,6 +963,45 @@ int rename_att_handler(iosystem_desc_t *ios) return PIO_NOERR; } +/** This function is run on the IO tasks to delete a netCDF + * attribute. */ +int delete_att_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int namelen, newnamelen; + char *name, *newname; + int mpierr; + int ret; + + LOG((1, "delete_att_handler")); + + /* Get the parameters for this function that the he comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if (!(name = malloc((namelen + 1) * sizeof(char)))) + return PIO_ENOMEM; + if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + LOG((2, "delete_att_handler namelen = %d name = %s ncid = %d varid = %d ", + namelen, name, ncid, varid)); + + /* Call the create file function. */ + if ((ret = PIOc_del_att(ncid, varid, name))) + return ret; + + /* Free resources. */ + free(name); + + LOG((1, "delete_att_handler succeeded!")); + return PIO_NOERR; +} + /** This function is run on the IO tasks. It reads or writes an array * of data to a netCDF variable. * @@ -1271,7 +1300,9 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) break; case PIO_MSG_RENAME_ATT: rename_att_handler(my_iosys); - LOG((1, "rename_att_handler returned")); + break; + case PIO_MSG_DEL_ATT: + delete_att_handler(my_iosys); break; case PIO_MSG_DEF_DIM: def_dim_handler(my_iosys); diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 0bc417c5f0a..7271cae2bd7 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -899,11 +899,11 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); - + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + /* Broadcast results. */ if (!ierr) { @@ -1104,11 +1104,11 @@ int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp) LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr)); } - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); - + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + /* Broadcast results. */ if (!ierr) { @@ -1335,7 +1335,6 @@ int PIOc_rename_att (int ncid, int varid, const char *name, if (ios->compmaster) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); - LOG((2, "PIOc_rename_att done sending msg")); if (!mpierr) mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -1398,9 +1397,10 @@ int PIOc_del_att(int ncid, int varid, const char *name) file_desc_t *file; /** Pointer to file information. */ int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ + int namelen = strlen(name); /** Length of name string. */ /* User must provide name of correct length. */ - if (!name || strlen(name) > NC_MAX_NAME) + if (!name || namelen > NC_MAX_NAME) return PIO_EINVAL; LOG((1, "PIOc_del_att ncid = %d varid = %d name = %s", ncid, varid, name)); @@ -1421,7 +1421,13 @@ int PIOc_del_att(int ncid, int varid, const char *name) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); } /* Handle MPI errors. */ @@ -1734,11 +1740,11 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) } } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; @@ -1927,11 +1933,11 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) } } - if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - ierr = check_netcdf(file, ierr, errstr,__LINE__); + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return PIO_EIO; + check_netcdf(file, ierr, __FILE__, __LINE__); + mpierr = MPI_Bcast(fill_value, 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index b2c2d2d3d70..d45e14431ae 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -462,12 +462,16 @@ main(int argc, char **argv) ERR(ret); /* Write an att and delete it. */ - /* if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data))) */ - /* ERR(ret); */ - /* if ((ret = PIOc_del_att(ncid, NC_GLOBAL, FIRST_ATT_NAME))) */ - /* ERR(ret); */ + nc_type myatttype; + if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data))) + ERR(ret); + if ((ret = PIOc_del_att(ncid, NC_GLOBAL, FIRST_ATT_NAME))) + ERR(ret); /* if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, NULL, NULL)) != PIO_ENOTATT) */ + /* { */ + /* printf("ret = %d\n", ret); */ /* ERR(ERR_AWFUL); */ + /* } */ /* Write some atts of different types. */ if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME, NC_SHORT, 1, &short_att_data))) From 28107470cb939ca087cbfb0c3f67b5c82e3c9930 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 10:32:14 -0400 Subject: [PATCH 071/184] continued async development --- src/clib/pio_nc_async.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 7271cae2bd7..0a3e1b1ec18 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1769,14 +1769,14 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp) { - iosystem_desc_t *ios; - file_desc_t *file; - int ierr = PIO_NOERR; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - int namelen; + int namelen = strlen(name); /* User must provide name and storage for varid. */ - if (!name || !varidp) + if (!name || !varidp || namelen > NC_MAX_NAME) { check_netcdf(file, PIO_EINVAL, __FILE__, __LINE__); return PIO_EINVAL; @@ -1796,14 +1796,12 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, if (!ios->ioproc) { int msg = PIO_MSG_DEF_VAR; + if(ios->compmaster) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + if (!mpierr) mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm); - namelen = strlen(name); - LOG((2, "bcasting namelen = %d name = %s\n", namelen, name)); - if (!ios->compmaster) - ios->compmaster = MPI_PROC_NULL; if (!mpierr) mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -1864,8 +1862,9 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, /* Broadcast results. */ if (!ierr) if (varidp) - mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm); - + if ((mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + return ierr; } From d9ef4b998257a38ad69ce47c03ddd40d26467591 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 10:42:43 -0400 Subject: [PATCH 072/184] more async changes --- src/clib/pio_nc_async.c | 74 +++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 0a3e1b1ec18..d95b70594f1 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1683,38 +1683,52 @@ int PIOc_redef (int ncid) */ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - int namelen; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - errstr = NULL; - ierr = PIO_NOERR; + /* User must provide name. */ + if (!name || strlen(name) > NC_MAX_NAME) + return PIO_EINVAL; - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - printf("%d PIOc_def_dim ncid = %d name = %s len = %d\n", my_rank, - ncid, name, len); + LOG((1, "PIOc_def_dim ncid = %d", ncid)); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_DEF_DIM; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - namelen = strlen(name); - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_DEF_DIM; + int namelen = strlen(name); + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF @@ -1744,9 +1758,13 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return PIO_EIO; check_netcdf(file, ierr, __FILE__, __LINE__); - - mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); + + /* Broadcast results to all tasks. Ignore NULL parameters. */ + if (!ierr) + if (idp) + if ((mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + return ierr; } From 7417325ae9e0393cf688e86c3639b022c675708e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 10:47:20 -0400 Subject: [PATCH 073/184] more async changes --- src/clib/pio_nc_async.c | 44 +++++++++++++---------------------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index d95b70594f1..bc1033958a9 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1791,10 +1791,9 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, file_desc_t *file; /** Pointer to file information. */ int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - int namelen = strlen(name); /* User must provide name and storage for varid. */ - if (!name || !varidp || namelen > NC_MAX_NAME) + if (!name || !varidp || strlen(name) > NC_MAX_NAME) { check_netcdf(file, PIO_EINVAL, __FILE__, __LINE__); return PIO_EINVAL; @@ -1814,6 +1813,7 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, if (!ios->ioproc) { int msg = PIO_MSG_DEF_VAR; + int namelen = strlen(name); if(ios->compmaster) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); @@ -1838,38 +1838,22 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, check_mpi(file, mpierr, __FILE__, __LINE__); } - /* IO tasks call the netCDF functions. */ + /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; - break; - case PIO_IOTYPE_NETCDF4C: - if (ios->io_rank == 0) - { - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); - if (!ierr) - ierr = nc_def_var_deflate(file->fh, *varidp, 0,1,1); - } - break; -#endif - case PIO_IOTYPE_NETCDF: - if (ios->io_rank == 0) - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); - break; -#endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + if (file->iotype == PIO_IOTYPE_PNETCDF) ierr = ncmpi_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); +#ifdef _NETCDF4 + /* For netCDF-4 serial files, turn on compression for this variable. */ + if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4C) + ierr = nc_def_var_deflate(file->fh, *varidp, 0, 1, 1); +#endif /* _NETCDF4 */ +#endif /* _NETCDF */ } /* Broadcast and check the return code. */ From b3d7b94b5b195ef55937d1b8d510b010f6e856e8 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 10:49:54 -0400 Subject: [PATCH 074/184] more async changes --- src/clib/pio_nc_async.c | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index bc1033958a9..b4d2430cfba 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1692,7 +1692,7 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) if (!name || strlen(name) > NC_MAX_NAME) return PIO_EINVAL; - LOG((1, "PIOc_def_dim ncid = %d", ncid)); + LOG((1, "PIOc_def_dim ncid = %d name = %s len = %d", ncid, name, len)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) @@ -1729,29 +1729,17 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) } /* If this is an IO task, then call the netCDF function. */ - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; - } - break; -#endif + if (ios->ioproc) + { #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_def_dim(file->fh, name, len, idp);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_def_dim(file->fh, name, len, idp);; +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_def_dim(file->fh, name, (size_t)len, idp); +#endif /* _NETCDF */ + LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } /* Broadcast and check the return code. */ From a4be288dfb0a073d76dbe667956224d5a1468559 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 10:54:45 -0400 Subject: [PATCH 075/184] more async changes --- src/clib/pio_nc_async.c | 86 ++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index b4d2430cfba..66d77d1c551 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -114,7 +114,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -1739,7 +1739,6 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) ierr = nc_def_dim(file->fh, name, (size_t)len, idp); #endif /* _NETCDF */ - LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } /* Broadcast and check the return code. */ @@ -1872,63 +1871,64 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) +int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep) { - int ierr; - int msg; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - iosystem_desc_t *ios; - file_desc_t *file; - char *errstr; - errstr = NULL; - ierr = PIO_NOERR; + LOG((1, "PIOc_inq ncid = %d", ncid)); - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VAR_FILL; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_VAR_FILL; + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_value);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + if (!mpierr) + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { +#ifdef _PNETCDF + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_valuep); +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_valuep); +#endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. Ignore NULL parameters. */ + if (!ierr) + if (fill_valuep) + if ((mpierr = MPI_Bcast(fill_valuep, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); - mpierr = MPI_Bcast(fill_value, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); return ierr; } From b61b5a3818c196b9de102b3af70d81c35edfd8eb Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 10:58:22 -0400 Subject: [PATCH 076/184] more async changes --- src/clib/pio_nc_async.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 66d77d1c551..dd3ec9775af 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1949,15 +1949,15 @@ int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep) */ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) { - iosystem_desc_t *ios; - file_desc_t *file; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ PIO_Offset attlen, typelen; nc_type atttype; - int ierr = PIO_NOERR; - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ /* User must provide a name and destination pointer. */ - if (!name || !ip) + if (!name || !ip || strlen(name) > NC_MAX_NAME) return PIO_EINVAL; LOG((1, "PIOc_get_att ncid %d varid %d name %s", ncid, varid, name)); @@ -1985,7 +1985,6 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) return ierr; } } - /* If async is in use, and this is not an IO task, bcast the * parameters and the attribute and type information we fetched. */ From a5d6df3304a6bf5ab09e155685cc86864de31c12 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 11:00:09 -0400 Subject: [PATCH 077/184] more async changes --- src/clib/pio_nc_async.c | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index dd3ec9775af..37a828dd6a6 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -2035,30 +2035,15 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att(file->fh, varid, name, ip); - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if (ios->io_rank == 0) - ierr = nc_get_att(file->fh, varid, name, ip); - break; -#endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + if (file->iotype == PIO_IOTYPE_PNETCDF) ierr = ncmpi_get_att(file->fh, varid, name, ip); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_get_att(file->fh, varid, name, ip); +#endif /* _NETCDF */ } - LOG((2, "PIOc_get_att called netcdf layer ierr = %d", ierr)); /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) @@ -2068,14 +2053,12 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) /* Broadcast results to all tasks. */ if (!ierr) { - LOG((2, "PIOc_get_att broadcasting att data")); if ((mpierr = MPI_Bcast(ip, (int)attlen * typelen, MPI_BYTE, ios->ioroot, ios->my_comm))) { check_mpi(file, mpierr, __FILE__, __LINE__); return PIO_EIO; } - LOG((2, "PIOc_get_att done broadcasting att data")); } return ierr; } From 7ec04777fedd1ae3ada51dbe435d7487841eae96 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 11:01:56 -0400 Subject: [PATCH 078/184] more async changes --- src/clib/pio_nc_async.c | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 37a828dd6a6..7ac27d02f9d 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -2152,34 +2152,22 @@ int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { - switch (file->iotype) - { -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att(file->fh, varid, name, xtype, (size_t)len, op); - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_att(file->fh, varid, name, xtype, (size_t)len, op); - } - break; -#endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + if (file->iotype == PIO_IOTYPE_PNETCDF) ierr = ncmpi_put_att(file->fh, varid, name, xtype, len, op); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_put_att(file->fh, varid, name, xtype, (size_t)len, op); +#endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + { + check_mpi(file, mpierr, __FILE__, __LINE__); return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); return ierr; From be7b7b41e1477f3b0fa42c08ea9147d016bf25c6 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 11:11:02 -0400 Subject: [PATCH 079/184] more async changes --- src/clib/pio_nc_async.c | 46 +++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 7ac27d02f9d..7498c426b1f 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -867,24 +867,38 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, return PIO_EBADID; ios = file->iosystem; - /* If using async and this is not an IO task, send parameter data - * over the intercomm. */ - if(ios->async_interface && !ios->ioproc) + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) { - char xtype_present = xtypep ? true : false; - char len_present = lenp ? true : false; - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - int namelen = strlen(name); - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } + if (!ios->ioproc) + { + char xtype_present = xtypep ? true : false; + char len_present = lenp ? true : false; + int namelen = strlen(name); + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); + } + /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { From cf375e13249a4de38018ee84448038043fe01396 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 11:31:52 -0400 Subject: [PATCH 080/184] more async changes --- src/clib/pio.h | 4 +- src/clib/pio_file.c | 2 +- src/clib/pio_nc_async.c | 194 ++++++++++++++++++++++------------------ 3 files changed, 112 insertions(+), 88 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index 596e10c0e29..ffcb2385566 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -413,8 +413,8 @@ int PIOc_inq_varnatts (int ncid, int varid, int *nattsp); int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp); int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_level); -int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, - int *deflatep, int *deflate_levelp); +int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep, + int *deflate_levelp); int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp); int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp); int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp); diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 03cce3e9c25..b3e8f9dc021 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -462,7 +462,7 @@ int PIOc_deletefile(const int iosysid, const char filename[]) /** * @name PIOc_sync */ -int PIOc_sync (int ncid) +int PIOc_sync(int ncid) { int ierr; int msg; diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 7498c426b1f..b3e31fa57ff 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -114,7 +114,10 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -269,7 +272,10 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -350,7 +356,10 @@ int PIOc_inq_format (int ncid, int *formatp) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + { + check_mpi(file, mpierr, __FILE__, __LINE__); return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -437,7 +446,10 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + { + check_mpi(file, mpierr, __FILE__, __LINE__); return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -555,7 +567,10 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results. */ @@ -660,7 +675,10 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast the results for non-null pointers. */ @@ -819,7 +837,10 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -915,7 +936,10 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + { + check_mpi(file, mpierr, __FILE__, __LINE__); return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results. */ @@ -1023,7 +1047,10 @@ int PIOc_inq_attname(int ncid, int varid, int attnum, char *name) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + { + check_mpi(file, mpierr, __FILE__, __LINE__); return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -1120,7 +1147,10 @@ int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + { + check_mpi(file, mpierr, __FILE__, __LINE__); return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results. */ @@ -1211,7 +1241,10 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + { + check_mpi(file, mpierr, __FILE__, __LINE__); return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); return ierr; @@ -1295,7 +1328,10 @@ int PIOc_rename_var(int ncid, int varid, const char *name) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + { + check_mpi(file, mpierr, __FILE__, __LINE__); return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); return ierr; @@ -1384,7 +1420,10 @@ int PIOc_rename_att (int ncid, int varid, const char *name, /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); LOG((2, "PIOc_rename_att succeeded")); @@ -1411,10 +1450,9 @@ int PIOc_del_att(int ncid, int varid, const char *name) file_desc_t *file; /** Pointer to file information. */ int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ - int namelen = strlen(name); /** Length of name string. */ /* User must provide name of correct length. */ - if (!name || namelen > NC_MAX_NAME) + if (!name || strlen(name) > NC_MAX_NAME) return PIO_EINVAL; LOG((1, "PIOc_del_att ncid = %d varid = %d name = %s", ncid, varid, name)); @@ -1430,6 +1468,7 @@ int PIOc_del_att(int ncid, int varid, const char *name) if (!ios->ioproc) { int msg = PIO_MSG_DEL_ATT; + int namelen = strlen(name); /** Length of name string. */ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); @@ -1465,7 +1504,10 @@ int PIOc_del_att(int ncid, int varid, const char *name) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); LOG((2, "PIOc_del_att succeeded")); @@ -1536,34 +1578,29 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); LOG((2, "PIOc_set_fill succeeded")); return ierr; } -/** - * @ingroup PIOc_enddef - * The PIO-C interface for the NetCDF function nc_enddef. - * - * This routine is called collectively by all tasks in the communicator - * ios.union_comm. For more information on the underlying NetCDF commmand - * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html - * - * @param ncid the ncid of the open file, obtained from - * PIOc_openfile() or PIOc_createfile(). - * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling - */ -int PIOc_enddef(int ncid) +/** This is an internal function that handles both PIOc_enddef and + * PIOc_redef. + * @param ncid the ncid of the file to enddef or redef + * @param is_enddef set to non-zero for enddef, 0 for redef. + * @returns PIO_NOERR on success, error code on failure. */ +int pioc_change_def(int ncid, int is_enddef) { iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ - LOG((1, "PIOc_enddef ncid = %d", ncid)); + LOG((1, "pioc_change_def ncid = %d is_enddef = %d", ncid, is_enddef)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) @@ -1575,7 +1612,7 @@ int PIOc_enddef(int ncid) { if (!ios->ioproc) { - int msg = PIO_MSG_ENDDEF; + int msg = is_enddef ? PIO_MSG_ENDDEF : PIO_MSG_REDEF; if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); @@ -1595,23 +1632,49 @@ int PIOc_enddef(int ncid) { #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_enddef(file->fh); + if (is_enddef) + ierr = ncmpi_enddef(file->fh); + else + ierr = ncmpi_redef(file->fh); #endif /* _PNETCDF */ #ifdef _NETCDF if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_enddef(file->fh); + if (is_enddef) + ierr = nc_enddef(file->fh); + else + ierr = nc_redef(file->fh); #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); - LOG((2, "PIOc_enddef succeeded")); return ierr; } +/** + * @ingroup PIOc_enddef + * The PIO-C interface for the NetCDF function nc_enddef. + * + * This routine is called collectively by all tasks in the communicator + * ios.union_comm. For more information on the underlying NetCDF commmand + * please read about this function in the NetCDF documentation at: + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * + * @param ncid the ncid of the open file, obtained from + * PIOc_openfile() or PIOc_createfile(). + * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling + */ +int PIOc_enddef(int ncid) +{ + return pioc_change_def(ncid, 1); +} + /** * @ingroup PIOc_redef * The PIO-C interface for the NetCDF function nc_redef. @@ -1625,60 +1688,9 @@ int PIOc_enddef(int ncid) * PIOc_openfile() or PIOc_createfile(). * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_redef (int ncid) +int PIOc_redef(int ncid) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ - - LOG((1, "PIOc_redef ncid = %d", ncid)); - - /* Find the info about this file. */ - if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; - ios = file->iosystem; - - /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) - { - if (!ios->ioproc) - { - int msg = PIO_MSG_ENDDEF; - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); - } - - /* If this is an IO task, then call the netCDF function. */ - if (ios->ioproc) - { -#ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_redef(file->fh); -#endif /* _PNETCDF */ -#ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_redef(file->fh); -#endif /* _NETCDF */ - } - - /* Broadcast and check the return code. */ - if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - check_netcdf(file, ierr, __FILE__, __LINE__); - - LOG((2, "PIOc_redef succeeded")); - return ierr; + return pioc_change_def(ncid, 0); } /** @@ -1757,7 +1769,10 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + { + check_mpi(file, mpierr2, __FILE__, __LINE__); return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -1859,7 +1874,10 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results. */ @@ -1934,7 +1952,10 @@ int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -2061,7 +2082,10 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + { + check_mpi(file, mpierr, __FILE__, __LINE__); return PIO_EIO; + } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. */ From 7e01a846314168cbe3a55977baa8531f794aaa82 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 11:41:33 -0400 Subject: [PATCH 081/184] got redef function working with async --- src/clib/pio_msg.c | 16 +++++++--------- tests/unit/test_intercomm.c | 8 ++++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 5c2f511da50..be2b75ea8ce 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -711,26 +711,23 @@ int sync_file_handler(iosystem_desc_t *ios) * @param ios pointer to the iosystem_desc_t. * @return PIO_NOERR for success, error code otherwise. */ -int enddef_file_handler(iosystem_desc_t *ios) +int change_def_file_handler(iosystem_desc_t *ios, int msg) { int ncid; int mpierr; int ret; - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - LOG((1, "%d enddef_file_handler\n", my_rank)); + LOG((1, "change_def_file_handler")); /* Get the parameters for this function that the comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - /* Call the sync file function. */ - if ((ret = PIOc_enddef(ncid))) - return ret; + /* Call the function. */ + ret = (msg == PIO_MSG_ENDDEF) ? PIOc_enddef(ncid) : PIOc_redef(ncid); - LOG((1, "%d enddef_file_handler succeeded!\n", my_rank)); + LOG((1, "change_def_file_handler succeeded!")); return PIO_NOERR; } @@ -1281,7 +1278,8 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) sync_file_handler(my_iosys); break; case PIO_MSG_ENDDEF: - enddef_file_handler(my_iosys); + case PIO_MSG_REDEF: + change_def_file_handler(my_iosys, msg); break; case PIO_MSG_OPEN_FILE: open_file_handler(my_iosys); diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index d45e14431ae..172c3ca760b 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -390,6 +390,14 @@ main(int argc, char **argv) if (verbose) printf("%d test_intercomm file created ncid = %d\n", my_rank, ncid); + /* /\* End define mode, then re-enter it. *\/ */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + if (verbose) + printf("%d test_intercomm calling redef\n", my_rank); + if ((ret = PIOc_redef(ncid))) + ERR(ret); + /* Test the inq_format function. */ int myformat; if ((ret = PIOc_inq_format(ncid, &myformat))) From 68c3068a99f9a1b3f4771cac34418053e3516f0e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 11:50:18 -0400 Subject: [PATCH 082/184] created internal function because pnetcdf does not have inq_type() --- src/clib/pio_nc_async.c | 70 +++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index b3e31fa57ff..30046e929aa 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -180,6 +180,45 @@ int PIOc_inq_unlimdim(int ncid, int *unlimdimidp) return PIOc_inq(ncid, NULL, NULL, unlimdimidp, NULL); } +/** Internal function to provide inq_type function for pnetcdf. */ +int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name, + PIO_Offset *sizep) +{ + int typelen; + char typename[NC_MAX_NAME + 1]; + + switch (xtype) + { + case NC_UBYTE: + case NC_BYTE: + case NC_CHAR: + typelen = 1; + break; + case NC_SHORT: + case NC_USHORT: + typelen = 2; + break; + case NC_UINT: + case NC_INT: + case NC_FLOAT: + typelen = 4; + break; + case NC_UINT64: + case NC_INT64: + case NC_DOUBLE: + typelen = 8; + break; + } + + /* If pointers were supplied, copy results. */ + if (sizep) + *sizep = typelen; + if (name) + strcpy(name, "some type"); + + return PIO_NOERR; +} + /** * @ingroup PIOc_typelen * The PIO-C interface for the NetCDF function nctypelen. @@ -227,41 +266,12 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) check_mpi(file, mpierr, __FILE__, __LINE__); } - /* If this is an IO task, then call the netCDF function. */ /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) - { - switch (xtype) - { - case NC_UBYTE: - case NC_BYTE: - case NC_CHAR: - typelen = 1; - break; - case NC_SHORT: - case NC_USHORT: - typelen = 2; - break; - case NC_UINT: - case NC_INT: - case NC_FLOAT: - typelen = 4; - break; - case NC_UINT64: - case NC_INT64: - case NC_DOUBLE: - typelen = 8; - break; - } - - if (sizep) - *sizep = typelen; - if (name) - strcpy(name, "some type"); - } + ierr = pioc_pnetcdf_inq_type(ncid, xtype, name, sizep); #endif /* _PNETCDF */ #ifdef _NETCDF if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) From 5f597a8b3813f9859654034f86796f3577407f40 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 12:03:08 -0400 Subject: [PATCH 083/184] further development of async code --- src/clib/pio_internal.h | 4 ++-- src/clib/pio_nc_async.c | 42 ++++++++++++++++------------------------- src/clib/pioc_support.c | 5 ++++- 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 3425517818a..412b3d99055 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -152,8 +152,8 @@ typedef struct pio_swapm_defaults void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk); void piomemerror(iosystem_desc_t ios, size_t req, char *fname, const int line); void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc); - void check_mpi(file_desc_t *file, const int mpierr, const char *filename, - const int line); + int check_mpi(file_desc_t *file, const int mpierr, const char *filename, + const int line); #ifdef BGQ void identity(MPI_Comm comm, int *iotask); diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 30046e929aa..c8e85600622 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -74,12 +74,11 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq netcdf Bcast unlimdimid_present = %d", unlimdimid_present)); } /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); + return check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -114,10 +113,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; - } + return check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -125,19 +121,19 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, { if (ndimsp) if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); if (nvarsp) if ((mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); if (ngattsp) if ((mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); if (unlimdimidp) if ((mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); } return ierr; @@ -225,7 +221,6 @@ int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name, */ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) { - int msg = PIO_MSG_INQ_TYPE; /** Message for async notification. */ iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ int ierr = PIO_NOERR; /** Return code from function calls. */ @@ -244,6 +239,7 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) { if (!ios->ioproc) { + int msg = PIO_MSG_INQ_TYPE; /** Message for async notification. */ char name_present = name ? true : false; char size_present = sizep ? true : false; @@ -262,7 +258,7 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); + return check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -282,10 +278,7 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; - } + return check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -297,13 +290,13 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) if (ios->iomaster) slen = strlen(name); if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); } if (sizep) if ((mpierr = MPI_Bcast(sizep , 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); } return ierr; @@ -339,14 +332,14 @@ int PIOc_inq_format (int ncid, int *formatp) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&format_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); } /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); + return check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -366,10 +359,7 @@ int PIOc_inq_format (int ncid, int *formatp) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; - } + return check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -377,7 +367,7 @@ int PIOc_inq_format (int ncid, int *formatp) { if (formatp) if ((mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); } return ierr; diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 579a9e64fc6..e08a019c517 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -193,8 +193,9 @@ void pioassert(_Bool expression, const char *msg, const char *fname, const int l @param mpierr the MPI return code to handle @param filename the name of the code file where error occured. @param line the line of code where error occured. + @return PIO_NOERR for no error, otherwise PIO_EIO. */ -void check_mpi(file_desc_t *file, const int mpierr, const char *filename, +int check_mpi(file_desc_t *file, const int mpierr, const char *filename, const int line) { if (mpierr) @@ -209,7 +210,9 @@ void check_mpi(file_desc_t *file, const int mpierr, const char *filename, /* Handle all MPI errors as PIO_EIO. */ check_netcdf(file, PIO_EIO, filename, line); + return PIO_EIO; } + return PIO_NOERR; } /** Check the result of a netCDF API call. From 46ba2cc6131823451225bd06044c3c5907acad60 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 17 May 2016 12:06:45 -0400 Subject: [PATCH 084/184] further development of async code --- src/clib/pio_nc_async.c | 57 ++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index c8e85600622..5d303a90be2 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -427,7 +427,7 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); + return check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -446,10 +446,7 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; - } + return check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ @@ -461,14 +458,14 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) if (ios->iomaster) slen = strlen(name); if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); } if (lenp) if ((mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); } return ierr; @@ -548,7 +545,7 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); + return check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -567,17 +564,14 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; - } + return check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results. */ if (!ierr) if (idp) if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); return ierr; } @@ -650,7 +644,7 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); + return check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -661,24 +655,23 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, if (file->iotype == PIO_IOTYPE_PNETCDF) { ierr = ncmpi_inq_varndims(file->fh, varid, &ndims); - ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + if (!ierr) + ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; } #endif /* _PNETCDF */ #ifdef _NETCDF if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) { ierr = nc_inq_varndims(file->fh, varid, &ndims); - ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp); + if (!ierr) + ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp); } #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; - } + return check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast the results for non-null pointers. */ @@ -690,30 +683,30 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, if(ios->iomaster) slen = strlen(name); if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); } if (xtypep) if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); if (ndimsp) { if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); file->varlist[varid].ndims = (*ndimsp); } if (dimidsp) { if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); } if (nattsp) if ((mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); } return ierr; @@ -781,13 +774,13 @@ int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) */ int PIOc_inq_varid (int ncid, const char *name, int *varidp) { - iosystem_desc_t *ios; - file_desc_t *file; - int ierr = PIO_NOERR; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ /* Caller must provide name. */ - if (!name) + if (!name || strlen(name) > NC_MAX_NAME) return PIO_EINVAL; /* Get file info based on ncid. */ From ce91865e4176d97b30f0d53d971492ebaebe2cdd Mon Sep 17 00:00:00 2001 From: katetc Date: Tue, 17 May 2016 14:29:01 -0600 Subject: [PATCH 085/184] Added documentation for the Unit and Performance tests in PIO2. Fixed some of the markup on the Examples page. --- doc/source/Examples.txt | 2 + doc/source/Testing.txt | 181 ++++++++++++++++++++++++++++++++++++++++ doc/source/base.txt | 2 + 3 files changed, 185 insertions(+) create mode 100644 doc/source/Testing.txt diff --git a/doc/source/Examples.txt b/doc/source/Examples.txt index f2af4ebcf66..e18d2926c7c 100644 --- a/doc/source/Examples.txt +++ b/doc/source/Examples.txt @@ -41,6 +41,8 @@ The Fortran examples are in the examples/f03 subdirectory. - examplePio.f90 A simple example showing a write, then read, of a 1D variable. +### Other Examples + PIO has been implemented in several geophysical component models, including the Community Atmosphere Model (CAM), the Community Land Model (CLM), the Parallel Ocean Program (POP), the Community Ice CodE (CICE), and coupler for used by CCSM4.0 (CPL7). We also provide diff --git a/doc/source/Testing.txt b/doc/source/Testing.txt new file mode 100644 index 00000000000..1d15f73b7ab --- /dev/null +++ b/doc/source/Testing.txt @@ -0,0 +1,181 @@ +/****************************************************************************** + * + * + * + * Copyright (C) 2009 + * + * Permission to use, copy, modify, and distribute this software and its + * documentation under the terms of the GNU General Public License is hereby + * granted. No representations are made about the suitability of this software + * for any purpose. It is provided "as is" without express or implied warranty. + * See the GNU General Public License for more details. + * + * Documents produced by Doxygen are derivative works derived from the + * input used in their production; they are not affected by this license. + * + */ /*! \page test Testing + +## Building PIO2 Tests + +To build both the Unit and Performance tests for PIO2, follow the general instructions for building PIO2 in either the [Installation](@ref install) page or the [Machine Walk-Through](@ref mach_walkthrough) page. During the Build step after (or instead of) the **make** command, type **make tests**. + +## PIO2 Unit Tests + +The Parallel IO library comes with more than 20 built-in unit tests to verify that the library is installed and working correctly. These tests utilize the _CMake_ and _CTest_ automation framework. Because the Parallel IO library is built for parallel applications, the unit tests should be run in a parallel environment. The simplest way to do this is to submit a PBS job to run the **ctest** command. + +For a library built into the example directory `/scratch/user/PIO_build/`, an example PBS script would be: + + #!/bin/bash + + #PBS -q normal + #PBS -l nodes=1:ppn=4 + #PBS -N piotests + #PBS -e piotests.e$PBS_JOBID + #PBS -o piotests.o$PBS_JOBID + + cd /scratch/user/PIO_build + ctest + +The output from the unit tests will be reported in the piotests.o$JOBID file. This should look something like: + +~~~~~~~~~~~~~~ +Test project /scratch/cluster/katec/PIO_build + Start 1: test_names + 1/24 Test #1: test_names ....................... Passed 0.60 sec + Start 2: test_nc4 + 2/24 Test #2: test_nc4 ......................... Passed 0.53 sec + Start 3: pio_unit_test + 3/24 Test #3: pio_unit_test .................... Passed 0.45 sec + Start 4: init_finialize_1_proc + 4/24 Test #4: init_finialize_1_proc ............ Passed 0.54 sec + Start 5: init_finialize_2_proc + 5/24 Test #5: init_finialize_2_proc ............ Passed 0.53 sec + Start 6: init_finalize_2_proc_with_args + 6/24 Test #6: init_finalize_2_proc_with_args ... Passed 0.55 sec + Start 7: pio_file_simple_tests + 7/24 Test #7: pio_file_simple_tests ............ Passed 0.58 sec + Start 8: pio_file_fail + 8/24 Test #8: pio_file_fail .................... Passed 0.62 sec + Start 9: ncdf_simple_tests + 9/24 Test #9: ncdf_simple_tests ................ Passed 0.60 sec + Start 10: ncdf_get_put_1proc +10/24 Test #10: ncdf_get_put_1proc ............... Passed 0.65 sec + Start 11: ncdf_get_put_2proc +11/24 Test #11: ncdf_get_put_2proc ............... Passed 0.63 sec + Start 12: ncdf_fail +12/24 Test #12: ncdf_fail ........................ Passed 0.52 sec + Start 13: pio_decomp_tests_1p +13/24 Test #13: pio_decomp_tests_1p .............. Passed 1.54 sec + Start 14: pio_decomp_tests_2p +14/24 Test #14: pio_decomp_tests_2p .............. Passed 1.99 sec + Start 15: pio_decomp_tests_3p +15/24 Test #15: pio_decomp_tests_3p .............. Passed 2.11 sec + Start 16: pio_decomp_tests_4p_1agg +16/24 Test #16: pio_decomp_tests_4p_1agg ......... Passed 2.12 sec + Start 17: pio_decomp_tests_4p_2agg +17/24 Test #17: pio_decomp_tests_4p_2agg ......... Passed 2.08 sec + Start 18: pio_decomp_tests_4p_3agg +18/24 Test #18: pio_decomp_tests_4p_3agg ......... Passed 2.08 sec + Start 19: pio_decomp_tests_4p_1iop +19/24 Test #19: pio_decomp_tests_4p_1iop ......... Passed 1.91 sec + Start 20: pio_decomp_tests_4p_2iop +20/24 Test #20: pio_decomp_tests_4p_2iop ......... Passed 2.50 sec + Start 21: pio_decomp_tests_4p_3iop +21/24 Test #21: pio_decomp_tests_4p_3iop ......... Passed 2.20 sec + Start 22: pio_decomp_tests_4p_2iop_2str +22/24 Test #22: pio_decomp_tests_4p_2iop_2str .... Passed 2.16 sec + Start 23: pio_decomp_tests_4p_2iop_1agg +23/24 Test #23: pio_decomp_tests_4p_2iop_1agg .... Passed 2.20 sec + Start 24: pio_decomp_fillval2 +24/24 Test #24: pio_decomp_fillval2 .............. Passed 0.60 sec + +100% tests passed, 0 tests failed out of 24 + +Total Test time (real) = 30.80 sec +~~~~~~~~~~~~~~ + +Another option would be to launch an interactive session, change into the build directory, and run the **ctest** command. + +On Yellowstone, the unit tests can run using the **execca** or **execgy** commands as: + + > setenv DAV_CORES 4 + > execca ctest + +## PIO2 Performance Test + +To run the performance tests, you will need to add two files to the **tests/performance** subdirectory of the PIO build directory. First, you will need a decomp file. You can download one from our google code page here: +https://svn-ccsm-piodecomps.cgd.ucar.edu/trunk/ . +You can use any of these files, and save them to your home or base work directory. Secondly, you will need to add a namelist file, named "pioperf.nl". Save this file in the directory with your **pioperf** executable (this is found in the **tests/performance** subdirectory of the PIO build directory). + + +The contents of the namelist file should look like: + + &pioperf + + decompfile = "/u/home/user/piodecomp30tasks01dims06.dat" + + pio_typenames = 'pnetcdf' + + niotasks = 30 + + rearrangers = 1 + + nvars = 2 + + / + +Here, the second line ("decompfile") points to the path for your decomp file (wherever you saved it). For the rest of the lines, each item added to the list adds another test to be run. For instance, to test all of the types of supported IO, your pio_typenames would look like: + + pio_typenames = 'pnetcdf','netcdf','netcdf4p','netcdf4c' + +HDF5 is netcdf4p, and Parallel-Netcdf is pnetcdf. + +To test with different numbers of IO tasks, you could do: + + niotasks = 30,15,5 + +(These tasks are the subset of the run tasks that are designated IO tasks) + +To test with both of the rearranger algorithms: + + rearrangers = 1,2 + +(Each rearranger is a different algorithm for converting from data in memory to data in a file on disk. The first one, BOX, is the older method from PIO1, the second, SUBSET, is a newer method that seems to be more efficient in large numbers of tasks) + +To test with different numbers of variables: + + nvars = 8,5,3,2 + +(The more variables you use, the higher data throughput goes, usually) + +To run, submit a job with 'pioperf' as the executable, and at least as many tasks as you have specified in the decomposition file. On yellowstone, a submit script could look like: + + #!/bin/tcsh + + #BSUB -P P00000000 # project code + #BSUB -W 00:10 # wall-clock time (hrs:mins) + #BSUB -n 30 # number of tasks in job + #BSUB -R "span[ptile=16]" # run 16 MPI tasks per node + #BSUB -J pio_perftest # job name + #BSUB -o pio_perftest.%J.out # output file name in which %J is replaced by the job ID + #BSUB -e pio_perftest.%J.err # error file name in which %J is replaced by the job ID + #BSUB -q small # queue + + #run the executable + mpirun.lsf /glade/p/work/katec/pio_work/pio_build/tests/performance/pioperf + +The result(s) will look like a line in the output file such as: +~~~~~~~~~~~~~~ +RESULT: write BOX 4 30 2 16.9905924688 +~~~~~~~~~~~~~~ + +You can decode this as: +1. Read/write describes the io operation performed +2. BOX/SUBSET is the algorithm for the rearranger (as described above) +3. 4 [1-4] is the io library used for the operation. The options here are [1] Parallel-netcdf [2] NetCDF3 [3] NetCDF4-Compressed [4] NetCDF4-Parallel +4. 30 [any number] is the number of io-specific tasks used in the operation. Must be less than the number of MPI tasks used in the test. +5. 2 [any number] is the number of variables read or written during the operation +6. 16.9905924688 [any number] is the Data Rate of the operation in MB/s. This is the important value for determining performance of the system. The higher this numbre is, the better the PIO2 library is performing for the given operation. + +_Last updated: 05-17-2016_ +*/ diff --git a/doc/source/base.txt b/doc/source/base.txt index 84079df79eb..2108786f13c 100644 --- a/doc/source/base.txt +++ b/doc/source/base.txt @@ -40,8 +40,10 @@ Please review the ChangeLog that is included with the distribution for up-to-dat - \ref intro - \ref install + - \ref mach_walkthrough - \ref decomp - \ref error + - \ref test - \ref examp - \ref faq - \ref api From cea8e392d6d2af0f398b1f8897d36ca1bbe8bea3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 18 May 2016 10:08:20 -0400 Subject: [PATCH 086/184] first pass at PIOc_put_vars_tc() --- src/clib/pio.h | 4 +- src/clib/pio_nc_async.c | 12 +- src/clib/pio_put_nc_async.c | 436 +++++++++++++++++++++++++++++------- 3 files changed, 362 insertions(+), 90 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index ffcb2385566..d7710f9f4ae 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -592,10 +592,11 @@ int PIOc_set_blocksize(const int newblocksize); int PIOc_put_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long long *op); int PIOc_put_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const double *op); int PIOc_put_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); + int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], nc_type xtype, const void *buf); int PIOc_put_var_uchar (int ncid, int varid, const unsigned char *op); int PIOc_put_var_long (int ncid, int varid, const long *op); int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op); - int PIOc_get_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], int *buf); + int PIOc_get_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], int *buf); int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset index[], float *buf); int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset index[], short *buf); int PIOc_get_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], int *buf); @@ -647,6 +648,7 @@ int PIOc_set_blocksize(const int newblocksize); int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], char *buf); int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset index[], unsigned char *buf); int PIOc_get_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); + int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], nc_type xtype, void *buf); int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf); int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf); int PIOc_get_var_schar (int ncid, int varid, signed char *buf); diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 5d303a90be2..a8db152c91f 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1828,7 +1828,7 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) - mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast(&(ncid), 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -1852,15 +1852,19 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, { #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); + ierr = ncmpi_def_var(ncid, name, xtype, ndims, dimidsp, varidp); #endif /* _PNETCDF */ #ifdef _NETCDF if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); + ierr = nc_def_var(ncid, name, xtype, ndims, dimidsp, varidp); #ifdef _NETCDF4 /* For netCDF-4 serial files, turn on compression for this variable. */ if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4C) - ierr = nc_def_var_deflate(file->fh, *varidp, 0, 1, 1); + ierr = nc_def_var_deflate(ncid, *varidp, 0, 1, 1); + + /* For netCDF-4 parallel files, set parallel access to collective. */ + if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4P) + ierr = nc_var_par_access(ncid, *varidp, NC_COLLECTIVE); #endif /* _NETCDF4 */ #endif /* _NETCDF */ } diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index dd9693a3abe..74a52328eeb 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -1,14 +1,357 @@ #include #include -/// -/// PIO interface to nc_put_vars_uchar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned char *op) +/** + * Internal PIO function which provides a type-neutral interface to + * nc_put_vars + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + + * @param start[] an array of start indicies (must have same number of + * entries as variable has dimensions). If NULL, indices of 0 will be + * used. + * + * @param count[] an array of counts (must have same number of entries + * as variable has dimensions). If NULL, counts matching the size of + * the variable will be used. + * + * @param stride[] an array of strides (must have same number of + * entries as variable has dimensions). If NULL, strides of 1 will be + * used. + * + * @param xtype the netCDF type of the data being passed in buf. Data + * will be automatically covnerted from this type to the type of the + * variable being written to. + * + * @param buf pointer to the data to be written. + * + * @return PIO_NOERR on success, error code otherwise. + */ +int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, nc_type xtype, const void *buf) +{ + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + int ndims; /** The number of dimensions in the variable. */ + int *dimids; /** The IDs of the dimensions for this variable. */ + PIO_Offset typelen; /** Size (in bytes) of the data type of data in buf. */ + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + LOG((1, "PIOc_put_vars_tc ncid = %d varid = %d start = %d count = %d " + "stride = %d xtype = %d", ncid, start, count, stride, xtype)); + + /* User must provide some data. */ + if (!buf) + return PIO_EINVAL; + + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) + return PIO_EBADID; + ios = file->iosystem; + + /* Run these on all tasks if async is not in use, but only on + * non-IO tasks if async is in use. */ + if (!ios->async_interface || !ios->ioproc) + { + /* Get the length of the data type. */ + if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &typelen))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Get the number of dims for this var. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + PIO_Offset dimlen[ndims]; + + /* If no count array was passed, we need to know the dimlens + * so we can calculate how many data elements are in the + * buf. */ + if (!count) + { + int dimid[ndims]; + + /* Get the dimids for this var. */ + if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Get the length of each dimension. */ + for (int vd = 0; vd < ndims; vd++) + if ((ierr = PIOc_inq_dimlen(ncid, dimid[vd], &dimlen[vd]))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + } + + /* Figure out the real start, count, and stride arrays. (The + * user may have passed in NULLs.) */ + PIO_Offset rstart[ndims], rcount[ndims], rstride[ndims]; + for (int vd = 0; vd < ndims; vd++) + { + rstart[vd] = start ? start[vd] : 0; + rcount[vd] = count ? count[vd] : dimlen[vd]; + rstride[vd] = stride ? stride[vd] : 1; + } + + /* How many elements in buf? */ + size_t num_elem = 1; + for (int vd = 0; vd < ndims; vd++) + num_elem *= (rcount[vd] - rstart[vd])/rstride[vd]; + LOG((2, "PIOc_put_vars_tc num_elem = %d", num_elem)); + } + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_PUT_VARS; + char start_present = start ? true : false; + char count_present = count ? true : false; + char stride_present = stride ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && start_present) + mpierr = MPI_Bcast(&start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && count_present) + mpierr = MPI_Bcast(&count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && stride_present) + mpierr = MPI_Bcast(&stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); + + /* /\* Broadcast values currently only known on computation tasks to IO tasks. *\/ */ + /* if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm))) */ + /* check_mpi(file, mpierr, __FILE__, __LINE__); */ + /* if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) */ + /* check_mpi(file, mpierr, __FILE__, __LINE__); */ + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { +#ifdef _PNETCDF + if (file->iotype == PIO_IOTYPE_PNETCDF) + { + vdesc = file->varlist + varid; + if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0) + vdesc->request = realloc(vdesc->request, + sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)); + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank == 0) + switch(xtype) + { + case NC_BYTE: + ierr = ncmpi_bput_vars_schar(ncid, varid, start, count, stride, buf, request); + break; + case NC_CHAR: + ierr = ncmpi_bput_vars_text(ncid, varid, start, count, stride, buf, request); + break; + case NC_SHORT: + ierr = ncmpi_bput_vars_short(ncid, varid, start, count, stride, buf, request); + break; + case NC_INT: + ierr = ncmpi_bput_vars_int(ncid, varid, start, count, stride, buf, request); + break; + case NC_FLOAT: + ierr = ncmpi_bput_vars_float(ncid, varid, start, count, stride, buf, request); + break; + case NC_DOUBLE: + ierr = ncmpi_bput_vars_double(ncid, varid, start, count, stride, buf, request); + break; + case NC_INT64: + ierr = ncmpi_bput_vars_longlong(ncid, varid, start, count, stride, buf, request); + break; + default: + LOG((0, "Unknown type for pnetcdf file! xtype = %d", xtype)); + + } + else + *request = PIO_REQ_NULL; + + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + } +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + switch(xtype) + { + case NC_BYTE: + ierr = nc_put_vars_schar(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_CHAR: + ierr = nc_put_vars_schar(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_SHORT: + ierr = nc_put_vars_short(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_INT: + ierr = nc_put_vars_int(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_FLOAT: + ierr = nc_put_vars_float(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_DOUBLE: + ierr = nc_put_vars_double(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; +#ifdef _NETCDF4 + case NC_UBYTE: + ierr = nc_put_vars_uchar(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_USHORT: + ierr = nc_put_vars_ushort(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_UINT: + ierr = nc_put_vars_uint(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_INT64: + ierr = nc_put_vars_longlong(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_UINT64: + ierr = nc_put_vars_ulonglong(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + /* case NC_STRING: */ + /* ierr = nc_put_vars_string(ncid, varid, (size_t *)start, (size_t *)count, */ + /* (ptrdiff_t *)stride, (void *)buf); */ + /* break; */ + default: + ierr = nc_put_vars(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); +#endif /* _NETCDF4 */ + } +#endif /* _NETCDF */ + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + return ierr; +} + +/** + * PIO interface to nc_put_vars + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + + * Refer to the + * netcdf documentation. */ +int PIOc_put_vars(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], + const PIO_Offset stride[], const void *buf, PIO_Offset bufcount, + MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset start[], + const PIO_Offset count[], const PIO_Offset stride[], + const unsigned char *op) { int ierr; int msg; @@ -4771,83 +5114,6 @@ int PIOc_put_vara_double (int ncid, int varid, const PIO_Offset start[], const P return ierr; } -/// -/// PIO interface to nc_put_vars -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} /// /// PIO interface to nc_put_var_uchar From 58b9466f8c56fb5c67823a4999eed5c10cda4609 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 18 May 2016 13:17:21 -0400 Subject: [PATCH 087/184] put_vars with async --- src/clib/pio_msg.c | 154 +++++++++++++++++++++++++++++++++--- src/clib/pio_put_nc_async.c | 13 ++- 2 files changed, 157 insertions(+), 10 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index be2b75ea8ce..ce26b78154e 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -579,6 +579,145 @@ int att_get_handler(iosystem_desc_t *ios) return PIO_NOERR; } +/** Handle var put operations. This code only runs on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. +*/ +int put_vars_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int mpierr; + int ierr; + char *name; + int namelen; + PIO_Offset typelen; /** Length (in bytes) of this type. */ + nc_type xtype; /** Type of the data being written. */ + char start_present, count_present, stride_present; + PIO_Offset *startp = NULL, *countp = NULL, *stridep = NULL; + int ndims; /** Number of dimensions. */ + void *buf; /** Buffer for data storage. */ + size_t num_elem; /** Number of data elements in the buffer. */ + + LOG((1, "put_vars_handler")); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + + /* Now we know how big to make these arrays. */ + PIO_Offset start[ndims], count[ndims], stride[ndims]; + + if ((mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if (!mpierr && start_present) + if ((mpierr = MPI_Bcast(start, ndims, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if (!mpierr && count_present) + if ((mpierr = MPI_Bcast(count, ndims, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if (!mpierr && stride_present) + if ((mpierr = MPI_Bcast(stride, ndims, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm))) + return PIO_EIO; + + /* Allocate room for our data. */ + if (!(buf = malloc(num_elem * typelen))) + return PIO_ENOMEM; + + /* Get the data. */ + if ((mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, 0, ios->intercomm))) + return PIO_EIO; + + LOG((1, "att_vars_handler ncid = %d varid = %d ndims = %d start_present = %d " + "count_present = %d stride_present = %d xtype = %d num_elem = %d typelen = %d", + ncid, varid, ndims, start_present, count_present, stride_present, xtype, + num_elem, typelen)); + + /* Set the non-NULL pointers. */ + if (start_present) + startp = start; + if (count_present) + countp = count; + if (stride_present) + stridep = stride; + + /* Call the function to write the data. */ + switch(xtype) + { + case NC_BYTE: + ierr = PIOc_put_vars_schar(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_CHAR: + ierr = PIOc_put_vars_schar(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_SHORT: + ierr = PIOc_put_vars_short(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_INT: + ierr = PIOc_put_vars_int(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_FLOAT: + ierr = PIOc_put_vars_float(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_DOUBLE: + ierr = PIOc_put_vars_double(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; +#ifdef _NETCDF4 + case NC_UBYTE: + ierr = PIOc_put_vars_uchar(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_USHORT: + ierr = PIOc_put_vars_ushort(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_UINT: + ierr = PIOc_put_vars_uint(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_INT64: + ierr = PIOc_put_vars_longlong(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_UINT64: + ierr = PIOc_put_vars_ulonglong(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + /* case NC_STRING: */ + /* ierr = PIOc_put_vars_string(ncid, varid, (size_t *)start, (size_t *)count, */ + /* (ptrdiff_t *)stride, (void *)buf); */ + /* break; */ + /* default:*/ + /* ierr = PIOc_put_vars(ncid, varid, (size_t *)start, (size_t *)count, */ + /* (ptrdiff_t *)stride, buf); */ +#endif /* _NETCDF4 */ + } + + return PIO_NOERR; +} + /** Do an inq_var on a netCDF variable. This function is only run on * IO tasks. * @@ -1338,6 +1477,12 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) case PIO_MSG_INQ_ATTID: inq_attid_handler(my_iosys); break; + case PIO_MSG_GET_VARS: + var_handler(my_iosys, msg); + break; + case PIO_MSG_PUT_VARS: + put_vars_handler(my_iosys); + break; case PIO_MSG_INITDECOMP_DOF: initdecomp_dof_handler(my_iosys); break; @@ -1350,15 +1495,6 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) case PIO_MSG_SETERRORHANDLING: seterrorhandling_handler(my_iosys); break; - case PIO_MSG_GET_VAR1: - var_handler(my_iosys, msg); - break; - case PIO_MSG_PUT_VAR1: - var_handler(my_iosys, msg); - break; - case PIO_MSG_PUT_VARA: - vara_handler(my_iosys, msg); - break; case PIO_MSG_FREEDECOMP: freedecomp_handler(my_iosys); break; diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 74a52328eeb..e5ae038a4c3 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -41,6 +41,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off int ndims; /** The number of dimensions in the variable. */ int *dimids; /** The IDs of the dimensions for this variable. */ PIO_Offset typelen; /** Size (in bytes) of the data type of data in buf. */ + size_t num_elem = 1; /** Number of data elements in the buffer. */ var_desc_t *vdesc; PIO_Offset usage; int *request; @@ -99,7 +100,6 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off } /* How many elements in buf? */ - size_t num_elem = 1; for (int vd = 0; vd < ndims; vd++) num_elem *= (rcount[vd] - rstart[vd])/rstride[vd]; LOG((2, "PIOc_put_vars_tc num_elem = %d", num_elem)); @@ -122,6 +122,8 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr && start_present) @@ -136,6 +138,15 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off mpierr = MPI_Bcast(&stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + + /* Send the data. */ + if (!mpierr) + mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, ios->compmaster, + ios->intercomm); } /* Handle MPI errors. */ From fdd7d8b966562352e88e775958d9175217516358 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 23 May 2016 16:45:30 -0400 Subject: [PATCH 088/184] more development of async code --- src/clib/pio_msg.c | 18 ++++++++++-------- src/clib/pio_put_nc_async.c | 7 ++++++- tests/unit/test_intercomm.c | 6 ++++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index ce26b78154e..126efaace43 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -617,17 +617,17 @@ int put_vars_handler(iosystem_desc_t *ios) if ((mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; if (!mpierr && start_present) - if ((mpierr = MPI_Bcast(start, ndims, MPI_CHAR, 0, ios->intercomm))) + if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm))) return PIO_EIO; if ((mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; if (!mpierr && count_present) - if ((mpierr = MPI_Bcast(count, ndims, MPI_CHAR, 0, ios->intercomm))) + if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm))) return PIO_EIO; if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; if (!mpierr && stride_present) - if ((mpierr = MPI_Bcast(stride, ndims, MPI_CHAR, 0, ios->intercomm))) + if ((mpierr = MPI_Bcast(stride, ndims, MPI_OFFSET, 0, ios->intercomm))) return PIO_EIO; if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; @@ -635,6 +635,10 @@ int put_vars_handler(iosystem_desc_t *ios) return PIO_EIO; if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm))) return PIO_EIO; + LOG((1, "put_vars_handler ncid = %d varid = %d ndims = %d start_present = %d " + "count_present = %d stride_present = %d xtype = %d num_elem = %d typelen = %d", + ncid, varid, ndims, start_present, count_present, stride_present, xtype, + num_elem, typelen)); /* Allocate room for our data. */ if (!(buf = malloc(num_elem * typelen))) @@ -643,12 +647,10 @@ int put_vars_handler(iosystem_desc_t *ios) /* Get the data. */ if ((mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, 0, ios->intercomm))) return PIO_EIO; - - LOG((1, "att_vars_handler ncid = %d varid = %d ndims = %d start_present = %d " - "count_present = %d stride_present = %d xtype = %d num_elem = %d typelen = %d", - ncid, varid, ndims, start_present, count_present, stride_present, xtype, - num_elem, typelen)); + for (int e = 0; e < num_elem; e++) + LOG((2, "element %d = %d", e, ((int *)buf)[e])); + /* Set the non-NULL pointers. */ if (start_present) startp = start; diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index e5ae038a4c3..9938ba09548 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -104,7 +104,8 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off num_elem *= (rcount[vd] - rstart[vd])/rstride[vd]; LOG((2, "PIOc_put_vars_tc num_elem = %d", num_elem)); } - + + sleep(2); /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { @@ -142,6 +143,9 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_put_vars_tc ncid = %d varid = %d ndims = %d start_present = %d " + "count_present = %d stride_present = %d xtype = %d num_elem = %d", ncid, varid, + ndims, start_present, count_present, stride_present, xtype, num_elem)); /* Send the data. */ if (!mpierr) @@ -149,6 +153,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off ios->intercomm); } + sleep(2); /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr2, __FILE__, __LINE__); diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 172c3ca760b..ca8f7aa7a3a 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -495,13 +495,15 @@ main(int argc, char **argv) if ((ret = PIOc_enddef(ncid))) ERR(ret); - /* Write some data. */ + /* /\* Write some data. *\/ */ /* for (int i = 0; i < LOCAL_DIM_LEN; i++) */ /* data[i] = my_rank; */ /* if (verbose) */ /* printf("%d test_intercomm writing data\n", my_rank); */ /* start[0] = !my_rank ? 0 : 2; */ - /* if ((ret = PIOc_put_vara_int(ncid, varid, start, count, data))) */ + /* count[0] = 2; */ + /* sleep(2); */ + /* if ((ret = PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_INT, data))) */ /* ERR(ret); */ /* Close the file. */ From d6bfc031bd36ccd98f9a1c8ecfad6eab9b542484 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 23 May 2016 18:10:33 -0400 Subject: [PATCH 089/184] more development of async changes --- src/clib/CMakeLists.txt | 2 +- src/clib/pio_msg.c | 65 +- src/clib/pio_put_nc_async.c | 2514 +++++------------------------------ tests/unit/test_intercomm.c | 30 +- 4 files changed, 383 insertions(+), 2228 deletions(-) diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt index 475aa6196a4..4344261aa28 100644 --- a/src/clib/CMakeLists.txt +++ b/src/clib/CMakeLists.txt @@ -23,7 +23,7 @@ set (PIO_GENNC_SRCS ${CMAKE_CURRENT_BINARY_DIR}/pio_put_nc.c ${CMAKE_CURRENT_BINARY_DIR}/pio_nc.c) if (PIO_ENABLE_ASYNC) - set (PIO_ADDL_SRCS pio_nc_async.c pio_put_nc_async.c pio_get_nc_async.c pio_msg.c) + set (PIO_ADDL_SRCS pio_nc_async.c pio_put_nc_async.c pio_get_nc_async.c pio_msg.c pio_varm.c) else () set (PIO_ADDL_SRCS ${PIO_GENNC_SRCS}) endif () diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 126efaace43..9c509398ca2 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -617,8 +617,11 @@ int put_vars_handler(iosystem_desc_t *ios) if ((mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; if (!mpierr && start_present) + { if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm))) return PIO_EIO; + LOG((1, "put_vars_handler getting start[0] = %d ndims = %d", start[0], ndims)); + } if ((mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; if (!mpierr && count_present) @@ -640,6 +643,16 @@ int put_vars_handler(iosystem_desc_t *ios) ncid, varid, ndims, start_present, count_present, stride_present, xtype, num_elem, typelen)); + for (int d = 0; d < ndims; d++) + { + if (start_present) + LOG((2, "start[%d] = %d\n", d, start[d])); + if (count_present) + LOG((2, "count[%d] = %d\n", d, count[d])); + if (stride_present) + LOG((2, "stride[%d] = %d\n", d, stride[d])); + } + /* Allocate room for our data. */ if (!(buf = malloc(num_elem * typelen))) return PIO_ENOMEM; @@ -663,57 +676,57 @@ int put_vars_handler(iosystem_desc_t *ios) switch(xtype) { case NC_BYTE: - ierr = PIOc_put_vars_schar(ncid, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf); + ierr = PIOc_put_vars_schar(ncid, varid, (size_t *)startp, (size_t *)countp, + (ptrdiff_t *)stridep, buf); break; case NC_CHAR: - ierr = PIOc_put_vars_schar(ncid, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf); + ierr = PIOc_put_vars_schar(ncid, varid, (size_t *)startp, (size_t *)countp, + (ptrdiff_t *)stridep, buf); break; case NC_SHORT: - ierr = PIOc_put_vars_short(ncid, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf); + ierr = PIOc_put_vars_short(ncid, varid, (size_t *)startp, (size_t *)countp, + (ptrdiff_t *)stridep, buf); break; case NC_INT: - ierr = PIOc_put_vars_int(ncid, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf); + ierr = PIOc_put_vars_int(ncid, varid, (size_t *)startp, (size_t *)countp, + (ptrdiff_t *)stridep, buf); break; case NC_FLOAT: - ierr = PIOc_put_vars_float(ncid, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf); + ierr = PIOc_put_vars_float(ncid, varid, (size_t *)startp, (size_t *)countp, + (ptrdiff_t *)stridep, buf); break; case NC_DOUBLE: - ierr = PIOc_put_vars_double(ncid, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf); + ierr = PIOc_put_vars_double(ncid, varid, (size_t *)startp, (size_t *)countp, + (ptrdiff_t *)stridep, buf); break; #ifdef _NETCDF4 case NC_UBYTE: - ierr = PIOc_put_vars_uchar(ncid, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf); + ierr = PIOc_put_vars_uchar(ncid, varid, (size_t *)startp, (size_t *)countp, + (ptrdiff_t *)stridep, buf); break; case NC_USHORT: - ierr = PIOc_put_vars_ushort(ncid, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf); + ierr = PIOc_put_vars_ushort(ncid, varid, (size_t *)startp, (size_t *)countp, + (ptrdiff_t *)stridep, buf); break; case NC_UINT: - ierr = PIOc_put_vars_uint(ncid, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf); + ierr = PIOc_put_vars_uint(ncid, varid, (size_t *)startp, (size_t *)countp, + (ptrdiff_t *)stridep, buf); break; case NC_INT64: - ierr = PIOc_put_vars_longlong(ncid, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf); + ierr = PIOc_put_vars_longlong(ncid, varid, (size_t *)startp, (size_t *)countp, + (ptrdiff_t *)stridep, buf); break; case NC_UINT64: - ierr = PIOc_put_vars_ulonglong(ncid, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf); + ierr = PIOc_put_vars_ulonglong(ncid, varid, (size_t *)startp, (size_t *)countp, + (ptrdiff_t *)stridep, buf); break; /* case NC_STRING: */ - /* ierr = PIOc_put_vars_string(ncid, varid, (size_t *)start, (size_t *)count, */ - /* (ptrdiff_t *)stride, (void *)buf); */ + /* ierr = PIOc_put_vars_string(ncid, varid, (size_t *)startp, (size_t *)countp, */ + /* (ptrdiff_t *)stridep, (void *)buf); */ /* break; */ /* default:*/ - /* ierr = PIOc_put_vars(ncid, varid, (size_t *)start, (size_t *)count, */ - /* (ptrdiff_t *)stride, buf); */ + /* ierr = PIOc_put_vars(ncid, varid, (size_t *)startp, (size_t *)countp, */ + /* (ptrdiff_t *)stridep, buf); */ #endif /* _NETCDF4 */ } diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 9938ba09548..95d31c826ce 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -1,3 +1,4 @@ +#include #include #include @@ -47,7 +48,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off int *request; LOG((1, "PIOc_put_vars_tc ncid = %d varid = %d start = %d count = %d " - "stride = %d xtype = %d", ncid, start, count, stride, xtype)); + "stride = %d xtype = %d", ncid, varid, start, count, stride, xtype)); /* User must provide some data. */ if (!buf) @@ -105,7 +106,6 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off LOG((2, "PIOc_put_vars_tc num_elem = %d", num_elem)); } - sleep(2); /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { @@ -119,6 +119,8 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if(ios->compmaster) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + /* Send the function parameters and associated informaiton + * to the msg handler. */ if (!mpierr) mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -128,15 +130,15 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if (!mpierr) mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr && start_present) - mpierr = MPI_Bcast(&start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr && count_present) - mpierr = MPI_Bcast(&count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr && stride_present) - mpierr = MPI_Bcast(&stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -146,14 +148,16 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off LOG((2, "PIOc_put_vars_tc ncid = %d varid = %d ndims = %d start_present = %d " "count_present = %d stride_present = %d xtype = %d num_elem = %d", ncid, varid, ndims, start_present, count_present, stride_present, xtype, num_elem)); + + for (int e = 0; e < num_elem; e++) + LOG((2, "PIOc_put_vars_tc element %d = %d", e, ((int *)buf)[e])); /* Send the data. */ if (!mpierr) - mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, ios->compmaster, + mpierr = MPI_Bcast((void *)buf, num_elem * typelen, MPI_BYTE, ios->compmaster, ios->intercomm); } - sleep(2); /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr2, __FILE__, __LINE__); @@ -365,9 +369,48 @@ int PIOc_put_vars(int ncid, int varid, const PIO_Offset start[], const PIO_Offse return ierr; } +/** Interface to netCDF data write function. */ int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned char *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UBYTE, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], + const PIO_Offset stride[], const unsigned short *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_USHORT, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], + const PIO_Offset stride[], const unsigned long long *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT64, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned int *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op) +{ + return PIOc_put_vars_tc(ncid, varid, NULL, NULL, NULL, NC_USHORT, op); +} + +/// +/// PIO interface to nc_put_var1_longlong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) { int ierr; int msg; @@ -384,7 +427,7 @@ int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset start[], if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_UCHAR; + msg = PIO_MSG_PUT_VAR1_LONGLONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -399,13 +442,13 @@ int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset start[], #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; } break; #endif @@ -420,7 +463,7 @@ int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset start[], request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_uchar(file->fh, varid, start, count, stride, op, request);; + ierr = ncmpi_bput_var1_longlong(file->fh, varid, index, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -438,14 +481,16 @@ int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset start[], return ierr; } -/// -/// PIO interface to nc_put_vars_ushort -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned short *op) +/** Interface to netCDF data write function. */ +int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset start[], + const PIO_Offset count[], const unsigned char *op) +{ + return PIOc_put_vars_uchar(ncid, varid, start, count, NULL, op); +} + + +/** Interface to netCDF data write function. */ +int PIOc_put_var1_long(int ncid, int varid, const PIO_Offset index[], const long *ip) { int ierr; int msg; @@ -462,7 +507,7 @@ int PIOc_put_vars_ushort (int ncid, int varid, const PIO_Offset start[], const P if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_USHORT; + msg = PIO_MSG_PUT_VAR1_LONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -477,13 +522,13 @@ int PIOc_put_vars_ushort (int ncid, int varid, const PIO_Offset start[], const P #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; } break; #endif @@ -498,7 +543,7 @@ int PIOc_put_vars_ushort (int ncid, int varid, const PIO_Offset start[], const P request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_ushort(file->fh, varid, start, count, stride, op, request);; + ierr = ncmpi_bput_var1_long(file->fh, varid, index, ip, request);; }else{ *request = PIO_REQ_NULL; } @@ -516,14 +561,33 @@ int PIOc_put_vars_ushort (int ncid, int varid, const PIO_Offset start[], const P return ierr; } +/** Interface to netCDF data write function. */ +int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_var_short(int ncid, int varid, const short *op) +{ + return PIOc_put_vars_short(ncid, varid, NULL, NULL, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], + const int *op) +{ + return PIOc_put_vars_int(ncid, varid, start, count, NULL, op); +} + /// -/// PIO interface to nc_put_vars_ulonglong +/// PIO interface to nc_put_var1_ushort /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned long long *op) +int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op) { int ierr; int msg; @@ -540,12 +604,12 @@ int PIOc_put_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], cons if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_ULONGLONG; + msg = PIO_MSG_PUT_VAR1_USHORT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -555,13 +619,13 @@ int PIOc_put_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], cons #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; } break; #endif @@ -576,7 +640,7 @@ int PIOc_put_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], cons request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_ulonglong(file->fh, varid, start, count, stride, op, request);; + ierr = ncmpi_bput_var1_ushort(file->fh, varid, index, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -595,13 +659,37 @@ int PIOc_put_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], cons } /// -/// PIO interface to nc_put_varm +/// PIO interface to nc_put_vara_text +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op) +{ + return PIOc_put_vars_text(ncid, varid, start, count, NULL, op); +} + +/// +/// PIO interface to nc_put_var_ulonglong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) +{ + return PIOc_put_vars_ulonglong(ncid, varid, NULL, NULL, NULL, op); +} + +/// +/// PIO interface to nc_put_var_int /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +int PIOc_put_var_int (int ncid, int varid, const int *op) { int ierr; int msg; @@ -618,12 +706,12 @@ int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offs if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARM; + msg = PIO_MSG_PUT_VAR_INT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -633,13 +721,13 @@ int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offs #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + ierr = nc_put_var_int(file->fh, varid, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + ierr = nc_put_var_int(file->fh, varid, op);; } break; #endif @@ -654,7 +742,7 @@ int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offs request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype, request);; + ierr = ncmpi_bput_var_int(file->fh, varid, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -673,13 +761,13 @@ int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offs } /// -/// PIO interface to nc_put_vars_uint +/// PIO interface to nc_put_var_longlong /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned int *op) +int PIOc_put_var_longlong (int ncid, int varid, const long long *op) { int ierr; int msg; @@ -696,12 +784,12 @@ int PIOc_put_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_UINT; + msg = PIO_MSG_PUT_VAR_LONGLONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -711,13 +799,13 @@ int PIOc_put_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_var_longlong(file->fh, varid, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_var_longlong(file->fh, varid, op);; } break; #endif @@ -732,7 +820,7 @@ int PIOc_put_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_uint(file->fh, varid, start, count, stride, op, request);; + ierr = ncmpi_bput_var_longlong(file->fh, varid, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -751,13 +839,13 @@ int PIOc_put_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO } /// -/// PIO interface to nc_put_varm_uchar +/// PIO interface to nc_put_var_schar /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned char *op) +int PIOc_put_var_schar (int ncid, int varid, const signed char *op) { int ierr; int msg; @@ -774,12 +862,12 @@ int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_UCHAR; + msg = PIO_MSG_PUT_VAR_SCHAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -789,13 +877,13 @@ int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PI #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + ierr = nc_put_var_schar(file->fh, varid, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + ierr = nc_put_var_schar(file->fh, varid, op);; } break; #endif @@ -810,7 +898,7 @@ int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PI request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_uchar(file->fh, varid, start, count, stride, imap, op, request);; + ierr = ncmpi_bput_var_schar(file->fh, varid, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -829,13 +917,13 @@ int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PI } /// -/// PIO interface to nc_put_var_ushort +/// PIO interface to nc_put_var_uint /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var_ushort (int ncid, int varid, const unsigned short *op) +int PIOc_put_var_uint (int ncid, int varid, const unsigned int *op) { int ierr; int msg; @@ -852,12 +940,12 @@ int PIOc_put_var_ushort (int ncid, int varid, const unsigned short *op) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_USHORT; + msg = PIO_MSG_PUT_VAR_UINT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -867,13 +955,13 @@ int PIOc_put_var_ushort (int ncid, int varid, const unsigned short *op) #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_ushort(file->fh, varid, op);; + ierr = nc_put_var_uint(file->fh, varid, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var_ushort(file->fh, varid, op);; + ierr = nc_put_var_uint(file->fh, varid, op);; } break; #endif @@ -888,7 +976,7 @@ int PIOc_put_var_ushort (int ncid, int varid, const unsigned short *op) request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var_ushort(file->fh, varid, op, request);; + ierr = ncmpi_bput_var_uint(file->fh, varid, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -907,13 +995,13 @@ int PIOc_put_var_ushort (int ncid, int varid, const unsigned short *op) } /// -/// PIO interface to nc_put_var1_longlong +/// PIO interface to nc_put_var /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) +int PIOc_put_var (int ncid, int varid, const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -930,12 +1018,12 @@ int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_LONGLONG; + msg = PIO_MSG_PUT_VAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -945,13 +1033,13 @@ int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var(file->fh, varid, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var(file->fh, varid, buf);; } break; #endif @@ -966,7 +1054,7 @@ int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_longlong(file->fh, varid, index, op, request);; + ierr = ncmpi_bput_var(file->fh, varid, buf, bufcount, buftype, request);; }else{ *request = PIO_REQ_NULL; } @@ -985,13 +1073,13 @@ int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const } /// -/// PIO interface to nc_put_vara_uchar +/// PIO interface to nc_put_vara_ushort /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned char *op) +int PIOc_put_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned short *op) { int ierr; int msg; @@ -1008,12 +1096,12 @@ int PIOc_put_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_UCHAR; + msg = PIO_MSG_PUT_VARA_USHORT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1023,13 +1111,13 @@ int PIOc_put_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PI #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, op);; + ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, op);; + ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);; } break; #endif @@ -1044,7 +1132,7 @@ int PIOc_put_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PI request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_uchar(file->fh, varid, start, count, op, request);; + ierr = ncmpi_bput_vara_ushort(file->fh, varid, start, count, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -1063,13 +1151,13 @@ int PIOc_put_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PI } /// -/// PIO interface to nc_put_varm_short +/// PIO interface to nc_put_vars_short /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const short *op) +int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const short *op) { int ierr; int msg; @@ -1086,12 +1174,12 @@ int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_SHORT; + msg = PIO_MSG_PUT_VARS_SHORT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1101,13 +1189,13 @@ int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PI #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; } break; #endif @@ -1122,7 +1210,7 @@ int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PI request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_short(file->fh, varid, start, count, stride, imap, op, request);; + ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, stride, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -1141,13 +1229,13 @@ int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PI } /// -/// PIO interface to nc_put_var1_long +/// PIO interface to nc_put_vara_uint /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var1_long (int ncid, int varid, const PIO_Offset index[], const long *ip) +int PIOc_put_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned int *op) { int ierr; int msg; @@ -1164,12 +1252,12 @@ int PIOc_put_var1_long (int ncid, int varid, const PIO_Offset index[], const lon if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_LONG; + msg = PIO_MSG_PUT_VARA_UINT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1179,13 +1267,13 @@ int PIOc_put_var1_long (int ncid, int varid, const PIO_Offset index[], const lon #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; + ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; + ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);; } break; #endif @@ -1200,7 +1288,7 @@ int PIOc_put_var1_long (int ncid, int varid, const PIO_Offset index[], const lon request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_long(file->fh, varid, index, ip, request);; + ierr = ncmpi_bput_vara_uint(file->fh, varid, start, count, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -1219,13 +1307,13 @@ int PIOc_put_var1_long (int ncid, int varid, const PIO_Offset index[], const lon } /// -/// PIO interface to nc_put_vars_long +/// PIO interface to nc_put_vara_schar /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long *op) +int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const signed char *op) { int ierr; int msg; @@ -1242,12 +1330,12 @@ int PIOc_put_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_LONG; + msg = PIO_MSG_PUT_VARA_SCHAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1257,13 +1345,13 @@ int PIOc_put_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);; } break; #endif @@ -1278,7 +1366,7 @@ int PIOc_put_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, stride, op, request);; + ierr = ncmpi_bput_vara_schar(file->fh, varid, start, count, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -1296,14 +1384,15 @@ int PIOc_put_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO return ierr; } + /// -/// PIO interface to nc_put_var_short +/// PIO interface to nc_put_var1_uchar /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var_short (int ncid, int varid, const short *op) +int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], const unsigned char *op) { int ierr; int msg; @@ -1320,12 +1409,12 @@ int PIOc_put_var_short (int ncid, int varid, const short *op) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_SHORT; + msg = PIO_MSG_PUT_VAR1_UCHAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1335,13 +1424,13 @@ int PIOc_put_var_short (int ncid, int varid, const short *op) #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_short(file->fh, varid, op);; + ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var_short(file->fh, varid, op);; + ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; } break; #endif @@ -1356,7 +1445,7 @@ int PIOc_put_var_short (int ncid, int varid, const short *op) request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var_short(file->fh, varid, op, request);; + ierr = ncmpi_bput_var1_uchar(file->fh, varid, index, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -1374,15 +1463,15 @@ int PIOc_put_var_short (int ncid, int varid, const short *op) return ierr; } + /// -/// PIO interface to nc_put_vara_int +/// PIO interface to nc_put_vars_schar /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], - const int *op) +int PIOc_put_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const signed char *op) { int ierr; int msg; @@ -1392,9 +1481,6 @@ int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset start[], const PIO_O var_desc_t *vdesc; PIO_Offset usage; int *request; - int size; - int ret; - int ndims; ierr = PIO_NOERR; @@ -1402,20 +1488,12 @@ int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset start[], const PIO_O if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_INT; + msg = PIO_MSG_PUT_VARS_SCHAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - if ((ret = PIOc_inq_varndims(ncid, varid, &ndims))) - return ret; - mpierr = MPI_Bcast((void *)start, ndims, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)count, ndims, MPI_INT, ios->compmaster, ios->intercomm); - for (int d = 0, size = 1; d < ndims; d++) - size *= count[d]; - mpierr = MPI_Bcast((void *)op, size, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1425,13 +1503,13 @@ int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset start[], const PIO_O #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, op);; + ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, op);; + ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; } break; #endif @@ -1446,7 +1524,7 @@ int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset start[], const PIO_O request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_int(file->fh, varid, start, count, op, request);; + ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, stride, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -1465,13 +1543,13 @@ int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset start[], const PIO_O } /// -/// PIO interface to nc_put_var1_ushort +/// PIO interface to nc_put_var1 /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op) +int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -1488,7 +1566,7 @@ int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const u if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_USHORT; + msg = PIO_MSG_PUT_VAR1; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1503,13 +1581,13 @@ int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const u #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -1524,7 +1602,7 @@ int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const u request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_ushort(file->fh, varid, index, op, request);; + ierr = ncmpi_bput_var1(file->fh, varid, index, buf, bufcount, buftype, request);; }else{ *request = PIO_REQ_NULL; } @@ -1543,13 +1621,13 @@ int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const u } /// -/// PIO interface to nc_put_vara_text +/// PIO interface to nc_put_vara_float /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op) +int PIOc_put_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const float *op) { int ierr; int msg; @@ -1566,7 +1644,7 @@ int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_TEXT; + msg = PIO_MSG_PUT_VARA_FLOAT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1581,13 +1659,13 @@ int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, op);; + ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, op);; + ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);; } break; #endif @@ -1602,7 +1680,7 @@ int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_text(file->fh, varid, start, count, op, request);; + ierr = ncmpi_bput_vara_float(file->fh, varid, start, count, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -1621,13 +1699,13 @@ int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO } /// -/// PIO interface to nc_put_varm_text +/// PIO interface to nc_put_var1_float /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const char *op) +int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], const float *op) { int ierr; int msg; @@ -1644,7 +1722,7 @@ int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_TEXT; + msg = PIO_MSG_PUT_VAR1_FLOAT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1659,13 +1737,13 @@ int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; } break; #endif @@ -1680,7 +1758,7 @@ int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_text(file->fh, varid, start, count, stride, imap, op, request);; + ierr = ncmpi_bput_var1_float(file->fh, varid, index, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -1698,14 +1776,15 @@ int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO return ierr; } + /// -/// PIO interface to nc_put_varm_ushort +/// PIO interface to nc_put_var1_text /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned short *op) +int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op) { int ierr; int msg; @@ -1722,7 +1801,7 @@ int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const P if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_USHORT; + msg = PIO_MSG_PUT_VAR1_TEXT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1737,13 +1816,13 @@ int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const P #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; } break; #endif @@ -1758,7 +1837,7 @@ int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const P request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_ushort(file->fh, varid, start, count, stride, imap, op, request);; + ierr = ncmpi_bput_var1_text(file->fh, varid, index, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -1777,13 +1856,13 @@ int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const P } /// -/// PIO interface to nc_put_var_ulonglong +/// PIO interface to nc_put_vars_text /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) +int PIOc_put_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const char *op) { int ierr; int msg; @@ -1800,7 +1879,7 @@ int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_ULONGLONG; + msg = PIO_MSG_PUT_VARS_TEXT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1815,13 +1894,13 @@ int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_ulonglong(file->fh, varid, op);; + ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var_ulonglong(file->fh, varid, op);; + ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; } break; #endif @@ -1836,7 +1915,7 @@ int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var_ulonglong(file->fh, varid, op, request);; + ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, stride, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -1854,14 +1933,15 @@ int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) return ierr; } + /// -/// PIO interface to nc_put_var_int +/// PIO interface to nc_put_vars_double /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var_int (int ncid, int varid, const int *op) +int PIOc_put_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const double *op) { int ierr; int msg; @@ -1878,7 +1958,7 @@ int PIOc_put_var_int (int ncid, int varid, const int *op) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_INT; + msg = PIO_MSG_PUT_VARS_DOUBLE; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1893,13 +1973,13 @@ int PIOc_put_var_int (int ncid, int varid, const int *op) #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_int(file->fh, varid, op);; + ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var_int(file->fh, varid, op);; + ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; } break; #endif @@ -1914,7 +1994,7 @@ int PIOc_put_var_int (int ncid, int varid, const int *op) request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var_int(file->fh, varid, op, request);; + ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, stride, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -1933,13 +2013,13 @@ int PIOc_put_var_int (int ncid, int varid, const int *op) } /// -/// PIO interface to nc_put_var_longlong +/// PIO interface to nc_put_vara_longlong /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var_longlong (int ncid, int varid, const long long *op) +int PIOc_put_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long long *op) { int ierr; int msg; @@ -1956,7 +2036,7 @@ int PIOc_put_var_longlong (int ncid, int varid, const long long *op) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_LONGLONG; + msg = PIO_MSG_PUT_VARA_LONGLONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1971,13 +2051,13 @@ int PIOc_put_var_longlong (int ncid, int varid, const long long *op) #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_longlong(file->fh, varid, op);; + ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var_longlong(file->fh, varid, op);; + ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);; } break; #endif @@ -1992,7 +2072,7 @@ int PIOc_put_var_longlong (int ncid, int varid, const long long *op) request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var_longlong(file->fh, varid, op, request);; + ierr = ncmpi_bput_vara_longlong(file->fh, varid, start, count, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -2011,13 +2091,13 @@ int PIOc_put_var_longlong (int ncid, int varid, const long long *op) } /// -/// PIO interface to nc_put_var_schar +/// PIO interface to nc_put_var_double /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var_schar (int ncid, int varid, const signed char *op) +int PIOc_put_var_double (int ncid, int varid, const double *op) { int ierr; int msg; @@ -2034,7 +2114,7 @@ int PIOc_put_var_schar (int ncid, int varid, const signed char *op) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_SCHAR; + msg = PIO_MSG_PUT_VAR_DOUBLE; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2049,1885 +2129,13 @@ int PIOc_put_var_schar (int ncid, int varid, const signed char *op) #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_schar(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_schar(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_schar(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_uint -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_uint (int ncid, int varid, const unsigned int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_uint(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_uint(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_uint(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var (int ncid, int varid, const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var(file->fh, varid, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_ushort -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_USHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_ushort(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_short -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const short *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_uint -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_uint(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_schar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const signed char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_schar(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_ulonglong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_ulonglong(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_uchar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], const unsigned char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_uchar(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_int -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_int(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_schar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const signed char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1 -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1(file->fh, varid, index, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_float(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_float(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_float(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_text -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_text(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_text -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_long -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_long(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vars_double -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const double *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_longlong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_longlong(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_double -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_double (int ncid, int varid, const double *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_double(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_double(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_double(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_float (int ncid, int varid, const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_float(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_float(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_float(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_ulonglong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_ulonglong(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_varm_uint -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_uint(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_var1_uint -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const unsigned int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var_double(file->fh, varid, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var_double(file->fh, varid, op);; } break; #endif @@ -3942,7 +2150,7 @@ int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const uns request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_uint(file->fh, varid, index, op, request);; + ierr = ncmpi_bput_var_double(file->fh, varid, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -3961,13 +2169,13 @@ int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const uns } /// -/// PIO interface to nc_put_var1_int +/// PIO interface to nc_put_var_float /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int *op) +int PIOc_put_var_float (int ncid, int varid, const float *op) { int ierr; int msg; @@ -3984,7 +2192,7 @@ int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_INT; + msg = PIO_MSG_PUT_VAR_FLOAT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3999,13 +2207,13 @@ int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var_float(file->fh, varid, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var_float(file->fh, varid, op);; } break; #endif @@ -4020,7 +2228,7 @@ int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_int(file->fh, varid, index, op, request);; + ierr = ncmpi_bput_var_float(file->fh, varid, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -4039,13 +2247,13 @@ int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int } /// -/// PIO interface to nc_put_vars_float +/// PIO interface to nc_put_var1_ulonglong /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const float *op) +int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op) { int ierr; int msg; @@ -4062,7 +2270,7 @@ int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_FLOAT; + msg = PIO_MSG_PUT_VAR1_ULONGLONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4077,13 +2285,13 @@ int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PI #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; } break; #endif @@ -4098,7 +2306,7 @@ int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PI request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, stride, op, request);; + ierr = ncmpi_bput_var1_ulonglong(file->fh, varid, index, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -4116,14 +2324,15 @@ int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PI return ierr; } + /// -/// PIO interface to nc_put_vara_short +/// PIO interface to nc_put_var1_uint /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const short *op) +int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const unsigned int *op) { int ierr; int msg; @@ -4140,7 +2349,7 @@ int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_SHORT; + msg = PIO_MSG_PUT_VAR1_UINT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4155,13 +2364,13 @@ int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PI #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);; + ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);; + ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; } break; #endif @@ -4176,7 +2385,7 @@ int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PI request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_short(file->fh, varid, start, count, op, request);; + ierr = ncmpi_bput_var1_uint(file->fh, varid, index, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -4195,13 +2404,13 @@ int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PI } /// -/// PIO interface to nc_put_var1_schar +/// PIO interface to nc_put_var1_int /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const signed char *op) +int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int *op) { int ierr; int msg; @@ -4218,7 +2427,7 @@ int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const si if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_SCHAR; + msg = PIO_MSG_PUT_VAR1_INT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4233,13 +2442,13 @@ int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const si #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; } break; #endif @@ -4254,7 +2463,7 @@ int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const si request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_schar(file->fh, varid, index, op, request);; + ierr = ncmpi_bput_var1_int(file->fh, varid, index, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -4273,13 +2482,13 @@ int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const si } /// -/// PIO interface to nc_put_vara_ulonglong +/// PIO interface to nc_put_vars_float /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned long long *op) +int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const float *op) { int ierr; int msg; @@ -4296,7 +2505,7 @@ int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], cons if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_ULONGLONG; + msg = PIO_MSG_PUT_VARS_FLOAT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4311,13 +2520,13 @@ int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], cons #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);; + ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);; + ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; } break; #endif @@ -4332,7 +2541,7 @@ int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], cons request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_ulonglong(file->fh, varid, start, count, op, request);; + ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, stride, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -4351,13 +2560,13 @@ int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], cons } /// -/// PIO interface to nc_put_varm_double +/// PIO interface to nc_put_vara_short /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const double *op) +int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const short *op) { int ierr; int msg; @@ -4374,7 +2583,7 @@ int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const P if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_DOUBLE; + msg = PIO_MSG_PUT_VARA_SHORT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4389,13 +2598,13 @@ int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const P #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);; } break; #endif @@ -4410,7 +2619,7 @@ int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const P request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_double(file->fh, varid, start, count, stride, imap, op, request);; + ierr = ncmpi_bput_vara_short(file->fh, varid, start, count, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -4429,13 +2638,13 @@ int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const P } /// -/// PIO interface to nc_put_vara +/// PIO interface to nc_put_var1_schar /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const signed char *op) { int ierr; int msg; @@ -4452,7 +2661,7 @@ int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offs if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARA; + msg = PIO_MSG_PUT_VAR1_SCHAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4467,13 +2676,13 @@ int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offs #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; } break; #endif @@ -4488,7 +2697,7 @@ int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offs request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vara(file->fh, varid, start, count, buf, bufcount, buftype, request);; + ierr = ncmpi_bput_var1_schar(file->fh, varid, index, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -4507,13 +2716,13 @@ int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offs } /// -/// PIO interface to nc_put_vara_long +/// PIO interface to nc_put_vara_ulonglong /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long *op) +int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned long long *op) { int ierr; int msg; @@ -4530,7 +2739,7 @@ int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_LONG; + msg = PIO_MSG_PUT_VARA_ULONGLONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4545,13 +2754,13 @@ int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);; + ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);; + ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);; } break; #endif @@ -4566,7 +2775,7 @@ int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_long(file->fh, varid, start, count, op, request);; + ierr = ncmpi_bput_vara_ulonglong(file->fh, varid, start, count, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -4584,14 +2793,15 @@ int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO return ierr; } + /// -/// PIO interface to nc_put_var1_double +/// PIO interface to nc_put_vara /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const double *op) +int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -4608,7 +2818,7 @@ int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const d if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_DOUBLE; + msg = PIO_MSG_PUT_VARA; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4623,13 +2833,13 @@ int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const d #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; } break; #endif @@ -4644,7 +2854,7 @@ int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const d request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_double(file->fh, varid, index, op, request);; + ierr = ncmpi_bput_vara(file->fh, varid, start, count, buf, bufcount, buftype, request);; }else{ *request = PIO_REQ_NULL; } @@ -4663,13 +2873,13 @@ int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const d } /// -/// PIO interface to nc_put_varm_schar +/// PIO interface to nc_put_vara_long /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const signed char *op) +int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long *op) { int ierr; int msg; @@ -4686,7 +2896,7 @@ int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_SCHAR; + msg = PIO_MSG_PUT_VARA_LONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4701,13 +2911,13 @@ int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PI #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);; } break; #endif @@ -4722,7 +2932,7 @@ int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PI request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_schar(file->fh, varid, start, count, stride, imap, op, request);; + ierr = ncmpi_bput_vara_long(file->fh, varid, start, count, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -4741,13 +2951,13 @@ int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PI } /// -/// PIO interface to nc_put_var_text +/// PIO interface to nc_put_var1_double /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_var_text (int ncid, int varid, const char *op) +int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const double *op) { int ierr; int msg; @@ -4764,7 +2974,7 @@ int PIOc_put_var_text (int ncid, int varid, const char *op) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_TEXT; + msg = PIO_MSG_PUT_VAR1_DOUBLE; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4779,13 +2989,13 @@ int PIOc_put_var_text (int ncid, int varid, const char *op) #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_text(file->fh, varid, op);; + ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var_text(file->fh, varid, op);; + ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; } break; #endif @@ -4800,7 +3010,7 @@ int PIOc_put_var_text (int ncid, int varid, const char *op) request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var_text(file->fh, varid, op, request);; + ierr = ncmpi_bput_var1_double(file->fh, varid, index, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -4819,13 +3029,13 @@ int PIOc_put_var_text (int ncid, int varid, const char *op) } /// -/// PIO interface to nc_put_vars_int +/// PIO interface to nc_put_var_text /// /// This routine is called collectively by all tasks in the communicator ios.union_comm. /// /// Refer to the netcdf documentation. /// -int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const int *op) +int PIOc_put_var_text (int ncid, int varid, const char *op) { int ierr; int msg; @@ -4842,7 +3052,7 @@ int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_ if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_INT; + msg = PIO_MSG_PUT_VAR_TEXT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4857,13 +3067,13 @@ int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_ #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_var_text(file->fh, varid, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; + ierr = nc_put_var_text(file->fh, varid, op);; } break; #endif @@ -4878,7 +3088,7 @@ int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_ request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, stride, op, request);; + ierr = ncmpi_bput_var_text(file->fh, varid, op, request);; }else{ *request = PIO_REQ_NULL; } @@ -4896,6 +3106,13 @@ int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_ return ierr; } +/** PIO interface to nc_put_vars_int */ +int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], + const PIO_Offset stride[], const int *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); +} + /// /// PIO interface to nc_put_var1_short /// @@ -5287,81 +3504,4 @@ int PIOc_put_var_long (int ncid, int varid, const long *op) return ierr; } -/// -/// PIO interface to nc_put_varm_longlong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARM_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_varm_longlong(file->fh, varid, start, count, stride, imap, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index ca8f7aa7a3a..6cb671b6ec3 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -19,9 +19,6 @@ /** The length of our test data. */ #define DIM_LEN 4 -/** The length of data on each (of 2) computational task. */ -#define LOCAL_DIM_LEN 2 - /** The name of the dimension in the netCDF output file. */ #define FIRST_DIM_NAME "jojo" #define DIM_NAME "dim_test_intercomm" @@ -379,7 +376,7 @@ main(int argc, char **argv) { int ncid, varid, dimid; PIO_Offset start[NDIM], count[NDIM] = {0}; - int data[LOCAL_DIM_LEN]; + int data[DIM_LEN]; /* Create a netCDF file with one dimension and one variable. */ if (verbose) @@ -495,16 +492,21 @@ main(int argc, char **argv) if ((ret = PIOc_enddef(ncid))) ERR(ret); - /* /\* Write some data. *\/ */ - /* for (int i = 0; i < LOCAL_DIM_LEN; i++) */ - /* data[i] = my_rank; */ - /* if (verbose) */ - /* printf("%d test_intercomm writing data\n", my_rank); */ - /* start[0] = !my_rank ? 0 : 2; */ - /* count[0] = 2; */ - /* sleep(2); */ - /* if ((ret = PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_INT, data))) */ - /* ERR(ret); */ + /* Write some data. For the PIOc_put/get functions, all + * data must be on compmaster before the function is + * called. Only compmaster's arguments are passed to the + * async msg handler. All other computation tasks are + * ignored. */ + for (int i = 0; i < DIM_LEN; i++) + data[i] = i; + if (verbose) + printf("%d test_intercomm writing data\n", my_rank); + if (verbose) + printf("%d test_intercomm writing data\n", my_rank); + start[0] = 0; + count[0] = DIM_LEN; + if ((ret = PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_INT, data))) + ERR(ret); /* Close the file. */ if (verbose) From 32e3165ce27d7b0010631313afbb110427197cfc Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 23 May 2016 18:48:02 -0400 Subject: [PATCH 090/184] further development of async --- src/clib/pio_put_nc_async.c | 3996 ++++++++++------------------------- 1 file changed, 1100 insertions(+), 2896 deletions(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 95d31c826ce..3a10a6dd2d5 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -12,15 +12,15 @@ * @param ncid identifies the netCDF file * @param varid the variable ID number - * @param start[] an array of start indicies (must have same number of + * @param *start an array of start indicies (must have same number of * entries as variable has dimensions). If NULL, indices of 0 will be * used. * - * @param count[] an array of counts (must have same number of entries + * @param *count an array of counts (must have same number of entries * as variable has dimensions). If NULL, counts matching the size of * the variable will be used. * - * @param stride[] an array of strides (must have same number of + * @param *stride an array of strides (must have same number of * entries as variable has dimensions). If NULL, strides of 1 will be * used. * @@ -235,7 +235,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off break; case NC_INT: ierr = nc_put_vars_int(ncid, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf); + (ptrdiff_t *)stride, buf); break; case NC_FLOAT: ierr = nc_put_vars_float(ncid, varid, (size_t *)start, (size_t *)count, @@ -266,10 +266,10 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off ierr = nc_put_vars_ulonglong(ncid, varid, (size_t *)start, (size_t *)count, (ptrdiff_t *)stride, buf); break; - /* case NC_STRING: */ - /* ierr = nc_put_vars_string(ncid, varid, (size_t *)start, (size_t *)count, */ - /* (ptrdiff_t *)stride, (void *)buf); */ - /* break; */ + /* case NC_STRING: */ + /* ierr = nc_put_vars_string(ncid, varid, (size_t *)start, (size_t *)count, */ + /* (ptrdiff_t *)stride, (void *)buf); */ + /* break; */ default: ierr = nc_put_vars(ncid, varid, (size_t *)start, (size_t *)count, (ptrdiff_t *)stride, buf); @@ -296,103 +296,106 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off * Refer to the * netcdf documentation. */ -int PIOc_put_vars(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], - const PIO_Offset stride[], const void *buf, PIO_Offset bufcount, +int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARS; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vars(file->fh, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vars(file->fh, varid, start, count, stride, buf, + bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } /** Interface to netCDF data write function. */ -int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset start[], - const PIO_Offset count[], const PIO_Offset stride[], +int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, const unsigned char *op) { return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UBYTE, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], - const PIO_Offset stride[], const unsigned short *op) +int PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const unsigned short *op) { return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_USHORT, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], - const PIO_Offset stride[], const unsigned long long *op) +int PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const unsigned long long *op) { return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT64, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned int *op) +int PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const unsigned int *op) { return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT, op); } @@ -412,78 +415,78 @@ int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op) /// int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_longlong(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_longlong(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } /** Interface to netCDF data write function. */ -int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset start[], - const PIO_Offset count[], const unsigned char *op) +int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const unsigned char *op) { return PIOc_put_vars_uchar(ncid, varid, start, count, NULL, op); } @@ -492,77 +495,78 @@ int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset start[], /** Interface to netCDF data write function. */ int PIOc_put_var1_long(int ncid, int varid, const PIO_Offset index[], const long *ip) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_long(file->fh, varid, index, ip, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_long(file->fh, varid, index, ip, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } /** Interface to netCDF data write function. */ -int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long *op) +int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const long *op) { return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); } @@ -574,2934 +578,1134 @@ int PIOc_put_var_short(int ncid, int varid, const short *op) } /** Interface to netCDF data write function. */ -int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], +int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const int *op) { return PIOc_put_vars_int(ncid, varid, start, count, NULL, op); } -/// -/// PIO interface to nc_put_var1_ushort -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], + const unsigned short *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_USHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_ushort(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_ushort(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } -/// -/// PIO interface to nc_put_vara_text -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op) +/** Interface to netCDF data write function. */ +int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const char *op) { return PIOc_put_vars_text(ncid, varid, start, count, NULL, op); } -/// -/// PIO interface to nc_put_var_ulonglong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// +/** Interface to netCDF data write function. */ int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) { return PIOc_put_vars_ulonglong(ncid, varid, NULL, NULL, NULL, op); } -/// -/// PIO interface to nc_put_var_int -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_int (int ncid, int varid, const int *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var_int(int ncid, int varid, const int *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_int(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_int(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_int(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; + return PIOc_put_vars_int(ncid, varid, NULL, NULL, NULL, op); } -/// -/// PIO interface to nc_put_var_longlong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_longlong (int ncid, int varid, const long long *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var_longlong(int ncid, int varid, const long long *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_longlong(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_longlong(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_longlong(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + return PIOc_put_vars_longlong(ncid, varid, NULL, NULL, NULL, op); +} - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +/** Interface to netCDF data write function. */ +int PIOc_put_var_schar(int ncid, int varid, const signed char *op) +{ + return PIOc_put_vars_schar(ncid, varid, NULL, NULL, NULL, op); +} - return ierr; +/** Interface to netCDF data write function. */ +int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op) +{ + return PIOc_put_vars_uint(ncid, varid, NULL, NULL, NULL, op); } -/// -/// PIO interface to nc_put_var_schar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_schar (int ncid, int varid, const signed char *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var(int ncid, int varid, const void *buf, PIO_Offset bufcount, + MPI_Datatype buftype) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_schar(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_schar(file->fh, varid, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_schar(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var(file->fh, varid, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } -/// -/// PIO interface to nc_put_var_uint -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_uint (int ncid, int varid, const unsigned int *op) +/** Interface to netCDF data write function. */ +int PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const unsigned short *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_uint(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_uint(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_uint(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + return PIOc_put_vars_ushort(ncid, varid, start, count, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, const short *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_SHORT, op); +} - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +/** Interface to netCDF data write function. */ +int PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const unsigned int *op) +{ + return PIOc_put_vars_uint(ncid, varid, start, count, NULL, op); +} - return ierr; +/** Interface to netCDF data write function. */ +int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const signed char *op) +{ + return PIOc_put_vars_schar(ncid, varid, start, count, NULL, op); } -/// -/// PIO interface to nc_put_var -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var (int ncid, int varid, const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) + +/** Interface to netCDF data write function. */ +int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], + const unsigned char *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var(file->fh, varid, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var(file->fh, varid, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_uchar(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } -/// -/// PIO interface to nc_put_vara_ushort -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned short *op) +/** Interface to netCDF data write function. */ +int PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const signed char *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_USHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_ushort(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_BYTE, op); } -/// -/// PIO interface to nc_put_vars_short -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const short *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, + PIO_Offset bufcount, MPI_Datatype buftype) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = PIO_NOERR; - return ierr; -} + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1; -/// -/// PIO interface to nc_put_vara_uint -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_uint(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; -} -/// -/// PIO interface to nc_put_vara_schar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const signed char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_schar(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - - -/// -/// PIO interface to nc_put_var1_uchar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], const unsigned char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_uchar(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; -/// -/// PIO interface to nc_put_vars_schar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const signed char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1(file->fh, varid, index, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } -/// -/// PIO interface to nc_put_var1 -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +/** Interface to netCDF data write function. */ +int PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const float *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1(file->fh, varid, index, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; + return PIOc_put_vars_float(ncid, varid, start, count, NULL, op); } -/// -/// PIO interface to nc_put_vara_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const float *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], + const float *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_float(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = PIO_NOERR; - return ierr; -} + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_FLOAT; -/// -/// PIO interface to nc_put_var1_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_float(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; -} - - -/// -/// PIO interface to nc_put_var1_text -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_text(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; - return ierr; -} + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; -/// -/// PIO interface to nc_put_vars_text -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_float(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } -/// -/// PIO interface to nc_put_vars_double -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const double *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/// -/// PIO interface to nc_put_vara_longlong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long long *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_longlong(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = PIO_NOERR; - return ierr; -} + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_TEXT; -/// -/// PIO interface to nc_put_var_double -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_double (int ncid, int varid, const double *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_double(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_double(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_double(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; -} - -/// -/// PIO interface to nc_put_var_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_float (int ncid, int varid, const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_float(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_float(file->fh, varid, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_float(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; - return ierr; -} + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; -/// -/// PIO interface to nc_put_var1_ulonglong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_ulonglong(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_text(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } - -/// -/// PIO interface to nc_put_var1_uint -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const unsigned int *op) +/** Interface to netCDF data write function. */ +int PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const char *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_uint(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_CHAR, op); } -/// -/// PIO interface to nc_put_var1_int -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int *op) +/** Interface to netCDF data write function. */ +int PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const double *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_int(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_DOUBLE, op); } -/// -/// PIO interface to nc_put_vars_float -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const float *op) +/** Interface to netCDF data write function. */ +int PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const long long *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; + return PIOc_put_vars_longlong(ncid, varid, start, count, NULL, op); } -/// -/// PIO interface to nc_put_vara_short -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const short *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var_double (int ncid, int varid, const double *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_short(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; + return PIOc_put_vars_double(ncid, varid, NULL, NULL, NULL, op); } -/// -/// PIO interface to nc_put_var1_schar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const signed char *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var_float(int ncid, int varid, const float *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_SCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_schar(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; + return PIOc_put_vars_float(ncid, varid, NULL, NULL, NULL, op); } -/// -/// PIO interface to nc_put_vara_ulonglong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned long long *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_ulonglong(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; - return ierr; -} + ierr = PIO_NOERR; + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_ULONGLONG; -/// -/// PIO interface to nc_put_vara -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara(file->fh, varid, start, count, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; -} - -/// -/// PIO interface to nc_put_vara_long -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_long(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_ulonglong(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } -/// -/// PIO interface to nc_put_var1_double -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const double *op) + +/** Interface to netCDF data write function. */ +int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const unsigned int *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_double(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_uint(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } -/// -/// PIO interface to nc_put_var_text -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_text (int ncid, int varid, const char *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_text(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_text(file->fh, varid, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_text(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_int(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } -/** PIO interface to nc_put_vars_int */ -int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], - const PIO_Offset stride[], const int *op) +int PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const float *op) { - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_FLOAT, op); } -/// -/// PIO interface to nc_put_var1_short -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_short (int ncid, int varid, const PIO_Offset index[], const short *op) +int PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const short *op) +{ + return PIOc_put_vars_short(ncid, varid, start, count, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset index[], const signed char *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_SHORT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_short(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_schar(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } -/// -/// PIO interface to nc_put_vars_longlong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long long *op) +/** Interface to netCDF data write function. */ +int PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const unsigned long long *op) +{ + return PIOc_put_vars_ulonglong(ncid, varid, start, count, NULL, op); +} + + +/** Interface to netCDF data write function. */ +int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const void *buf, + PIO_Offset bufcount, MPI_Datatype buftype) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARA; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars_longlong(file->fh, varid, start, count, stride, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_vara(file->fh, varid, start, count, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } -/// -/// PIO interface to nc_put_vara_double -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const double *op) +/** Interface to netCDF data write function. */ +int PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const long *op) +{ + return PIOc_put_vars_long(ncid, varid, start, count, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset index[], const double *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARA_DOUBLE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vara_double(file->fh, varid, start, count, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_double(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} - return ierr; +/** Interface to netCDF data write function. */ +int PIOc_put_var_text(int ncid, int varid, const char *op) +{ + return PIOc_put_vars_text(ncid, varid, NULL, NULL, NULL, op); } +/** PIO interface to nc_put_vars_int */ +int PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const int *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); +} -/// -/// PIO interface to nc_put_var_uchar -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_uchar (int ncid, int varid, const unsigned char *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset index[], const short *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VAR1_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_uchar(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_uchar(file->fh, varid, op);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_uchar(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_var1_short(file->fh, varid, index, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; + return ierr; } -/// -/// PIO interface to nc_put_var_long -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var_long (int ncid, int varid, const long *op) +/** Interface to netCDF data write function. */ +int PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const long long *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR_LONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var_long(file->fh, varid, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var_long(file->fh, varid, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var_long(file->fh, varid, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT64, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const double *op) +{ + return PIOc_put_vars_double(ncid, varid, start, count, NULL, op); +} - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - return ierr; +/** Interface to netCDF data write function. */ +int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op) +{ + return PIOc_put_vars_uchar(ncid, varid, NULL, NULL, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_var_long(int ncid, int varid, const long *op) +{ + return PIOc_put_vars_long(ncid, varid, NULL, NULL, NULL, op); } From 4d335b3732817c9175002eb2a9dadcbf46c93ade Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 23 May 2016 18:49:27 -0400 Subject: [PATCH 091/184] added file for varm functions --- src/clib/pio_varm.c | 1013 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1013 insertions(+) create mode 100644 src/clib/pio_varm.c diff --git a/src/clib/pio_varm.c b/src/clib/pio_varm.c new file mode 100644 index 00000000000..63c90aac518 --- /dev/null +++ b/src/clib/pio_varm.c @@ -0,0 +1,1013 @@ +#include +#include +#include + +/// +/// PIO interface to nc_put_varm +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_uchar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_UCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_uchar(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_short +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_SHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_short(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} +/// +/// PIO interface to nc_put_varm_text +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_TEXT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_text(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_ushort +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned short *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_USHORT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_ushort(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_ulonglong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_ULONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_ulonglong(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} +/// +/// PIO interface to nc_put_varm_int +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_INT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_int(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_float +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const float *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_FLOAT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_float(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} +/// +/// PIO interface to nc_put_varm_long +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_LONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_long(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_uint +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned int *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_UINT; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_uint(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_double +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const double *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_DOUBLE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_double(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} +/// +/// PIO interface to nc_put_varm_schar +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const signed char *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_SCHAR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_schar(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} + +/// +/// PIO interface to nc_put_varm_longlong +/// +/// This routine is called collectively by all tasks in the communicator ios.union_comm. +/// +/// Refer to the netcdf documentation. +/// +int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + var_desc_t *vdesc; + PIO_Offset usage; + int *request; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_PUT_VARM_LONGLONG; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); + ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + vdesc = file->varlist + varid; + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + } + request = vdesc->request+vdesc->nreqs; + + if(ios->io_rank==0){ + ierr = ncmpi_bput_varm_longlong(file->fh, varid, start, count, stride, imap, op, request);; + }else{ + *request = PIO_REQ_NULL; + } + vdesc->nreqs++; + flush_output_buffer(file, false, 0); + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + return ierr; +} From 7119c0bb62b3a318a1c3511cf9c6baa50de337dc Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 23 May 2016 18:52:04 -0400 Subject: [PATCH 092/184] return error for async use with varm --- src/clib/pio_varm.c | 117 ++++++++++++++++++-------------------------- 1 file changed, 48 insertions(+), 69 deletions(-) diff --git a/src/clib/pio_varm.c b/src/clib/pio_varm.c index 63c90aac518..d13b2ac41c1 100644 --- a/src/clib/pio_varm.c +++ b/src/clib/pio_varm.c @@ -28,12 +28,9 @@ int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offs ios = file->iosystem; msg = PIO_MSG_PUT_VARM; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; if(ios->ioproc){ switch(file->iotype){ @@ -106,12 +103,9 @@ int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PI ios = file->iosystem; msg = PIO_MSG_PUT_VARM_UCHAR; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; if(ios->ioproc){ switch(file->iotype){ @@ -184,12 +178,9 @@ int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PI ios = file->iosystem; msg = PIO_MSG_PUT_VARM_SHORT; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; if(ios->ioproc){ switch(file->iotype){ @@ -261,11 +252,10 @@ int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO ios = file->iosystem; msg = PIO_MSG_PUT_VARM_TEXT; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; + if(ios->ioproc){ @@ -339,11 +329,10 @@ int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const P ios = file->iosystem; msg = PIO_MSG_PUT_VARM_USHORT; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; + if(ios->ioproc){ @@ -417,11 +406,10 @@ int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], cons ios = file->iosystem; msg = PIO_MSG_PUT_VARM_ULONGLONG; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; + if(ios->ioproc){ @@ -494,11 +482,10 @@ int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_ ios = file->iosystem; msg = PIO_MSG_PUT_VARM_INT; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; + if(ios->ioproc){ @@ -572,11 +559,10 @@ int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PI ios = file->iosystem; msg = PIO_MSG_PUT_VARM_FLOAT; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; + if(ios->ioproc){ @@ -649,11 +635,10 @@ int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO ios = file->iosystem; msg = PIO_MSG_PUT_VARM_LONG; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; + if(ios->ioproc){ @@ -727,11 +712,10 @@ int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO ios = file->iosystem; msg = PIO_MSG_PUT_VARM_UINT; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; + if(ios->ioproc){ @@ -805,11 +789,10 @@ int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const P ios = file->iosystem; msg = PIO_MSG_PUT_VARM_DOUBLE; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; + if(ios->ioproc){ @@ -882,11 +865,10 @@ int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PI ios = file->iosystem; msg = PIO_MSG_PUT_VARM_SCHAR; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; + if(ios->ioproc){ @@ -960,12 +942,9 @@ int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const ios = file->iosystem; msg = PIO_MSG_PUT_VARM_LONGLONG; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - + /* Sorry, but varm functions are not supported by the async interface. */ + if(ios->async_interface) + return PIO_EINVAL; if(ios->ioproc){ switch(file->iotype){ From 2fe1f56eb847adeaf75b58f8965d1c33de05116a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 24 May 2016 09:15:56 -0400 Subject: [PATCH 093/184] cleaned up put_var1 functions --- src/clib/pio_nc_async.c | 1718 +++++++++++++++++------------------ src/clib/pio_put_nc_async.c | 1376 ++++++++-------------------- 2 files changed, 1218 insertions(+), 1876 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index a8db152c91f..93b59ae2721 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -36,7 +36,7 @@ * PIOc_Set_File_Error_Handling */ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, - int *unlimdimidp) + int *unlimdimidp) { iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ @@ -47,93 +47,93 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_INQ; /** Message for async notification. */ - char ndims_present = ndimsp ? true : false; - char nvars_present = nvarsp ? true : false; - char ngatts_present = ngattsp ? true : false; - char unlimdimid_present = unlimdimidp ? true : false; - - if (ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ; /** Message for async notification. */ + char ndims_present = ndimsp ? true : false; + char nvars_present = nvarsp ? true : false; + char ngatts_present = ngattsp ? true : false; + char unlimdimid_present = unlimdimidp ? true : false; + + if (ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype == PIO_IOTYPE_NETCDF && file->do_io) - { - /* Should not be necessary to do this - nc_inq should - * handle null pointers. This has been reported as a bug - * to netCDF developers. */ - int tmp_ndims, tmp_nvars, tmp_ngatts, tmp_unlimdimid; - ierr = nc_inq(ncid, &tmp_ndims, &tmp_nvars, &tmp_ngatts, &tmp_unlimdimid); - if (ndimsp) - *ndimsp = tmp_ndims; - if (nvarsp) - *nvarsp = tmp_nvars; - if (ngattsp) - *ngattsp = tmp_ngatts; - if (unlimdimidp) - *unlimdimidp = tmp_unlimdimid; - } else if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + if (file->iotype == PIO_IOTYPE_NETCDF && file->do_io) + { + /* Should not be necessary to do this - nc_inq should + * handle null pointers. This has been reported as a bug + * to netCDF developers. */ + int tmp_ndims, tmp_nvars, tmp_ngatts, tmp_unlimdimid; + ierr = nc_inq(ncid, &tmp_ndims, &tmp_nvars, &tmp_ngatts, &tmp_unlimdimid); + if (ndimsp) + *ndimsp = tmp_ndims; + if (nvarsp) + *nvarsp = tmp_nvars; + if (ngattsp) + *ngattsp = tmp_ngatts; + if (unlimdimidp) + *unlimdimidp = tmp_unlimdimid; + } else if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); #endif /* _NETCDF */ - LOG((2, "PIOc_inq netcdf call returned %d", ierr)); + LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ if (!ierr) { - if (ndimsp) - if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - - if (nvarsp) - if ((mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - - if (ngattsp) - if ((mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - - if (unlimdimidp) - if ((mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + if (ndimsp) + if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + + if (nvarsp) + if ((mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + + if (ngattsp) + if ((mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + + if (unlimdimidp) + if ((mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); } return ierr; @@ -178,7 +178,7 @@ int PIOc_inq_unlimdim(int ncid, int *unlimdimidp) /** Internal function to provide inq_type function for pnetcdf. */ int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name, - PIO_Offset *sizep) + PIO_Offset *sizep) { int typelen; char typename[NC_MAX_NAME + 1]; @@ -188,29 +188,29 @@ int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name, case NC_UBYTE: case NC_BYTE: case NC_CHAR: - typelen = 1; - break; + typelen = 1; + break; case NC_SHORT: case NC_USHORT: - typelen = 2; - break; + typelen = 2; + break; case NC_UINT: case NC_INT: case NC_FLOAT: - typelen = 4; - break; + typelen = 4; + break; case NC_UINT64: case NC_INT64: case NC_DOUBLE: - typelen = 8; - break; + typelen = 8; + break; } /* If pointers were supplied, copy results. */ if (sizep) - *sizep = typelen; + *sizep = typelen; if (name) - strcpy(name, "some type"); + strcpy(name, "some type"); return PIO_NOERR; } @@ -231,72 +231,72 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_INQ_TYPE; /** Message for async notification. */ - char name_present = name ? true : false; - char size_present = sizep ? true : false; - - if (ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_TYPE; /** Message for async notification. */ + char name_present = name ? true : false; + char size_present = sizep ? true : false; + + if (ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = pioc_pnetcdf_inq_type(ncid, xtype, name, sizep); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = pioc_pnetcdf_inq_type(ncid, xtype, name, sizep); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_inq_type(ncid, xtype, name, (size_t *)sizep); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_type(ncid, xtype, name, (size_t *)sizep); #endif /* _NETCDF */ - LOG((2, "PIOc_inq_type netcdf call returned %d", ierr)); + LOG((2, "PIOc_inq_type netcdf call returned %d", ierr)); } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ if (!ierr) { - if (name) - { - int slen; - if (ios->iomaster) - slen = strlen(name); - if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - } - if (sizep) - if ((mpierr = MPI_Bcast(sizep , 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + if (name) + { + int slen; + if (ios->iomaster) + slen = strlen(name); + if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + } + if (sizep) + if ((mpierr = MPI_Bcast(sizep , 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); } return ierr; @@ -317,57 +317,57 @@ int PIOc_inq_format (int ncid, int *formatp) /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_INQ_FORMAT; - char format_present = formatp ? true : false; - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_FORMAT; + char format_present = formatp ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&format_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&format_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_inq_format(file->fh, formatp); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq_format(file->fh, formatp); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_inq_format(file->fh, formatp); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_format(file->fh, formatp); #endif /* _NETCDF */ - LOG((2, "PIOc_inq netcdf call returned %d", ierr)); + LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ if (!ierr) { - if (formatp) - if ((mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + if (formatp) + if ((mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); } return ierr; @@ -398,74 +398,74 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) /* Get the file info, based on the ncid. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_INQ_DIM; - char name_present = name ? true : false; - char len_present = lenp ? true : false; - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq netcdf Bcast name_present = %d", name_present)); - if (!mpierr) - mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq netcdf Bcast len_present = %d", len_present)); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_DIM; + char name_present = name ? true : false; + char len_present = lenp ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_inq netcdf Bcast name_present = %d", name_present)); + if (!mpierr) + mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_inq netcdf Bcast len_present = %d", len_present)); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);; + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);; #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ if (!ierr) { - if (name) - { - int slen; - if (ios->iomaster) - slen = strlen(name); - if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - } - - if (lenp) - if ((mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + if (name) + { + int slen; + if (ios->iomaster) + slen = strlen(name); + if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + } + + if (lenp) + if ((mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); } return ierr; @@ -512,66 +512,66 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) /* Name must be provided. */ if (!name) - return PIO_EINVAL; + return PIO_EINVAL; LOG((1, "PIOc_inq_dimid name = %s", name)); /* Get the file info, based on the ncid. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If using async, and not an IO task, then send parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_INQ_DIMID; - char id_present = idp ? true : false; - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - int namelen = strlen(name); - if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_DIMID; + char id_present = idp ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen = strlen(name); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* IO tasks call the netCDF functions. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_inq_dimid(file->fh, name, idp);; + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq_dimid(file->fh, name, idp);; #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_inq_dimid(file->fh, name, idp);; + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_dimid(file->fh, name, idp);; #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results. */ if (!ierr) - if (idp) - if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + if (idp) + if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); return ierr; } @@ -593,7 +593,7 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, - int *dimidsp, int *nattsp) + int *dimidsp, int *nattsp) { iosystem_desc_t *ios; file_desc_t *file; @@ -605,108 +605,108 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, /* Get the file info, based on the ncid. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_INQ_VAR; - char name_present = name ? true : false; - char xtype_present = xtypep ? true : false; - char ndims_present = ndimsp ? true : false; - char dimids_present = dimidsp ? true : false; - char natts_present = nattsp ? true : false; - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d " - "dimids_present = %d, natts_present = %d nattsp = %d", - name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp)); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); - } - + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_VAR; + char name_present = name ? true : false; + char xtype_present = xtypep ? true : false; + char ndims_present = ndimsp ? true : false; + char dimids_present = dimidsp ? true : false; + char natts_present = nattsp ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d " + "dimids_present = %d, natts_present = %d nattsp = %d", + name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp)); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); + } + /* Call the netCDF layer. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - { - ierr = ncmpi_inq_varndims(file->fh, varid, &ndims); - if (!ierr) - ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; - } + if (file->iotype == PIO_IOTYPE_PNETCDF) + { + ierr = ncmpi_inq_varndims(file->fh, varid, &ndims); + if (!ierr) + ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + } #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - { - ierr = nc_inq_varndims(file->fh, varid, &ndims); - if (!ierr) - ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp); - } + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + { + ierr = nc_inq_varndims(file->fh, varid, &ndims); + if (!ierr) + ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp); + } #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast the results for non-null pointers. */ if (!ierr) { - if (name) - { - int slen; - if(ios->iomaster) - slen = strlen(name); - if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - } - if (xtypep) - if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - - if (ndimsp) - { - if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - file->varlist[varid].ndims = (*ndimsp); - } - if (dimidsp) - { - if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - } - if (nattsp) - if ((mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + if (name) + { + int slen; + if(ios->iomaster) + slen = strlen(name); + if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + } + if (xtypep) + if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + + if (ndimsp) + { + if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + file->varlist[varid].ndims = (*ndimsp); + } + if (dimidsp) + { + if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + } + if (nattsp) + if ((mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); } return ierr; @@ -781,65 +781,65 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) /* Caller must provide name. */ if (!name || strlen(name) > NC_MAX_NAME) - return PIO_EINVAL; + return PIO_EINVAL; /* Get file info based on ncid. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; LOG((1, "PIOc_inq_varid ncid = %d name = %s", ncid, name)); if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_INQ_VARID; - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - int namelen; - namelen = strlen(name); - if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_VARID; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen; + namelen = strlen(name); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_inq_varid(file->fh, name, varidp);; + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq_varid(file->fh, name, varidp);; #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_inq_varid(file->fh, name, varidp); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_varid(file->fh, name, varidp); #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ if (varidp) - if ((mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); return ierr; } @@ -861,7 +861,7 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, - PIO_Offset *lenp) + PIO_Offset *lenp) { int msg = PIO_MSG_INQ_ATT; iosystem_desc_t *ios; @@ -871,79 +871,79 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, /* Caller must provide a name. */ if (!name) - return PIO_EINVAL; + return PIO_EINVAL; LOG((1, "PIOc_inq_att ncid = %d varid = %d xtpyep = %d lenp = %d", - ncid, varid, xtypep, lenp)); + ncid, varid, xtypep, lenp)); /* Find file based on ncid. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - char xtype_present = xtypep ? true : false; - char len_present = lenp ? true : false; - int namelen = strlen(name); - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (!ios->ioproc) + { + char xtype_present = xtypep ? true : false; + char len_present = lenp ? true : false; + int namelen = strlen(name); + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp); #endif /* _NETCDF */ - LOG((2, "PIOc_inq netcdf call returned %d", ierr)); + LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results. */ if (!ierr) { - if(xtypep) - if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - if(lenp) - if ((mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + if(xtypep) + if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if(lenp) + if ((mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); } return ierr; @@ -990,73 +990,73 @@ int PIOc_inq_attname(int ncid, int varid, int attnum, char *name) int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ LOG((1, "PIOc_inq_attname ncid = %d varid = %d attnum = %d", ncid, varid, - attnum)); + attnum)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_INQ_ATTNAME; - char name_present = name ? true : false; - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_ATTNAME; + char name_present = name ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);; + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);; #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_inq_attname(file->fh, varid, attnum, name);; + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_attname(file->fh, varid, attnum, name);; #endif /* _NETCDF */ - LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr)); + LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr)); } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ if (!ierr) - if (name) - { - int namelen = strlen(name); - if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->ioroot, - ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - } + if (name) + { + int namelen = strlen(name); + if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->ioroot, + ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + } return ierr; } @@ -1085,73 +1085,73 @@ int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp) /* User must provide name shorter than NC_MAX_NAME +1. */ if (!name || strlen(name) > NC_MAX_NAME) - return PIO_EINVAL; + return PIO_EINVAL; LOG((1, "PIOc_inq_attid ncid = %d varid = %d name = %s", ncid, varid, name)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_INQ_ATTID; - int namelen = strlen(name); - char id_present = idp ? true : false; - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_ATTID; + int namelen = strlen(name); + char id_present = idp ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_inq_attid(file->fh, varid, name, idp);; + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq_attid(file->fh, varid, name, idp);; #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_inq_attid(file->fh, varid, name, idp);; + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_attid(file->fh, varid, name, idp);; #endif /* _NETCDF */ - LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr)); + LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr)); } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results. */ if (!ierr) { - if (idp) - if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + if (idp) + if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); } return ierr; @@ -1179,42 +1179,42 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name) /* User must provide name of correct length. */ if (!name || strlen(name) > NC_MAX_NAME) - return PIO_EINVAL; + return PIO_EINVAL; LOG((1, "PIOc_rename_dim ncid = %d dimid = %d name = %s", ncid, dimid, name)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_RENAME_DIM; /** Message for async notification. */ - int namelen = strlen(name); + if (!ios->ioproc) + { + int msg = PIO_MSG_RENAME_DIM; /** Message for async notification. */ + int namelen = strlen(name); - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_rename_dim Bcast file->fh = %d dimid = %d namelen = %d name = %s", - file->fh, dimid, namelen, name)); - } + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_rename_dim Bcast file->fh = %d dimid = %d namelen = %d name = %s", + file->fh, dimid, namelen, name)); + } - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -1222,21 +1222,21 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name) if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_rename_dim(file->fh, dimid, name); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_rename_dim(file->fh, dimid, name); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_rename_dim(file->fh, dimid, name);; + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_rename_dim(file->fh, dimid, name);; #endif /* _NETCDF */ - LOG((2, "PIOc_inq netcdf call returned %d", ierr)); + LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); @@ -1266,42 +1266,42 @@ int PIOc_rename_var(int ncid, int varid, const char *name) /* User must provide name of correct length. */ if (!name || strlen(name) > NC_MAX_NAME) - return PIO_EINVAL; + return PIO_EINVAL; LOG((1, "PIOc_rename_var ncid = %d varid = %d name = %s", ncid, varid, name)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_RENAME_VAR; /** Message for async notification. */ - int namelen = strlen(name); + if (!ios->ioproc) + { + int msg = PIO_MSG_RENAME_VAR; /** Message for async notification. */ + int namelen = strlen(name); - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_rename_var Bcast file->fh = %d varid = %d namelen = %d name = %s", - file->fh, varid, namelen, name)); - } + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_rename_var Bcast file->fh = %d varid = %d namelen = %d name = %s", + file->fh, varid, namelen, name)); + } - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -1309,21 +1309,21 @@ int PIOc_rename_var(int ncid, int varid, const char *name) if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_rename_var(file->fh, varid, name); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_rename_var(file->fh, varid, name); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_rename_var(file->fh, varid, name);; + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_rename_var(file->fh, varid, name);; #endif /* _NETCDF */ - LOG((2, "PIOc_inq netcdf call returned %d", ierr)); + LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); @@ -1346,7 +1346,7 @@ int PIOc_rename_var(int ncid, int varid, const char *name) * PIOc_Set_File_Error_Handling */ int PIOc_rename_att (int ncid, int varid, const char *name, - const char *newname) + const char *newname) { iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ @@ -1355,67 +1355,67 @@ int PIOc_rename_att (int ncid, int varid, const char *name, /* User must provide names of correct length. */ if (!name || strlen(name) > NC_MAX_NAME || - !newname || strlen(newname) > NC_MAX_NAME) - return PIO_EINVAL; + !newname || strlen(newname) > NC_MAX_NAME) + return PIO_EINVAL; LOG((1, "PIOc_rename_att ncid = %d varid = %d name = %s newname = %s", - ncid, varid, name, newname)); + ncid, varid, name, newname)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_RENAME_ATT; /** Message for async notification. */ - int namelen = strlen(name); - int newnamelen = strlen(newname); - - if (ios->compmaster) - mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((char *)newname, newnamelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (!ios->ioproc) + { + int msg = PIO_MSG_RENAME_ATT; /** Message for async notification. */ + int namelen = strlen(name); + int newnamelen = strlen(newname); + + if (ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((char *)newname, newnamelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_rename_att(file->fh, varid, name, newname); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_rename_att(file->fh, varid, name, newname); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_rename_att(file->fh, varid, name, newname); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_rename_att(file->fh, varid, name, newname); #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); @@ -1446,60 +1446,60 @@ int PIOc_del_att(int ncid, int varid, const char *name) /* User must provide name of correct length. */ if (!name || strlen(name) > NC_MAX_NAME) - return PIO_EINVAL; + return PIO_EINVAL; LOG((1, "PIOc_del_att ncid = %d varid = %d name = %s", ncid, varid, name)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_DEL_ATT; - int namelen = strlen(name); /** Length of name string. */ + if (!ios->ioproc) + { + int msg = PIO_MSG_DEL_ATT; + int namelen = strlen(name); /** Length of name string. */ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_del_att(file->fh, varid, name); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_del_att(file->fh, varid, name); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_del_att(file->fh, varid, name); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_del_att(file->fh, varid, name); #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); @@ -1528,31 +1528,31 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ LOG((1, "PIOc_set_fill ncid = %d fillmode = %d old_modep = %d", ncid, fillmode, - old_modep)); + old_modep)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_SET_FILL; + if (!ios->ioproc) + { + int msg = PIO_MSG_SET_FILL; - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - } + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + } - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -1560,20 +1560,20 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_set_fill(file->fh, fillmode, old_modep); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_set_fill(file->fh, fillmode, old_modep); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_set_fill(file->fh, fillmode, old_modep); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_set_fill(file->fh, fillmode, old_modep); #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); @@ -1597,53 +1597,53 @@ int pioc_change_def(int ncid, int is_enddef) /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = is_enddef ? PIO_MSG_ENDDEF : PIO_MSG_REDEF; + if (!ios->ioproc) + { + int msg = is_enddef ? PIO_MSG_ENDDEF : PIO_MSG_REDEF; - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - } + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + } - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - if (is_enddef) - ierr = ncmpi_enddef(file->fh); - else - ierr = ncmpi_redef(file->fh); + if (file->iotype == PIO_IOTYPE_PNETCDF) + if (is_enddef) + ierr = ncmpi_enddef(file->fh); + else + ierr = ncmpi_redef(file->fh); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - if (is_enddef) - ierr = nc_enddef(file->fh); - else - ierr = nc_redef(file->fh); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + if (is_enddef) + ierr = nc_enddef(file->fh); + else + ierr = nc_redef(file->fh); #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); @@ -1709,70 +1709,70 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) /* User must provide name. */ if (!name || strlen(name) > NC_MAX_NAME) - return PIO_EINVAL; + return PIO_EINVAL; LOG((1, "PIOc_def_dim ncid = %d name = %s len = %d", ncid, name, len)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_DEF_DIM; - int namelen = strlen(name); + if (!ios->ioproc) + { + int msg = PIO_MSG_DEF_DIM; + int namelen = strlen(name); - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if (!mpierr) - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); - } + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + } - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_def_dim(file->fh, name, len, idp);; + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_def_dim(file->fh, name, len, idp);; #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_def_dim(file->fh, name, (size_t)len, idp); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_def_dim(file->fh, name, (size_t)len, idp); #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr2, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr2, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ if (!ierr) - if (idp) - if ((mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + if (idp) + if ((mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); return ierr; } @@ -1794,7 +1794,7 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) * PIOc_Set_File_Error_Handling */ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, - const int *dimidsp, int *varidp) + const int *dimidsp, int *varidp) { iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ @@ -1804,67 +1804,67 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, /* User must provide name and storage for varid. */ if (!name || !varidp || strlen(name) > NC_MAX_NAME) { - check_netcdf(file, PIO_EINVAL, __FILE__, __LINE__); - return PIO_EINVAL; + check_netcdf(file, PIO_EINVAL, __FILE__, __LINE__); + return PIO_EINVAL; } /* Get the file information. */ if (!(file = pio_get_file_from_id(ncid))) { - check_netcdf(file, PIO_EBADID, __FILE__, __LINE__); - return PIO_EBADID; + check_netcdf(file, PIO_EBADID, __FILE__, __LINE__); + return PIO_EBADID; } ios = file->iosystem; /* If using async, and not an IO task, then send parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_DEF_VAR; - int namelen = strlen(name); - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&(ncid), 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (!ios->ioproc) + { + int msg = PIO_MSG_DEF_VAR; + int namelen = strlen(name); + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&(ncid), 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_def_var(ncid, name, xtype, ndims, dimidsp, varidp); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_def_var(ncid, name, xtype, ndims, dimidsp, varidp); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_def_var(ncid, name, xtype, ndims, dimidsp, varidp); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_def_var(ncid, name, xtype, ndims, dimidsp, varidp); #ifdef _NETCDF4 - /* For netCDF-4 serial files, turn on compression for this variable. */ - if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4C) - ierr = nc_def_var_deflate(ncid, *varidp, 0, 1, 1); + /* For netCDF-4 serial files, turn on compression for this variable. */ + if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4C) + ierr = nc_def_var_deflate(ncid, *varidp, 0, 1, 1); - /* For netCDF-4 parallel files, set parallel access to collective. */ - if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4P) - ierr = nc_var_par_access(ncid, *varidp, NC_COLLECTIVE); + /* For netCDF-4 parallel files, set parallel access to collective. */ + if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4P) + ierr = nc_var_par_access(ncid, *varidp, NC_COLLECTIVE); #endif /* _NETCDF4 */ #endif /* _NETCDF */ } @@ -1872,16 +1872,16 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results. */ if (!ierr) - if (varidp) - if ((mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + if (varidp) + if ((mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); return ierr; } @@ -1911,55 +1911,55 @@ int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep) /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_INQ_VAR_FILL; + if (!ios->ioproc) + { + int msg = PIO_MSG_INQ_VAR_FILL; - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - if (!mpierr) - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } + if (!mpierr) + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_valuep); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_valuep); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_valuep); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_valuep); #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ if (!ierr) - if (fill_valuep) - if ((mpierr = MPI_Bcast(fill_valuep, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + if (fill_valuep) + if ((mpierr = MPI_Bcast(fill_valuep, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); return ierr; } @@ -1990,98 +1990,98 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) /* User must provide a name and destination pointer. */ if (!name || !ip || strlen(name) > NC_MAX_NAME) - return PIO_EINVAL; + return PIO_EINVAL; LOG((1, "PIOc_get_att ncid %d varid %d name %s", ncid, varid, name)); /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* Run these on all tasks if async is not in use, but only on * non-IO tasks if async is in use. */ if (!ios->async_interface || !ios->ioproc) { - /* Get the type and length of the attribute. */ - if ((ierr = PIOc_inq_att(file->fh, varid, name, &atttype, &attlen))) - { - check_netcdf(file, ierr, __FILE__, __LINE__); - return ierr; - } + /* Get the type and length of the attribute. */ + if ((ierr = PIOc_inq_att(file->fh, varid, name, &atttype, &attlen))) + { + check_netcdf(file, ierr, __FILE__, __LINE__); + return ierr; + } - /* Get the length (in bytes) of the type. */ - if ((ierr = PIOc_inq_type(file->fh, atttype, NULL, &typelen))) - { - check_netcdf(file, ierr, __FILE__, __LINE__); - return ierr; - } + /* Get the length (in bytes) of the type. */ + if ((ierr = PIOc_inq_type(file->fh, atttype, NULL, &typelen))) + { + check_netcdf(file, ierr, __FILE__, __LINE__); + return ierr; + } } /* If async is in use, and this is not an IO task, bcast the * parameters and the attribute and type information we fetched. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_GET_ATT; - - /* Send the message to IO master. */ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - /* Send the function parameters. */ - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - int namelen = strlen(name); - if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&atttype, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); - - /* Broadcast values currently only known on computation tasks to IO tasks. */ - LOG((2, "PIOc_get_att bcast from comproot = %d attlen = %d typelen = %d", ios->comproot, attlen, typelen)); - if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((2, "PIOc_get_att bcast complete attlen = %d typelen = %d", attlen, typelen)); - } - + if (!ios->ioproc) + { + int msg = PIO_MSG_GET_ATT; + + /* Send the message to IO master. */ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + /* Send the function parameters. */ + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen = strlen(name); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&atttype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); + + /* Broadcast values currently only known on computation tasks to IO tasks. */ + LOG((2, "PIOc_get_att bcast from comproot = %d attlen = %d typelen = %d", ios->comproot, attlen, typelen)); + if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + LOG((2, "PIOc_get_att bcast complete attlen = %d typelen = %d", attlen, typelen)); + } + /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_get_att(file->fh, varid, name, ip); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_get_att(file->fh, varid, name, ip); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_get_att(file->fh, varid, name, ip); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_get_att(file->fh, varid, name, ip); #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); @@ -2089,11 +2089,11 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) if (!ierr) { if ((mpierr = MPI_Bcast(ip, (int)attlen * typelen, MPI_BYTE, ios->ioroot, - ios->my_comm))) - { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; - } + ios->my_comm))) + { + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; + } } return ierr; } @@ -2113,7 +2113,7 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const void *op) + PIO_Offset len, const void *op) { iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ @@ -2125,83 +2125,83 @@ int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; + return PIO_EBADID; ios = file->iosystem; /* Run these on all tasks if async is not in use, but only on * non-IO tasks if async is in use. */ if (!ios->async_interface || !ios->ioproc) { - /* Get the length (in bytes) of the type. */ - if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &typelen))) - { - check_netcdf(file, ierr, __FILE__, __LINE__); - return ierr; - } - LOG((2, "PIOc_put_att typelen = %d", ncid, typelen)); + /* Get the length (in bytes) of the type. */ + if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &typelen))) + { + check_netcdf(file, ierr, __FILE__, __LINE__); + return ierr; + } + LOG((2, "PIOc_put_att typelen = %d", ncid, typelen)); } /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { - if (!ios->ioproc) - { - int msg = PIO_MSG_PUT_ATT; - - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); - int namelen = strlen(name); - if (!mpierr) - mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)op, len * typelen, MPI_BYTE, ios->compmaster, - ios->intercomm); - LOG((2, "PIOc_put_att finished bcast ncid = %d varid = %d namelen = %d name = %s " - "len = %d typelen = %d", file->fh, varid, namelen, name, len, typelen)); - } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); - - /* Broadcast values currently only known on computation tasks to IO tasks. */ - LOG((2, "PIOc_put_att bcast from comproot = %d typelen = %d", ios->comproot, typelen)); - if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + if (!ios->ioproc) + { + int msg = PIO_MSG_PUT_ATT; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + int namelen = strlen(name); + if (!mpierr) + mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)op, len * typelen, MPI_BYTE, ios->compmaster, + ios->intercomm); + LOG((2, "PIOc_put_att finished bcast ncid = %d varid = %d namelen = %d name = %s " + "len = %d typelen = %d", file->fh, varid, namelen, name, len, typelen)); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); + + /* Broadcast values currently only known on computation tasks to IO tasks. */ + LOG((2, "PIOc_put_att bcast from comproot = %d typelen = %d", ios->comproot, typelen)); + if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_put_att(file->fh, varid, name, xtype, len, op); + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_put_att(file->fh, varid, name, xtype, len, op); #endif /* _PNETCDF */ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_put_att(file->fh, varid, name, xtype, (size_t)len, op); + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_put_att(file->fh, varid, name, xtype, (size_t)len, op); #endif /* _NETCDF */ } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; + check_mpi(file, mpierr, __FILE__, __LINE__); + return PIO_EIO; } check_netcdf(file, ierr, __FILE__, __LINE__); @@ -2330,7 +2330,7 @@ int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) * The PIO-C interface for the NetCDF function nc_put_att_schar. */ int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const signed char *op) + PIO_Offset len, const signed char *op) { return PIOc_put_att(ncid, varid, name, xtype, len, op); } @@ -2340,7 +2340,7 @@ int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype, * The PIO-C interface for the NetCDF function nc_put_att_long. */ int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const long *op) + PIO_Offset len, const long *op) { return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } @@ -2350,7 +2350,7 @@ int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype, * The PIO-C interface for the NetCDF function nc_put_att_int. */ int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const int *op) + PIO_Offset len, const int *op) { return PIOc_put_att(ncid, varid, name, xtype, len, op); } @@ -2360,7 +2360,7 @@ int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype, * The PIO-C interface for the NetCDF function nc_put_att_uchar. */ int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const unsigned char *op) + PIO_Offset len, const unsigned char *op) { return PIOc_put_att(ncid, varid, name, xtype, len, op); } @@ -2370,7 +2370,7 @@ int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype, * The PIO-C interface for the NetCDF function nc_put_att_longlong. */ int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const long long *op) + PIO_Offset len, const long long *op) { return PIOc_put_att(ncid, varid, name, xtype, len, op); } @@ -2380,7 +2380,7 @@ int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype, * The PIO-C interface for the NetCDF function nc_put_att_uint. */ int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const unsigned int *op) + PIO_Offset len, const unsigned int *op) { return PIOc_put_att(ncid, varid, name, xtype, len, op); } @@ -2390,7 +2390,7 @@ int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, * The PIO-C interface for the NetCDF function nc_put_att_ubyte. */ int PIOc_put_att_ubyte(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const unsigned char *op) + PIO_Offset len, const unsigned char *op) { return PIOc_put_att(ncid, varid, name, xtype, len, op); } @@ -2400,7 +2400,7 @@ int PIOc_put_att_ubyte(int ncid, int varid, const char *name, nc_type xtype, * The PIO-C interface for the NetCDF function nc_put_att_float. */ int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const float *op) + PIO_Offset len, const float *op) { return PIOc_put_att(ncid, varid, name, xtype, len, op); } @@ -2410,7 +2410,7 @@ int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype, * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. */ int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const unsigned long long *op) + PIO_Offset len, const unsigned long long *op) { return PIOc_put_att(ncid, varid, name, xtype, len, op); } @@ -2420,7 +2420,7 @@ int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype, * The PIO-C interface for the NetCDF function nc_put_att_ushort. */ int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const unsigned short *op) + PIO_Offset len, const unsigned short *op) { return PIOc_put_att(ncid, varid, name, xtype, len, op); } @@ -2430,7 +2430,7 @@ int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype, * The PIO-C interface for the NetCDF function nc_put_att_text. */ int PIOc_put_att_text(int ncid, int varid, const char *name, - PIO_Offset len, const char *op) + PIO_Offset len, const char *op) { return PIOc_put_att(ncid, varid, name, NC_CHAR, len, op); } @@ -2440,7 +2440,7 @@ int PIOc_put_att_text(int ncid, int varid, const char *name, * The PIO-C interface for the NetCDF function nc_put_att_short. */ int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const short *op) + PIO_Offset len, const short *op) { return PIOc_put_att(ncid, varid, name, xtype, len, op); } @@ -2450,7 +2450,7 @@ int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype, * The PIO-C interface for the NetCDF function nc_put_att_double. */ int PIOc_put_att_double(int ncid, int varid, const char *name, nc_type xtype, - PIO_Offset len, const double *op) + PIO_Offset len, const double *op) { return PIOc_put_att(ncid, varid, name, xtype, len, op); } diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 3a10a6dd2d5..4d80cf88989 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -287,90 +287,6 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off return ierr; } -/** - * PIO interface to nc_put_vars - * - * This routine is called collectively by all tasks in the - * communicator ios.union_comm. - - * Refer to the - * netcdf documentation. */ -int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const void *buf, PIO_Offset bufcount, - MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VARS; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_vars(file->fh, varid, (size_t *)start, (size_t *)count, - (ptrdiff_t *)stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_vars(file->fh, varid, start, count, stride, buf, - bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - /** Interface to netCDF data write function. */ int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, @@ -406,966 +322,292 @@ int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op) return PIOc_put_vars_tc(ncid, varid, NULL, NULL, NULL, NC_USHORT, op); } -/// -/// PIO interface to nc_put_var1_longlong -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// -int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op) { + int ndims; int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_LONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_longlong(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; + return PIOc_put_vars_text(ncid, varid, index, count, NULL, op); +} - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; +/** Interface to netCDF data write function. */ +int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], + const unsigned char *op) +{ + int ndims; + int ierr; - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_longlong(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; - return ierr; + return PIOc_put_vars_uchar(ncid, varid, index, count, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const unsigned char *op) +int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset index[], const signed char *op) { - return PIOc_put_vars_uchar(ncid, varid, start, count, NULL, op); -} + int ndims; + int ierr; + + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; + + return PIOc_put_vars_schar(ncid, varid, index, count, NULL, op); +} /** Interface to netCDF data write function. */ -int PIOc_put_var1_long(int ncid, int varid, const PIO_Offset index[], const long *ip) +int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], + const unsigned short *op) { + int ndims; int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - ierr = PIO_NOERR; + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_LONG; + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } + return PIOc_put_vars_short(ncid, varid, index, count, NULL, op); +} +/** Interface to netCDF data write function. */ +int PIOc_put_var1_uint(int ncid, int varid, const PIO_Offset index[], const unsigned int *op) +{ + int ndims; + int ierr; - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_long(file->fh, varid, (size_t *) index, ip);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_long(file->fh, varid, index, ip, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + return PIOc_put_vars_uint(ncid, varid, index, count, NULL, op); +} - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +/** Interface to netCDF data write function. */ +int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset index[], const short *op) +{ + int ndims; + int ierr; - return ierr; + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; + + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; + + return PIOc_put_vars_short(ncid, varid, index, count, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const long *op) +int PIOc_put_var1_int(int ncid, int varid, const PIO_Offset index[], const int *op) { - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); + int ndims; + int ierr; + + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; + + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; + + return PIOc_put_vars_int(ncid, varid, index, count, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var_short(int ncid, int varid, const short *op) +int PIOc_put_var1_float(int ncid, int varid, const PIO_Offset index[], const float *op) { - return PIOc_put_vars_short(ncid, varid, NULL, NULL, NULL, op); + int ndims; + int ierr; + + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; + + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; + + return PIOc_put_vars_float(ncid, varid, index, count, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const int *op) +int PIOc_put_var1_long(int ncid, int varid, const PIO_Offset index[], const long *op) { - return PIOc_put_vars_int(ncid, varid, start, count, NULL, op); + int ndims; + int ierr; + + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; + + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; + + return PIOc_put_vars_long(ncid, varid, index, count, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], - const unsigned short *op) +int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset index[], const double *op) { + int ndims; int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - ierr = PIO_NOERR; + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_USHORT; + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_ushort(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_ushort(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const char *op) -{ - return PIOc_put_vars_text(ncid, varid, start, count, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) -{ - return PIOc_put_vars_ulonglong(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_vars_double(ncid, varid, index, count, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var_int(int ncid, int varid, const int *op) +int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op) { - return PIOc_put_vars_int(ncid, varid, NULL, NULL, NULL, op); -} + int ndims; + int ierr; -/** Interface to netCDF data write function. */ -int PIOc_put_var_longlong(int ncid, int varid, const long long *op) -{ - return PIOc_put_vars_longlong(ncid, varid, NULL, NULL, NULL, op); -} + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; -/** Interface to netCDF data write function. */ -int PIOc_put_var_schar(int ncid, int varid, const signed char *op) -{ - return PIOc_put_vars_schar(ncid, varid, NULL, NULL, NULL, op); -} + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; -/** Interface to netCDF data write function. */ -int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op) -{ - return PIOc_put_vars_uint(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_vars_ulonglong(ncid, varid, index, count, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var(int ncid, int varid, const void *buf, PIO_Offset bufcount, - MPI_Datatype buftype) +int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) { + int ndims; int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - if(ios->io_rank==0){ - ierr = ncmpi_bput_var(file->fh, varid, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; - return ierr; + return PIOc_put_vars_longlong(ncid, varid, index, count, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const unsigned short *op) +int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const unsigned char *op) { - return PIOc_put_vars_ushort(ncid, varid, start, count, NULL, op); + return PIOc_put_vars_uchar(ncid, varid, start, count, NULL, op); } -/** Interface to netCDF data write function. */ -int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const PIO_Offset *stride, const short *op) -{ - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_SHORT, op); -} /** Interface to netCDF data write function. */ -int PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const unsigned int *op) +int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const long *op) { - return PIOc_put_vars_uint(ncid, varid, start, count, NULL, op); + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const signed char *op) +int PIOc_put_var_short(int ncid, int varid, const short *op) { - return PIOc_put_vars_schar(ncid, varid, start, count, NULL, op); + return PIOc_put_vars_short(ncid, varid, NULL, NULL, NULL, op); } - /** Interface to netCDF data write function. */ -int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], - const unsigned char *op) +int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const int *op) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_UCHAR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_uchar(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_uchar(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; + return PIOc_put_vars_int(ncid, varid, start, count, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const signed char *op) +int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const char *op) { - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_BYTE, op); + return PIOc_put_vars_text(ncid, varid, start, count, NULL, op); } -/** Interface to netCDF data write function. */ -int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, - PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1(file->fh, varid, index, buf, bufcount, buftype, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const float *op) -{ - return PIOc_put_vars_float(ncid, varid, start, count, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], - const float *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_FLOAT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_float(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_float(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - - -/** Interface to netCDF data write function. */ -int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_TEXT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_text(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_text(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const char *op) -{ - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_CHAR, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const double *op) -{ - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_DOUBLE, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const long long *op) -{ - return PIOc_put_vars_longlong(ncid, varid, start, count, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var_double (int ncid, int varid, const double *op) -{ - return PIOc_put_vars_double(ncid, varid, NULL, NULL, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var_float(int ncid, int varid, const float *op) -{ - return PIOc_put_vars_float(ncid, varid, NULL, NULL, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_ULONGLONG; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_ulonglong(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_ulonglong(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - - -/** Interface to netCDF data write function. */ -int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const unsigned int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_UINT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_uint(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_uint(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - return ierr; -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int *op) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - var_desc_t *vdesc; - PIO_Offset usage; - int *request; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_INT; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_put_var1_int(file->fh, varid, (size_t *) index, op);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - vdesc = file->varlist + varid; - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - } - request = vdesc->request+vdesc->nreqs; - - if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_int(file->fh, varid, index, op, request);; - }else{ - *request = PIO_REQ_NULL; - } - vdesc->nreqs++; - flush_output_buffer(file, false, 0); - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +/** Interface to netCDF data write function. */ +int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) +{ + return PIOc_put_vars_ulonglong(ncid, varid, NULL, NULL, NULL, op); +} - return ierr; +/** Interface to netCDF data write function. */ +int PIOc_put_var_int(int ncid, int varid, const int *op) +{ + return PIOc_put_vars_int(ncid, varid, NULL, NULL, NULL, op); } -int PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const float *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var_longlong(int ncid, int varid, const long long *op) { - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_FLOAT, op); + return PIOc_put_vars_longlong(ncid, varid, NULL, NULL, NULL, op); } -int PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const short *op) +/** Interface to netCDF data write function. */ +int PIOc_put_var_schar(int ncid, int varid, const signed char *op) { - return PIOc_put_vars_short(ncid, varid, start, count, NULL, op); + return PIOc_put_vars_schar(ncid, varid, NULL, NULL, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset index[], const signed char *op) +int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op) +{ + return PIOc_put_vars_uint(ncid, varid, NULL, NULL, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_var(int ncid, int varid, const void *buf, PIO_Offset bufcount, + MPI_Datatype buftype) { int ierr; int msg; @@ -1382,7 +624,7 @@ int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset index[], const sig if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_SCHAR; + msg = PIO_MSG_PUT_VAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1397,13 +639,13 @@ int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset index[], const sig #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var(file->fh, varid, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var1_schar(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var(file->fh, varid, buf);; } break; #endif @@ -1418,7 +660,7 @@ int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset index[], const sig request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_schar(file->fh, varid, index, op, request);; + ierr = ncmpi_bput_var(file->fh, varid, buf, bufcount, buftype, request);; }else{ *request = PIO_REQ_NULL; } @@ -1436,6 +678,94 @@ int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset index[], const sig return ierr; } +/** Interface to netCDF data write function. */ +int PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const unsigned short *op) +{ + return PIOc_put_vars_ushort(ncid, varid, start, count, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, const short *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_SHORT, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const unsigned int *op) +{ + return PIOc_put_vars_uint(ncid, varid, start, count, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const signed char *op) +{ + return PIOc_put_vars_schar(ncid, varid, start, count, NULL, op); +} + + +/** Interface to netCDF data write function. */ +int PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const signed char *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_BYTE, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const float *op) +{ + return PIOc_put_vars_float(ncid, varid, start, count, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const char *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_CHAR, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const double *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_DOUBLE, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const long long *op) +{ + return PIOc_put_vars_longlong(ncid, varid, start, count, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_var_double (int ncid, int varid, const double *op) +{ + return PIOc_put_vars_double(ncid, varid, NULL, NULL, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_var_float(int ncid, int varid, const float *op) +{ + return PIOc_put_vars_float(ncid, varid, NULL, NULL, NULL, op); +} + +int PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const float *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_FLOAT, op); +} + +int PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const short *op) +{ + return PIOc_put_vars_short(ncid, varid, start, count, NULL, op); +} + /** Interface to netCDF data write function. */ int PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const unsigned long long *op) @@ -1445,8 +775,64 @@ int PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start, const /** Interface to netCDF data write function. */ -int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const void *buf, - PIO_Offset bufcount, MPI_Datatype buftype) +int PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const long *op) +{ + return PIOc_put_vars_long(ncid, varid, start, count, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_var_text(int ncid, int varid, const char *op) +{ + return PIOc_put_vars_text(ncid, varid, NULL, NULL, NULL, op); +} + +/** PIO interface to nc_put_vars_int */ +int PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const int *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const long long *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT64, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const double *op) +{ + return PIOc_put_vars_double(ncid, varid, start, count, NULL, op); +} + + +/** Interface to netCDF data write function. */ +int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op) +{ + return PIOc_put_vars_uchar(ncid, varid, NULL, NULL, NULL, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_var_long(int ncid, int varid, const long *op) +{ + return PIOc_put_vars_long(ncid, varid, NULL, NULL, NULL, op); +} + +/** + * PIO interface to nc_put_vars + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + + * Refer to the + * netcdf documentation. */ +int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const void *buf, PIO_Offset bufcount, + MPI_Datatype buftype) { int ierr; int msg; @@ -1463,7 +849,7 @@ int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VARA; + msg = PIO_MSG_PUT_VARS; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1478,13 +864,14 @@ int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_put_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_put_vars(file->fh, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf);; } break; #endif @@ -1499,7 +886,8 @@ int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_vara(file->fh, varid, start, count, buf, bufcount, buftype, request);; + ierr = ncmpi_bput_vars(file->fh, varid, start, count, stride, buf, + bufcount, buftype, request);; }else{ *request = PIO_REQ_NULL; } @@ -1518,14 +906,8 @@ int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset } /** Interface to netCDF data write function. */ -int PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const long *op) -{ - return PIOc_put_vars_long(ncid, varid, start, count, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset index[], const double *op) +int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, + PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -1542,7 +924,7 @@ int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset index[], const do if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_DOUBLE; + msg = PIO_MSG_PUT_VAR1; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1557,13 +939,13 @@ int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset index[], const do #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var1_double(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_var1(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -1578,7 +960,7 @@ int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset index[], const do request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_double(file->fh, varid, index, op, request);; + ierr = ncmpi_bput_var1(file->fh, varid, index, buf, bufcount, buftype, request);; }else{ *request = PIO_REQ_NULL; } @@ -1597,20 +979,8 @@ int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset index[], const do } /** Interface to netCDF data write function. */ -int PIOc_put_var_text(int ncid, int varid, const char *op) -{ - return PIOc_put_vars_text(ncid, varid, NULL, NULL, NULL, op); -} - -/** PIO interface to nc_put_vars_int */ -int PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const int *op) -{ - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset index[], const short *op) +int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const void *buf, + PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -1627,7 +997,7 @@ int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset index[], const sho if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_VAR1_SHORT; + msg = PIO_MSG_PUT_VARA; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1642,13 +1012,13 @@ int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset index[], const sho #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE); - ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_var1_short(file->fh, varid, (size_t *) index, op);; + ierr = nc_put_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; } break; #endif @@ -1663,7 +1033,7 @@ int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset index[], const sho request = vdesc->request+vdesc->nreqs; if(ios->io_rank==0){ - ierr = ncmpi_bput_var1_short(file->fh, varid, index, op, request);; + ierr = ncmpi_bput_vara(file->fh, varid, start, count, buf, bufcount, buftype, request);; }else{ *request = PIO_REQ_NULL; } @@ -1681,31 +1051,3 @@ int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset index[], const sho return ierr; } -/** Interface to netCDF data write function. */ -int PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const long long *op) -{ - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT64, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const double *op) -{ - return PIOc_put_vars_double(ncid, varid, start, count, NULL, op); -} - - -/** Interface to netCDF data write function. */ -int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op) -{ - return PIOc_put_vars_uchar(ncid, varid, NULL, NULL, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var_long(int ncid, int varid, const long *op) -{ - return PIOc_put_vars_long(ncid, varid, NULL, NULL, NULL, op); -} - - From 98eed4dd7836f6b6449b7700250a796345b3404d Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 24 May 2016 09:38:50 -0400 Subject: [PATCH 094/184] futher cleanup of var1 functions --- src/clib/pio_put_nc_async.c | 572 ++++++++++++++---------------------- 1 file changed, 225 insertions(+), 347 deletions(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 4d80cf88989..ba653397e0b 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -23,7 +23,7 @@ * @param *stride an array of strides (must have same number of * entries as variable has dimensions). If NULL, strides of 1 will be * used. - * + * * @param xtype the netCDF type of the data being passed in buf. Data * will be automatically covnerted from this type to the type of the * variable being written to. @@ -33,7 +33,7 @@ * @return PIO_NOERR on success, error code otherwise. */ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, nc_type xtype, const void *buf) + const PIO_Offset *stride, nc_type xtype, const void *buf) { iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ @@ -79,11 +79,11 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if (!count) { int dimid[ndims]; - + /* Get the dimids for this var. */ if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid))) return check_netcdf(file, ierr, __FILE__, __LINE__); - + /* Get the length of each dimension. */ for (int vd = 0; vd < ndims; vd++) if ((ierr = PIOc_inq_dimlen(ncid, dimid[vd], &dimlen[vd]))) @@ -116,7 +116,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off char count_present = count ? true : false; char stride_present = stride ? true : false; - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); /* Send the function parameters and associated informaiton @@ -134,11 +134,11 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if (!mpierr) mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr && count_present) - mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr && stride_present) - mpierr = MPI_Bcast((PIO_Offset *)stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((PIO_Offset *)stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -151,7 +151,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off for (int e = 0; e < num_elem; e++) LOG((2, "PIOc_put_vars_tc element %d = %d", e, ((int *)buf)[e])); - + /* Send the data. */ if (!mpierr) mpierr = MPI_Bcast((void *)buf, num_elem * typelen, MPI_BYTE, ios->compmaster, @@ -160,7 +160,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off /* Handle MPI errors. */ if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr2, __FILE__, __LINE__); + return check_mpi(file, mpierr2, __FILE__, __LINE__); check_mpi(file, mpierr, __FILE__, __LINE__); /* /\* Broadcast values currently only known on computation tasks to IO tasks. *\/ */ @@ -169,7 +169,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off /* if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) */ /* check_mpi(file, mpierr, __FILE__, __LINE__); */ } - + /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { @@ -178,7 +178,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off { vdesc = file->varlist + varid; if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0) - vdesc->request = realloc(vdesc->request, + vdesc->request = realloc(vdesc->request, sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)); request = vdesc->request+vdesc->nreqs; @@ -208,14 +208,14 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off break; default: LOG((0, "Unknown type for pnetcdf file! xtype = %d", xtype)); - + } else *request = PIO_REQ_NULL; - + vdesc->nreqs++; flush_output_buffer(file, false, 0); - } + } #endif /* _PNETCDF */ #ifdef _NETCDF if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) @@ -245,7 +245,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off ierr = nc_put_vars_double(ncid, varid, (size_t *)start, (size_t *)count, (ptrdiff_t *)stride, buf); break; -#ifdef _NETCDF4 +#ifdef _NETCDF4 case NC_UBYTE: ierr = nc_put_vars_uchar(ncid, varid, (size_t *)start, (size_t *)count, (ptrdiff_t *)stride, buf); @@ -273,7 +273,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off default: ierr = nc_put_vars(ncid, varid, (size_t *)start, (size_t *)count, (ptrdiff_t *)stride, buf); -#endif /* _NETCDF4 */ +#endif /* _NETCDF4 */ } #endif /* _NETCDF */ } @@ -287,117 +287,94 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off return ierr; } +/** Interface to netCDF data write function. */ +int PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const char *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_CHAR, op); +} + /** Interface to netCDF data write function. */ int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, - const unsigned char *op) + const unsigned char *op) { - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UBYTE, op); + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UBYTE, op); +} + +/** Interface to netCDF data write function. */ +int PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const signed char *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_BYTE, op); } /** Interface to netCDF data write function. */ int PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const unsigned short *op) + const PIO_Offset *stride, const unsigned short *op) { - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_USHORT, op); + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_USHORT, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const unsigned long long *op) +int PIOc_put_vars_short(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, const short *op) { - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT64, op); + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_SHORT, op); } /** Interface to netCDF data write function. */ int PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const unsigned int *op) + const PIO_Offset *stride, const unsigned int *op) { - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT, op); + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT, op); } -/** Interface to netCDF data write function. */ -int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op) +/** PIO interface to nc_put_vars_int */ +int PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const int *op) { - return PIOc_put_vars_tc(ncid, varid, NULL, NULL, NULL, NC_USHORT, op); + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op) +int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const long *op) { - int ndims; - int ierr; - - /* Find the number of dimensions. */ - if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) - return ierr; - - /* Set up count array. */ - PIO_Offset count[ndims]; - for (int c = 0; c < ndims; c++) - count[c] = 1; - - return PIOc_put_vars_text(ncid, varid, index, count, NULL, op); + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], - const unsigned char *op) +int PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const float *op) { - int ndims; - int ierr; - - /* Find the number of dimensions. */ - if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) - return ierr; - - /* Set up count array. */ - PIO_Offset count[ndims]; - for (int c = 0; c < ndims; c++) - count[c] = 1; - - return PIOc_put_vars_uchar(ncid, varid, index, count, NULL, op); + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_FLOAT, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset index[], const signed char *op) +int PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const long long *op) { - int ndims; - int ierr; - - /* Find the number of dimensions. */ - if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) - return ierr; - - /* Set up count array. */ - PIO_Offset count[ndims]; - for (int c = 0; c < ndims; c++) - count[c] = 1; - - return PIOc_put_vars_schar(ncid, varid, index, count, NULL, op); + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT64, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], - const unsigned short *op) +int PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const double *op) { - int ndims; - int ierr; - - /* Find the number of dimensions. */ - if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) - return ierr; - - /* Set up count array. */ - PIO_Offset count[ndims]; - for (int c = 0; c < ndims; c++) - count[c] = 1; + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_DOUBLE, op); +} - return PIOc_put_vars_short(ncid, varid, index, count, NULL, op); +/** Interface to netCDF data write function. */ +int PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, const unsigned long long *op) +{ + return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT64, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_uint(int ncid, int varid, const PIO_Offset index[], const unsigned int *op) +int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype, + const void *op) { int ndims; int ierr; @@ -411,203 +388,248 @@ int PIOc_put_var1_uint(int ncid, int varid, const PIO_Offset index[], const unsi for (int c = 0; c < ndims; c++) count[c] = 1; - return PIOc_put_vars_uint(ncid, varid, index, count, NULL, op); + return PIOc_put_vars_tc(ncid, varid, index, count, NULL, xtype, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset index[], const short *op) +int PIOc_put_var1_text(int ncid, int varid, const PIO_Offset *index, const char *op) { - int ndims; - int ierr; - - /* Find the number of dimensions. */ - if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) - return ierr; - - /* Set up count array. */ - PIO_Offset count[ndims]; - for (int c = 0; c < ndims; c++) - count[c] = 1; - - return PIOc_put_vars_short(ncid, varid, index, count, NULL, op); + return PIOc_put_var1_tc(ncid, varid, index, NC_CHAR, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_int(int ncid, int varid, const PIO_Offset index[], const int *op) +int PIOc_put_var1_uchar(int ncid, int varid, const PIO_Offset *index, + const unsigned char *op) { - int ndims; - int ierr; + return PIOc_put_var1_tc(ncid, varid, index, NC_UBYTE, op); +} - /* Find the number of dimensions. */ - if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) - return ierr; +/** Interface to netCDF data write function. */ +int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset *index, + const signed char *op) +{ + return PIOc_put_var1_tc(ncid, varid, index, NC_BYTE, op); +} - /* Set up count array. */ - PIO_Offset count[ndims]; - for (int c = 0; c < ndims; c++) - count[c] = 1; +/** Interface to netCDF data write function. */ +int PIOc_put_var1_ushort(int ncid, int varid, const PIO_Offset *index, + const unsigned short *op) +{ + return PIOc_put_var1_tc(ncid, varid, index, NC_USHORT, op); +} - return PIOc_put_vars_int(ncid, varid, index, count, NULL, op); +/** Interface to netCDF data write function. */ +int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset *index, + const short *op) +{ + return PIOc_put_var1_tc(ncid, varid, index, NC_SHORT, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_float(int ncid, int varid, const PIO_Offset index[], const float *op) +int PIOc_put_var1_uint(int ncid, int varid, const PIO_Offset *index, + const unsigned int *op) { - int ndims; - int ierr; + return PIOc_put_var1_tc(ncid, varid, index, NC_UINT, op); +} - /* Find the number of dimensions. */ - if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) - return ierr; +/** Interface to netCDF data write function. */ +int PIOc_put_var1_int(int ncid, int varid, const PIO_Offset *index, const int *op) +{ + return PIOc_put_var1_tc(ncid, varid, index, NC_INT, op); +} - /* Set up count array. */ - PIO_Offset count[ndims]; - for (int c = 0; c < ndims; c++) - count[c] = 1; +/** Interface to netCDF data write function. */ +int PIOc_put_var1_float(int ncid, int varid, const PIO_Offset *index, const float *op) +{ + return PIOc_put_var1_tc(ncid, varid, index, NC_FLOAT, op); +} - return PIOc_put_vars_float(ncid, varid, index, count, NULL, op); +/** Interface to netCDF data write function. */ +int PIOc_put_var1_long(int ncid, int varid, const PIO_Offset *index, const long *op) +{ + return PIOc_put_var1_tc(ncid, varid, index, NC_LONG, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_long(int ncid, int varid, const PIO_Offset index[], const long *op) +int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset *index, + const double *op) { - int ndims; - int ierr; + return PIOc_put_var1_tc(ncid, varid, index, NC_DOUBLE, op); +} - /* Find the number of dimensions. */ - if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) - return ierr; +/** Interface to netCDF data write function. */ +int PIOc_put_var1_ulonglong(int ncid, int varid, const PIO_Offset *index, + const unsigned long long *op) +{ + return PIOc_put_var1_tc(ncid, varid, index, NC_UINT64, op); +} - /* Set up count array. */ - PIO_Offset count[ndims]; - for (int c = 0; c < ndims; c++) - count[c] = 1; +/** Interface to netCDF data write function. */ +int PIOc_put_var1_longlong(int ncid, int varid, const PIO_Offset *index, + const long long *op) +{ + return PIOc_put_var1_tc(ncid, varid, index, NC_INT64, op); +} - return PIOc_put_vars_long(ncid, varid, index, count, NULL, op); +/** Interface to netCDF data write function. */ +int PIOc_put_vara_text(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const char *op) +{ + return PIOc_put_vars_text(ncid, varid, start, count, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset index[], const double *op) +int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const unsigned char *op) { - int ndims; - int ierr; + return PIOc_put_vars_uchar(ncid, varid, start, count, NULL, op); +} - /* Find the number of dimensions. */ - if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) - return ierr; +/** Interface to netCDF data write function. */ +int PIOc_put_vara_schar(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const signed char *op) +{ + return PIOc_put_vars_schar(ncid, varid, start, count, NULL, op); +} - /* Set up count array. */ - PIO_Offset count[ndims]; - for (int c = 0; c < ndims; c++) - count[c] = 1; +/** Interface to netCDF data write function. */ +int PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const unsigned short *op) +{ + return PIOc_put_vars_ushort(ncid, varid, start, count, NULL, op); +} - return PIOc_put_vars_double(ncid, varid, index, count, NULL, op); +/** Interface to netCDF data write function. */ +int PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const short *op) +{ + return PIOc_put_vars_short(ncid, varid, start, count, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op) +int PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const unsigned int *op) { - int ndims; - int ierr; + return PIOc_put_vars_uint(ncid, varid, start, count, NULL, op); +} - /* Find the number of dimensions. */ - if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) - return ierr; +/** Interface to netCDF data write function. */ +int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const int *op) +{ + return PIOc_put_vars_int(ncid, varid, start, count, NULL, op); +} - /* Set up count array. */ - PIO_Offset count[ndims]; - for (int c = 0; c < ndims; c++) - count[c] = 1; +/** Interface to netCDF data write function. */ +int PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const long *op) +{ + return PIOc_put_vars_long(ncid, varid, start, count, NULL, op); +} - return PIOc_put_vars_ulonglong(ncid, varid, index, count, NULL, op); +/** Interface to netCDF data write function. */ +int PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const float *op) +{ + return PIOc_put_vars_float(ncid, varid, start, count, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) +int PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const unsigned long long *op) { - int ndims; - int ierr; + return PIOc_put_vars_ulonglong(ncid, varid, start, count, NULL, op); +} - /* Find the number of dimensions. */ - if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) - return ierr; +/** Interface to netCDF data write function. */ +int PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const long long *op) +{ + return PIOc_put_vars_longlong(ncid, varid, start, count, NULL, op); +} - /* Set up count array. */ - PIO_Offset count[ndims]; - for (int c = 0; c < ndims; c++) - count[c] = 1; +/** Interface to netCDF data write function. */ +int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const double *op) +{ + return PIOc_put_vars_double(ncid, varid, start, count, NULL, op); +} - return PIOc_put_vars_longlong(ncid, varid, index, count, NULL, op); +/** Interface to netCDF data write function. */ +int PIOc_put_var_text(int ncid, int varid, const char *op) +{ + return PIOc_put_vars_text(ncid, varid, NULL, NULL, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const unsigned char *op) +int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op) { - return PIOc_put_vars_uchar(ncid, varid, start, count, NULL, op); + return PIOc_put_vars_uchar(ncid, varid, NULL, NULL, NULL, op); } +/** Interface to netCDF data write function. */ +int PIOc_put_var_schar(int ncid, int varid, const signed char *op) +{ + return PIOc_put_vars_schar(ncid, varid, NULL, NULL, NULL, op); +} /** Interface to netCDF data write function. */ -int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const long *op) +int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op) { - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); + return PIOc_put_vars_tc(ncid, varid, NULL, NULL, NULL, NC_USHORT, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var_short(int ncid, int varid, const short *op) +int PIOc_put_var_short(int ncid, int varid, const short *op) { return PIOc_put_vars_short(ncid, varid, NULL, NULL, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const int *op) +int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op) { - return PIOc_put_vars_int(ncid, varid, start, count, NULL, op); + return PIOc_put_vars_uint(ncid, varid, NULL, NULL, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const char *op) +int PIOc_put_var_int(int ncid, int varid, const int *op) { - return PIOc_put_vars_text(ncid, varid, start, count, NULL, op); + return PIOc_put_vars_int(ncid, varid, NULL, NULL, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op) +int PIOc_put_var_long(int ncid, int varid, const long *op) { - return PIOc_put_vars_ulonglong(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_vars_long(ncid, varid, NULL, NULL, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var_int(int ncid, int varid, const int *op) +int PIOc_put_var_float(int ncid, int varid, const float *op) { - return PIOc_put_vars_int(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_vars_float(ncid, varid, NULL, NULL, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var_longlong(int ncid, int varid, const long long *op) +int PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op) { - return PIOc_put_vars_longlong(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_vars_ulonglong(ncid, varid, NULL, NULL, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var_schar(int ncid, int varid, const signed char *op) +int PIOc_put_var_longlong(int ncid, int varid, const long long *op) { - return PIOc_put_vars_schar(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_vars_longlong(ncid, varid, NULL, NULL, NULL, op); } /** Interface to netCDF data write function. */ -int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op) +int PIOc_put_var_double(int ncid, int varid, const double *op) { - return PIOc_put_vars_uint(ncid, varid, NULL, NULL, NULL, op); + return PIOc_put_vars_double(ncid, varid, NULL, NULL, NULL, op); } /** Interface to netCDF data write function. */ int PIOc_put_var(int ncid, int varid, const void *buf, PIO_Offset bufcount, - MPI_Datatype buftype) + MPI_Datatype buftype) { int ierr; int msg; @@ -627,7 +649,7 @@ int PIOc_put_var(int ncid, int varid, const void *buf, PIO_Offset bufcount, msg = PIO_MSG_PUT_VAR; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -654,7 +676,7 @@ int PIOc_put_var(int ncid, int varid, const void *buf, PIO_Offset bufcount, vdesc = file->varlist + varid; if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, + vdesc->request = realloc(vdesc->request, sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); } request = vdesc->request+vdesc->nreqs; @@ -678,149 +700,6 @@ int PIOc_put_var(int ncid, int varid, const void *buf, PIO_Offset bufcount, return ierr; } -/** Interface to netCDF data write function. */ -int PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const unsigned short *op) -{ - return PIOc_put_vars_ushort(ncid, varid, start, count, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const PIO_Offset *stride, const short *op) -{ - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_SHORT, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const unsigned int *op) -{ - return PIOc_put_vars_uint(ncid, varid, start, count, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const signed char *op) -{ - return PIOc_put_vars_schar(ncid, varid, start, count, NULL, op); -} - - -/** Interface to netCDF data write function. */ -int PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const signed char *op) -{ - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_BYTE, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const float *op) -{ - return PIOc_put_vars_float(ncid, varid, start, count, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const char *op) -{ - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_CHAR, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const double *op) -{ - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_DOUBLE, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const long long *op) -{ - return PIOc_put_vars_longlong(ncid, varid, start, count, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var_double (int ncid, int varid, const double *op) -{ - return PIOc_put_vars_double(ncid, varid, NULL, NULL, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var_float(int ncid, int varid, const float *op) -{ - return PIOc_put_vars_float(ncid, varid, NULL, NULL, NULL, op); -} - -int PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const float *op) -{ - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_FLOAT, op); -} - -int PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const short *op) -{ - return PIOc_put_vars_short(ncid, varid, start, count, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const unsigned long long *op) -{ - return PIOc_put_vars_ulonglong(ncid, varid, start, count, NULL, op); -} - - -/** Interface to netCDF data write function. */ -int PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const long *op) -{ - return PIOc_put_vars_long(ncid, varid, start, count, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var_text(int ncid, int varid, const char *op) -{ - return PIOc_put_vars_text(ncid, varid, NULL, NULL, NULL, op); -} - -/** PIO interface to nc_put_vars_int */ -int PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const int *op) -{ - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, const long long *op) -{ - return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT64, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start, - const PIO_Offset *count, const double *op) -{ - return PIOc_put_vars_double(ncid, varid, start, count, NULL, op); -} - - -/** Interface to netCDF data write function. */ -int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op) -{ - return PIOc_put_vars_uchar(ncid, varid, NULL, NULL, NULL, op); -} - -/** Interface to netCDF data write function. */ -int PIOc_put_var_long(int ncid, int varid, const long *op) -{ - return PIOc_put_vars_long(ncid, varid, NULL, NULL, NULL, op); -} - /** * PIO interface to nc_put_vars * @@ -832,7 +711,7 @@ int PIOc_put_var_long(int ncid, int varid, const long *op) * netcdf documentation. */ int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, const void *buf, PIO_Offset bufcount, - MPI_Datatype buftype) + MPI_Datatype buftype) { int ierr; int msg; @@ -852,7 +731,7 @@ int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset msg = PIO_MSG_PUT_VARS; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -880,7 +759,7 @@ int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset vdesc = file->varlist + varid; if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, + vdesc->request = realloc(vdesc->request, sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); } request = vdesc->request+vdesc->nreqs; @@ -906,8 +785,8 @@ int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset } /** Interface to netCDF data write function. */ -int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, - PIO_Offset bufcount, MPI_Datatype buftype) +int PIOc_put_var1(int ncid, int varid, const PIO_Offset *index, const void *buf, + PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -927,7 +806,7 @@ int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *bu msg = PIO_MSG_PUT_VAR1; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -954,7 +833,7 @@ int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *bu vdesc = file->varlist + varid; if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, + vdesc->request = realloc(vdesc->request, sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); } request = vdesc->request+vdesc->nreqs; @@ -980,7 +859,7 @@ int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *bu /** Interface to netCDF data write function. */ int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const void *buf, - PIO_Offset bufcount, MPI_Datatype buftype) + PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -1000,7 +879,7 @@ int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset msg = PIO_MSG_PUT_VARA; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -1027,7 +906,7 @@ int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset vdesc = file->varlist + varid; if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, + vdesc->request = realloc(vdesc->request, sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); } request = vdesc->request+vdesc->nreqs; @@ -1050,4 +929,3 @@ int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset return ierr; } - From de1c4e7ea5d066aecff9ac082cb387ddf0899f4e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 26 May 2016 15:14:43 -0400 Subject: [PATCH 095/184] got get_vars working --- src/clib/pio_get_nc_async.c | 7508 ++++++++++++++++------------------- src/clib/pio_msg.c | 203 +- src/clib/pio_nc_async.c | 5 +- src/clib/pio_varm.c | 998 +++++ tests/unit/test_intercomm.c | 15 +- 5 files changed, 4547 insertions(+), 4182 deletions(-) diff --git a/src/clib/pio_get_nc_async.c b/src/clib/pio_get_nc_async.c index 0da690aaf21..f3cbc0f5a9e 100644 --- a/src/clib/pio_get_nc_async.c +++ b/src/clib/pio_get_nc_async.c @@ -1,4988 +1,4198 @@ +#include #include #include -int PIOc_get_var1_schar (int ncid, int varid, const PIO_Offset index[], signed char *buf) +int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, nc_type xtype, void *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_SCHAR; - ibuftype = MPI_CHAR; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + int ndims; /** The number of dimensions in the variable. */ + int *dimids; /** The IDs of the dimensions for this variable. */ + PIO_Offset typelen; /** Size (in bytes) of the data type of data in buf. */ + size_t num_elem = 1; /** Number of data elements in the buffer. */ + int bcast = false; + + LOG((1, "PIOc_get_vars_tc ncid = %d varid = %d start = %d count = %d " + "stride = %d xtype = %d", ncid, varid, start, count, stride, xtype)); + + /* User must provide a place to put some data. */ + if (!buf) + return PIO_EINVAL; + + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) + return PIO_EBADID; + ios = file->iosystem; + + /* Run these on all tasks if async is not in use, but only on + * non-IO tasks if async is in use. */ + if (!ios->async_interface || !ios->ioproc) + { + /* Get the length of the data type. */ + if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &typelen))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Get the number of dims for this var. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + PIO_Offset dimlen[ndims]; + + /* If no count array was passed, we need to know the dimlens + * so we can calculate how many data elements are in the + * buf. */ + if (!count) + { + int dimid[ndims]; + + /* Get the dimids for this var. */ + if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Get the length of each dimension. */ + for (int vd = 0; vd < ndims; vd++) + if ((ierr = PIOc_inq_dimlen(ncid, dimid[vd], &dimlen[vd]))) + return check_netcdf(file, ierr, __FILE__, __LINE__); + } + + /* Figure out the real start, count, and stride arrays. (The + * user may have passed in NULLs.) */ + PIO_Offset rstart[ndims], rcount[ndims], rstride[ndims]; + for (int vd = 0; vd < ndims; vd++) + { + rstart[vd] = start ? start[vd] : 0; + rcount[vd] = count ? count[vd] : dimlen[vd]; + rstride[vd] = stride ? stride[vd] : 1; + } + + /* How many elements in buf? */ + for (int vd = 0; vd < ndims; vd++) + num_elem *= (rcount[vd] - rstart[vd])/rstride[vd]; + LOG((2, "PIOc_put_vars_tc num_elem = %d", num_elem)); + } + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_GET_VARS; + char start_present = start ? true : false; + char count_present = count ? true : false; + char stride_present = stride ? true : false; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + /* Send the function parameters and associated informaiton + * to the msg handler. */ + if (!mpierr) + mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && start_present) + mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && count_present) + mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr && stride_present) + mpierr = MPI_Bcast((PIO_Offset *)stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_get_vars_tc ncid = %d varid = %d ndims = %d start_present = %d " + "count_present = %d stride_present = %d xtype = %d num_elem = %d", ncid, varid, + ndims, start_present, count_present, stride_present, xtype, num_elem)); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + check_mpi(file, mpierr, __FILE__, __LINE__); + + /* Broadcast values currently only known on computation tasks to IO tasks. */ + if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) + check_mpi(file, mpierr, __FILE__, __LINE__); + } + + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + if (file->iotype == PIO_IOTYPE_PNETCDF) + { #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_schar(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_schar_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + LOG((1, "PNET_READ_AND_BCAST")); + ncmpi_begin_indep_data(file->fh); + if (ios->iomaster) + { + switch(xtype) + { + case NC_BYTE: + ierr = ncmpi_get_vars_schar(ncid, varid, start, count, stride, buf); + break; + case NC_CHAR: + ierr = ncmpi_get_vars_text(ncid, varid, start, count, stride, buf); + break; + case NC_SHORT: + ierr = ncmpi_get_vars_short(ncid, varid, start, count, stride, buf); + break; + case NC_INT: + ierr = ncmpi_get_vars_int(ncid, varid, start, count, stride, buf); + break; + case NC_FLOAT: + ierr = ncmpi_get_vars_float(ncid, varid, start, count, stride, buf); + break; + case NC_DOUBLE: + ierr = ncmpi_get_vars_double(ncid, varid, start, count, stride, buf); + break; + case NC_INT64: + ierr = ncmpi_get_vars_longlong(ncid, varid, start, count, stride, buf); + break; + default: + LOG((0, "Unknown type for pnetcdf file! xtype = %d", xtype)); + } + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else /* PNET_READ_AND_BCAST */ + LOG((1, "not PNET_READ_AND_BCAST")); + switch(xtype) + { + case NC_BYTE: + ierr = ncmpi_get_vars_schar_all(ncid, varid, start, count, stride, buf); + break; + case NC_CHAR: + ierr = ncmpi_get_vars_text_all(ncid, varid, start, count, stride, buf); + break; + case NC_SHORT: + ierr = ncmpi_get_vars_short_all(ncid, varid, start, count, stride, buf); + break; + case NC_INT: + ierr = ncmpi_get_vars_int_all(ncid, varid, start, count, stride, buf); + for (int i = 0; i < 4; i++) + LOG((2, "((int *)buf)[%d] = %d", i, ((int *)buf)[0])); + break; + case NC_FLOAT: + ierr = ncmpi_get_vars_float_all(ncid, varid, start, count, stride, buf); + break; + case NC_DOUBLE: + ierr = ncmpi_get_vars_double_all(ncid, varid, start, count, stride, buf); + break; + case NC_INT64: + ierr = ncmpi_get_vars_longlong_all(ncid, varid, start, count, stride, buf); + break; + default: + LOG((0, "Unknown type for pnetcdf file! xtype = %d", xtype)); + } +#endif /* PNET_READ_AND_BCAST */ + } +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + switch(xtype) + { + case NC_BYTE: + ierr = nc_get_vars_schar(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_CHAR: + ierr = nc_get_vars_schar(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_SHORT: + ierr = nc_get_vars_short(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_INT: + ierr = nc_get_vars_int(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_FLOAT: + ierr = nc_get_vars_float(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_DOUBLE: + ierr = nc_get_vars_double(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; +#ifdef _NETCDF4 + case NC_UBYTE: + ierr = nc_get_vars_uchar(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_USHORT: + ierr = nc_get_vars_ushort(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_UINT: + ierr = nc_get_vars_uint(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_INT64: + ierr = nc_get_vars_longlong(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + case NC_UINT64: + ierr = nc_get_vars_ulonglong(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); + break; + /* case NC_STRING: */ + /* ierr = nc_get_vars_string(ncid, varid, (size_t *)start, (size_t *)count, */ + /* (ptrdiff_t *)stride, (void *)buf); */ + /* break; */ + default: + ierr = nc_get_vars(ncid, varid, (size_t *)start, (size_t *)count, + (ptrdiff_t *)stride, buf); +#endif /* _NETCDF4 */ + } +#endif /* _NETCDF */ } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Send the data. */ + LOG((2, "PIOc_get_vars_tc bcasting data num_elem = %d typelen = %d", num_elem, + typelen)); + if (!mpierr) + mpierr = MPI_Bcast((void *)buf, num_elem * typelen, MPI_BYTE, ios->ioroot, + ios->my_comm); + return ierr; +} - return ierr; +int PIOc_get_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, int *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_INT, buf); } -int PIOc_get_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned long long *buf) +int PIOc_get_var1_schar (int ncid, int varid, const PIO_Offset *index, signed char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_SCHAR; + ibuftype = MPI_CHAR; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_ulonglong(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_schar(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vars_ulonglong_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_var1_schar_all(file->fh, varid, index, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned char *buf) +int PIOc_get_vars_ulonglong (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, unsigned long long *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_uchar(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_uchar_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], signed char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_SCHAR; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_schar(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_ulonglong(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_varm_schar_all(file->fh, varid, start, count, stride, imap, buf);; + ierr = ncmpi_get_vars_ulonglong_all(file->fh, varid, start, count, stride, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], short *buf) +int PIOc_get_vars_short (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, short *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_SHORT; - ibuftype = MPI_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_SHORT; + ibuftype = MPI_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_short(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_short(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vars_short_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_vars_short_all(file->fh, varid, start, count, stride, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } int PIOc_get_var_double (int ncid, int varid, double *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_DOUBLE; - ibuftype = MPI_DOUBLE; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_DOUBLE; + ibuftype = MPI_DOUBLE; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_double(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_double(file->fh, varid, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_double(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_double(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_double(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_double(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var_double_all(file->fh, varid, buf);; + ierr = ncmpi_get_var_double_all(file->fh, varid, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], double *buf) +int PIOc_get_vara_double (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, double *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_DOUBLE; - ibuftype = MPI_DOUBLE; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_DOUBLE; + ibuftype = MPI_DOUBLE; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_double(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_double(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_double_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_vara_double_all(file->fh, varid, start, count, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } int PIOc_get_var_int (int ncid, int varid, int *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_INT; - ibuftype = MPI_INT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_INT; + ibuftype = MPI_INT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_int(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_int(file->fh, varid, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_int(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_int(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_int(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_int(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var_int_all(file->fh, varid, buf);; + ierr = ncmpi_get_var_int_all(file->fh, varid, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } int PIOc_get_var_ushort (int ncid, int varid, unsigned short *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_ushort(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_ushort(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_ushort(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_ushort_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = PIO_NOERR; - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_TEXT; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_ushort(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_ushort(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_text(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_ushort(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_text_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_var_ushort_all(file->fh, varid, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], int *buf) +int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_INT; - ibuftype = MPI_INT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_TEXT; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_int(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_text(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_int_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_vara_text_all(file->fh, varid, start, count, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset index[], float *buf) +int PIOc_get_vara_int (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, int *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_FLOAT; - ibuftype = MPI_FLOAT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_INT; + ibuftype = MPI_INT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_float(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_int(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var1_float_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_vara_int_all(file->fh, varid, start, count, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset index[], short *buf) +int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset *index, float *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_SHORT; - ibuftype = MPI_SHORT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_FLOAT; + ibuftype = MPI_FLOAT; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_short(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_float(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var1_short_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_var1_float_all(file->fh, varid, index, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], int *buf) +int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset *index, short *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_INT; - ibuftype = MPI_INT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_SHORT; + ibuftype = MPI_SHORT; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_int(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_short(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vars_int_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_var1_short_all(file->fh, varid, index, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } int PIOc_get_var_text (int ncid, int varid, char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_TEXT; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_text(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_text(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_text(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_text_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_TEXT; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = PIO_NOERR; - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], double *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_DOUBLE; - ibuftype = MPI_DOUBLE; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_text(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_text(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_double(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_text(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_varm_double_all(file->fh, varid, start, count, stride, imap, buf);; + ierr = ncmpi_get_var_text_all(file->fh, varid, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], signed char *buf) +int PIOc_get_vars_schar (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, signed char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_SCHAR; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_SCHAR; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_schar(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_schar(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vars_schar_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_vars_schar_all(file->fh, varid, start, count, stride, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned short *buf) +int PIOc_get_vara_ushort (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, unsigned short *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_ushort(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_ushort(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_ushort_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_vara_ushort_all(file->fh, varid, start, count, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var1_ushort (int ncid, int varid, const PIO_Offset index[], unsigned short *buf) +int PIOc_get_var1_ushort (int ncid, int varid, const PIO_Offset *index, unsigned short *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_ushort(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_ushort(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var1_ushort_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_var1_ushort_all(file->fh, varid, index, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } int PIOc_get_var_float (int ncid, int varid, float *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_FLOAT; - ibuftype = MPI_FLOAT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_FLOAT; + ibuftype = MPI_FLOAT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_float(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_float(file->fh, varid, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_float(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_float(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_float(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_float(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var_float_all(file->fh, varid, buf);; + ierr = ncmpi_get_var_float_all(file->fh, varid, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned char *buf) +int PIOc_get_vars_uchar (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, unsigned char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_uchar(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_uchar(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vars_uchar_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_vars_uchar_all(file->fh, varid, start, count, stride, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var(file->fh, varid, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var(file->fh, varid, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var(file->fh, varid, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var_all(file->fh, varid, buf, bufcount, buftype);; + ierr = ncmpi_get_var_all(file->fh, varid, buf, bufcount, buftype);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset index[], long long *buf) +int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset *index, long long *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_LONGLONG; - ibuftype = MPI_LONG_LONG; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_LONGLONG; + ibuftype = MPI_LONG_LONG; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_longlong(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_longlong(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var1_longlong_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_var1_longlong_all(file->fh, varid, index, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned short *buf) +int PIOc_get_vars_ushort (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, unsigned short *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_ushort(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_ushort(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vars_ushort_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_vars_ushort_all(file->fh, varid, start, count, stride, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } int PIOc_get_var_long (int ncid, int varid, long *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_LONG; - ibuftype = MPI_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_LONG; + ibuftype = MPI_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_long(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_long(file->fh, varid, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_long(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_long(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_long(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_long(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var_long_all(file->fh, varid, buf);; + ierr = ncmpi_get_var_long_all(file->fh, varid, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset index[], double *buf) +int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset *index, double *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_DOUBLE; - ibuftype = MPI_DOUBLE; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_DOUBLE; + ibuftype = MPI_DOUBLE; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_double(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_double(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var1_double_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_var1_double_all(file->fh, varid, index, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned int *buf) +int PIOc_get_vara_uint (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, unsigned int *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_UINT; - ibuftype = MPI_UNSIGNED; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_UINT; + ibuftype = MPI_UNSIGNED; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_uint(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_uint(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_uint_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_vara_uint_all(file->fh, varid, start, count, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long long *buf) +int PIOc_get_vars_longlong (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, long long *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_LONGLONG; - ibuftype = MPI_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_LONGLONG; + ibuftype = MPI_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_longlong(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_longlong(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vars_longlong_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_vars_longlong_all(file->fh, varid, start, count, stride, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } int PIOc_get_var_longlong (int ncid, int varid, long long *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_LONGLONG; - ibuftype = MPI_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_LONGLONG; + ibuftype = MPI_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_longlong(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_longlong(file->fh, varid, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_longlong(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_longlong(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_longlong(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_longlong(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var_longlong_all(file->fh, varid, buf);; + ierr = ncmpi_get_var_longlong_all(file->fh, varid, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], short *buf) +int PIOc_get_vara_short (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, short *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_SHORT; - ibuftype = MPI_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_SHORT; + ibuftype = MPI_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_short(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_short(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_short_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_vara_short_all(file->fh, varid, start, count, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long *buf) +int PIOc_get_vara_long (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, long *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_LONG; - ibuftype = MPI_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_LONG; + ibuftype = MPI_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_long(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_long(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_long_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_vara_long_all(file->fh, varid, start, count, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var1_int (int ncid, int varid, const PIO_Offset index[], int *buf) +int PIOc_get_var1_int (int ncid, int varid, const PIO_Offset *index, int *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_INT; - ibuftype = MPI_INT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_INT; + ibuftype = MPI_INT; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_int(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_int(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var1_int_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_var1_int_all(file->fh, varid, index, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], unsigned long long *buf) +int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset *index, unsigned long long *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_ulonglong(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_ulonglong(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var1_ulonglong_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_var1_ulonglong_all(file->fh, varid, index, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } int PIOc_get_var_uchar (int ncid, int varid, unsigned char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_uchar(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_uchar(file->fh, varid, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_uchar(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_uchar(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_uchar(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_uchar(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var_uchar_all(file->fh, varid, buf);; + ierr = ncmpi_get_var_uchar_all(file->fh, varid, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned char *buf) +int PIOc_get_vara_uchar (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, unsigned char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_uchar(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_uchar(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_uchar_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_vara_uchar_all(file->fh, varid, start, count, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], float *buf) +int PIOc_get_vars_float (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, float *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_FLOAT; - ibuftype = MPI_FLOAT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_FLOAT; + ibuftype = MPI_FLOAT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_float(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_float(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vars_float_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_vars_float_all(file->fh, varid, start, count, stride, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long *buf) +int PIOc_get_vars_long (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, long *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_LONG; - ibuftype = MPI_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_LONG; + ibuftype = MPI_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_long(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_long(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vars_long_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_vars_long_all(file->fh, varid, start, count, stride, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var1 (int ncid, int varid, const PIO_Offset index[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +int PIOc_get_var1 (int ncid, int varid, const PIO_Offset *index, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1(file->fh, varid, index, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1(file->fh, varid, index, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var1_all(file->fh, varid, index, buf, bufcount, buftype);; + ierr = ncmpi_get_var1_all(file->fh, varid, index, buf, bufcount, buftype);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_UINT; - ibuftype = MPI_UNSIGNED; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_uint(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_uint(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_uint(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_uint_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_UINT; + ibuftype = MPI_UNSIGNED; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; } - } + ierr = PIO_NOERR; - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_uint(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_uint(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara(file->fh, varid, start, count, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_uint(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_all(file->fh, varid, start, count, buf, bufcount, buftype);; + ierr = ncmpi_get_var_uint_all(file->fh, varid, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], signed char *buf) +int PIOc_get_vara (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_SCHAR; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_schar(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara(file->fh, varid, start, count, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_schar_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_vara_all(file->fh, varid, start, count, buf, bufcount, buftype);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset index[], unsigned int *buf) +int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, signed char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_UINT; - ibuftype = MPI_UNSIGNED; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_uint(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_uint_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_SCHAR; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_UINT; - ibuftype = MPI_UNSIGNED; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_uint(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_schar(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vars_uint_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_vara_schar_all(file->fh, varid, start, count, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], float *buf) +int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset *index, unsigned int *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_FLOAT; - ibuftype = MPI_FLOAT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_UINT; + ibuftype = MPI_UNSIGNED; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_float(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_uint(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_float_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_var1_uint_all(file->fh, varid, index, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], char *buf) +int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, unsigned int *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_TEXT; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_text(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_text_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_UINT; + ibuftype = MPI_UNSIGNED; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset index[], char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_TEXT; - ibuftype = MPI_CHAR; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_text(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_uint(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var1_text_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_vars_uint_all(file->fh, varid, start, count, stride, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], int *buf) +int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, float *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_INT; - ibuftype = MPI_INT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_int(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_int_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_FLOAT; + ibuftype = MPI_FLOAT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_UINT; - ibuftype = MPI_UNSIGNED; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_uint(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_float(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_varm_uint_all(file->fh, varid, start, count, stride, imap, buf);; + ierr = ncmpi_get_vara_float_all(file->fh, varid, start, count, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset *index, char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_TEXT; + ibuftype = MPI_CHAR; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_text(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_varm_all(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);; + ierr = ncmpi_get_var1_text_all(file->fh, varid, index, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], double *buf) +int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, double *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_DOUBLE; - ibuftype = MPI_DOUBLE; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_double(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_double_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_DOUBLE; + ibuftype = MPI_DOUBLE; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_LONGLONG; - ibuftype = MPI_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_longlong(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_double(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_longlong_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_vars_double_all(file->fh, varid, start, count, stride, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) +int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, long long *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_ulonglong(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_ulonglong(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_ulonglong(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_ulonglong_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_LONGLONG; + ibuftype = MPI_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_ulonglong(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_longlong(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vara_ulonglong_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_vara_longlong_all(file->fh, varid, start, count, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var_short (int ncid, int varid, short *buf) +int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_SHORT; - ibuftype = MPI_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_short(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_short(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_short(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_short_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; } - } + ierr = PIO_NOERR; - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], float *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_FLOAT; - ibuftype = MPI_FLOAT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_ulonglong(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_ulonglong(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_float(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_ulonglong(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_varm_float_all(file->fh, varid, start, count, stride, imap, buf);; + ierr = ncmpi_get_var_ulonglong_all(file->fh, varid, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset index[], long *buf) +int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, unsigned long long *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_LONG; - ibuftype = MPI_LONG; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_long(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_long_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARA_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_LONG; - ibuftype = MPI_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_long(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vara_ulonglong(file->fh, varid, start, count, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_varm_long_all(file->fh, varid, start, count, stride, imap, buf);; + ierr = ncmpi_get_vara_ulonglong_all(file->fh, varid, start, count, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned short *buf) +int PIOc_get_var_short (int ncid, int varid, short *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_ushort(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_varm_ushort_all(file->fh, varid, start, count, stride, imap, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_SHORT; + ibuftype = MPI_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; } - } + ierr = PIO_NOERR; - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_LONGLONG; - ibuftype = MPI_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_short(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_short(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_longlong(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_short(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_varm_longlong_all(file->fh, varid, start, count, stride, imap, buf);; + ierr = ncmpi_get_var_short_all(file->fh, varid, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], char *buf) +int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset *index, long *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_TEXT; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_LONG; + ibuftype = MPI_LONG; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_text(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_long(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vars_text_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_var1_long_all(file->fh, varid, index, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset index[], unsigned char *buf) +int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_uchar(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_uchar_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS_TEXT; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } - return ierr; -} -int PIOc_get_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars_text(file->fh, varid, start, count, stride, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_vars_all(file->fh, varid, start, count, stride, buf, bufcount, buftype);; + ierr = ncmpi_get_vars_text_all(file->fh, varid, start, count, stride, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf) +int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset *index, unsigned char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_SHORT; - ibuftype = MPI_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR1_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ibufcnt = 1; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_short(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var1_uchar(file->fh, varid, index, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_varm_short_all(file->fh, varid, start, count, stride, imap, buf);; + ierr = ncmpi_get_var1_uchar_all(file->fh, varid, index, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } -int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf) +int PIOc_get_vars (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARM_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARS; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_vars(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_varm_ulonglong(file->fh, varid, start, count, stride, imap, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_vars(file->fh, varid, start, count, stride, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_varm_ulonglong_all(file->fh, varid, start, count, stride, imap, buf);; + ierr = ncmpi_get_vars_all(file->fh, varid, start, count, stride, buf, bufcount, buftype);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } int PIOc_get_var_schar (int ncid, int varid, signed char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_SCHAR; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VAR_SCHAR; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + int dimid[ndims]; + PIO_Offset dimsize; + ibufcnt = 1; + PIOc_inq_vardimid(file->fh, varid, dimid); + for(int i=0;ifh, dimid[i], &dimsize); + ibufcnt *= dimsize; + } + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_schar(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_schar(file->fh, varid, buf);; - } - break; + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_var_schar(file->fh, varid, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_var_schar(file->fh, varid, buf);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_PNETCDF: #ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_schar(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_var_schar(file->fh, varid, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; #else - ierr = ncmpi_get_var_schar_all(file->fh, varid, buf);; + ierr = ncmpi_get_var_schar_all(file->fh, varid, buf);; #endif - break; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } - return ierr; + return ierr; } diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 9c509398ca2..be5c2676f7b 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -40,8 +40,8 @@ int inq_type_handler(iosystem_desc_t *ios) return PIO_EIO; if ((mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - LOG((2, "inq_type_handler got parameters ncid = %d datatype = %d", ncid, xtype)); + /* Handle null pointer issues. */ if (name_present) namep = name; if (size_present) @@ -51,8 +51,6 @@ int inq_type_handler(iosystem_desc_t *ios) if ((ret = PIOc_inq_type(ncid, xtype, namep, sizep))) return ret; - if (sizep) - LOG((2, "inq_type_handler size = %d", *sizep)); LOG((1, "inq_type_handler succeeded!")); return PIO_NOERR; } @@ -676,63 +674,208 @@ int put_vars_handler(iosystem_desc_t *ios) switch(xtype) { case NC_BYTE: - ierr = PIOc_put_vars_schar(ncid, varid, (size_t *)startp, (size_t *)countp, - (ptrdiff_t *)stridep, buf); + ierr = PIOc_put_vars_schar(ncid, varid, startp, countp, stridep, buf); break; case NC_CHAR: - ierr = PIOc_put_vars_schar(ncid, varid, (size_t *)startp, (size_t *)countp, - (ptrdiff_t *)stridep, buf); + ierr = PIOc_put_vars_schar(ncid, varid, startp, countp, stridep, buf); break; case NC_SHORT: - ierr = PIOc_put_vars_short(ncid, varid, (size_t *)startp, (size_t *)countp, - (ptrdiff_t *)stridep, buf); + ierr = PIOc_put_vars_short(ncid, varid, startp, countp, stridep, buf); break; case NC_INT: - ierr = PIOc_put_vars_int(ncid, varid, (size_t *)startp, (size_t *)countp, - (ptrdiff_t *)stridep, buf); + ierr = PIOc_put_vars_int(ncid, varid, startp, countp, + stridep, buf); break; case NC_FLOAT: - ierr = PIOc_put_vars_float(ncid, varid, (size_t *)startp, (size_t *)countp, - (ptrdiff_t *)stridep, buf); + ierr = PIOc_put_vars_float(ncid, varid, startp, countp, + stridep, buf); break; case NC_DOUBLE: - ierr = PIOc_put_vars_double(ncid, varid, (size_t *)startp, (size_t *)countp, - (ptrdiff_t *)stridep, buf); + ierr = PIOc_put_vars_double(ncid, varid, startp, countp, + stridep, buf); break; #ifdef _NETCDF4 case NC_UBYTE: - ierr = PIOc_put_vars_uchar(ncid, varid, (size_t *)startp, (size_t *)countp, - (ptrdiff_t *)stridep, buf); + ierr = PIOc_put_vars_uchar(ncid, varid, startp, countp, + stridep, buf); break; case NC_USHORT: - ierr = PIOc_put_vars_ushort(ncid, varid, (size_t *)startp, (size_t *)countp, - (ptrdiff_t *)stridep, buf); + ierr = PIOc_put_vars_ushort(ncid, varid, startp, countp, + stridep, buf); break; case NC_UINT: - ierr = PIOc_put_vars_uint(ncid, varid, (size_t *)startp, (size_t *)countp, - (ptrdiff_t *)stridep, buf); + ierr = PIOc_put_vars_uint(ncid, varid, startp, countp, + stridep, buf); break; case NC_INT64: - ierr = PIOc_put_vars_longlong(ncid, varid, (size_t *)startp, (size_t *)countp, - (ptrdiff_t *)stridep, buf); + ierr = PIOc_put_vars_longlong(ncid, varid, startp, countp, + stridep, buf); break; case NC_UINT64: - ierr = PIOc_put_vars_ulonglong(ncid, varid, (size_t *)startp, (size_t *)countp, - (ptrdiff_t *)stridep, buf); + ierr = PIOc_put_vars_ulonglong(ncid, varid, startp, countp, + stridep, buf); break; /* case NC_STRING: */ - /* ierr = PIOc_put_vars_string(ncid, varid, (size_t *)startp, (size_t *)countp, */ - /* (ptrdiff_t *)stridep, (void *)buf); */ + /* ierr = PIOc_put_vars_string(ncid, varid, startp, countp, */ + /* stridep, (void *)buf); */ /* break; */ /* default:*/ - /* ierr = PIOc_put_vars(ncid, varid, (size_t *)startp, (size_t *)countp, */ - /* (ptrdiff_t *)stridep, buf); */ + /* ierr = PIOc_put_vars(ncid, varid, startp, countp, */ + /* stridep, buf); */ #endif /* _NETCDF4 */ } return PIO_NOERR; } +/** Handle var get operations. This code only runs on IO tasks. + * + * @param ios pointer to the iosystem_desc_t. + * @return PIO_NOERR for success, error code otherwise. +*/ +int get_vars_handler(iosystem_desc_t *ios) +{ + int ncid; + int varid; + int mpierr; + int ierr; + char *name; + int namelen; + PIO_Offset typelen; /** Length (in bytes) of this type. */ + nc_type xtype; /** Type of the data being written. */ + char start_present, count_present, stride_present; + PIO_Offset *startp = NULL, *countp = NULL, *stridep = NULL; + int ndims; /** Number of dimensions. */ + void *buf; /** Buffer for data storage. */ + size_t num_elem; /** Number of data elements in the buffer. */ + + LOG((1, "get_vars_handler")); + + /* Get the parameters for this function that the the comp master + * task is broadcasting. */ + if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + + /* Now we know how big to make these arrays. */ + PIO_Offset start[ndims], count[ndims], stride[ndims]; + + if ((mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if (!mpierr && start_present) + { + if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm))) + return PIO_EIO; + LOG((1, "put_vars_handler getting start[0] = %d ndims = %d", start[0], ndims)); + } + if ((mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if (!mpierr && count_present) + if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm))) + return PIO_EIO; + if (!mpierr && stride_present) + if ((mpierr = MPI_Bcast(stride, ndims, MPI_OFFSET, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, 0, ios->intercomm))) + return PIO_EIO; + if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm))) + return PIO_EIO; + LOG((1, "get_vars_handler ncid = %d varid = %d ndims = %d start_present = %d " + "count_present = %d stride_present = %d xtype = %d num_elem = %d typelen = %d", + ncid, varid, ndims, start_present, count_present, stride_present, xtype, + num_elem, typelen)); + + for (int d = 0; d < ndims; d++) + { + if (start_present) + LOG((2, "start[%d] = %d\n", d, start[d])); + if (count_present) + LOG((2, "count[%d] = %d\n", d, count[d])); + if (stride_present) + LOG((2, "stride[%d] = %d\n", d, stride[d])); + } + + /* Allocate room for our data. */ + if (!(buf = malloc(num_elem * typelen))) + return PIO_ENOMEM; + + /* Set the non-NULL pointers. */ + if (start_present) + startp = start; + if (count_present) + countp = count; + if (stride_present) + stridep = stride; + + /* Call the function to read the data. */ + switch(xtype) + { + case NC_BYTE: + ierr = PIOc_get_vars_schar(ncid, varid, startp, countp, + stridep, buf); + break; + case NC_CHAR: + ierr = PIOc_get_vars_schar(ncid, varid, startp, countp, + stridep, buf); + break; + case NC_SHORT: + ierr = PIOc_get_vars_short(ncid, varid, startp, countp, + stridep, buf); + break; + case NC_INT: + ierr = PIOc_get_vars_int(ncid, varid, startp, countp, + stridep, buf); + break; + case NC_FLOAT: + ierr = PIOc_get_vars_float(ncid, varid, startp, countp, + stridep, buf); + break; + case NC_DOUBLE: + ierr = PIOc_get_vars_double(ncid, varid, startp, countp, + stridep, buf); + break; +#ifdef _NETCDF4 + case NC_UBYTE: + ierr = PIOc_get_vars_uchar(ncid, varid, startp, countp, + stridep, buf); + break; + case NC_USHORT: + ierr = PIOc_get_vars_ushort(ncid, varid, startp, countp, + stridep, buf); + break; + case NC_UINT: + ierr = PIOc_get_vars_uint(ncid, varid, startp, countp, + stridep, buf); + break; + case NC_INT64: + ierr = PIOc_get_vars_longlong(ncid, varid, startp, countp, + stridep, buf); + break; + case NC_UINT64: + ierr = PIOc_get_vars_ulonglong(ncid, varid, startp, countp, + stridep, buf); + break; + /* case NC_STRING: */ + /* ierr = PIOc_get_vars_string(ncid, varid, startp, countp, */ + /* stridep, (void *)buf); */ + /* break; */ + /* default:*/ + /* ierr = PIOc_get_vars(ncid, varid, startp, countp, */ + /* stridep, buf); */ +#endif /* _NETCDF4 */ + } + + LOG((1, "get_vars_handler succeeded!")); + return PIO_NOERR; +} + /** Do an inq_var on a netCDF variable. This function is only run on * IO tasks. * @@ -1493,7 +1636,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) inq_attid_handler(my_iosys); break; case PIO_MSG_GET_VARS: - var_handler(my_iosys, msg); + get_vars_handler(my_iosys); break; case PIO_MSG_PUT_VARS: put_vars_handler(my_iosys); diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 93b59ae2721..b05a2637ffd 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -291,8 +291,9 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) slen = strlen(name); if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); + if (!mpierr) + if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); } if (sizep) if ((mpierr = MPI_Bcast(sizep , 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) diff --git a/src/clib/pio_varm.c b/src/clib/pio_varm.c index d13b2ac41c1..3110a4333f3 100644 --- a/src/clib/pio_varm.c +++ b/src/clib/pio_varm.c @@ -990,3 +990,1001 @@ int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const return ierr; } + +int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_UCHAR; + ibuftype = MPI_UNSIGNED_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_uchar(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_uchar_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], signed char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_SCHAR; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_schar(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_schar_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], double *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_DOUBLE; + ibuftype = MPI_DOUBLE; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_double(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_double_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], char *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_TEXT; + ibuftype = MPI_CHAR; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_text(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_text_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_INT; + ibuftype = MPI_INT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_int(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_int_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned int *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_UINT; + ibuftype = MPI_UNSIGNED; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_uint(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_uint_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM; + ibufcnt = bufcount; + ibuftype = buftype; + ierr = PIO_NOERR; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_all(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], float *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_FLOAT; + ibuftype = MPI_FLOAT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_float(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_float_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_LONG; + ibuftype = MPI_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_long(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_long_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_ushort(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_ushort_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_LONGLONG; + ibuftype = MPI_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_longlong(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_longlong_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_SHORT; + ibuftype = MPI_SHORT; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_short(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_short_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + +int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf) +{ + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + MPI_Datatype ibuftype; + int ndims; + int ibufcnt; + bool bcast = false; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_GET_VARM_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; + ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + ibufcnt = 1; + for(int i=0;iasync_interface && ! ios->ioproc){ + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + + if(ios->ioproc){ + switch(file->iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + bcast = true; + if(ios->iomaster){ + ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, buf);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: +#ifdef PNET_READ_AND_BCAST + ncmpi_begin_indep_data(file->fh); + if(ios->iomaster){ + ierr = ncmpi_get_varm_ulonglong(file->fh, varid, start, count, stride, imap, buf);; + }; + ncmpi_end_indep_data(file->fh); + bcast=true; +#else + ierr = ncmpi_get_varm_ulonglong_all(file->fh, varid, start, count, stride, imap, buf);; +#endif + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + + if(ios->async_interface || bcast || + (ios->num_iotasks < ios->num_comptasks)){ + MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); + } + + return ierr; +} + diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 6cb671b6ec3..fd999ce91db 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -99,6 +99,19 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) NC_NOWRITE))) ERR(ret); + /* Try to read the data. */ + PIO_Offset start[NDIM] = {0}, count[NDIM] = {DIM_LEN}; + int data_in[DIM_LEN]; + if ((ret = PIOc_get_vars_tc(ncid, 0, start, count, NULL, NC_INT, data_in))) + ERR(ret); + for (int i = 0; i < DIM_LEN; i++) + { + if (verbose) + printf("%d test_intercomm read data_in[%d] = %d\n", my_rank, i, data_in[i]); + if (data_in[i] != i) + ERR(ERR_AWFUL); + } + /* Find the number of dimensions, variables, and global attributes.*/ if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) ERR(ret); @@ -372,7 +385,7 @@ main(int argc, char **argv) if (comp_task) { for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) -/* for (int fmt = 0; fmt < 1; fmt++) */ +/* for (int fmt = 1; fmt < 2; fmt++) */ { int ncid, varid, dimid; PIO_Offset start[NDIM], count[NDIM] = {0}; From e7eced0e159fbfa744a80bdd89ecab5f989d895e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 26 May 2016 16:07:31 -0400 Subject: [PATCH 096/184] got get_var/vara working --- src/clib/pio_get_nc_async.c | 3168 ++++------------------------------- 1 file changed, 316 insertions(+), 2852 deletions(-) diff --git a/src/clib/pio_get_nc_async.c b/src/clib/pio_get_nc_async.c index f3cbc0f5a9e..da55a082e2e 100644 --- a/src/clib/pio_get_nc_async.c +++ b/src/clib/pio_get_nc_async.c @@ -3,7 +3,7 @@ #include int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, nc_type xtype, void *buf) + const PIO_Offset *stride, nc_type xtype, void *buf) { iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ @@ -272,2630 +272,219 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off /* Send the data. */ LOG((2, "PIOc_get_vars_tc bcasting data num_elem = %d typelen = %d", num_elem, - typelen)); + typelen)); if (!mpierr) mpierr = MPI_Bcast((void *)buf, num_elem * typelen, MPI_BYTE, ios->ioroot, ios->my_comm); return ierr; } -int PIOc_get_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, - const PIO_Offset *stride, int *buf) +int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, char *buf) { - return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_INT, buf); + return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_CHAR, buf); } -int PIOc_get_var1_schar (int ncid, int varid, const PIO_Offset *index, signed char *buf) +int PIOc_get_vars_uchar(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, unsigned char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_SCHAR; - ibuftype = MPI_CHAR; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_schar(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_schar_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; + return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_UBYTE, buf); } -int PIOc_get_vars_ulonglong (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, unsigned long long *buf) +int PIOc_get_vars_schar (int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, signed char *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } + return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_BYTE, buf); +} +int PIOc_get_vars_ushort(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, unsigned short *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_USHORT, buf); +} - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_ulonglong(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_ulonglong_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } +int PIOc_get_vars_short(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, short *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_SHORT, buf); +} - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +int PIOc_get_vars_uint(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, unsigned int *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_UINT, buf); +} - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } +int PIOc_get_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, + const PIO_Offset *stride, int *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_INT, buf); +} - return ierr; +int PIOc_get_vars_long(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, long *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_LONG, buf); } -int PIOc_get_vars_short (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, short *buf) +int PIOc_get_vars_float(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, float *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; + return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_FLOAT, buf); +} - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_SHORT; - ibuftype = MPI_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } +int PIOc_get_vars_ulonglong(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, + unsigned long long *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_UINT64, buf); +} +int PIOc_get_vars_longlong (int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, const PIO_Offset *stride, long long *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_UINT64, buf); +} - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_short(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_short_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } +int PIOc_get_vara_text(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, char *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_CHAR, buf); +} - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +int PIOc_get_vara_uchar(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, unsigned char *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_UBYTE, buf); +} - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } +int PIOc_get_vara_schar(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, signed char *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_BYTE, buf); +} - return ierr; +int PIOc_get_vara_ushort(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, unsigned short *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_USHORT, buf); } -int PIOc_get_var_double (int ncid, int varid, double *buf) +int PIOc_get_vara_short(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, short *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; + return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_SHORT, buf); +} - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_DOUBLE; - ibuftype = MPI_DOUBLE; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; +int PIOc_get_vara_long(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, long *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_LONG, buf); +} - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } +int PIOc_get_vara_uint(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, unsigned int *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_UINT, buf); +} +int PIOc_get_vara_int(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, int *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_INT, buf); +} - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_double(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_double(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_double(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_double_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_double (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, double *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_DOUBLE; - ibuftype = MPI_DOUBLE; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_double(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_double(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_double_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_int (int ncid, int varid, int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_INT; - ibuftype = MPI_INT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_int(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_int(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_int(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_int_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_ushort (int ncid, int varid, unsigned short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_ushort(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_ushort(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_ushort(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_ushort_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_TEXT; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_text(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_text(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_text_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_int (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_INT; - ibuftype = MPI_INT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_int(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_int(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_int_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset *index, float *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_FLOAT; - ibuftype = MPI_FLOAT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_float(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_float_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset *index, short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_SHORT; - ibuftype = MPI_SHORT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_short(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_short_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_text (int ncid, int varid, char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_TEXT; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_text(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_text(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_text(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_text_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_schar (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, signed char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_SCHAR; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_schar(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_schar_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_ushort (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, unsigned short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_ushort(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_ushort(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_ushort_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_ushort (int ncid, int varid, const PIO_Offset *index, unsigned short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_ushort(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_ushort_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_float (int ncid, int varid, float *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_FLOAT; - ibuftype = MPI_FLOAT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_float(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_float(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_float(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_float_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_uchar (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, unsigned char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_uchar(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_uchar_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var(file->fh, varid, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_all(file->fh, varid, buf, bufcount, buftype);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset *index, long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_LONGLONG; - ibuftype = MPI_LONG_LONG; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_longlong(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_longlong_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_ushort (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, unsigned short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_ushort(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_ushort_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_long (int ncid, int varid, long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_LONG; - ibuftype = MPI_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_long(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_long(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_long(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_long_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset *index, double *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_DOUBLE; - ibuftype = MPI_DOUBLE; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_double(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_double_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_uint (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, unsigned int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_UINT; - ibuftype = MPI_UNSIGNED; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_uint(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_uint(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_uint_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_longlong (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_LONGLONG; - ibuftype = MPI_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_longlong(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_longlong_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_longlong (int ncid, int varid, long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_LONGLONG; - ibuftype = MPI_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_longlong(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_longlong(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_longlong(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_longlong_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_short (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_SHORT; - ibuftype = MPI_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_short(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_short(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_short_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_long (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_LONG; - ibuftype = MPI_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_long(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_long(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_long_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_int (int ncid, int varid, const PIO_Offset *index, int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_INT; - ibuftype = MPI_INT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_int(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_int_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset *index, unsigned long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_ulonglong(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_ulonglong_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_uchar (int ncid, int varid, unsigned char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_uchar(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_uchar(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_uchar(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_uchar_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vara_uchar (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, unsigned char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara_uchar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara_uchar(file->fh, varid, start, count, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_uchar_all(file->fh, varid, start, count, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_float (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, float *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_FLOAT; - ibuftype = MPI_FLOAT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_float(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_float(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_float_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_vars_long (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARS_LONG; - ibuftype = MPI_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vars_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vars_long(file->fh, varid, start, count, stride, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vars_long_all(file->fh, varid, start, count, stride, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1 (int ncid, int varid, const PIO_Offset *index, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +int PIOc_get_vara_float(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, float *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1(file->fh, varid, index, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_all(file->fh, varid, index, buf, bufcount, buftype);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_FLOAT, buf); +} - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +int PIOc_get_vara_double(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, double *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_DOUBLE, buf); +} - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } +int PIOc_get_vara_ulonglong(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, unsigned long long *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_UINT64, buf); +} - return ierr; +int PIOc_get_vara_longlong(int ncid, int varid, const PIO_Offset *start, + const PIO_Offset *count, long long *buf) +{ + return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_INT64, buf); +} + +int PIOc_get_var_text(int ncid, int varid, char *buf) +{ + return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_CHAR, buf); +} + +int PIOc_get_var_uchar(int ncid, int varid, unsigned char *buf) +{ + return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UBYTE, buf); +} + +int PIOc_get_var_schar(int ncid, int varid, signed char *buf) +{ + return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_BYTE, buf); +} + +int PIOc_get_var_ushort(int ncid, int varid, unsigned short *buf) +{ + return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_USHORT, buf); +} + +int PIOc_get_var_short(int ncid, int varid, short *buf) +{ + return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_SHORT, buf); +} + +int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) +{ + return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UINT, buf); +} + +int PIOc_get_var_int(int ncid, int varid, int *buf) +{ + return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_INT, buf); } -int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) +int PIOc_get_var_long (int ncid, int varid, long *buf) +{ + return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_LONG, buf); +} + +int PIOc_get_var_float(int ncid, int varid, float *buf) +{ + return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_FLOAT, buf); +} + +int PIOc_get_var_double(int ncid, int varid, double *buf) +{ + return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_DOUBLE, buf); +} + +int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) +{ + return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UINT64, buf); +} + +int PIOc_get_var_longlong (int ncid, int varid, long long *buf) +{ + return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_INT64, buf); +} + +int PIOc_get_var1_schar (int ncid, int varid, const PIO_Offset *index, signed char *buf) { int ierr; int msg; @@ -2911,21 +500,13 @@ int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VAR_UINT; - ibuftype = MPI_UNSIGNED; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; + msg = PIO_MSG_GET_VAR1_SCHAR; + ibuftype = MPI_CHAR; ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } ierr = PIO_NOERR; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -2936,14 +517,14 @@ int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_uint(file->fh, varid, buf);; + ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_var_uint(file->fh, varid, buf);; + ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -2952,12 +533,12 @@ int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_var_uint(file->fh, varid, buf);; + ierr = ncmpi_get_var1_schar(file->fh, varid, index, buf);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_var_uint_all(file->fh, varid, buf);; + ierr = ncmpi_get_var1_schar_all(file->fh, varid, index, buf);; #endif break; #endif @@ -2968,7 +549,7 @@ int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -2976,7 +557,7 @@ int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) return ierr; } -int PIOc_get_vara (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset *index, float *buf) { int ierr; int msg; @@ -2992,13 +573,13 @@ int PIOc_get_vara (int ncid, int varid, const PIO_Offset *start, const PIO_Offse if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VARA; - ibufcnt = bufcount; - ibuftype = buftype; + msg = PIO_MSG_GET_VAR1_FLOAT; + ibuftype = MPI_FLOAT; + ibufcnt = 1; ierr = PIO_NOERR; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3009,14 +590,14 @@ int PIOc_get_vara (int ncid, int varid, const PIO_Offset *start, const PIO_Offse #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -3025,12 +606,12 @@ int PIOc_get_vara (int ncid, int varid, const PIO_Offset *start, const PIO_Offse #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_vara(file->fh, varid, start, count, buf, bufcount, buftype);; + ierr = ncmpi_get_var1_float(file->fh, varid, index, buf);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_vara_all(file->fh, varid, start, count, buf, bufcount, buftype);; + ierr = ncmpi_get_var1_float_all(file->fh, varid, index, buf);; #endif break; #endif @@ -3041,7 +622,7 @@ int PIOc_get_vara (int ncid, int varid, const PIO_Offset *start, const PIO_Offse ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3049,7 +630,7 @@ int PIOc_get_vara (int ncid, int varid, const PIO_Offset *start, const PIO_Offse return ierr; } -int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, signed char *buf) +int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset *index, short *buf) { int ierr; int msg; @@ -3065,17 +646,13 @@ int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset *start, const PIO if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VARA_SCHAR; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + msg = PIO_MSG_GET_VAR1_SHORT; + ibuftype = MPI_SHORT; ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3086,14 +663,14 @@ int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset *start, const PIO #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_vara_schar(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -3102,12 +679,12 @@ int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset *start, const PIO #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_vara_schar(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_var1_short(file->fh, varid, index, buf);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_vara_schar_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_var1_short_all(file->fh, varid, index, buf);; #endif break; #endif @@ -3118,7 +695,7 @@ int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset *start, const PIO ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3126,7 +703,8 @@ int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset *start, const PIO return ierr; } -int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset *index, unsigned int *buf) + +int PIOc_get_var1_ushort (int ncid, int varid, const PIO_Offset *index, unsigned short *buf) { int ierr; int msg; @@ -3142,13 +720,13 @@ int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset *index, unsigned i if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_UINT; - ibuftype = MPI_UNSIGNED; + msg = PIO_MSG_GET_VAR1_USHORT; + ibuftype = MPI_UNSIGNED_SHORT; ibufcnt = 1; ierr = PIO_NOERR; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3159,14 +737,14 @@ int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset *index, unsigned i #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; + ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; + ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -3175,12 +753,12 @@ int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset *index, unsigned i #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_var1_uint(file->fh, varid, index, buf);; + ierr = ncmpi_get_var1_ushort(file->fh, varid, index, buf);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_var1_uint_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_var1_ushort_all(file->fh, varid, index, buf);; #endif break; #endif @@ -3191,7 +769,7 @@ int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset *index, unsigned i ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3199,7 +777,8 @@ int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset *index, unsigned i return ierr; } -int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, unsigned int *buf) + +int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -3215,17 +794,13 @@ int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset *start, const PIO_ if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VARS_UINT; - ibuftype = MPI_UNSIGNED; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3236,14 +811,14 @@ int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset *start, const PIO_ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + ierr = nc_get_var(file->fh, varid, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_vars_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + ierr = nc_get_var(file->fh, varid, buf);; } break; #endif @@ -3252,12 +827,12 @@ int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset *start, const PIO_ #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_vars_uint(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_var(file->fh, varid, buf, bufcount, buftype);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_vars_uint_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_var_all(file->fh, varid, buf, bufcount, buftype);; #endif break; #endif @@ -3268,7 +843,7 @@ int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset *start, const PIO_ ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3276,7 +851,7 @@ int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset *start, const PIO_ return ierr; } -int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, float *buf) +int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset *index, long long *buf) { int ierr; int msg; @@ -3292,17 +867,13 @@ int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset *start, const PIO if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VARA_FLOAT; - ibuftype = MPI_FLOAT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + msg = PIO_MSG_GET_VAR1_LONGLONG; + ibuftype = MPI_LONG_LONG; ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3313,14 +884,14 @@ int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset *start, const PIO #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_vara_float(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -3329,12 +900,12 @@ int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset *start, const PIO #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_vara_float(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_var1_longlong(file->fh, varid, index, buf);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_vara_float_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_var1_longlong_all(file->fh, varid, index, buf);; #endif break; #endif @@ -3345,7 +916,7 @@ int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset *start, const PIO ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3353,7 +924,7 @@ int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset *start, const PIO return ierr; } -int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset *index, char *buf) +int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset *index, double *buf) { int ierr; int msg; @@ -3369,13 +940,13 @@ int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset *index, char *buf) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_TEXT; - ibuftype = MPI_CHAR; + msg = PIO_MSG_GET_VAR1_DOUBLE; + ibuftype = MPI_DOUBLE; ibufcnt = 1; ierr = PIO_NOERR; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3386,14 +957,14 @@ int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset *index, char *buf) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; + ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; + ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -3402,12 +973,12 @@ int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset *index, char *buf) #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_var1_text(file->fh, varid, index, buf);; + ierr = ncmpi_get_var1_double(file->fh, varid, index, buf);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_var1_text_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_var1_double_all(file->fh, varid, index, buf);; #endif break; #endif @@ -3418,7 +989,7 @@ int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset *index, char *buf) ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3426,7 +997,8 @@ int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset *index, char *buf) return ierr; } -int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, double *buf) + +int PIOc_get_var1_int (int ncid, int varid, const PIO_Offset *index, int *buf) { int ierr; int msg; @@ -3442,17 +1014,13 @@ int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset *start, const PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VARS_DOUBLE; - ibuftype = MPI_DOUBLE; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + msg = PIO_MSG_GET_VAR1_INT; + ibuftype = MPI_INT; ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3463,14 +1031,14 @@ int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset *start, const PI #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_vars_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -3479,12 +1047,12 @@ int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset *start, const PI #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_vars_double(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_var1_int(file->fh, varid, index, buf);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_vars_double_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_var1_int_all(file->fh, varid, index, buf);; #endif break; #endif @@ -3495,7 +1063,7 @@ int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset *start, const PI ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3503,7 +1071,7 @@ int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset *start, const PI return ierr; } -int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, long long *buf) +int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset *index, unsigned long long *buf) { int ierr; int msg; @@ -3519,17 +1087,13 @@ int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset *start, const if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VARA_LONGLONG; - ibuftype = MPI_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + msg = PIO_MSG_GET_VAR1_ULONGLONG; + ibuftype = MPI_UNSIGNED_LONG_LONG; ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3540,14 +1104,14 @@ int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset *start, const #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_vara_longlong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -3556,12 +1120,12 @@ int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset *start, const #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_vara_longlong(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_var1_ulonglong(file->fh, varid, index, buf);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_vara_longlong_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_var1_ulonglong_all(file->fh, varid, index, buf);; #endif break; #endif @@ -3572,7 +1136,7 @@ int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset *start, const ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3580,7 +1144,8 @@ int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset *start, const return ierr; } -int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) + +int PIOc_get_var1 (int ncid, int varid, const PIO_Offset *index, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -3596,21 +1161,13 @@ int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VAR_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } + msg = PIO_MSG_GET_VAR1; + ibufcnt = bufcount; + ibuftype = buftype; ierr = PIO_NOERR; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3621,14 +1178,14 @@ int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_ulonglong(file->fh, varid, buf);; + ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_var_ulonglong(file->fh, varid, buf);; + ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -3637,12 +1194,12 @@ int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_var_ulonglong(file->fh, varid, buf);; + ierr = ncmpi_get_var1(file->fh, varid, index, buf, bufcount, buftype);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_var_ulonglong_all(file->fh, varid, buf);; + ierr = ncmpi_get_var1_all(file->fh, varid, index, buf, bufcount, buftype);; #endif break; #endif @@ -3653,7 +1210,7 @@ int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3661,7 +1218,8 @@ int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) return ierr; } -int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, unsigned long long *buf) + +int PIOc_get_vara (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -3677,17 +1235,13 @@ int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset *start, const if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VARA_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3698,14 +1252,14 @@ int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset *start, const #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_vara_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, buf);; + ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; } break; #endif @@ -3714,12 +1268,12 @@ int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset *start, const #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_vara_ulonglong(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_vara(file->fh, varid, start, count, buf, bufcount, buftype);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_vara_ulonglong_all(file->fh, varid, start, count, buf);; + ierr = ncmpi_get_vara_all(file->fh, varid, start, count, buf, bufcount, buftype);; #endif break; #endif @@ -3730,7 +1284,7 @@ int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset *start, const ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3738,7 +1292,8 @@ int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset *start, const return ierr; } -int PIOc_get_var_short (int ncid, int varid, short *buf) + +int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset *index, unsigned int *buf) { int ierr; int msg; @@ -3754,21 +1309,13 @@ int PIOc_get_var_short (int ncid, int varid, short *buf) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VAR_SHORT; - ibuftype = MPI_SHORT; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; + msg = PIO_MSG_GET_VAR1_UINT; + ibuftype = MPI_UNSIGNED; ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } ierr = PIO_NOERR; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3779,14 +1326,14 @@ int PIOc_get_var_short (int ncid, int varid, short *buf) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_short(file->fh, varid, buf);; + ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_var_short(file->fh, varid, buf);; + ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -3795,12 +1342,12 @@ int PIOc_get_var_short (int ncid, int varid, short *buf) #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_var_short(file->fh, varid, buf);; + ierr = ncmpi_get_var1_uint(file->fh, varid, index, buf);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_var_short_all(file->fh, varid, buf);; + ierr = ncmpi_get_var1_uint_all(file->fh, varid, index, buf);; #endif break; #endif @@ -3811,7 +1358,7 @@ int PIOc_get_var_short (int ncid, int varid, short *buf) ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3819,7 +1366,7 @@ int PIOc_get_var_short (int ncid, int varid, short *buf) return ierr; } -int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset *index, long *buf) +int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset *index, char *buf) { int ierr; int msg; @@ -3835,13 +1382,13 @@ int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset *index, long *buf) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_LONG; - ibuftype = MPI_LONG; + msg = PIO_MSG_GET_VAR1_TEXT; + ibuftype = MPI_CHAR; ibufcnt = 1; ierr = PIO_NOERR; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3852,14 +1399,14 @@ int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset *index, long *buf) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; + ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; + ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -3868,12 +1415,12 @@ int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset *index, long *buf) #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_var1_long(file->fh, varid, index, buf);; + ierr = ncmpi_get_var1_text(file->fh, varid, index, buf);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_var1_long_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_var1_text_all(file->fh, varid, index, buf);; #endif break; #endif @@ -3884,7 +1431,7 @@ int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset *index, long *buf) ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3892,7 +1439,9 @@ int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset *index, long *buf) return ierr; } -int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, char *buf) + + +int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset *index, long *buf) { int ierr; int msg; @@ -3908,17 +1457,13 @@ int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset *start, const PIO_ if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VARS_TEXT; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); + msg = PIO_MSG_GET_VAR1_LONG; + ibuftype = MPI_LONG; ibufcnt = 1; - for(int i=0;iasync_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -3929,14 +1474,14 @@ int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset *start, const PIO_ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_vars_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, buf);; + ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -3945,12 +1490,12 @@ int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset *start, const PIO_ #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_vars_text(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_var1_long(file->fh, varid, index, buf);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_vars_text_all(file->fh, varid, start, count, stride, buf);; + ierr = ncmpi_get_var1_long_all(file->fh, varid, index, buf);; #endif break; #endif @@ -3961,7 +1506,7 @@ int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset *start, const PIO_ ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -3969,7 +1514,7 @@ int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset *start, const PIO_ return ierr; } -int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset *index, unsigned char *buf) +int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset *index, unsigned char *buf) { int ierr; int msg; @@ -3991,7 +1536,7 @@ int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset *index, unsigned ierr = PIO_NOERR; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -4034,7 +1579,7 @@ int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset *index, unsigned ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } @@ -4042,7 +1587,7 @@ int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset *index, unsigned return ierr; } -int PIOc_get_vars (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +int PIOc_get_vars (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -4064,7 +1609,7 @@ int PIOc_get_vars (int ncid, int varid, const PIO_Offset *start, const PIO_Offse ierr = PIO_NOERR; if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); } @@ -4107,88 +1652,7 @@ int PIOc_get_vars (int ncid, int varid, const PIO_Offset *start, const PIO_Offse ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var_schar (int ncid, int varid, signed char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR_SCHAR; - ibuftype = MPI_CHAR; - ierr = PIOc_inq_varndims(file->fh, varid, &ndims); - int dimid[ndims]; - PIO_Offset dimsize; - ibufcnt = 1; - PIOc_inq_vardimid(file->fh, varid, dimid); - for(int i=0;ifh, dimid[i], &dimsize); - ibufcnt *= dimsize; - } - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var_schar(file->fh, varid, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var_schar(file->fh, varid, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var_schar(file->fh, varid, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var_schar_all(file->fh, varid, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || + if(ios->async_interface || bcast || (ios->num_iotasks < ios->num_comptasks)){ MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); } From 696d403b85ef340408903fa2aeb0a129d90c03d0 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 26 May 2016 16:51:29 -0400 Subject: [PATCH 097/184] got get_var1 working --- src/clib/pio_get_nc_async.c | 959 ++++-------------------------------- 1 file changed, 83 insertions(+), 876 deletions(-) diff --git a/src/clib/pio_get_nc_async.c b/src/clib/pio_get_nc_async.c index da55a082e2e..610f0a6cfd3 100644 --- a/src/clib/pio_get_nc_async.c +++ b/src/clib/pio_get_nc_async.c @@ -484,153 +484,88 @@ int PIOc_get_var_longlong (int ncid, int varid, long long *buf) return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_INT64, buf); } -int PIOc_get_var1_schar (int ncid, int varid, const PIO_Offset *index, signed char *buf) +int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype, + void *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_SCHAR; - ibuftype = MPI_CHAR; - ibufcnt = 1; - ierr = PIO_NOERR; + int ierr; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } + /* Find the number of dimensions. */ + if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) + return ierr; + /* Set up count array. */ + PIO_Offset count[ndims]; + for (int c = 0; c < ndims; c++) + count[c] = 1; - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_schar(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_schar(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_schar_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + return PIOc_get_vars_tc(ncid, varid, index, count, NULL, xtype, buf); +} - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +int PIOc_get_var1_text(int ncid, int varid, const PIO_Offset *index, char *buf) +{ + return PIOc_get_var1_tc(ncid, varid, index, NC_CHAR, buf); +} - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } +int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset *index, unsigned char *buf) +{ + return PIOc_get_var1_tc(ncid, varid, index, NC_UBYTE, buf); +} - return ierr; +int PIOc_get_var1_schar(int ncid, int varid, const PIO_Offset *index, signed char *buf) +{ + return PIOc_get_var1_tc(ncid, varid, index, NC_BYTE, buf); } -int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset *index, float *buf) +int PIOc_get_var1_ushort(int ncid, int varid, const PIO_Offset *index, unsigned short *buf) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; + return PIOc_get_var1_tc(ncid, varid, index, NC_USHORT, buf); +} - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_FLOAT; - ibuftype = MPI_FLOAT; - ibufcnt = 1; - ierr = PIO_NOERR; +int PIOc_get_var1_short(int ncid, int varid, const PIO_Offset *index, short *buf) +{ + return PIOc_get_var1_tc(ncid, varid, index, NC_SHORT, buf); +} - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } +int PIOc_get_var1_uint(int ncid, int varid, const PIO_Offset *index, unsigned int *buf) +{ + return PIOc_get_var1_tc(ncid, varid, index, NC_UINT, buf); +} +int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset *index, long *buf) +{ + return PIOc_get_var1_tc(ncid, varid, index, NC_LONG, buf); +} - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_float(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_float(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_float_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } +int PIOc_get_var1_int(int ncid, int varid, const PIO_Offset *index, int *buf) +{ + return PIOc_get_var1_tc(ncid, varid, index, NC_INT, buf); +} - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +int PIOc_get_var1_float(int ncid, int varid, const PIO_Offset *index, float *buf) +{ + return PIOc_get_var1_tc(ncid, varid, index, NC_FLOAT, buf); +} - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } +int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset *index, double *buf) +{ + return PIOc_get_var1_tc(ncid, varid, index, NC_DOUBLE, buf); +} - return ierr; +int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset *index, + unsigned long long *buf) +{ + return PIOc_get_var1_tc(ncid, varid, index, NC_INT64, buf); } + -int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset *index, short *buf) +int PIOc_get_var1_longlong(int ncid, int varid, const PIO_Offset *index, + long long *buf) +{ + return PIOc_get_var1_tc(ncid, varid, index, NC_INT64, buf); +} + +int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -646,9 +581,9 @@ int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset *index, short *bu if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_SHORT; - ibuftype = MPI_SHORT; - ibufcnt = 1; + msg = PIO_MSG_GET_VAR; + ibufcnt = bufcount; + ibuftype = buftype; ierr = PIO_NOERR; if(ios->async_interface && ! ios->ioproc){ @@ -663,14 +598,14 @@ int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset *index, short *bu #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; + ierr = nc_get_var(file->fh, varid, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_var1_short(file->fh, varid, (size_t *) index, buf);; + ierr = nc_get_var(file->fh, varid, buf);; } break; #endif @@ -679,12 +614,12 @@ int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset *index, short *bu #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_var1_short(file->fh, varid, index, buf);; + ierr = ncmpi_get_var(file->fh, varid, buf, bufcount, buftype);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_var1_short_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_var_all(file->fh, varid, buf, bufcount, buftype);; #endif break; #endif @@ -704,81 +639,11 @@ int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset *index, short *bu } -int PIOc_get_var1_ushort (int ncid, int varid, const PIO_Offset *index, unsigned short *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_USHORT; - ibuftype = MPI_UNSIGNED_SHORT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_ushort(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_ushort(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_ushort_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - return ierr; -} -int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) +int PIOc_get_var1 (int ncid, int varid, const PIO_Offset *index, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -794,7 +659,7 @@ int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datat if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VAR; + msg = PIO_MSG_GET_VAR1; ibufcnt = bufcount; ibuftype = buftype; ierr = PIO_NOERR; @@ -811,14 +676,14 @@ int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datat #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var(file->fh, varid, buf);; + ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_var(file->fh, varid, buf);; + ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; } break; #endif @@ -827,12 +692,12 @@ int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datat #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_var(file->fh, varid, buf, bufcount, buftype);; + ierr = ncmpi_get_var1(file->fh, varid, index, buf, bufcount, buftype);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_var_all(file->fh, varid, buf, bufcount, buftype);; + ierr = ncmpi_get_var1_all(file->fh, varid, index, buf, bufcount, buftype);; #endif break; #endif @@ -851,7 +716,7 @@ int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datat return ierr; } -int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset *index, long long *buf) +int PIOc_get_vara (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { int ierr; int msg; @@ -867,9 +732,9 @@ int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset *index, long l if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_LONGLONG; - ibuftype = MPI_LONG_LONG; - ibufcnt = 1; + msg = PIO_MSG_GET_VARA; + ibufcnt = bufcount; + ibuftype = buftype; ierr = PIO_NOERR; if(ios->async_interface && ! ios->ioproc){ @@ -884,14 +749,14 @@ int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset *index, long l #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; + ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: bcast = true; if(ios->iomaster){ - ierr = nc_get_var1_longlong(file->fh, varid, (size_t *) index, buf);; + ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; } break; #endif @@ -900,12 +765,12 @@ int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset *index, long l #ifdef PNET_READ_AND_BCAST ncmpi_begin_indep_data(file->fh); if(ios->iomaster){ - ierr = ncmpi_get_var1_longlong(file->fh, varid, index, buf);; + ierr = ncmpi_get_vara(file->fh, varid, start, count, buf, bufcount, buftype);; }; ncmpi_end_indep_data(file->fh); bcast=true; #else - ierr = ncmpi_get_var1_longlong_all(file->fh, varid, index, buf);; + ierr = ncmpi_get_vara_all(file->fh, varid, start, count, buf, bufcount, buftype);; #endif break; #endif @@ -924,668 +789,10 @@ int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset *index, long l return ierr; } -int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset *index, double *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_DOUBLE; - ibuftype = MPI_DOUBLE; - ibufcnt = 1; - ierr = PIO_NOERR; - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_double(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_double(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_double_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - - -int PIOc_get_var1_int (int ncid, int varid, const PIO_Offset *index, int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_INT; - ibuftype = MPI_INT; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_int(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_int(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_int_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset *index, unsigned long long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_ULONGLONG; - ibuftype = MPI_UNSIGNED_LONG_LONG; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_ulonglong(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_ulonglong(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_ulonglong_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - - -int PIOc_get_var1 (int ncid, int varid, const PIO_Offset *index, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1(file->fh, varid, index, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_all(file->fh, varid, index, buf, bufcount, buftype);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - - -int PIOc_get_vara (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VARA; - ibufcnt = bufcount; - ibuftype = buftype; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_vara(file->fh, varid, (size_t *) start, (size_t *) count, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_vara(file->fh, varid, start, count, buf, bufcount, buftype);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_vara_all(file->fh, varid, start, count, buf, bufcount, buftype);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - - -int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset *index, unsigned int *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_UINT; - ibuftype = MPI_UNSIGNED; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_uint(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_uint(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_uint_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset *index, char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_TEXT; - ibuftype = MPI_CHAR; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_text(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_text(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_text_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - - - -int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset *index, long *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_LONG; - ibuftype = MPI_LONG; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_long(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_long(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_long_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} - -int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset *index, unsigned char *buf) -{ - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - MPI_Datatype ibuftype; - int ndims; - int ibufcnt; - bool bcast = false; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_GET_VAR1_UCHAR; - ibuftype = MPI_UNSIGNED_CHAR; - ibufcnt = 1; - ierr = PIO_NOERR; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); - } - - - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index, buf);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - bcast = true; - if(ios->iomaster){ - ierr = nc_get_var1_uchar(file->fh, varid, (size_t *) index, buf);; - } - break; -#endif -#ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: -#ifdef PNET_READ_AND_BCAST - ncmpi_begin_indep_data(file->fh); - if(ios->iomaster){ - ierr = ncmpi_get_var1_uchar(file->fh, varid, index, buf);; - }; - ncmpi_end_indep_data(file->fh); - bcast=true; -#else - ierr = ncmpi_get_var1_uchar_all(file->fh, varid, index, buf);; -#endif - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ios->async_interface || bcast || - (ios->num_iotasks < ios->num_comptasks)){ - MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm); - } - - return ierr; -} int PIOc_get_vars (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, void *buf, PIO_Offset bufcount, MPI_Datatype buftype) { From 1ad2762b10a1784fc2e8d5ae99ed17c45151afdf Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 27 May 2016 09:59:14 -0400 Subject: [PATCH 098/184] response to code review comments --- src/clib/pio_file.c | 36 +++++++++++---------- src/clib/pio_msg.c | 76 --------------------------------------------- 2 files changed, 20 insertions(+), 92 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index b3e8f9dc021..68ae7741a87 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -145,22 +145,26 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, break; } - // If we failed to open a file due to an incompatible type of NetCDF, try it - // once with just plain old basic NetCDF -/* #ifdef _NETCDF */ -/* if((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF)) { */ -/* if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); */ -/* // reset ierr on all tasks */ -/* ierr = PIO_NOERR; */ -/* // reset file markers for NETCDF on all tasks */ -/* file->iotype = PIO_IOTYPE_NETCDF; */ - -/* // open netcdf file serially on main task */ -/* if(ios->io_rank==0){ */ -/* ierr = nc_open(filename, file->mode, &(file->fh)); } */ - -/* } */ -/* #endif */ + // If we failed to open a file due to an incompatible type of + // NetCDF, try it once with just plain old basic NetCDF. This is a + // dangerous thing to do, since it may result in models running + // with sequential access, while the user thinks they are getting + // parallel IO. It would be better if the users who want this took + // this action in their own code. +#ifdef _NETCDF + if((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF)) { + if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); + // reset ierr on all tasks + ierr = PIO_NOERR; + // reset file markers for NETCDF on all tasks + file->iotype = PIO_IOTYPE_NETCDF; + + // open netcdf file serially on main task + if(ios->io_rank==0){ + ierr = nc_open(filename, file->mode, &(file->fh)); } + + } +#endif } /* Broadcast and check the return code. */ diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index be5c2676f7b..f506a4af03f 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -1296,82 +1296,6 @@ int delete_att_handler(iosystem_desc_t *ios) return PIO_NOERR; } -/** This function is run on the IO tasks. It reads or writes an array - * of data to a netCDF variable. - * - * @param ios pointer to the iosystem_desc_t data. - * @param msg the message sent my the comp root task. - * - * @return PIO_NOERR for success, error code otherwise. */ -int vara_handler(iosystem_desc_t *ios, int msg) -{ - int ncid; - int len, namelen; - int iotype; - char *name; - int mode; - int mpierr; - int ret; - int varid; - nc_type xtype; - int ndims; - int *dimids; - void *data; - int size_in_bytes; - PIO_Offset *count, *start; - - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - LOG((1, "%d def_var_handler comproot = %d\n", my_rank, ios->comproot)); - - if (msg == PIO_MSG_PUT_VARA) - { - /* Get the parameters for this function that the he comp master - * task is broadcasting. */ - if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) - return PIO_EIO; - if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm))) - return PIO_EIO; - if ((ret = PIOc_inq_varndims(ncid, varid, &ndims))) - return ret; - if (!(start = malloc(ndims * sizeof(int)))) - return PIO_ENOMEM; - if (!(count = malloc(ndims * sizeof(int)))) - return PIO_ENOMEM; - if ((mpierr = MPI_Bcast(start, ndims, MPI_INT, 0, ios->intercomm))) - return PIO_EIO; - if ((mpierr = MPI_Bcast(count, ndims, MPI_INT, 0, ios->intercomm))) - return PIO_EIO; - int size = 1; - for (int d = 0; d < ndims; d++) - size *= count[d]; - size_in_bytes = size * sizeof(int); - if (!(data = malloc(size_in_bytes))) - return PIO_ENOMEM; - if ((mpierr = MPI_Bcast(data, size_in_bytes, MPI_INT, 0, - ios->intercomm))) - return PIO_EIO; - LOG((2, " def_var_handler got parameters namelen = %d " - "name = %s len = %d ncid = %d", namelen, name, len, ncid)); - - /* Call the create file function. */ - if ((ret = PIOc_put_vara_int(ncid, varid, start, count, data))) - return ret; - - /* Free resources. */ - free(start); - free(count); - free(data); - } - else - { - - } - - LOG((1, "%d vara_handler succeeded!\n", my_rank)); - return PIO_NOERR; -} - /** This function is run on the IO tasks to open a netCDF file. * * @param ios pointer to the iosystem_desc_t data. From 9ebc91744053c093a9501bd74b51f9cdc9dc9155 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 27 May 2016 10:58:13 -0400 Subject: [PATCH 099/184] clean up --- src/clib/pio_file.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 68ae7741a87..c08273dcfa2 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -146,11 +146,7 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, } // If we failed to open a file due to an incompatible type of - // NetCDF, try it once with just plain old basic NetCDF. This is a - // dangerous thing to do, since it may result in models running - // with sequential access, while the user thinks they are getting - // parallel IO. It would be better if the users who want this took - // this action in their own code. + // NetCDF, try it once with just plain old basic NetCDF. #ifdef _NETCDF if((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF)) { if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); From 3a205bb949e17c39e0f9872a90d3b0ef14b218be Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 27 May 2016 10:06:07 -0600 Subject: [PATCH 100/184] turn on async for yellowstone --- ctest/CTestEnvironment-nwsc.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctest/CTestEnvironment-nwsc.cmake b/ctest/CTestEnvironment-nwsc.cmake index 29884997dc8..bc0f5194e99 100644 --- a/ctest/CTestEnvironment-nwsc.cmake +++ b/ctest/CTestEnvironment-nwsc.cmake @@ -1,7 +1,7 @@ #============================================================================== # # This file sets the environment variables needed to configure and build -# on the NCAR Wyoming Supercomputing Center systems +# on the NCAR Wyoming Supercomputing Center systems # (yellowstone/caldera/geyser). # #============================================================================== @@ -10,7 +10,7 @@ # set with existing environment variables: NETCDF, PNETCDF, HDF5, etc. # Define the extra CMake configure options -set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE") +set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_ASYNC=TRUE") # If MPISERIAL environment variable is set, then enable MPISERIAL if (DEFINED ENV{MPISERIAL}) From f1f9a06ffc253429f570b1ba8d2515a632bd77b4 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 27 May 2016 17:26:11 -0400 Subject: [PATCH 101/184] removed use of MPI_Comm_create_group --- tests/unit/test_intercomm.c | 52 ++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index fd999ce91db..fcd59edaff4 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -307,9 +307,11 @@ main(int argc, char **argv) MPIERR(ret); /* Check that a valid number of processors was specified. */ - if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || - ntasks == 8 || ntasks == 16)) + if (ntasks != 4) + { fprintf(stderr, "test_intercomm Number of processors must be exactly 4!\n"); + ERR(ERR_AWFUL); + } if (verbose) printf("%d: test_intercomm ParallelIO Library test_intercomm running on %d processors.\n", my_rank, ntasks); @@ -325,46 +327,41 @@ main(int argc, char **argv) #define COMPONENT_COUNT 1 MPI_Comm comp_comms; MPI_Comm io_comm; - MPI_Group io_group; - MPI_Group comp_group; /* Tasks 0 and 1 will be computational. Tasks 2 and 3 will be I/O * tasks. */ - - // Get the group of processes in MPI_COMM_WORLD - MPI_Group world_group; - MPI_Comm_group(MPI_COMM_WORLD, &world_group); int comp_task; + int color = my_rank < 2 ? 0 : 1; // Determine color based on row + + // Split the communicator based on the color and use the + // original rank for ordering + MPI_Comm row_comm; + MPI_Comm_split(MPI_COMM_WORLD, color, my_rank, &row_comm); + + int row_rank, row_size; + MPI_Comm_rank(row_comm, &row_rank); + MPI_Comm_size(row_comm, &row_size); + + printf("WORLD RANK: %d \t ROW RANK/SIZE: %d/%d\n", + my_rank, row_rank, row_size); if (my_rank == 0 || my_rank == 1) { /* We will define comp_comm. The io_comm will get null. */ io_comm = MPI_COMM_NULL; - int n = 2; - const int ranks[2] = {0, 1}; - - /* Construct a group with ranks 0, 1 in world_group. */ - MPI_Group_incl(world_group, n, ranks, &comp_group); - MPI_Comm_create_group(MPI_COMM_WORLD, comp_group, 0, &comp_comms); - if (verbose) - printf("%d test_intercomm included in comp_group.\n", my_rank); - + comp_comms = row_comm; comp_task = 1; + if (verbose) + printf("%d added to the comp_comm\n", my_rank); } else { /* We will define io_comm. The comp_comms array will get nulls. */ comp_comms = MPI_COMM_NULL; - int n = 2; - const int ranks[2] = {2, 3}; - - /* Construct a group with ranks 2, 3 in world_group. */ - MPI_Group_incl(world_group, n, ranks, &io_group); - MPI_Comm_create_group(MPI_COMM_WORLD, io_group, 0, &io_comm); - if (verbose) - printf("%d test_intercomm included in io_group.\n", my_rank); - + io_comm = row_comm; comp_task = 0; + if (verbose) + printf("%d added to the io_comm\n", my_rank); } /* Turn on logging. */ @@ -544,15 +541,12 @@ main(int argc, char **argv) /* Free local MPI resources. */ if (verbose) printf("%d test_intercomm Freeing local MPI resources...\n", my_rank); - MPI_Group_free(&world_group); if (comp_task) { - MPI_Group_free(&comp_group); MPI_Comm_free(&comp_comms); } else { - MPI_Group_free(&io_group); MPI_Comm_free(&io_comm); } From 30e787863e8d3044f698300fdf7ab5e49449863c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sun, 29 May 2016 17:30:04 -0400 Subject: [PATCH 102/184] took out log statement that was causing too much output --- src/clib/pio_put_nc_async.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index ba653397e0b..b980a27d849 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -149,9 +149,6 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off "count_present = %d stride_present = %d xtype = %d num_elem = %d", ncid, varid, ndims, start_present, count_present, stride_present, xtype, num_elem)); - for (int e = 0; e < num_elem; e++) - LOG((2, "PIOc_put_vars_tc element %d = %d", e, ((int *)buf)[e])); - /* Send the data. */ if (!mpierr) mpierr = MPI_Bcast((void *)buf, num_elem * typelen, MPI_BYTE, ios->compmaster, From a9c9029653759743db1233c1d38276f0352f55c9 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 30 May 2016 07:56:58 -0400 Subject: [PATCH 103/184] took out logging statement --- src/clib/pio_msg.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index f506a4af03f..7b110f3ae44 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -659,8 +659,8 @@ int put_vars_handler(iosystem_desc_t *ios) if ((mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, 0, ios->intercomm))) return PIO_EIO; - for (int e = 0; e < num_elem; e++) - LOG((2, "element %d = %d", e, ((int *)buf)[e])); + /* for (int e = 0; e < num_elem; e++) */ + /* LOG((2, "element %d = %d", e, ((int *)buf)[e])); */ /* Set the non-NULL pointers. */ if (start_present) From 30be388cb3e57a9c4a721bc08457eefd3811e699 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 31 May 2016 14:35:38 -0400 Subject: [PATCH 104/184] trying to fix problem with put on some platforms --- src/clib/pio_put_nc_async.c | 5 ++++- src/flib/pio.F90 | 1 + src/flib/pio_nf.F90 | 38 +++++++++++++++++++++++++++++++++---- tests/unit/basic_tests.F90 | 2 ++ 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index b980a27d849..4c760ac60f2 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -156,9 +156,12 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off } /* Handle MPI errors. */ + LOG((2, "checking mpierr = %d", mpierr)); if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + check_mpi(file, mpierr, __FILE__, __LINE__); + LOG((2, "checked mpierr = %d", mpierr)); /* /\* Broadcast values currently only known on computation tasks to IO tasks. *\/ */ /* if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm))) */ diff --git a/src/flib/pio.F90 b/src/flib/pio.F90 index 2f3781f018e..c415f7a6fb3 100644 --- a/src/flib/pio.F90 +++ b/src/flib/pio.F90 @@ -53,6 +53,7 @@ module pio PIO_def_var , & PIO_def_var_deflate , & PIO_redef , & + PIO_set_log_level, & PIO_inquire_variable , & PIO_inquire_dimension, & PIO_set_chunk_cache, & diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90 index 5c798f834ec..e409f1ccd57 100644 --- a/src/flib/pio_nf.F90 +++ b/src/flib/pio_nf.F90 @@ -36,7 +36,8 @@ module pio_nf pio_get_chunk_cache , & pio_set_var_chunk_cache , & pio_get_var_chunk_cache , & - pio_redef + pio_redef , & + pio_set_log_level ! pio_copy_att to be done interface pio_def_var @@ -187,6 +188,11 @@ module pio_nf redef_id end interface + interface pio_set_log_level + module procedure & + set_log_level + end interface + interface pio_inquire module procedure & inquire_desc , & @@ -648,18 +654,42 @@ integer function redef_desc(File) result(ierr) type (File_desc_t) , intent(inout) :: File ierr = redef_id(file%fh) end function redef_desc + +!> +!! @defgroup PIO_set_log_level +!< +!> +!! @public +!! @ingroup PIO_set_log_level +!! @brief Sets the logging level. Only takes effect if PIO was built with +!! PIO_ENABLE_LOGGING=On +!! @details +!! @param log_level the logging level. +!! @retval ierr @copydoc error_return +!< + integer function set_log_level(log_level) result(ierr) + integer, intent(in) :: log_level + interface + integer(C_INT) function PIOc_set_log_level(log_level) & + bind(C ,name="PIOc_set_log_level") + use iso_c_binding + integer(C_INT), value :: log_level + end function PIOc_set_log_level + end interface + ierr = PIOc_set_log_level(log_level) + end function set_log_level !> !! @public !! @ingroup PIO_redef !! @brief Wrapper for the C function \ref PIOc_redef . !< integer function redef_id(ncid) result(ierr) - integer ,intent(in) :: ncid + integer, intent(in) :: ncid interface integer(C_INT) function PIOc_redef(ncid) & - bind(C ,name="PIOc_redef") + bind(C, name="PIOc_redef") use iso_c_binding - integer(C_INT) , value :: ncid + integer(C_INT), value :: ncid end function PIOc_redef end interface ierr = PIOc_redef(ncid) diff --git a/tests/unit/basic_tests.F90 b/tests/unit/basic_tests.F90 index de41cef69d4..c97b955d39d 100644 --- a/tests/unit/basic_tests.F90 +++ b/tests/unit/basic_tests.F90 @@ -276,6 +276,7 @@ Subroutine test_open(test_id, err_msg) call mpi_abort(MPI_COMM_WORLD,0,ret_val) end if + PIO_set_log_level(2) ret_val = PIO_inq_unlimdim(pio_file, unlimdimid) if(unlimdimid /= -1) then err_msg = "Error in inq_unlimdim" @@ -283,6 +284,7 @@ Subroutine test_open(test_id, err_msg) print *,__FILE__,__LINE__,iotype, trim(err_msg) call mpi_abort(MPI_COMM_WORLD,0,ret_val) end if + PIO_set_log_level(0) From 751adeed63f53924b0b235109631e34fb3d7feb3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 31 May 2016 15:31:47 -0400 Subject: [PATCH 105/184] took out use of log in fortran test --- tests/unit/basic_tests.F90 | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/unit/basic_tests.F90 b/tests/unit/basic_tests.F90 index c97b955d39d..7a8d14e8bda 100644 --- a/tests/unit/basic_tests.F90 +++ b/tests/unit/basic_tests.F90 @@ -276,7 +276,6 @@ Subroutine test_open(test_id, err_msg) call mpi_abort(MPI_COMM_WORLD,0,ret_val) end if - PIO_set_log_level(2) ret_val = PIO_inq_unlimdim(pio_file, unlimdimid) if(unlimdimid /= -1) then err_msg = "Error in inq_unlimdim" @@ -284,8 +283,6 @@ Subroutine test_open(test_id, err_msg) print *,__FILE__,__LINE__,iotype, trim(err_msg) call mpi_abort(MPI_COMM_WORLD,0,ret_val) end if - PIO_set_log_level(0) - ! Close file From 5d1dba08d3d227b81d9dd321327929ad3e4269b5 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 1 Jun 2016 17:26:33 -0400 Subject: [PATCH 106/184] fixed error handling of MPI errors on transfer of parameter data to pio_msg --- src/clib/pio_get_nc_async.c | 5 +- src/clib/pio_msg.c | 20 +++--- src/clib/pio_nc_async.c | 123 ++++++++++++++++++++++-------------- src/clib/pio_put_nc_async.c | 13 ++-- 4 files changed, 91 insertions(+), 70 deletions(-) diff --git a/src/clib/pio_get_nc_async.c b/src/clib/pio_get_nc_async.c index 610f0a6cfd3..5536b18d614 100644 --- a/src/clib/pio_get_nc_async.c +++ b/src/clib/pio_get_nc_async.c @@ -119,9 +119,10 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); /* Broadcast values currently only known on computation tasks to IO tasks. */ if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 7b110f3ae44..0e6ee373c2d 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -895,9 +895,7 @@ int inq_var_handler(iosystem_desc_t *ios) int ndims, dimids[NC_MAX_DIMS], natts; int ret; - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - LOG((1, "%d inq_var_handler\n", my_rank)); + LOG((1, "inq_var_handler")); /* Get the parameters for this function that the the comp master * task is broadcasting. */ @@ -915,9 +913,9 @@ int inq_var_handler(iosystem_desc_t *ios) return PIO_EIO; if ((mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - printf("%d inq_var_handler ncid = %d varid = %d name_present = %d xtype_present = %d ndims_present = %d " - "dimids_present = %d natts_present = %d\n", - my_rank, ncid, varid, name_present, xtype_present, ndims_present, dimids_present, natts_present); + LOG((2,"inq_var_handler ncid = %d varid = %d name_present = %d xtype_present = %d ndims_present = %d " + "dimids_present = %d natts_present = %d\n", + ncid, varid, name_present, xtype_present, ndims_present, dimids_present, natts_present)); /* Set the non-NULL pointers. */ if (name_present) @@ -935,6 +933,9 @@ int inq_var_handler(iosystem_desc_t *ios) if ((ret = PIOc_inq_var(ncid, varid, namep, xtypep, ndimsp, dimidsp, nattsp))) return ret; + if (ndims_present) + LOG((2, "inq_var_handler ndims = %d", ndims)); + return PIO_NOERR; } @@ -985,15 +986,13 @@ int sync_file_handler(iosystem_desc_t *ios) int mpierr; int ret; - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - LOG((1, "%d sync_file_handler\n", my_rank)); + LOG((1, "sync_file_handler")); /* Get the parameters for this function that the comp master * task is broadcasting. */ if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm))) return PIO_EIO; - LOG((1, "%d sync_file_handler got parameter ncid = %d\n", my_rank, ncid)); + LOG((1, "sync_file_handler got parameter ncid = %d", ncid)); /* Call the sync file function. */ if ((ret = PIOc_sync(ncid))) @@ -1786,6 +1785,7 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, int iam; int io_leader, comp_leader; int root; + MPI_Group io_grp, comm_grp, union_grp; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index b05a2637ffd..19070f778f3 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -77,9 +77,10 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -257,9 +258,10 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -339,9 +341,10 @@ int PIOc_inq_format (int ncid, int *formatp) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -427,9 +430,10 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -545,9 +549,10 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* IO tasks call the netCDF functions. */ @@ -644,9 +649,10 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* Call the netCDF layer. */ @@ -670,6 +676,9 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, #endif /* _NETCDF */ } + if (ndimsp) + LOG((2, "PIOc_inq_var ndims = %d", *ndimsp)); + /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); @@ -697,6 +706,7 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); file->varlist[varid].ndims = (*ndimsp); + LOG((2, "PIOc_inq_var Bcast ndims = %d", *ndimsp)); } if (dimidsp) { @@ -811,9 +821,10 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -909,9 +920,10 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -1020,9 +1032,10 @@ int PIOc_inq_attname(int ncid, int varid, int attnum, char *name) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -1120,9 +1133,10 @@ int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -1213,9 +1227,10 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -1300,9 +1315,10 @@ int PIOc_rename_var(int ncid, int varid, const char *name) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -1394,9 +1410,10 @@ int PIOc_rename_att (int ncid, int varid, const char *name, } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -1478,9 +1495,10 @@ int PIOc_del_att(int ncid, int varid, const char *name) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + check_mpi(file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -1551,9 +1569,10 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } @@ -1616,9 +1635,10 @@ int pioc_change_def(int ncid, int is_enddef) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -1743,9 +1763,10 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -1843,9 +1864,10 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -1930,9 +1952,10 @@ int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. */ @@ -2052,9 +2075,10 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); /* Broadcast values currently only known on computation tasks to IO tasks. */ LOG((2, "PIOc_get_att bcast from comproot = %d attlen = %d typelen = %d", ios->comproot, attlen, typelen)); @@ -2175,9 +2199,10 @@ int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, } /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); - check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); /* Broadcast values currently only known on computation tasks to IO tasks. */ LOG((2, "PIOc_put_att bcast from comproot = %d typelen = %d", ios->comproot, typelen)); diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 4c760ac60f2..3e76eb4396f 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -156,18 +156,13 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off } /* Handle MPI errors. */ - LOG((2, "checking mpierr = %d", mpierr)); - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + LOG((2, "PIOc_put_vars_tc bcasting mpierr = %d", mpierr)); + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) return check_mpi(file, mpierr2, __FILE__, __LINE__); + LOG((2, "PIOc_put_vars_tc checking mpierr = %d", mpierr)); if (mpierr) check_mpi(file, mpierr, __FILE__, __LINE__); - LOG((2, "checked mpierr = %d", mpierr)); - - /* /\* Broadcast values currently only known on computation tasks to IO tasks. *\/ */ - /* if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm))) */ - /* check_mpi(file, mpierr, __FILE__, __LINE__); */ - /* if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) */ - /* check_mpi(file, mpierr, __FILE__, __LINE__); */ + LOG((2, "PIOc_put_vars_tc checked mpierr = %d", mpierr)); } /* If this is an IO task, then call the netCDF function. */ From 7b92cdb5454c78269b968a17e72f4e6af0400981 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 2 Jun 2016 06:30:39 -0400 Subject: [PATCH 107/184] more log messages to find bug --- src/clib/pio_msg.c | 8 +++----- src/clib/pio_put_nc_async.c | 9 +++++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 0e6ee373c2d..d8fc89da06f 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -236,9 +236,7 @@ int inq_dim_handler(iosystem_desc_t *ios, int msg) int mpierr; int ret; - int my_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - LOG((1, "inq_dim_handler\n", my_rank)); + LOG((1, "inq_dim_handler")); /* Get the parameters for this function that the the comp master * task is broadcasting. */ @@ -250,8 +248,8 @@ int inq_dim_handler(iosystem_desc_t *ios, int msg) return PIO_EIO; if ((mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, 0, ios->intercomm))) return PIO_EIO; - printf("%d inq_handler name_present = %d len_present = %d\n", - name_present, len_present); + LOG((2, "inq_handler name_present = %d len_present = %d", name_present, + len_present)); /* Set the non-null pointers. */ if (name_present) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 3e76eb4396f..3fde2c0dde8 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -156,10 +156,8 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off } /* Handle MPI errors. */ - LOG((2, "PIOc_put_vars_tc bcasting mpierr = %d", mpierr)); if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) return check_mpi(file, mpierr2, __FILE__, __LINE__); - LOG((2, "PIOc_put_vars_tc checking mpierr = %d", mpierr)); if (mpierr) check_mpi(file, mpierr, __FILE__, __LINE__); LOG((2, "PIOc_put_vars_tc checked mpierr = %d", mpierr)); @@ -171,6 +169,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) { + LOG((2, "PIOc_put_vars_tc calling pnetcdf function")); vdesc = file->varlist + varid; if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0) vdesc->request = realloc(vdesc->request, @@ -214,6 +213,9 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off #endif /* _PNETCDF */ #ifdef _NETCDF if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + { + LOG((2, "PIOc_put_vars_tc calling netcdf function file->iotype = %d", + file->iotype)); switch(xtype) { case NC_BYTE: @@ -270,14 +272,17 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off (ptrdiff_t *)stride, buf); #endif /* _NETCDF4 */ } + } #endif /* _NETCDF */ } /* Broadcast and check the return code. */ + LOG((2, "PIOc_put_vars_tc bcasting netcdf return code %d", ierr)); if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); if (ierr) return check_netcdf(file, ierr, __FILE__, __LINE__); + LOG((2, "PIOc_put_vars_tc bcast netcdf return code %d complete", ierr)); return ierr; } From 0cd713cd614eff1803d6ea96cbf26862c1debdbb Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 3 Jun 2016 11:28:59 -0400 Subject: [PATCH 108/184] added temporary mpi_intercomm_merge() function for MPI_SERIAL builds --- CMakeLists.txt | 6 ++++++ src/clib/pio_msg.c | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b13a09dbebb..ecf7c2ad9e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,12 @@ else() set(ENABLE_LOGGING 0) endif() +if(PIO_USE_MPISERIAL) + set(USE_MPI_SERIAL 1) +else() + set(USE_MPI_SERIAL 0) +endif() + #===== Library Variables ===== set (PIO_FILESYSTEM_HINTS IGNORE CACHE STRING "Filesystem hints (lustre or gpfs)") diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index d8fc89da06f..6207a9fd273 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -11,6 +11,17 @@ #include #include +/* MPI serial builds stub out MPI functions so that the MPI code can + * work on one processor. This function is missing from our serial MPI + * implementation, so it is included here. This can be removed after + * it is added to the MPI serial library. */ +#ifdef USE_MPI_SERIAL +int MPI_Intercomm_merge(MPI_Comm intercomm, int high, MPI_Comm *newintracomm) +{ + return MPI_SUCCESS; +} +#endif /* USE_MPI_SERIAL */ + #ifdef PIO_ENABLE_LOGGING extern int my_rank; extern int pio_log_level; @@ -1779,14 +1790,11 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, iosystem_desc_t *my_iosys; int ierr = PIO_NOERR; int mpierr; - int my_rank; int iam; int io_leader, comp_leader; int root; MPI_Group io_grp, comm_grp, union_grp; - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - /* Allocate struct to hold io system info for each component. */ if (!(iosys = (iosystem_desc_t *) calloc(1, sizeof(iosystem_desc_t) * component_count))) ierr = PIO_ENOMEM; @@ -2069,6 +2077,9 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, my_iosys->async_interface = true; /* For debug purposes, print the contents of the struct. */ + /*int my_rank;*/ + /* MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);*/ + /* for (int t = 0; t < my_iosys->num_iotasks + my_iosys->num_comptasks; t++) */ /* { */ /* MPI_Barrier(my_iosys->union_comm); */ @@ -2090,7 +2101,6 @@ int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, /* If there was an error, make sure all tasks see it. */ if (ierr) { - /*mpierr = MPI_Bcast(&ierr, 1, MPI_INT, iosys->my_rank, iosys->intercomm);*/ mpierr = MPI_Bcast(&ierr, 1, MPI_INT, 0, iosys->intercomm); CheckMPIReturn(mpierr, __FILE__, __LINE__); if (mpierr) From 8585feb03caae9943ae3ad0249ca7e200da6c2d6 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 3 Jun 2016 13:07:42 -0400 Subject: [PATCH 109/184] temporarily turned off pnetcdf in test_intercomm.c --- tests/unit/test_intercomm.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index fcd59edaff4..c39e44f677b 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -381,8 +381,7 @@ main(int argc, char **argv) * and when the do, they should go straight to finalize. */ if (comp_task) { - for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) -/* for (int fmt = 1; fmt < 2; fmt++) */ + for (int fmt = 1; fmt < NUM_NETCDF_FLAVORS; fmt++) { int ncid, varid, dimid; PIO_Offset start[NDIM], count[NDIM] = {0}; From b62a9bb22e2763bfaa18716a2a74011ce141c9f6 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 3 Jun 2016 13:50:58 -0400 Subject: [PATCH 110/184] more logging statements --- src/clib/pio_put_nc_async.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 3fde2c0dde8..b1df3a9cc93 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -175,6 +175,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off vdesc->request = realloc(vdesc->request, sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)); request = vdesc->request+vdesc->nreqs; + LOG((2, "PIOc_put_vars_tc request = %d", vdesc->request)); if(ios->io_rank == 0) switch(xtype) @@ -208,6 +209,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off *request = PIO_REQ_NULL; vdesc->nreqs++; + LOG((2, "PIOc_put_vars_tc flushing output buffer")); flush_output_buffer(file, false, 0); } #endif /* _PNETCDF */ From 4ff0a3682ba7ab5a488da3371f5708bc18d7a53c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 3 Jun 2016 14:26:37 -0400 Subject: [PATCH 111/184] commented out extra declaration --- src/clib/pio_msg.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 6207a9fd273..385450cdba1 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -15,12 +15,12 @@ * work on one processor. This function is missing from our serial MPI * implementation, so it is included here. This can be removed after * it is added to the MPI serial library. */ -#ifdef USE_MPI_SERIAL -int MPI_Intercomm_merge(MPI_Comm intercomm, int high, MPI_Comm *newintracomm) -{ - return MPI_SUCCESS; -} -#endif /* USE_MPI_SERIAL */ +/* #ifdef USE_MPI_SERIAL */ +/* int MPI_Intercomm_merge(MPI_Comm intercomm, int high, MPI_Comm *newintracomm) */ +/* { */ +/* return MPI_SUCCESS; */ +/* } */ +/* #endif /\* USE_MPI_SERIAL *\/ */ #ifdef PIO_ENABLE_LOGGING extern int my_rank; From 4e50dd072eefe5d84e4009bb751b074d4b1d32aa Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sat, 4 Jun 2016 07:11:39 -0400 Subject: [PATCH 112/184] changes to fix cdash tests, added logging to fortran --- src/clib/pio_nc_async.c | 33 +++++++++++++++++++++++++++------ src/flib/pio_nf.F90 | 16 ++++++++-------- tests/unit/basic_tests.F90 | 12 ++++++++++-- tests/unit/driver.F90 | 1 + tests/unit/test_intercomm.c | 3 +++ 5 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 19070f778f3..27146747c06 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -74,6 +74,8 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); + LOG((2, "PIOc_inq ncid = %d ndims_present = %d nvars_present = %d ngatts_present = %d unlimdimid_present = %d", + ncid, ndims_present, nvars_present, ngatts_present, unlimdimid_present)); } /* Handle MPI errors. */ @@ -88,16 +90,27 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, { #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + { + LOG((2, "PIOc_inq calling ncmpi_inq unlimdimidp = %d", unlimdimidp)); + ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + LOG((2, "PIOc_inq called ncmpi_inq")); + if (unlimdimidp) + LOG((2, "PIOc_inq returned from ncmpi_inq unlimdimid = %d", *unlimdimidp)); + } #endif /* _PNETCDF */ #ifdef _NETCDF if (file->iotype == PIO_IOTYPE_NETCDF && file->do_io) { + LOG((2, "PIOc_inq calling classic nc_inq")); /* Should not be necessary to do this - nc_inq should * handle null pointers. This has been reported as a bug * to netCDF developers. */ int tmp_ndims, tmp_nvars, tmp_ngatts, tmp_unlimdimid; + LOG((2, "PIOc_inq calling classic nc_inq")); ierr = nc_inq(ncid, &tmp_ndims, &tmp_nvars, &tmp_ngatts, &tmp_unlimdimid); + LOG((2, "PIOc_inq calling classic nc_inq")); + if (unlimdimidp) + LOG((2, "classic tmp_unlimdimid = %d", tmp_unlimdimid)); if (ndimsp) *ndimsp = tmp_ndims; if (nvarsp) @@ -106,16 +119,23 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, *ngattsp = tmp_ngatts; if (unlimdimidp) *unlimdimidp = tmp_unlimdimid; - } else if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + if (unlimdimidp) + LOG((2, "classic unlimdimid = %d", *unlimdimidp)); + } + else if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + { + LOG((2, "PIOc_inq calling netcdf-4 nc_inq")); + ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp); + } #endif /* _NETCDF */ LOG((2, "PIOc_inq netcdf call returned %d", ierr)); } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - check_netcdf(file, ierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); + if (mpierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ if (!ierr) @@ -174,7 +194,8 @@ int PIOc_inq_natts(int ncid, int *ngattsp) */ int PIOc_inq_unlimdim(int ncid, int *unlimdimidp) { - return PIOc_inq(ncid, NULL, NULL, unlimdimidp, NULL); + LOG((1, "PIOc_inq_unlimdim ncid = %d unlimdimidp = %d", ncid, unlimdimidp)); + return PIOc_inq(ncid, NULL, NULL, NULL, unlimdimidp); } /** Internal function to provide inq_type function for pnetcdf. */ diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90 index e409f1ccd57..8aa74d06f9f 100644 --- a/src/flib/pio_nf.F90 +++ b/src/flib/pio_nf.F90 @@ -181,7 +181,8 @@ module pio_nf module procedure & enddef_desc , & enddef_id - end interface + end interface pio_enddef + interface pio_redef module procedure & redef_desc , & @@ -191,7 +192,7 @@ module pio_nf interface pio_set_log_level module procedure & set_log_level - end interface + end interface pio_set_log_level interface pio_inquire module procedure & @@ -659,11 +660,10 @@ end function redef_desc !! @defgroup PIO_set_log_level !< !> -!! @public !! @ingroup PIO_set_log_level -!! @brief Sets the logging level. Only takes effect if PIO was built with +!! Sets the logging level. Only takes effect if PIO was built with !! PIO_ENABLE_LOGGING=On -!! @details +!! !! @param log_level the logging level. !! @retval ierr @copydoc error_return !< @@ -671,7 +671,7 @@ integer function set_log_level(log_level) result(ierr) integer, intent(in) :: log_level interface integer(C_INT) function PIOc_set_log_level(log_level) & - bind(C ,name="PIOc_set_log_level") + bind(C, name="PIOc_set_log_level") use iso_c_binding integer(C_INT), value :: log_level end function PIOc_set_log_level @@ -1663,7 +1663,7 @@ end function set_chunk_cache !> !! @public -!! @ingroup PIO_set_chunk_cache +!! @ingroup PIO_get_chunk_cache !! @brief Gets current settings for chunk cache (only relevant for netCDF4/HDF5 files.) !< integer function get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, & @@ -1693,7 +1693,7 @@ end function get_chunk_cache !> !! @public -!! @ingroup PIO_set_chunk_cache +!! @ingroup PIO_set_var_chunk_cache !! @brief Changes chunk cache settings for a variable in a netCDF-4/HDF5 file. !< integer function set_var_chunk_cache_id(file, varid, chunk_cache_size, & diff --git a/tests/unit/basic_tests.F90 b/tests/unit/basic_tests.F90 index 7a8d14e8bda..6988301e4c3 100644 --- a/tests/unit/basic_tests.F90 +++ b/tests/unit/basic_tests.F90 @@ -147,6 +147,12 @@ Subroutine test_open(test_id, err_msg) integer :: unlimdimid type(var_desc_t) :: pio_var + ! These will be used to set chunk cache sizes in netCDF-4/HDF5 + ! files. + integer(kind=PIO_OFFSET_KIND) :: chunk_cache_size + integer(kind=PIO_OFFSET_KIND) :: chunk_cache_nelems + real :: chunk_cache_preemption + err_msg = "no_error" dims(1) = 3*ntasks compdof = 3*my_rank+(/1,2,3/) ! Where in the global array each task writes @@ -276,15 +282,17 @@ Subroutine test_open(test_id, err_msg) call mpi_abort(MPI_COMM_WORLD,0,ret_val) end if + ret_val = PIO_set_log_level(3) ret_val = PIO_inq_unlimdim(pio_file, unlimdimid) if(unlimdimid /= -1) then err_msg = "Error in inq_unlimdim" + print *, "Error in inq_unlimdim ", unlimdimid call PIO_closefile(pio_file) print *,__FILE__,__LINE__,iotype, trim(err_msg) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + !call mpi_abort(MPI_COMM_WORLD,0,ret_val) end if + ret_val = PIO_set_log_level(0) - ! Close file call PIO_closefile(pio_file) end if diff --git a/tests/unit/driver.F90 b/tests/unit/driver.F90 index 7dbcfada7ed..f6cac452f6c 100644 --- a/tests/unit/driver.F90 +++ b/tests/unit/driver.F90 @@ -26,6 +26,7 @@ Program pio_unit_test_driver #if defined( _NETCDF4) && defined(LOGGING) integer, external :: nc_set_log_level2 #endif + integer ret_val ! Set up MPI call MPI_Init(ierr) call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index c39e44f677b..14abbcdf24b 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -139,6 +139,9 @@ check_file(int iosysid, int format, char *filename, int my_rank, int verbose) ERR(ret); if (unlimdimid != -1) ERR(ERR_WRONG); + /* Should succeed, do nothing. */ + if ((ret = PIOc_inq_unlimdim(ncid, NULL))) + ERR(ret); /* Check out the dimension. */ if ((ret = PIOc_inq_dim(ncid, 0, dimname, &dimlen))) From e42ad0b82544899d0a698e9286b42c4e8ab5b598 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Sat, 4 Jun 2016 07:23:32 -0400 Subject: [PATCH 113/184] turned pnetcdf testing back on for test_intercomm.c --- tests/unit/basic_tests.F90 | 3 +-- tests/unit/test_intercomm.c | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/basic_tests.F90 b/tests/unit/basic_tests.F90 index 6988301e4c3..5568d7b13fe 100644 --- a/tests/unit/basic_tests.F90 +++ b/tests/unit/basic_tests.F90 @@ -286,10 +286,9 @@ Subroutine test_open(test_id, err_msg) ret_val = PIO_inq_unlimdim(pio_file, unlimdimid) if(unlimdimid /= -1) then err_msg = "Error in inq_unlimdim" - print *, "Error in inq_unlimdim ", unlimdimid call PIO_closefile(pio_file) print *,__FILE__,__LINE__,iotype, trim(err_msg) - !call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD,0,ret_val) end if ret_val = PIO_set_log_level(0) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 14abbcdf24b..4b0588167d6 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -384,7 +384,7 @@ main(int argc, char **argv) * and when the do, they should go straight to finalize. */ if (comp_task) { - for (int fmt = 1; fmt < NUM_NETCDF_FLAVORS; fmt++) + for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) { int ncid, varid, dimid; PIO_Offset start[NDIM], count[NDIM] = {0}; From 2f2bb4496affa9b0bcfcd4006acfb34fd3732224 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 6 Jun 2016 03:41:23 -0600 Subject: [PATCH 114/184] tryhing to isolate put problem --- src/clib/pio_put_nc_async.c | 1 + tests/unit/test_intercomm.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index b1df3a9cc93..010f2c322df 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -211,6 +211,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off vdesc->nreqs++; LOG((2, "PIOc_put_vars_tc flushing output buffer")); flush_output_buffer(file, false, 0); + LOG((2, "PIOc_put_vars_tc flushed output buffer")); } #endif /* _PNETCDF */ #ifdef _NETCDF diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 4b0588167d6..14abbcdf24b 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -384,7 +384,7 @@ main(int argc, char **argv) * and when the do, they should go straight to finalize. */ if (comp_task) { - for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) + for (int fmt = 1; fmt < NUM_NETCDF_FLAVORS; fmt++) { int ncid, varid, dimid; PIO_Offset start[NDIM], count[NDIM] = {0}; From 378ac20fac1abb87d90a474cd8b8e01c9bcddddf Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 6 Jun 2016 03:51:06 -0600 Subject: [PATCH 115/184] tryhing to isolate put problem --- src/clib/pio_msg.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 385450cdba1..2463306dfe1 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -519,11 +519,13 @@ int att_put_handler(iosystem_desc_t *ios) /* Call the function to read the attribute. */ if ((ierr = PIOc_put_att(ncid, varid, name, atttype, attlen, op))) return ierr; + LOG((2, "put_handler called PIOc_put_att, ierr = %d", ierr)); /* Free resources. */ free(name); free(op); + LOG((2, "put_handler complete!")); return PIO_NOERR; } From b2f19be3e4f79336fa91dfd65131724e1737b96d Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 6 Jun 2016 04:19:21 -0600 Subject: [PATCH 116/184] tryhing to isolate put problem --- src/clib/pio_get_nc_async.c | 3 +++ src/clib/pio_put_nc_async.c | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/clib/pio_get_nc_async.c b/src/clib/pio_get_nc_async.c index 5536b18d614..acfaa6d248e 100644 --- a/src/clib/pio_get_nc_async.c +++ b/src/clib/pio_get_nc_async.c @@ -140,6 +140,9 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off #ifdef PNET_READ_AND_BCAST LOG((1, "PNET_READ_AND_BCAST")); ncmpi_begin_indep_data(file->fh); + + /* Only the IO master does the IO, so we are not really + * getting parallel IO here. */ if (ios->iomaster) { switch(xtype) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 010f2c322df..080779df818 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -178,6 +178,8 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off LOG((2, "PIOc_put_vars_tc request = %d", vdesc->request)); if(ios->io_rank == 0) + { + LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf call")); switch(xtype) { case NC_BYTE: @@ -203,8 +205,9 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off break; default: LOG((0, "Unknown type for pnetcdf file! xtype = %d", xtype)); - } + LOG((2, "PIOc_put_vars_tc io_rank 0 done with pnetcdf call")); + } else *request = PIO_REQ_NULL; From 8a640f371fb176aea8382229ae94bbbf0fab3014 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 6 Jun 2016 04:24:30 -0600 Subject: [PATCH 117/184] no longer run test_intercomm for serial builds --- tests/unit/CMakeLists.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 3d26a75f18c..f4f49793221 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -43,7 +43,7 @@ if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU") PRIVATE -ffree-line-length-none) endif() -if (PIO_ENABLE_ASYNC) +if (PIO_ENABLE_ASYNC AND NOT PIO_USE_MPI_SERIAL) add_executable (test_intercomm EXCLUDE_FROM_ALL test_intercomm.c) target_link_libraries (test_intercomm pioc) add_dependencies (tests test_intercomm) @@ -67,10 +67,6 @@ add_dependencies (tests pio_unit_test) set (DEFAULT_TEST_TIMEOUT 240) if (PIO_USE_MPISERIAL) - if (PIO_ENABLE_ASYNC) - add_test(NAME test_intercomm - COMMAND test_intercomm) - endif () add_test(NAME test_names COMMAND test_names) add_test(NAME test_nc4 From 5d0d400ce0b4e12656f7c3682630bd49f8b041bc Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 6 Jun 2016 15:49:55 -0600 Subject: [PATCH 118/184] changed type of num_elems from size_t to PIO_Offset --- src/clib/pio_get_nc_async.c | 2 +- src/clib/pio_msg.c | 4 ++-- src/clib/pio_put_nc_async.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/clib/pio_get_nc_async.c b/src/clib/pio_get_nc_async.c index acfaa6d248e..d7511c1bc14 100644 --- a/src/clib/pio_get_nc_async.c +++ b/src/clib/pio_get_nc_async.c @@ -12,7 +12,7 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off int ndims; /** The number of dimensions in the variable. */ int *dimids; /** The IDs of the dimensions for this variable. */ PIO_Offset typelen; /** Size (in bytes) of the data type of data in buf. */ - size_t num_elem = 1; /** Number of data elements in the buffer. */ + PIO_Offset num_elem = 1; /** Number of data elements in the buffer. */ int bcast = false; LOG((1, "PIOc_get_vars_tc ncid = %d varid = %d start = %d count = %d " diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 2463306dfe1..1bcf2dc023f 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -607,7 +607,7 @@ int put_vars_handler(iosystem_desc_t *ios) PIO_Offset *startp = NULL, *countp = NULL, *stridep = NULL; int ndims; /** Number of dimensions. */ void *buf; /** Buffer for data storage. */ - size_t num_elem; /** Number of data elements in the buffer. */ + PIO_Offset num_elem; /** Number of data elements in the buffer. */ LOG((1, "put_vars_handler")); @@ -758,7 +758,7 @@ int get_vars_handler(iosystem_desc_t *ios) PIO_Offset *startp = NULL, *countp = NULL, *stridep = NULL; int ndims; /** Number of dimensions. */ void *buf; /** Buffer for data storage. */ - size_t num_elem; /** Number of data elements in the buffer. */ + PIO_Offset num_elem; /** Number of data elements in the buffer. */ LOG((1, "get_vars_handler")); diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 080779df818..af6cdd9762e 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -42,7 +42,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off int ndims; /** The number of dimensions in the variable. */ int *dimids; /** The IDs of the dimensions for this variable. */ PIO_Offset typelen; /** Size (in bytes) of the data type of data in buf. */ - size_t num_elem = 1; /** Number of data elements in the buffer. */ + PIO_Offset num_elem = 1; /** Number of data elements in the buffer. */ var_desc_t *vdesc; PIO_Offset usage; int *request; From 611b76deb4d38e7e02f8a874a8c0f21da5e34fd8 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 7 Jun 2016 04:42:17 -0600 Subject: [PATCH 119/184] more logging to try and get enddef working on caldera --- src/clib/pio_nc_async.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 27146747c06..2feb4191b6d 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1640,7 +1640,8 @@ int pioc_change_def(int ncid, int is_enddef) if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - + LOG((2, "pioc_change_def found file")); + /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { @@ -1649,10 +1650,11 @@ int pioc_change_def(int ncid, int is_enddef) int msg = is_enddef ? PIO_MSG_ENDDEF : PIO_MSG_REDEF; if(ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + LOG((2, "pioc_change_def ncid = %d", file->fh)); } /* Handle MPI errors. */ @@ -1665,6 +1667,7 @@ int pioc_change_def(int ncid, int is_enddef) /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { + LOG((2, "pioc_change_def calling netcdf function")); #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) if (is_enddef) @@ -1679,15 +1682,15 @@ int pioc_change_def(int ncid, int is_enddef) else ierr = nc_redef(file->fh); #endif /* _NETCDF */ + LOG((2, "pioc_change_def ierr = %d", ierr)); } /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - { - check_mpi(file, mpierr, __FILE__, __LINE__); - return PIO_EIO; - } - check_netcdf(file, ierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + LOG((2, "pioc_change_def succeeded")); return ierr; } From 5651c78d5e8c5839a879be541adba91ba1918070 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 7 Jun 2016 04:50:46 -0600 Subject: [PATCH 120/184] now MPI_abort does not overwrite ret_val --- tests/unit/basic_tests.F90 | 42 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/unit/basic_tests.F90 b/tests/unit/basic_tests.F90 index 5568d7b13fe..0c07a61a911 100644 --- a/tests/unit/basic_tests.F90 +++ b/tests/unit/basic_tests.F90 @@ -33,7 +33,7 @@ Subroutine test_create(test_id, err_msg) ! Local Vars character(len=str_len) :: filename - integer :: iotype, ret_val, pio_dim + integer :: iotype, ret_val, ret_val2, pio_dim err_msg = "no_error" @@ -51,7 +51,7 @@ Subroutine test_create(test_id, err_msg) ! Error in PIO_createfile print *,' ret_val = ', ret_val err_msg = "Could not create " // trim(filename) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if call mpi_barrier(mpi_comm_world,ret_val) @@ -62,7 +62,7 @@ Subroutine test_create(test_id, err_msg) ! Error in PIO_enddef err_msg = "Could not end define mode" call PIO_closefile(pio_file) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if end if call PIO_closefile(pio_file) @@ -73,7 +73,7 @@ Subroutine test_create(test_id, err_msg) if (ret_val .ne. PIO_NOERR) then ! Error in PIO_openfile err_msg = "Could not open " // trim(filename) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if ! Close file @@ -85,7 +85,7 @@ Subroutine test_create(test_id, err_msg) if (ret_val .ne. PIO_NOERR) then ! Error in PIO_createfile err_msg = "Could not clobber " // trim(filename) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if ! Leave define mode @@ -94,7 +94,7 @@ Subroutine test_create(test_id, err_msg) ! Error in PIO_enddef err_msg = "Could not end define mode in clobbered file" call PIO_closefile(pio_file) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if ! Close file @@ -113,7 +113,7 @@ Subroutine test_create(test_id, err_msg) err_msg = "Was able to clobber file despite PIO_NOCLOBBER" ret_val = PIO_enddef(pio_file) call PIO_closefile(pio_file) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if end if @@ -137,7 +137,7 @@ Subroutine test_open(test_id, err_msg) ! Local Vars character(len=str_len) :: filename - integer :: iotype, ret_val + integer :: iotype, ret_val, ret_val2 ! Data used to test writing integer, dimension(3) :: data_buffer, compdof @@ -172,7 +172,7 @@ Subroutine test_open(test_id, err_msg) ! Error in PIO_openfile err_msg = "Successfully opened file that doesn't exist" call PIO_closefile(pio_file) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if ! Open existing file, write data to it (for binary file, need to create new file) @@ -184,7 +184,7 @@ Subroutine test_open(test_id, err_msg) if (ret_val .ne. PIO_NOERR) then ! Error in PIO_openfile (or PIO_createfile) err_msg = "Could not open " // trim(filename) // " in write mode" - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if ! Enter define mode for netcdf files @@ -194,7 +194,7 @@ Subroutine test_open(test_id, err_msg) ! Error in PIO_redef err_msg = "Could not enter redef mode" call PIO_closefile(pio_file) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if ! Define a new dimension N @@ -203,7 +203,7 @@ Subroutine test_open(test_id, err_msg) ! Error in PIO_def_dim err_msg = "Could not define dimension N" call PIO_closefile(pio_file) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if ! Define a new variable foo @@ -213,7 +213,7 @@ Subroutine test_open(test_id, err_msg) ! Error in PIO_def_var err_msg = "Could not define variable foo" call PIO_closefile(pio_file) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if ! Leave define mode @@ -223,7 +223,7 @@ Subroutine test_open(test_id, err_msg) print *,__FILE__,__LINE__,ret_val err_msg = "Could not end define mode" call PIO_closefile(pio_file) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if end if @@ -235,7 +235,7 @@ Subroutine test_open(test_id, err_msg) ! Error in PIO_write_darray err_msg = "Could not write data" call PIO_closefile(pio_file) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if ! Close file @@ -248,7 +248,7 @@ Subroutine test_open(test_id, err_msg) if (ret_val .ne. PIO_NOERR) then ! Error opening file err_msg = "Could not open file in NoWrite mode" - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if ! Try to write (should fail) @@ -260,7 +260,7 @@ Subroutine test_open(test_id, err_msg) ! Error in PIO_write_darray err_msg = "Wrote to file opened in NoWrite mode" call PIO_closefile(pio_file) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if call mpi_barrier(MPI_COMM_WORLD,ret_val) @@ -273,13 +273,13 @@ Subroutine test_open(test_id, err_msg) err_msg = "Error in read_darray" call PIO_closefile(pio_file) print *,__FILE__,__LINE__,err_msg - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if if(any(data_buffer /= my_rank)) then err_msg = "Error reading data" call PIO_closefile(pio_file) print *,__FILE__,__LINE__,iotype, trim(err_msg), data_buffer - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if ret_val = PIO_set_log_level(3) @@ -288,7 +288,7 @@ Subroutine test_open(test_id, err_msg) err_msg = "Error in inq_unlimdim" call PIO_closefile(pio_file) print *,__FILE__,__LINE__,iotype, trim(err_msg) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if ret_val = PIO_set_log_level(0) @@ -309,7 +309,7 @@ Subroutine test_open(test_id, err_msg) ! Error in PIO_openfile err_msg = "Opened a non-netcdf file as netcdf" call PIO_closefile(pio_file) - call mpi_abort(MPI_COMM_WORLD,0,ret_val) + call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) end if end if From c659512a97db1749bc117deef7b2b2859a51c936 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 7 Jun 2016 06:53:29 -0600 Subject: [PATCH 121/184] clean up error handling --- tests/unit/basic_tests.F90 | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/basic_tests.F90 b/tests/unit/basic_tests.F90 index 0c07a61a911..daad01babca 100644 --- a/tests/unit/basic_tests.F90 +++ b/tests/unit/basic_tests.F90 @@ -191,7 +191,6 @@ Subroutine test_open(test_id, err_msg) if (is_netcdf(iotype)) then ret_val = PIO_redef(pio_file) if (ret_val .ne. PIO_NOERR) then - ! Error in PIO_redef err_msg = "Could not enter redef mode" call PIO_closefile(pio_file) call mpi_abort(MPI_COMM_WORLD, 0, ret_val2) From 5c49a3a4fbafa3c5a72ee68cdabc92f8365cc56a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 7 Jun 2016 07:22:56 -0600 Subject: [PATCH 122/184] cleanup of error handling in file functions --- src/clib/pio_file.c | 821 +++++++++++++++++++++------------------- src/clib/pio_nc_async.c | 2 +- 2 files changed, 425 insertions(+), 398 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index c08273dcfa2..6e9d5c9d38d 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -16,177 +16,181 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, const char *filename, const int mode) { - int msg = PIO_MSG_OPEN_FILE; - size_t len; - iosystem_desc_t *ios; - file_desc_t *file; - int ierr = PIO_NOERR; - int mpierr = MPI_SUCCESS; - - LOG((1, "PIOc_openfile iosysid = %d", iosysid)); - - /* Get the IO system info from the iosysid. */ - if (!(ios = pio_get_iosystem_from_id(iosysid))) - { - printf("bad iosysid %d\n",iosysid); - return PIO_EBADID; - } - - /* User must provide valid input for these parameters. */ - if (!ncidp || !iotype || !filename) - return PIO_EINVAL; - if (*iotype < PIO_IOTYPE_PNETCDF || *iotype > PIO_IOTYPE_NETCDF4P) - return PIO_ENOMEM; - - /* Allocate space for the file info. */ - if (!(file = (file_desc_t *) malloc(sizeof(*file)))) - return PIO_ENOMEM; - - /* Fill in some file values. */ - file->iotype = *iotype; - file->next = NULL; - file->iosystem = ios; - file->mode = mode; - for(int i=0; ivarlist[i].record = -1; - file->varlist[i].ndims = -1; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + + LOG((1, "PIOc_openfile iosysid = %d", iosysid)); + + /* User must provide valid input for these parameters. */ + if (!ncidp || !iotype || !filename) + return PIO_EINVAL; + if (*iotype < PIO_IOTYPE_PNETCDF || *iotype > PIO_IOTYPE_NETCDF4P) + return PIO_ENOMEM; + + /* Get the IO system info from the iosysid. */ + if (!(ios = pio_get_iosystem_from_id(iosysid))) + { + LOG((0, "PIOc_openfile got bad iosysid %d",iosysid)); + return PIO_EBADID; + } + + /* Allocate space for the file info. */ + if (!(file = (file_desc_t *) malloc(sizeof(*file)))) + return PIO_ENOMEM; + + /* Fill in some file values. */ + file->iotype = *iotype; + file->next = NULL; + file->iosystem = ios; + file->mode = mode; + for (int i = 0; i < PIO_MAX_VARS; i++) + { + file->varlist[i].record = -1; + file->varlist[i].ndims = -1; #ifdef _PNETCDF - file->varlist[i].request = NULL; - file->varlist[i].nreqs=0; + file->varlist[i].request = NULL; + file->varlist[i].nreqs=0; #endif - file->varlist[i].fillbuf = NULL; - file->varlist[i].iobuf = NULL; - } - - file->buffer.validvars=0; - file->buffer.vid=NULL; - file->buffer.data=NULL; - file->buffer.next=NULL; - file->buffer.frame=NULL; - file->buffer.fillvalue=NULL; - - /** Set to true if this task should participate in IO (only true for - * one task with netcdf serial files. */ - if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF || - ios->io_rank == 0) - file->do_io = 1; - else - file->do_io = 0; + file->varlist[i].fillbuf = NULL; + file->varlist[i].iobuf = NULL; + } + + file->buffer.validvars = 0; + file->buffer.vid = NULL; + file->buffer.data = NULL; + file->buffer.next = NULL; + file->buffer.frame = NULL; + file->buffer.fillvalue = NULL; + + /** Set to true if this task should participate in IO (only true for + * one task with netcdf serial files. */ + if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF || + ios->io_rank == 0) + file->do_io = 1; + else + file->do_io = 0; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) - { - if (!ios->ioproc) - { - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - len = strlen(filename); - if (!mpierr) - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); - check_mpi(file, mpierr, __FILE__, __LINE__); - } + if (ios->async_interface) + { + int msg = PIO_MSG_OPEN_FILE; + size_t len = strlen(filename); + + if (!ios->ioproc) + { + /* Send the message to the message handler. */ + if (ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + /* Send the parameters of the function call. */ + if (!mpierr) + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); + } /* If this is an IO task, then call the netCDF function. */ - if (ios->ioproc) - { - switch (file->iotype) + if (ios->ioproc) { + switch (file->iotype) + { #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: + case PIO_IOTYPE_NETCDF4P: #ifdef _MPISERIAL - ierr = nc_open(filename, file->mode, &(file->fh)); + ierr = nc_open(filename, file->mode, &(file->fh)); #else - file->mode = file->mode | NC_MPIIO; - ierr = nc_open_par(filename, file->mode, ios->io_comm, ios->info, &file->fh); + file->mode = file->mode | NC_MPIIO; + ierr = nc_open_par(filename, file->mode, ios->io_comm, ios->info, &file->fh); #endif - break; + break; - case PIO_IOTYPE_NETCDF4C: - file->mode = file->mode | NC_NETCDF4; - // *** Note the INTENTIONAL FALLTHROUGH *** + case PIO_IOTYPE_NETCDF4C: + file->mode = file->mode | NC_NETCDF4; + // *** Note the INTENTIONAL FALLTHROUGH *** #endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_open(filename, file->mode, &file->fh); - } - break; + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_open(filename, file->mode, &file->fh); + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &file->fh); - - // This should only be done with a file opened to append - if (ierr == PIO_NOERR && (file->mode & PIO_WRITE)) - { - if(ios->iomaster) - printf("%d Setting IO buffer %ld\n",__LINE__,PIO_BUFFER_SIZE_LIMIT); - ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT ); - } - break; + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &file->fh); + + // This should only be done with a file opened to append + if (ierr == PIO_NOERR && (file->mode & PIO_WRITE)) + { + if(ios->iomaster) + LOG((1, "%d Setting IO buffer %ld", __LINE__, PIO_BUFFER_SIZE_LIMIT)); + ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT); + } + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - break; - } + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + break; + } - // If we failed to open a file due to an incompatible type of - // NetCDF, try it once with just plain old basic NetCDF. + // If we failed to open a file due to an incompatible type of + // NetCDF, try it once with just plain old basic NetCDF. #ifdef _NETCDF - if((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF)) { - if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); - // reset ierr on all tasks - ierr = PIO_NOERR; - // reset file markers for NETCDF on all tasks - file->iotype = PIO_IOTYPE_NETCDF; + if((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF)) { + if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); + // reset ierr on all tasks + ierr = PIO_NOERR; + // reset file markers for NETCDF on all tasks + file->iotype = PIO_IOTYPE_NETCDF; - // open netcdf file serially on main task - if(ios->io_rank==0){ - ierr = nc_open(filename, file->mode, &(file->fh)); } + // open netcdf file serially on main task + if(ios->io_rank==0){ + ierr = nc_open(filename, file->mode, &(file->fh)); } - } + } #endif - } - - /* Broadcast and check the return code. */ - if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return PIO_EIO; - check_netcdf(file, ierr, __FILE__, __LINE__); - - /* Broadcast results to all tasks. Ignore NULL parameters. */ - if (!ierr) - { - if ((mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm))) - return PIO_EIO; + } + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. Ignore NULL parameters. */ + if (!ierr) + { + if ((mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); - if ((mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm))) - return PIO_EIO; + if ((mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); - *ncidp = file->fh; - pio_add_to_file_list(file); - } + *ncidp = file->fh; + pio_add_to_file_list(file); + } - if (ios->io_rank==0){ - printf("Open file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr); -// if(file->fh==5) print_trace(stdout); - } + if (ios->io_rank == 0) + LOG((1, "Open file %s %d", filename, file->fh)); - return ierr; + return ierr; } /** @@ -202,131 +206,154 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, */ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, - const char filename[], const int mode) + const char filename[], const int mode) { - int ierr; - int msg; - int mpierr; - - size_t len; - iosystem_desc_t *ios; - file_desc_t *file; - - - ierr = PIO_NOERR; - - ios = pio_get_iosystem_from_id(iosysid); - file = (file_desc_t *) malloc(sizeof(file_desc_t)); - file->next = NULL; - file->iosystem = ios; - file->iotype = *iotype; - - file->buffer.validvars=0; - file->buffer.data=NULL; - file->buffer.next=NULL; - file->buffer.vid=NULL; - file->buffer.ioid=-1; - file->buffer.frame=NULL; - file->buffer.fillvalue=NULL; - - for(int i=0; ivarlist[i].record = -1; - file->varlist[i].ndims = -1; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + + /* User must provide valid input for these parameters. */ + if (!ncidp || !iotype || !filename || strlen(filename) > NC_MAX_NAME) + return PIO_EINVAL; + + /* Get the IO system info from the iosysid. */ + if (!(ios = pio_get_iosystem_from_id(iosysid))) + return PIO_EBADID; + + /* Allocate space for the file info. */ + if (!(file = (file_desc_t *)malloc(sizeof(file_desc_t)))) + return PIO_ENOMEM; + + /* Fill in some file values. */ + file->next = NULL; + file->iosystem = ios; + file->iotype = *iotype; + + file->buffer.validvars = 0; + file->buffer.data = NULL; + file->buffer.next = NULL; + file->buffer.vid = NULL; + file->buffer.ioid = -1; + file->buffer.frame = NULL; + file->buffer.fillvalue = NULL; + + for(int i = 0; i < PIO_MAX_VARS; i++) + { + file->varlist[i].record = -1; + file->varlist[i].ndims = -1; #ifdef _PNETCDF - file->varlist[i].request = NULL; - file->varlist[i].nreqs=0; + file->varlist[i].request = NULL; + file->varlist[i].nreqs=0; #endif - file->varlist[i].fillbuf = NULL; - file->varlist[i].iobuf = NULL; - } + file->varlist[i].fillbuf = NULL; + file->varlist[i].iobuf = NULL; + } - msg = PIO_MSG_CREATE_FILE; - file->mode = mode; + file->mode = mode; - /** Set to true if this task should participate in IO (only true for - * one task with netcdf serial files. */ - if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF || - ios->io_rank == 0) - file->do_io = 1; - else - file->do_io = 0; + /** Set to true if this task should participate in IO (only true for + * one task with netcdf serial files. */ + if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF || + ios->io_rank == 0) + file->do_io = 1; + else + file->do_io = 0; /* If async is in use, and this is not an IO task, bcast the parameters. */ - if (ios->async_interface) - { - if (!ios->ioproc) - { - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); - len = strlen(filename); - if (!mpierr) - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); - if (!mpierr) - mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); - } - - /* Handle MPI errors. */ - mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm); - check_mpi(file, mpierr, __FILE__, __LINE__); - } + if (ios->async_interface) + { + int msg = PIO_MSG_CREATE_FILE; + size_t len = strlen(filename); + + if (!ios->ioproc) + { + /* Send the message to the message handler. */ + if (ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + /* Send the parameters of the function call. */ + if (!mpierr) + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); + } - if(ios->ioproc){ - switch(file->iotype){ + if (ios->ioproc) + { + switch (file->iotype) + { #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - // The 64 bit options are not compatable with hdf5 format files - // printf("%d %d %d %d %d \n",__LINE__,file->mode,PIO_64BIT_DATA, PIO_64BIT_OFFSET, NC_MPIIO); - file->mode = file->mode | NC_MPIIO | NC_NETCDF4; - //printf("%s %d %d %d\n",__FILE__,__LINE__,file->mode, NC_MPIIO| NC_NETCDF4); - ierr = nc_create_par(filename, file->mode, ios->io_comm,ios->info , &(file->fh)); - break; - case PIO_IOTYPE_NETCDF4C: - file->mode = file->mode | NC_NETCDF4; + case PIO_IOTYPE_NETCDF4P: + // The 64 bit options are not compatable with hdf5 format files + // printf("%d %d %d %d %d \n",__LINE__,file->mode,PIO_64BIT_DATA, PIO_64BIT_OFFSET, NC_MPIIO); + file->mode = file->mode | NC_MPIIO | NC_NETCDF4; + //printf("%s %d %d %d\n",__FILE__,__LINE__,file->mode, NC_MPIIO| NC_NETCDF4); + ierr = nc_create_par(filename, file->mode, ios->io_comm,ios->info , &(file->fh)); + break; + case PIO_IOTYPE_NETCDF4C: + file->mode = file->mode | NC_NETCDF4; #endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_create(filename, file->mode, &(file->fh)); - } - break; + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_create(filename, file->mode, &(file->fh)); + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_create(ios->io_comm, filename, file->mode, ios->info, &(file->fh)); - if(ierr == PIO_NOERR){ - if(ios->io_rank==0){ - printf("%d Setting IO buffer size on all iotasks to %ld\n",ios->io_rank,PIO_BUFFER_SIZE_LIMIT); - } - int oldfill; - ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT ); - // ierr = ncmpi_set_fill(file->fh, NC_FILL, &oldfill); - } - break; + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_create(ios->io_comm, filename, file->mode, ios->info, &(file->fh)); + if(ierr == PIO_NOERR){ + if(ios->io_rank==0){ + printf("%d Setting IO buffer size on all iotasks to %ld\n",ios->io_rank,PIO_BUFFER_SIZE_LIMIT); + } + int oldfill; + ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT ); + // ierr = ncmpi_set_fill(file->fh, NC_FILL, &oldfill); + } + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - - if(ierr == PIO_NOERR){ - mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm); - file->mode = file->mode | PIO_WRITE; // This flag is implied by netcdf create functions but we need to know if its set - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm); - *ncidp = file->fh; - pio_add_to_file_list(file); - *ncidp = file->fh; - } - if(ios->io_rank==0){ - printf("Create file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr); -// if(file->fh==5) print_trace(stdout); - } - return ierr; + + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + + /* Broadcast results to all tasks. Ignore NULL parameters. */ + if (!ierr) + { + if ((mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + file->mode = file->mode | PIO_WRITE; // This flag is implied by netcdf create functions but we need to know if its set + + if ((mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + + *ncidp = file->fh; + pio_add_to_file_list(file); + } + + if (ios->io_rank == 0) + LOG((1, "Create file %s %d", filename, file->fh)); + + return ierr; } /** @@ -336,71 +363,71 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, */ int PIOc_closefile(int ncid) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_CLOSE_FILE; - if((file->mode & PIO_WRITE)){ - PIOc_sync(ncid); - } - - /* If async is in use and this is a comp tasks, then the compmaster - * sends a msg to the pio_msg_handler running on the IO master and - * waiting for a message. Then broadcast the ncid over the intercomm - * to the IO tasks. */ - if(ios->async_interface && !ios->ioproc){ - if(ios->comp_rank==0) { - mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_CLOSE_FILE; + if((file->mode & PIO_WRITE)){ + PIOc_sync(ncid); } - mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm); - } - if(ios->ioproc){ - switch(file->iotype){ + /* If async is in use and this is a comp tasks, then the compmaster + * sends a msg to the pio_msg_handler running on the IO master and + * waiting for a message. Then broadcast the ncid over the intercomm + * to the IO tasks. */ + if(ios->async_interface && !ios->ioproc){ + if(ios->comp_rank==0) { + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + } + mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm); + } + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_close(file->fh); - break; - case PIO_IOTYPE_NETCDF4C: + case PIO_IOTYPE_NETCDF4P: + ierr = nc_close(file->fh); + break; + case PIO_IOTYPE_NETCDF4C: #endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_close(file->fh); - } - break; + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_close(file->fh); + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - if((file->mode & PIO_WRITE)){ - ierr = ncmpi_buffer_detach(file->fh); - } - ierr = ncmpi_close(file->fh); - break; + case PIO_IOTYPE_PNETCDF: + if((file->mode & PIO_WRITE)){ + ierr = ncmpi_buffer_detach(file->fh); + } + ierr = ncmpi_close(file->fh); + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } - } - if(ios->io_rank==0){ - printf("Close file %d \n",file->fh); + if(ios->io_rank==0){ + printf("Close file %d \n",file->fh); // if(file->fh==5) print_trace(stdout); - } + } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - int iret = pio_delete_file_from_list(ncid); + int iret = pio_delete_file_from_list(ncid); - return ierr; + return ierr; } /** @@ -411,45 +438,45 @@ int PIOc_closefile(int ncid) */ int PIOc_deletefile(const int iosysid, const char filename[]) { - int ierr; - int msg; - int mpierr; - int chkerr; - iosystem_desc_t *ios; - size_t len; - - ierr = PIO_NOERR; - ios = pio_get_iosystem_from_id(iosysid); - - if(ios == NULL) - return PIO_EBADID; - - msg = PIO_MSG_DELETE_FILE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - len = strlen(filename); - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); - } - // The barriers are needed to assure that no task is trying to operate on the file while it is being deleted. - if(ios->ioproc){ - MPI_Barrier(ios->io_comm); + int ierr; + int msg; + int mpierr; + int chkerr; + iosystem_desc_t *ios; + size_t len; + + ierr = PIO_NOERR; + ios = pio_get_iosystem_from_id(iosysid); + + if(ios == NULL) + return PIO_EBADID; + + msg = PIO_MSG_DELETE_FILE; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->comp_rank==0) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + len = strlen(filename); + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } + // The barriers are needed to assure that no task is trying to operate on the file while it is being deleted. + if(ios->ioproc){ + MPI_Barrier(ios->io_comm); #ifdef _NETCDF - if(ios->io_rank==0) - ierr = nc_delete(filename); + if(ios->io_rank==0) + ierr = nc_delete(filename); #else #ifdef _PNETCDF - ierr = ncmpi_delete(filename, ios->info); + ierr = ncmpi_delete(filename, ios->info); #endif #endif - MPI_Barrier(ios->io_comm); - } - // Special case - always broadcast the return from the - MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + MPI_Barrier(ios->io_comm); + } + // Special case - always broadcast the return from the + MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm); - return ierr; + return ierr; } /// @@ -460,77 +487,77 @@ int PIOc_deletefile(const int iosysid, const char filename[]) /// Refer to the netcdf documentation. /// /** -* @name PIOc_sync -*/ + * @name PIOc_sync + */ int PIOc_sync(int ncid) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; - wmulti_buffer *wmb, *twmb; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; - ios = file->iosystem; - msg = PIO_MSG_SYNC; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank == 0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - if((file->mode & PIO_WRITE)){ - // cn_buffer_report( *ios, true); - wmb = &(file->buffer); - while(wmb != NULL){ - // printf("%s %d %d %d\n",__FILE__,__LINE__,wmb->ioid, wmb->validvars); - if(wmb->validvars>0){ - flush_buffer(ncid, wmb, true); - } - twmb = wmb; - wmb = wmb->next; - if(twmb == &(file->buffer)){ - twmb->ioid=-1; - twmb->next=NULL; - }else{ - brel(twmb); - } + int ierr; + int msg; + int mpierr; + iosystem_desc_t *ios; + file_desc_t *file; + wmulti_buffer *wmb, *twmb; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if(file == NULL) + return PIO_EBADID; + ios = file->iosystem; + msg = PIO_MSG_SYNC; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->comp_rank == 0) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } - flush_output_buffer(file, true, 0); - if(ios->ioproc){ - switch(file->iotype){ + if((file->mode & PIO_WRITE)){ + // cn_buffer_report( *ios, true); + wmb = &(file->buffer); + while(wmb != NULL){ + // printf("%s %d %d %d\n",__FILE__,__LINE__,wmb->ioid, wmb->validvars); + if(wmb->validvars>0){ + flush_buffer(ncid, wmb, true); + } + twmb = wmb; + wmb = wmb->next; + if(twmb == &(file->buffer)){ + twmb->ioid=-1; + twmb->next=NULL; + }else{ + brel(twmb); + } + } + flush_output_buffer(file, true, 0); + + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_sync(file->fh);; - break; - case PIO_IOTYPE_NETCDF4C: + case PIO_IOTYPE_NETCDF4P: + ierr = nc_sync(file->fh);; + break; + case PIO_IOTYPE_NETCDF4C: #endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_sync(file->fh);; - } - break; + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_sync(file->fh);; + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_sync(file->fh);; - break; + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_sync(file->fh);; + break; #endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - } + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); - } - return ierr; + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + } + return ierr; } diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 2feb4191b6d..7fab862ee80 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -134,7 +134,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); - if (mpierr) + if (ierr) return check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast results to all tasks. Ignore NULL parameters. */ From ab93ed0faf5ba784eab792b1eb11f2785e538181 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 7 Jun 2016 07:28:22 -0600 Subject: [PATCH 123/184] cleanup of error handling in file functions --- src/clib/pio_file.c | 59 ++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 6e9d5c9d38d..31fc8ee285e 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -363,19 +363,17 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, */ int PIOc_closefile(int ncid) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - ierr = PIO_NOERR; - file = pio_get_file_from_id(ncid); - if(file == NULL) - return PIO_EBADID; + /* Find the info about this file. */ + if (!(file = pio_get_file_from_id(ncid))) + return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_CLOSE_FILE; + if((file->mode & PIO_WRITE)){ PIOc_sync(ncid); } @@ -384,15 +382,31 @@ int PIOc_closefile(int ncid) * sends a msg to the pio_msg_handler running on the IO master and * waiting for a message. Then broadcast the ncid over the intercomm * to the IO tasks. */ - if(ios->async_interface && !ios->ioproc){ - if(ios->comp_rank==0) { - mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_CLOSE_FILE; + + if(ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); } - mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm); + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } - if(ios->ioproc){ - switch(file->iotype){ + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { + switch (file->iotype) + { #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: @@ -418,14 +432,15 @@ int PIOc_closefile(int ncid) ierr = iotype_error(file->iotype,__FILE__,__LINE__); } } - if(ios->io_rank==0){ - printf("Close file %d \n",file->fh); -// if(file->fh==5) print_trace(stdout); - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); - int iret = pio_delete_file_from_list(ncid); + /* Delete file from our list of open files. */ + pio_delete_file_from_list(ncid); return ierr; } From 4a391ca40fd0d653be83b35118915b25c9b261cf Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 03:53:06 -0600 Subject: [PATCH 124/184] more logging statements to find bug --- src/clib/pio_nc_async.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 7fab862ee80..aa48139e41e 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -1648,16 +1648,17 @@ int pioc_change_def(int ncid, int is_enddef) if (!ios->ioproc) { int msg = is_enddef ? PIO_MSG_ENDDEF : PIO_MSG_REDEF; - + LOG((2, "sending message msg = %d", msg)); if(ios->compmaster) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); - LOG((2, "pioc_change_def ncid = %d", file->fh)); + LOG((2, "pioc_change_def ncid = %d mpierr = %d", file->fh, mpierr)); } /* Handle MPI errors. */ + LOG((2, "pioc_change_def handling MPI errors")); if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) check_mpi(file, mpierr2, __FILE__, __LINE__); if (mpierr) @@ -1665,6 +1666,7 @@ int pioc_change_def(int ncid, int is_enddef) } /* If this is an IO task, then call the netCDF function. */ + LOG((2, "pioc_change_def ios->ioproc = %d", ios->ioproc)); if (ios->ioproc) { LOG((2, "pioc_change_def calling netcdf function")); @@ -1686,6 +1688,7 @@ int pioc_change_def(int ncid, int is_enddef) } /* Broadcast and check the return code. */ + LOG((2, "pioc_change_def bcasting return code ierr = %d", ierr)); if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); if (ierr) From 07bf495624054c16537a1bd600d09656fa82be39 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 03:54:06 -0600 Subject: [PATCH 125/184] turned pnetcdf back on in test_intercomm --- tests/unit/test_intercomm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 14abbcdf24b..4b0588167d6 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -384,7 +384,7 @@ main(int argc, char **argv) * and when the do, they should go straight to finalize. */ if (comp_task) { - for (int fmt = 1; fmt < NUM_NETCDF_FLAVORS; fmt++) + for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) { int ncid, varid, dimid; PIO_Offset start[NDIM], count[NDIM] = {0}; From 26298e654db171039a8c4e0ae3711484e88af061 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 11:00:27 -0600 Subject: [PATCH 126/184] more log messages --- tests/unit/test_intercomm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 4b0588167d6..36fb2a72ad2 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -368,7 +368,7 @@ main(int argc, char **argv) } /* Turn on logging. */ - if ((ret = PIOc_set_log_level(2))) + if ((ret = PIOc_set_log_level(3))) ERR(ret); /* Initialize the async setup. */ From c871a94842fe210254962c40715c5088c81c88bb Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 11:01:49 -0600 Subject: [PATCH 127/184] more log messages --- tests/unit/test_intercomm.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 36fb2a72ad2..e6f5376b797 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -400,8 +400,10 @@ main(int argc, char **argv) printf("%d test_intercomm file created ncid = %d\n", my_rank, ncid); /* /\* End define mode, then re-enter it. *\/ */ + sleep(1); if ((ret = PIOc_enddef(ncid))) ERR(ret); + sleep(1); if (verbose) printf("%d test_intercomm calling redef\n", my_rank); if ((ret = PIOc_redef(ncid))) From 7b4f744a6d5748fb754c81e71b0e12b13da7502c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 11:15:48 -0600 Subject: [PATCH 128/184] more log messages --- src/clib/pio_msg.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 1bcf2dc023f..7bc1503adea 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -1504,13 +1504,16 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) break; case PIO_MSG_CREATE_FILE: create_file_handler(my_iosys); + LOG((2, "returned from create_file_handler")); break; case PIO_MSG_SYNC: sync_file_handler(my_iosys); break; case PIO_MSG_ENDDEF: case PIO_MSG_REDEF: + LOG((2, "calling change_def_file_handler")); change_def_file_handler(my_iosys, msg); + LOG((2, "returned from change_def_file_handler")); break; case PIO_MSG_OPEN_FILE: open_file_handler(my_iosys); From 7f1793579b277c7f59efd770202fead75849728b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 11:24:42 -0600 Subject: [PATCH 129/184] more log messages --- src/clib/pio_msg.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 7bc1503adea..b6313d6e751 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -1602,16 +1602,19 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) } /* If an error was returned by the handler, do something! */ + LOG((3, "pio_msg_handler checking error")); if (ret) { LOG((0, "hander returned error code %d", ret)); MPI_Finalize(); } + LOG((3, "pio_msg_handler getting ready to listen")); /* Unless finalize was called, listen for another msg from the * component whose message we just handled. */ if (!io_rank && msg != -1) { + LOG((3, "pio_msg_handler about to Irecv")); my_iosys = &iosys[index]; mpierr = MPI_Irecv(&msg, 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG, my_iosys->union_comm, &req[index]); From 887f6514abb154d8485047ecb7d6cd054cabe7a8 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 13:54:28 -0600 Subject: [PATCH 130/184] more log messages --- src/clib/pio_msg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index b6313d6e751..65b300f445d 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -1602,7 +1602,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) } /* If an error was returned by the handler, do something! */ - LOG((3, "pio_msg_handler checking error")); + LOG((3, "pio_msg_handler checking error ret = %d", ret)); if (ret) { LOG((0, "hander returned error code %d", ret)); From fcd502d2d6e0f28bbe049305c7a9e7c70afb94bf Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 14:05:35 -0600 Subject: [PATCH 131/184] more log messages --- src/clib/pio_msg.c | 2 +- tests/unit/test_intercomm.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index 65b300f445d..e6220d850cd 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -1446,7 +1446,7 @@ int pio_msg_handler(int io_rank, int component_count, iosystem_desc_t *iosys) MPI_Status status; int index; int mpierr; - int ret; + int ret = PIO_NOERR; int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index e6f5376b797..36fb2a72ad2 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -400,10 +400,8 @@ main(int argc, char **argv) printf("%d test_intercomm file created ncid = %d\n", my_rank, ncid); /* /\* End define mode, then re-enter it. *\/ */ - sleep(1); if ((ret = PIOc_enddef(ncid))) ERR(ret); - sleep(1); if (verbose) printf("%d test_intercomm calling redef\n", my_rank); if ((ret = PIOc_redef(ncid))) From 70238ae30b263858e600f0ec1432e65e05e939c3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 14:13:23 -0600 Subject: [PATCH 132/184] more log messages --- src/clib/pio_put_nc_async.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index af6cdd9762e..7bfc0bedfba 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -192,7 +192,9 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off ierr = ncmpi_bput_vars_short(ncid, varid, start, count, stride, buf, request); break; case NC_INT: + LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf for int")); ierr = ncmpi_bput_vars_int(ncid, varid, start, count, stride, buf, request); + LOG((2, "PIOc_put_vars_tc io_rank 0 done with pnetcdf call for int ierr = %d", ierr)); break; case NC_FLOAT: ierr = ncmpi_bput_vars_float(ncid, varid, start, count, stride, buf, request); From f8453253207e4a55f7a9507f56f50f542136a150 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 14:24:38 -0600 Subject: [PATCH 133/184] more log messages --- src/clib/pio_put_nc_async.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 7bfc0bedfba..da6e7ec1086 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -179,7 +179,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if(ios->io_rank == 0) { - LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf call")); + LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf call xtype = %d", xtype)); switch(xtype) { case NC_BYTE: @@ -192,7 +192,8 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off ierr = ncmpi_bput_vars_short(ncid, varid, start, count, stride, buf, request); break; case NC_INT: - LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf for int")); + LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf for int")); + sleep(1); ierr = ncmpi_bput_vars_int(ncid, varid, start, count, stride, buf, request); LOG((2, "PIOc_put_vars_tc io_rank 0 done with pnetcdf call for int ierr = %d", ierr)); break; From abde5d28479e1449b4e3306b2d378072090cb4cc Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 14:39:39 -0600 Subject: [PATCH 134/184] more log messages --- src/clib/pio_put_nc_async.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index da6e7ec1086..b264fbd9662 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -163,6 +163,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off LOG((2, "PIOc_put_vars_tc checked mpierr = %d", mpierr)); } + sleep(1); /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { @@ -180,6 +181,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if(ios->io_rank == 0) { LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf call xtype = %d", xtype)); + LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf call xtype = %d this time I mean it!", xtype)); switch(xtype) { case NC_BYTE: @@ -193,7 +195,6 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off break; case NC_INT: LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf for int")); - sleep(1); ierr = ncmpi_bput_vars_int(ncid, varid, start, count, stride, buf, request); LOG((2, "PIOc_put_vars_tc io_rank 0 done with pnetcdf call for int ierr = %d", ierr)); break; From 09d5b65ea7811feb6dd96d781b5da73da62ec63a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 14:51:12 -0600 Subject: [PATCH 135/184] more log messages --- src/clib/pio_get_nc_async.c | 2 +- src/clib/pio_put_nc_async.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/clib/pio_get_nc_async.c b/src/clib/pio_get_nc_async.c index d7511c1bc14..7636dd6251e 100644 --- a/src/clib/pio_get_nc_async.c +++ b/src/clib/pio_get_nc_async.c @@ -71,7 +71,7 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off /* How many elements in buf? */ for (int vd = 0; vd < ndims; vd++) num_elem *= (rcount[vd] - rstart[vd])/rstride[vd]; - LOG((2, "PIOc_put_vars_tc num_elem = %d", num_elem)); + LOG((2, "PIOc_get_vars_tc num_elem = %d", num_elem)); } /* If async is in use, and this is not an IO task, bcast the parameters. */ diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index b264fbd9662..87f84fd534f 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -180,7 +180,6 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if(ios->io_rank == 0) { - LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf call xtype = %d", xtype)); LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf call xtype = %d this time I mean it!", xtype)); switch(xtype) { From 08b7a7e184a6197d2e24713a6df842010657dc69 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 14:57:35 -0600 Subject: [PATCH 136/184] more log messages --- src/clib/pio_put_nc_async.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 87f84fd534f..ccf8e69cba6 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -181,6 +181,8 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if(ios->io_rank == 0) { LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf call xtype = %d this time I mean it!", xtype)); + LOG((2, "PIOc_put_vars_tc ncid = %d varid = %d start[0] = %d count[0] = %d stride = %d", + ncid, varid, start[0], count[0], stride)); switch(xtype) { case NC_BYTE: From b3a83a35c72a7a694521c74f188d02ed5a60c6bb Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 15:08:04 -0600 Subject: [PATCH 137/184] more log messages --- src/clib/pio_put_nc_async.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index ccf8e69cba6..54e6b91a517 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -170,6 +170,18 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off #ifdef _PNETCDF if (file->iotype == PIO_IOTYPE_PNETCDF) { + PIO_Offset *fake_stride; + + if (!stride_present) + { + if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + for (int d = 0; d < ndims; d++) + fake_stride[d] = 1; + } + else + fake_stride = stride; + LOG((2, "PIOc_put_vars_tc calling pnetcdf function")); vdesc = file->varlist + varid; if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0) @@ -186,27 +198,27 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off switch(xtype) { case NC_BYTE: - ierr = ncmpi_bput_vars_schar(ncid, varid, start, count, stride, buf, request); + ierr = ncmpi_bput_vars_schar(ncid, varid, start, count, fake_stride, buf, request); break; case NC_CHAR: - ierr = ncmpi_bput_vars_text(ncid, varid, start, count, stride, buf, request); + ierr = ncmpi_bput_vars_text(ncid, varid, start, count, fake_stride, buf, request); break; case NC_SHORT: - ierr = ncmpi_bput_vars_short(ncid, varid, start, count, stride, buf, request); + ierr = ncmpi_bput_vars_short(ncid, varid, start, count, fake_stride, buf, request); break; case NC_INT: LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf for int")); - ierr = ncmpi_bput_vars_int(ncid, varid, start, count, stride, buf, request); + ierr = ncmpi_bput_vars_int(ncid, varid, start, count, fake_stride, buf, request); LOG((2, "PIOc_put_vars_tc io_rank 0 done with pnetcdf call for int ierr = %d", ierr)); break; case NC_FLOAT: - ierr = ncmpi_bput_vars_float(ncid, varid, start, count, stride, buf, request); + ierr = ncmpi_bput_vars_float(ncid, varid, start, count, fake_stride, buf, request); break; case NC_DOUBLE: - ierr = ncmpi_bput_vars_double(ncid, varid, start, count, stride, buf, request); + ierr = ncmpi_bput_vars_double(ncid, varid, start, count, fake_stride, buf, request); break; case NC_INT64: - ierr = ncmpi_bput_vars_longlong(ncid, varid, start, count, stride, buf, request); + ierr = ncmpi_bput_vars_longlong(ncid, varid, start, count, fake_stride, buf, request); break; default: LOG((0, "Unknown type for pnetcdf file! xtype = %d", xtype)); @@ -220,6 +232,10 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off LOG((2, "PIOc_put_vars_tc flushing output buffer")); flush_output_buffer(file, false, 0); LOG((2, "PIOc_put_vars_tc flushed output buffer")); + + /* Free malloced resources. */ + if (!stride_present) + free(fake_stride); } #endif /* _PNETCDF */ #ifdef _NETCDF From 50ab03bcb795f2328acbdb2a7ccae009d5843045 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 15:13:04 -0600 Subject: [PATCH 138/184] more log messages --- src/clib/pio_put_nc_async.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 54e6b91a517..d6c4a52a47d 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -43,6 +43,9 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off int *dimids; /** The IDs of the dimensions for this variable. */ PIO_Offset typelen; /** Size (in bytes) of the data type of data in buf. */ PIO_Offset num_elem = 1; /** Number of data elements in the buffer. */ + char start_present = start ? true : false; /* Is start non-NULL? */ + char count_present = count ? true : false; /* Is count non-NULL? */ + char stride_present = stride ? true : false; /* Is stride non-NULL? */ var_desc_t *vdesc; PIO_Offset usage; int *request; @@ -112,9 +115,6 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if (!ios->ioproc) { int msg = PIO_MSG_PUT_VARS; - char start_present = start ? true : false; - char count_present = count ? true : false; - char stride_present = stride ? true : false; if(ios->compmaster) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); @@ -163,7 +163,6 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off LOG((2, "PIOc_put_vars_tc checked mpierr = %d", mpierr)); } - sleep(1); /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { @@ -180,7 +179,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off fake_stride[d] = 1; } else - fake_stride = stride; + fake_stride = (PIO_Offset *)stride; LOG((2, "PIOc_put_vars_tc calling pnetcdf function")); vdesc = file->varlist + varid; From 386b6415966a375d1c452b6314a89dacb0583d69 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 8 Jun 2016 15:14:15 -0600 Subject: [PATCH 139/184] compensate for poor handling of NULL for stride by pnetcdf --- src/clib/pio_put_nc_async.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index d6c4a52a47d..75dddbbd059 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -191,7 +191,6 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if(ios->io_rank == 0) { - LOG((2, "PIOc_put_vars_tc io_rank 0 doing pnetcdf call xtype = %d this time I mean it!", xtype)); LOG((2, "PIOc_put_vars_tc ncid = %d varid = %d start[0] = %d count[0] = %d stride = %d", ncid, varid, start[0], count[0], stride)); switch(xtype) From 612998ffa7a95e85410498e15e0bdeace1ab9619 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 9 Jun 2016 13:22:26 -0600 Subject: [PATCH 140/184] more logging --- src/clib/pio_put_nc_async.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 75dddbbd059..f0ed5e1bbb1 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -173,6 +173,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if (!stride_present) { + LOG((2, "stride not present")); if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset)))) return PIO_ENOMEM; for (int d = 0; d < ndims; d++) @@ -183,16 +184,18 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off LOG((2, "PIOc_put_vars_tc calling pnetcdf function")); vdesc = file->varlist + varid; - if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0) + if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0) vdesc->request = realloc(vdesc->request, sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)); - request = vdesc->request+vdesc->nreqs; + request = vdesc->request + vdesc->nreqs; LOG((2, "PIOc_put_vars_tc request = %d", vdesc->request)); if(ios->io_rank == 0) { LOG((2, "PIOc_put_vars_tc ncid = %d varid = %d start[0] = %d count[0] = %d stride = %d", - ncid, varid, start[0], count[0], stride)); + ncid, varid, start[0], count[0], stride)); + for (int d = 0; d < ndims; d++) + LOG((2, "start[%d] = %d count[%d] = %d stride[%d] = %d", d, start[d], d, count[d], d, stride[d])); switch(xtype) { case NC_BYTE: From 329777ef82794a58b6cb1331d354dcacebf88b15 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 9 Jun 2016 13:29:58 -0600 Subject: [PATCH 141/184] more logging --- src/clib/pio_put_nc_async.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index f0ed5e1bbb1..94217901950 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -192,8 +192,8 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if(ios->io_rank == 0) { - LOG((2, "PIOc_put_vars_tc ncid = %d varid = %d start[0] = %d count[0] = %d stride = %d", - ncid, varid, start[0], count[0], stride)); + LOG((2, "PIOc_put_vars_tc ncid = %d varid = %d start[0] = %d count[0] = %d fake_stride[0] = %d", + ncid, varid, start[0], count[0], fake_stride[0])); for (int d = 0; d < ndims; d++) LOG((2, "start[%d] = %d count[%d] = %d stride[%d] = %d", d, start[d], d, count[d], d, stride[d])); switch(xtype) From be9af21569c1e140475a5c5111ba9dad9ee7eb91 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 9 Jun 2016 13:47:45 -0600 Subject: [PATCH 142/184] stopped faking the stride for puts --- src/clib/pio_put_nc_async.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 94217901950..7cd476b7100 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -174,10 +174,11 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if (!stride_present) { LOG((2, "stride not present")); - if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset)))) - return PIO_ENOMEM; - for (int d = 0; d < ndims; d++) - fake_stride[d] = 1; + /* if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset)))) */ + /* return PIO_ENOMEM; */ + /* for (int d = 0; d < ndims; d++) */ + /* fake_stride[d] = 1; */ + fake_stride = (PIO_Offset *)stride; } else fake_stride = (PIO_Offset *)stride; @@ -190,12 +191,13 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off request = vdesc->request + vdesc->nreqs; LOG((2, "PIOc_put_vars_tc request = %d", vdesc->request)); - if(ios->io_rank == 0) + /* Only the IO master actually does the call. */ + if (ios->iomaster) { - LOG((2, "PIOc_put_vars_tc ncid = %d varid = %d start[0] = %d count[0] = %d fake_stride[0] = %d", - ncid, varid, start[0], count[0], fake_stride[0])); - for (int d = 0; d < ndims; d++) - LOG((2, "start[%d] = %d count[%d] = %d stride[%d] = %d", d, start[d], d, count[d], d, stride[d])); +/* LOG((2, "PIOc_put_vars_tc ncid = %d varid = %d start[0] = %d count[0] = %d fake_stride[0] = %d", + ncid, varid, start[0], count[0], fake_stride[0]));*/ + /* for (int d = 0; d < ndims; d++) */ + /* LOG((2, "start[%d] = %d count[%d] = %d stride[%d] = %d", d, start[d], d, count[d], d, stride[d])); */ switch(xtype) { case NC_BYTE: From 985ba7266e7bb93cdd70da4fd136c088da6385bc Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 10 Jun 2016 08:49:01 -0600 Subject: [PATCH 143/184] more log messages --- src/clib/pio_put_nc_async.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 7cd476b7100..74a59570118 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -46,6 +46,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off char start_present = start ? true : false; /* Is start non-NULL? */ char count_present = count ? true : false; /* Is count non-NULL? */ char stride_present = stride ? true : false; /* Is stride non-NULL? */ + PIO_Offset *rstart, *rcount, *rstride; var_desc_t *vdesc; PIO_Offset usage; int *request; @@ -66,13 +67,20 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off * non-IO tasks if async is in use. */ if (!ios->async_interface || !ios->ioproc) { - /* Get the length of the data type. */ - if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &typelen))) - return check_netcdf(file, ierr, __FILE__, __LINE__); - /* Get the number of dims for this var. */ if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) return check_netcdf(file, ierr, __FILE__, __LINE__); + } + + /* Broadcase the number of dimensions to all tasks. */ + + /* Run these on all tasks if async is not in use, but only on + * non-IO tasks if async is in use. */ + if (!ios->async_interface || !ios->ioproc) + { + /* Get the length of the data type. */ + if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &typelen))) + return check_netcdf(file, ierr, __FILE__, __LINE__); PIO_Offset dimlen[ndims]; @@ -93,9 +101,16 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off return check_netcdf(file, ierr, __FILE__, __LINE__); } + /* Allocate memory for these arrays, now that we know ndims. */ + if (!(rstart = malloc(ndims * sizeof(PIO_Offset)))) + return check_netcdf(file, PIO_ENOMEM, __FILE__, __LINE__); + if (!(rcount = malloc(ndims * sizeof(PIO_Offset)))) + return check_netcdf(file, PIO_ENOMEM, __FILE__, __LINE__); + if (!(rstride = malloc(ndims * sizeof(PIO_Offset)))) + return check_netcdf(file, PIO_ENOMEM, __FILE__, __LINE__); + /* Figure out the real start, count, and stride arrays. (The * user may have passed in NULLs.) */ - PIO_Offset rstart[ndims], rcount[ndims], rstride[ndims]; for (int vd = 0; vd < ndims; vd++) { rstart[vd] = start ? start[vd] : 0; @@ -174,11 +189,10 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if (!stride_present) { LOG((2, "stride not present")); - /* if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset)))) */ - /* return PIO_ENOMEM; */ - /* for (int d = 0; d < ndims; d++) */ - /* fake_stride[d] = 1; */ - fake_stride = (PIO_Offset *)stride; + if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + for (int d = 0; d < ndims; d++) + fake_stride[d] = 1; } else fake_stride = (PIO_Offset *)stride; From 82d9faa1b833c84bc9e04348f75408fdade60819 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 10 Jun 2016 11:04:28 -0600 Subject: [PATCH 144/184] working on put issue --- src/clib/pio_nc_async.c | 19 +++++++++++-------- src/clib/pio_put_nc_async.c | 13 ++++++------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index aa48139e41e..73322b86d18 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -664,9 +664,9 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d " - "dimids_present = %d, natts_present = %d nattsp = %d", - name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp)); + LOG((2, "PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d " + "dimids_present = %d, natts_present = %d nattsp = %d", + name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp)); } /* Handle MPI errors. */ @@ -698,12 +698,13 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, } if (ndimsp) - LOG((2, "PIOc_inq_var ndims = %d", *ndimsp)); + LOG((2, "PIOc_inq_var ndims = %d ierr = %d", *ndimsp, ierr)); /* Broadcast and check the return code. */ if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); - check_netcdf(file, ierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); /* Broadcast the results for non-null pointers. */ if (!ierr) @@ -724,9 +725,11 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, if (ndimsp) { + if (ios->ioroot) + LOG((2, "PIOc_inq_var about to Bcast ndims = %d ios->ioroot = %d", *ndimsp, ios->ioroot)); if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm))) return check_mpi(file, mpierr, __FILE__, __LINE__); - file->varlist[varid].ndims = (*ndimsp); + file->varlist[varid].ndims = *ndimsp; LOG((2, "PIOc_inq_var Bcast ndims = %d", *ndimsp)); } if (dimidsp) @@ -2110,9 +2113,9 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) /* Broadcast values currently only known on computation tasks to IO tasks. */ LOG((2, "PIOc_get_att bcast from comproot = %d attlen = %d typelen = %d", ios->comproot, attlen, typelen)); if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm))) - check_mpi(file, mpierr, __FILE__, __LINE__); + return check_mpi(file, mpierr, __FILE__, __LINE__); LOG((2, "PIOc_get_att bcast complete attlen = %d typelen = %d", attlen, typelen)); } diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 74a59570118..055acc247ff 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -70,14 +70,7 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off /* Get the number of dims for this var. */ if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims))) return check_netcdf(file, ierr, __FILE__, __LINE__); - } - - /* Broadcase the number of dimensions to all tasks. */ - /* Run these on all tasks if async is not in use, but only on - * non-IO tasks if async is in use. */ - if (!ios->async_interface || !ios->ioproc) - { /* Get the length of the data type. */ if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &typelen))) return check_netcdf(file, ierr, __FILE__, __LINE__); @@ -176,6 +169,12 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off if (mpierr) check_mpi(file, mpierr, __FILE__, __LINE__); LOG((2, "PIOc_put_vars_tc checked mpierr = %d", mpierr)); + + /* Broadcast values currently only known on computation tasks to IO tasks. */ + LOG((2, "PIOc_put_vars_tc bcast from comproot")); + if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + LOG((2, "PIOc_put_vars_tc complete bcast from comproot ndims = %d", ndims)); } /* If this is an IO task, then call the netCDF function. */ From 617c65aac12a5431200e92bed0f5c6f16c459844 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 Jun 2016 08:02:59 -0600 Subject: [PATCH 145/184] cleanup --- src/clib/pio_file.c | 99 +++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 44 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 31fc8ee285e..9f9cf1873f0 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -260,7 +260,8 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, else file->do_io = 0; - /* If async is in use, and this is not an IO task, bcast the parameters. */ + /* If async is in use, and this is not an IO task, bcast the + * parameters. */ if (ios->async_interface) { int msg = PIO_MSG_CREATE_FILE; @@ -358,7 +359,7 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, /** ** @ingroup PIO_closefile - ** @brief close a file previously opened with PIO + ** Close a file previously opened with PIO ** @param ncid: the file pointer */ int PIOc_closefile(int ncid) @@ -368,7 +369,6 @@ int PIOc_closefile(int ncid) int ierr = PIO_NOERR; /** Return code from function calls. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; @@ -453,29 +453,36 @@ int PIOc_closefile(int ncid) */ int PIOc_deletefile(const int iosysid, const char filename[]) { - int ierr; - int msg; - int mpierr; - int chkerr; - iosystem_desc_t *ios; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + int msg = PIO_MSG_DELETE_FILE; size_t len; - ierr = PIO_NOERR; - ios = pio_get_iosystem_from_id(iosysid); - - if(ios == NULL) + /* Get the IO system info from the id. */ + if (!(ios = pio_get_iosystem_from_id(iosysid))) return PIO_EBADID; - msg = PIO_MSG_DELETE_FILE; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - len = strlen(filename); - mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); - mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + /* If async is in use, send message to IO master task. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + if(ios->comp_rank==0) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + len = strlen(filename); + if (!mpierr) + mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (!mpierr) + mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); + } } - // The barriers are needed to assure that no task is trying to operate on the file while it is being deleted. + + /* If this is an IO task, then call the netCDF function. The + * barriers are needed to assure that no task is trying to operate + * on the file while it is being deleted. */ if(ios->ioproc){ MPI_Barrier(ios->io_comm); #ifdef _NETCDF @@ -488,46 +495,50 @@ int PIOc_deletefile(const int iosysid, const char filename[]) #endif MPI_Barrier(ios->io_comm); } + // Special case - always broadcast the return from the MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm); return ierr; } -/// -/// PIO interface to nc_sync -/// -/// This routine is called collectively by all tasks in the communicator ios.union_comm. -/// -/// Refer to the netcdf documentation. -/// /** - * @name PIOc_sync + * PIO interface to nc_sync This routine is called collectively by all + * tasks in the communicator ios.union_comm. + * + * Refer to the netcdf documentation. */ int PIOc_sync(int ncid) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ wmulti_buffer *wmb, *twmb; - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL) + /* Get the file info from the ncid. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_SYNC; - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank == 0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + /* If async is in use, send message to IO master tasks. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = PIO_MSG_SYNC; + + if(ios->comp_rank == 0) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } } - if((file->mode & PIO_WRITE)){ + if (file->mode & PIO_WRITE) + { // cn_buffer_report( *ios, true); wmb = &(file->buffer); while(wmb != NULL){ From 6c09c564f334cc8124234deb2ecf61eefde4475e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 Jun 2016 09:13:15 -0600 Subject: [PATCH 146/184] more cleanup --- src/clib/pio_file.c | 65 +++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 9f9cf1873f0..44bd07888f3 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -1,18 +1,18 @@ #include #include #include -/** - ** @public - ** @ingroup PIO_openfile - ** @brief open an existing file using pio - ** @details Input parameters are read on comp task 0 and ignored elsewhere. - ** @param iosysid : A defined pio system descriptor (input) - ** @param ncidp : A pio file descriptor (output) - ** @param iotype : A pio output format (input) - ** @param filename : The filename to open - ** @param mode : The netcdf mode for the open operation - */ +/* Open an existing file using pio + * @public + * @ingroup PIO_openfile + * + * @details Input parameters are read on comp task 0 and ignored elsewhere. + * @param iosysid : A defined pio system descriptor (input) + * @param ncidp : A pio file descriptor (output) + * @param iotype : A pio output format (input) + * @param filename : The filename to open + * @param mode : The netcdf mode for the open operation + */ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, const char *filename, const int mode) { @@ -193,16 +193,17 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, return ierr; } -/** - ** @public - ** @ingroup PIO_createfile - ** @brief open a new file using pio - ** @details Input parameters are read on comp task 0 and ignored elsewhere. - ** @param iosysid : A defined pio system descriptor (input) - ** @param ncidp : A pio file descriptor (output) - ** @param iotype : A pio output format (input) - ** @param filename : The filename to open - ** @param mode : The netcdf mode for the open operation +/* Open a new file using pio. Input parameters are read on comp task + * 0 and ignored elsewhere. + * + * @public + * @ingroup PIO_createfile + * + * @param iosysid : A defined pio system descriptor (input) + * @param ncidp : A pio file descriptor (output) + * @param iotype : A pio output format (input) + * @param filename : The filename to open + * @param mode : The netcdf mode for the open operation */ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, @@ -357,10 +358,10 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, return ierr; } -/** - ** @ingroup PIO_closefile - ** Close a file previously opened with PIO - ** @param ncid: the file pointer +/* Close a file previously opened with PIO. + * @ingroup PIO_closefile + * + * @param ncid: the file pointer */ int PIOc_closefile(int ncid) { @@ -374,9 +375,9 @@ int PIOc_closefile(int ncid) return PIO_EBADID; ios = file->iosystem; - if((file->mode & PIO_WRITE)){ + /* Sync changes before closing. */ + if (file->mode & PIO_WRITE) PIOc_sync(ncid); - } /* If async is in use and this is a comp tasks, then the compmaster * sends a msg to the pio_msg_handler running on the IO master and @@ -445,11 +446,11 @@ int PIOc_closefile(int ncid) return ierr; } -/** - ** @ingroup PIO_deletefile - ** @brief Delete a file - ** @param iosysid : a pio system handle - ** @param filename : a filename +/* Delete a file. + * @ingroup PIO_deletefile + * + * @param iosysid : a pio system handle + * @param filename : a filename */ int PIOc_deletefile(const int iosysid, const char filename[]) { From cadc3327f08187d9f4b189271c6cb51ed0e50780 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 Jun 2016 10:25:22 -0600 Subject: [PATCH 147/184] breaking branch to test cdash building of branches --- src/clib/pio_file.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 44bd07888f3..0fdaa9e54dd 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -2,7 +2,7 @@ #include #include -/* Open an existing file using pio +/* Open an existing file using pio. * @public * @ingroup PIO_openfile * @@ -205,7 +205,6 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, * @param filename : The filename to open * @param mode : The netcdf mode for the open operation */ - int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, const char filename[], const int mode) { @@ -479,6 +478,12 @@ int PIOc_deletefile(const int iosysid, const char filename[]) if (!mpierr) mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. The @@ -519,6 +524,7 @@ int PIOc_sync(int ncid) int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ wmulti_buffer *wmb, *twmb; + return PIO_EBADID; /* Get the file info from the ncid. */ if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; @@ -533,8 +539,10 @@ int PIOc_sync(int ncid) if(ios->comp_rank == 0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, + ios->intercomm); } } From 995ebc5a61f5e52a8c3336d6d46379e089ed1e8d Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 Jun 2016 10:53:54 -0600 Subject: [PATCH 148/184] more cleanup --- src/clib/pio_file.c | 50 ++++++++++++++++++++++++++----------- tests/unit/test_intercomm.c | 4 +-- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 0fdaa9e54dd..19282111472 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -31,10 +31,7 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, /* Get the IO system info from the iosysid. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) - { - LOG((0, "PIOc_openfile got bad iosysid %d",iosysid)); return PIO_EBADID; - } /* Allocate space for the file info. */ if (!(file = (file_desc_t *) malloc(sizeof(*file)))) @@ -451,14 +448,17 @@ int PIOc_closefile(int ncid) * @param iosysid : a pio system handle * @param filename : a filename */ -int PIOc_deletefile(const int iosysid, const char filename[]) +int PIOc_deletefile(const int iosysid, const char *filename) { iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ int ierr = PIO_NOERR; /** Return code from function calls. */ + int deleted = 0; /** Becomes true when file is deleted. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - int msg = PIO_MSG_DELETE_FILE; - size_t len; + + /* Filename must be provided. */ + if (!filename || strlen(filename) > NC_MAX_NAME) + return PIO_EINVAL; /* Get the IO system info from the id. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) @@ -469,10 +469,12 @@ int PIOc_deletefile(const int iosysid, const char filename[]) { if (!ios->ioproc) { - if(ios->comp_rank==0) + int msg = PIO_MSG_DELETE_FILE; + size_t len = strlen(filename); + + if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - len = strlen(filename); if (!mpierr) mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) @@ -489,21 +491,27 @@ int PIOc_deletefile(const int iosysid, const char filename[]) /* If this is an IO task, then call the netCDF function. The * barriers are needed to assure that no task is trying to operate * on the file while it is being deleted. */ - if(ios->ioproc){ + if (ios->ioproc) + { MPI_Barrier(ios->io_comm); #ifdef _NETCDF - if(ios->io_rank==0) + if (ios->iomaster) ierr = nc_delete(filename); + deleted++; #else #ifdef _PNETCDF - ierr = ncmpi_delete(filename, ios->info); + if (!deleted) + ierr = ncmpi_delete(filename, ios->info); #endif #endif MPI_Barrier(ios->io_comm); } - // Special case - always broadcast the return from the - MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm); + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); return ierr; } @@ -524,7 +532,6 @@ int PIOc_sync(int ncid) int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ wmulti_buffer *wmb, *twmb; - return PIO_EBADID; /* Get the file info from the ncid. */ if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; @@ -537,13 +544,19 @@ int PIOc_sync(int ncid) { int msg = PIO_MSG_SYNC; - if(ios->comp_rank == 0) + if (ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); } + + /* Handle MPI errors. */ + /* if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) */ + /* return check_mpi(file, mpierr2, __FILE__, __LINE__); */ + /* if (mpierr) */ + /* return check_mpi(file, mpierr, __FILE__, __LINE__); */ } if (file->mode & PIO_WRITE) @@ -593,6 +606,13 @@ int PIOc_sync(int ncid) ierr = check_netcdf(file, ierr, __FILE__,__LINE__); } + + /* Broadcast and check the return code. */ + /* if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) */ + /* return check_mpi(file, mpierr, __FILE__, __LINE__); */ + /* if (ierr) */ + /* return check_netcdf(file, ierr, __FILE__, __LINE__); */ + return ierr; } diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index 36fb2a72ad2..fa680201f39 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -531,8 +531,8 @@ main(int argc, char **argv) ERR(ret); /* Now delete the file. */ - /* if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) */ - /* ERR(ret); */ + if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) + ERR(ret); /* if ((ret = PIOc_openfile(iosysid, &ncid, &format[fmt], filename[fmt], */ /* NC_NOWRITE)) != PIO_ENFILE) */ /* ERR(ERR_AWFUL); */ From c1df0fb0002000a72377e7bb4d871fd42c4d9075 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 Jun 2016 14:00:35 -0600 Subject: [PATCH 149/184] more cleanup --- src/clib/pio_file.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 19282111472..2cebfa578bf 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -562,18 +562,23 @@ int PIOc_sync(int ncid) if (file->mode & PIO_WRITE) { // cn_buffer_report( *ios, true); - wmb = &(file->buffer); - while(wmb != NULL){ + wmb = &file->buffer; + while(wmb) + { // printf("%s %d %d %d\n",__FILE__,__LINE__,wmb->ioid, wmb->validvars); - if(wmb->validvars>0){ + if (wmb->validvars > 0) + { flush_buffer(ncid, wmb, true); } twmb = wmb; wmb = wmb->next; - if(twmb == &(file->buffer)){ - twmb->ioid=-1; - twmb->next=NULL; - }else{ + if (twmb == &file->buffer) + { + twmb->ioid = -1; + twmb->next = NULL; + } + else + { brel(twmb); } } From 87e32a2dc78015a9dff539c2d7295f650443fe76 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 Jun 2016 14:06:39 -0600 Subject: [PATCH 150/184] more cleanup --- src/clib/pio_file.c | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 2cebfa578bf..93599bed6ca 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -584,29 +584,17 @@ int PIOc_sync(int ncid) } flush_output_buffer(file, true, 0); - if(ios->ioproc){ - switch(file->iotype){ -#ifdef _NETCDF -#ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_sync(file->fh);; - break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank==0){ - ierr = nc_sync(file->fh);; - } - break; -#endif + /* If this is an IO task, then call the netCDF function. */ + if (ios->ioproc) + { #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_sync(file->fh);; - break; -#endif - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } + if (file->iotype == PIO_IOTYPE_PNETCDF) + ierr = ncmpi_sync(file->fh); +#endif /* _PNETCDF */ +#ifdef _NETCDF + if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) + ierr = nc_sync(file->fh); +#endif /* _NETCDF */ } ierr = check_netcdf(file, ierr, __FILE__,__LINE__); From bf1ed730fc5c3c41aba4ff44a5e54265d79adcf8 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 Jun 2016 14:43:14 -0600 Subject: [PATCH 151/184] more cleanup --- src/clib/pio_file.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 93599bed6ca..e52f9423736 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -550,6 +550,8 @@ int PIOc_sync(int ncid) if (!mpierr) mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } /* Handle MPI errors. */ From 2559cf4f87d765a2993446b25d67591adad7f4d8 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 14 Jun 2016 10:24:52 -0600 Subject: [PATCH 152/184] added logging statement to debug cdash problem --- tests/unit/driver.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/driver.F90 b/tests/unit/driver.F90 index f6cac452f6c..d12d1d29cc2 100644 --- a/tests/unit/driver.F90 +++ b/tests/unit/driver.F90 @@ -63,7 +63,7 @@ Program pio_unit_test_driver ! Ignore namelist values if PIO not built with correct options ! (i.e. don't test pnetcdf if not built with pnetcdf) - + ret_val = PIO_set_log_level(2) #ifndef _NETCDF if (ltest_netcdf) then write(*,"(A,1x,A)") "WARNING: can not test netcdf files because PIO", & From 1b7ffb68e389d323fb15781d4650bf5f7f8c3049 Mon Sep 17 00:00:00 2001 From: Imperial Fleet Commander Tarth Date: Tue, 14 Jun 2016 10:33:54 -0600 Subject: [PATCH 153/184] Revert "Ejh cleanup6" --- src/clib/pio_file.c | 117 +++++++++++++++--------------------- tests/unit/test_intercomm.c | 4 +- 2 files changed, 49 insertions(+), 72 deletions(-) diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index e52f9423736..44bd07888f3 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -2,7 +2,7 @@ #include #include -/* Open an existing file using pio. +/* Open an existing file using pio * @public * @ingroup PIO_openfile * @@ -31,7 +31,10 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, /* Get the IO system info from the iosysid. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) + { + LOG((0, "PIOc_openfile got bad iosysid %d",iosysid)); return PIO_EBADID; + } /* Allocate space for the file info. */ if (!(file = (file_desc_t *) malloc(sizeof(*file)))) @@ -202,6 +205,7 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, * @param filename : The filename to open * @param mode : The netcdf mode for the open operation */ + int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, const char filename[], const int mode) { @@ -448,17 +452,14 @@ int PIOc_closefile(int ncid) * @param iosysid : a pio system handle * @param filename : a filename */ -int PIOc_deletefile(const int iosysid, const char *filename) +int PIOc_deletefile(const int iosysid, const char filename[]) { iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; /** Pointer to file information. */ int ierr = PIO_NOERR; /** Return code from function calls. */ - int deleted = 0; /** Becomes true when file is deleted. */ int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - - /* Filename must be provided. */ - if (!filename || strlen(filename) > NC_MAX_NAME) - return PIO_EINVAL; + int msg = PIO_MSG_DELETE_FILE; + size_t len; /* Get the IO system info from the id. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) @@ -469,49 +470,35 @@ int PIOc_deletefile(const int iosysid, const char *filename) { if (!ios->ioproc) { - int msg = PIO_MSG_DELETE_FILE; - size_t len = strlen(filename); - - if(ios->compmaster) + if(ios->comp_rank==0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + len = strlen(filename); if (!mpierr) mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm); } - - /* Handle MPI errors. */ - if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) - return check_mpi(file, mpierr2, __FILE__, __LINE__); - if (mpierr) - return check_mpi(file, mpierr, __FILE__, __LINE__); } /* If this is an IO task, then call the netCDF function. The * barriers are needed to assure that no task is trying to operate * on the file while it is being deleted. */ - if (ios->ioproc) - { + if(ios->ioproc){ MPI_Barrier(ios->io_comm); #ifdef _NETCDF - if (ios->iomaster) + if(ios->io_rank==0) ierr = nc_delete(filename); - deleted++; #else #ifdef _PNETCDF - if (!deleted) - ierr = ncmpi_delete(filename, ios->info); + ierr = ncmpi_delete(filename, ios->info); #endif #endif MPI_Barrier(ios->io_comm); } - /* Broadcast and check the return code. */ - if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) - return check_mpi(file, mpierr, __FILE__, __LINE__); - if (ierr) - return check_netcdf(file, ierr, __FILE__, __LINE__); + // Special case - always broadcast the return from the + MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm); return ierr; } @@ -544,70 +531,60 @@ int PIOc_sync(int ncid) { int msg = PIO_MSG_SYNC; - if (ios->compmaster) + if(ios->comp_rank == 0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - - if (!mpierr) - mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, - ios->intercomm); - if (mpierr) - return check_mpi(file, mpierr, __FILE__, __LINE__); + + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } - - /* Handle MPI errors. */ - /* if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) */ - /* return check_mpi(file, mpierr2, __FILE__, __LINE__); */ - /* if (mpierr) */ - /* return check_mpi(file, mpierr, __FILE__, __LINE__); */ } if (file->mode & PIO_WRITE) { // cn_buffer_report( *ios, true); - wmb = &file->buffer; - while(wmb) - { + wmb = &(file->buffer); + while(wmb != NULL){ // printf("%s %d %d %d\n",__FILE__,__LINE__,wmb->ioid, wmb->validvars); - if (wmb->validvars > 0) - { + if(wmb->validvars>0){ flush_buffer(ncid, wmb, true); } twmb = wmb; wmb = wmb->next; - if (twmb == &file->buffer) - { - twmb->ioid = -1; - twmb->next = NULL; - } - else - { + if(twmb == &(file->buffer)){ + twmb->ioid=-1; + twmb->next=NULL; + }else{ brel(twmb); } } flush_output_buffer(file, true, 0); - /* If this is an IO task, then call the netCDF function. */ - if (ios->ioproc) - { -#ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF) - ierr = ncmpi_sync(file->fh); -#endif /* _PNETCDF */ + if(ios->ioproc){ + switch(file->iotype){ #ifdef _NETCDF - if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) - ierr = nc_sync(file->fh); -#endif /* _NETCDF */ +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + ierr = nc_sync(file->fh);; + break; + case PIO_IOTYPE_NETCDF4C: +#endif + case PIO_IOTYPE_NETCDF: + if(ios->io_rank==0){ + ierr = nc_sync(file->fh);; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + ierr = ncmpi_sync(file->fh);; + break; +#endif + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } } ierr = check_netcdf(file, ierr, __FILE__,__LINE__); } - - /* Broadcast and check the return code. */ - /* if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) */ - /* return check_mpi(file, mpierr, __FILE__, __LINE__); */ - /* if (ierr) */ - /* return check_netcdf(file, ierr, __FILE__, __LINE__); */ - return ierr; } diff --git a/tests/unit/test_intercomm.c b/tests/unit/test_intercomm.c index fa680201f39..36fb2a72ad2 100644 --- a/tests/unit/test_intercomm.c +++ b/tests/unit/test_intercomm.c @@ -531,8 +531,8 @@ main(int argc, char **argv) ERR(ret); /* Now delete the file. */ - if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) - ERR(ret); + /* if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) */ + /* ERR(ret); */ /* if ((ret = PIOc_openfile(iosysid, &ncid, &format[fmt], filename[fmt], */ /* NC_NOWRITE)) != PIO_ENFILE) */ /* ERR(ERR_AWFUL); */ From 08dc6ffc1236cb594a708742e5d7a7cba9b7766f Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 15 Jun 2016 09:18:53 -0600 Subject: [PATCH 154/184] documentation and spacing cleanup --- src/clib/pio_darray.c | 3261 +++++++++++++++++++++-------------------- tests/unit/test_nc4.c | 12 +- 2 files changed, 1704 insertions(+), 1569 deletions(-) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index 75a451cba59..11ecd8582c7 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -13,545 +13,574 @@ #include #include + #define PIO_WRITE_BUFFERING 1 -PIO_Offset PIO_BUFFER_SIZE_LIMIT=10485760; // 10MB default limit -bufsize PIO_CNBUFFER_LIMIT=33554432; -static void *CN_bpool=NULL; -static PIO_Offset maxusage=0; -/** @brief Set the pio buffer size limit, this is the size of the data buffer on the IO nodes. +PIO_Offset PIO_BUFFER_SIZE_LIMIT = 10485760; // 10MB default limit +bufsize PIO_CNBUFFER_LIMIT = 33554432; +static void *CN_bpool = NULL; +static PIO_Offset maxusage = 0; + +/** Set the pio buffer size limit. This is the size of the data buffer + * on the IO nodes. * + * The pio_buffer_size_limit will only apply to files opened after + * the setting is changed. * - * The pio_buffer_size_limit will only apply to files opened after the setting is changed. - * @param limit the size of the buffer on the IO nodes - * @return The previous limit setting. + * @param limit the size of the buffer on the IO nodes + * @return The previous limit setting. */ - PIO_Offset PIOc_set_buffer_size_limit(const PIO_Offset limit) - { - PIO_Offset oldsize; - oldsize = PIO_BUFFER_SIZE_LIMIT; - if(limit>0) - PIO_BUFFER_SIZE_LIMIT=limit; - return(oldsize); - } - -/** @brief Initialize the compute buffer to size PIO_CNBUFFER_LIMIT +PIO_Offset PIOc_set_buffer_size_limit(const PIO_Offset limit) +{ + PIO_Offset oldsize; + oldsize = PIO_BUFFER_SIZE_LIMIT; + if (limit>0) + PIO_BUFFER_SIZE_LIMIT = limit; + return(oldsize); +} + +/** Initialize the compute buffer to size PIO_CNBUFFER_LIMIT. + * + * This routine initializes the compute buffer pool if the bget memory + * management is used. * - * This routine initializes the compute buffer pool if the bget memory management is used. * @param ios the iosystem descriptor which will use the new buffer */ - void compute_buffer_init(iosystem_desc_t ios) { #ifndef PIO_USE_MALLOC - if(CN_bpool == NULL){ - CN_bpool = malloc( PIO_CNBUFFER_LIMIT ); - if(CN_bpool==NULL){ - char errmsg[180]; - sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d: try reducing PIO_CNBUFFER_LIMIT\n",PIO_CNBUFFER_LIMIT,ios.comp_rank); - piodie(errmsg,__FILE__,__LINE__); - } - bpool( CN_bpool, PIO_CNBUFFER_LIMIT); - if(CN_bpool==NULL){ - char errmsg[180]; - sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d: try reducing PIO_CNBUFFER_LIMIT\n",PIO_CNBUFFER_LIMIT,ios.comp_rank); - piodie(errmsg,__FILE__,__LINE__); + if (CN_bpool == NULL){ + CN_bpool = malloc( PIO_CNBUFFER_LIMIT ); + if (CN_bpool==NULL){ + char errmsg[180]; + sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d: try reducing PIO_CNBUFFER_LIMIT\n",PIO_CNBUFFER_LIMIT,ios.comp_rank); + piodie(errmsg,__FILE__,__LINE__); + } + bpool( CN_bpool, PIO_CNBUFFER_LIMIT); + if (CN_bpool==NULL){ + char errmsg[180]; + sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d: try reducing PIO_CNBUFFER_LIMIT\n",PIO_CNBUFFER_LIMIT,ios.comp_rank); + piodie(errmsg,__FILE__,__LINE__); + } + bectl(NULL, malloc, free, PIO_CNBUFFER_LIMIT); } - bectl(NULL, malloc, free, PIO_CNBUFFER_LIMIT); - } #endif } /** @ingroup PIO_write_darray - * @brief Write a single distributed field to output. This routine is only used if aggregation is off. - * @param[in] file: a pointer to the open file descriptor for the file that will be written to - * @param[in] iodesc: a pointer to the defined iodescriptor for the buffer - * @param[in] vid: the variable id to be written - * @param[in] IOBUF: the buffer to be written from this mpi task - * @param[in] fillvalue: the optional fillvalue to be used for missing data in this buffer + * + * Write a single distributed field to output. This routine is only + * used if aggregation is off. + * + * @param[in] file: a pointer to the open file descriptor for the file + * that will be written to + * + * @param[in] iodesc: a pointer to the defined iodescriptor for the buffer + * + * @param[in] vid: the variable id to be written + * + * @param[in] IOBUF: the buffer to be written from this mpi task + * + * @param[in] fillvalue: the optional fillvalue to be used for missing + * data in this buffer */ - int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF, void *fillvalue) - { - iosystem_desc_t *ios; - var_desc_t *vdesc; - int ndims; - int ierr; - int i; - int msg; - int mpierr; - int dsize; - MPI_Status status; - PIO_Offset usage; - int fndims; - PIO_Offset tdsize; +int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, + void *IOBUF, void *fillvalue) +{ + iosystem_desc_t *ios; + var_desc_t *vdesc; + int ndims; + int ierr; + int i; + int msg; + int mpierr; + int dsize; + MPI_Status status; + PIO_Offset usage; + int fndims; + PIO_Offset tdsize; #ifdef TIMING - GPTLstart("PIO:write_darray_nc"); + GPTLstart("PIO:write_darray_nc"); #endif - tdsize=0; - ierr = PIO_NOERR; - - ios = file->iosystem; - if(ios == NULL){ - fprintf(stderr,"Failed to find iosystem handle \n"); - return PIO_EBADID; - } - vdesc = (file->varlist)+vid; - - if(vdesc == NULL){ - fprintf(stderr,"Failed to find variable handle %d\n",vid); - return PIO_EBADID; - } - ndims = iodesc->ndims; - msg = 0; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - ierr = PIOc_inq_varndims(file->fh, vid, &fndims); - - - if(ios->ioproc){ - io_region *region; - int ncid = file->fh; - int regioncnt; - int rrcnt; - void *bufptr; - void *tmp_buf=NULL; - int tsize; - size_t start[fndims]; - size_t count[fndims]; - int buflen, j; - - PIO_Offset *startlist[iodesc->maxregions]; - PIO_Offset *countlist[iodesc->maxregions]; - MPI_Type_size(iodesc->basetype, &tsize); - - region = iodesc->firstregion; - - if(vdesc->record >= 0 && ndimsiosystem; + if (ios == NULL){ + fprintf(stderr,"Failed to find iosystem handle \n"); + return PIO_EBADID; + } + vdesc = (file->varlist)+vid; + + if (vdesc == NULL){ + fprintf(stderr,"Failed to find variable handle %d\n",vid); + return PIO_EBADID; + } + ndims = iodesc->ndims; + msg = 0; + + if (ios->async_interface && ! ios->ioproc){ + if (ios->comp_rank==0) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + ierr = PIOc_inq_varndims(file->fh, vid, &fndims); + + if (ios->ioproc){ + io_region *region; + int ncid = file->fh; + int regioncnt; + int rrcnt; + void *bufptr; + void *tmp_buf = NULL; + int tsize; + size_t start[fndims]; + size_t count[fndims]; + int buflen, j; + + PIO_Offset *startlist[iodesc->maxregions]; + PIO_Offset *countlist[iodesc->maxregions]; + MPI_Type_size(iodesc->basetype, &tsize); + + region = iodesc->firstregion; + + if (vdesc->record >= 0 && ndimsiotype == PIO_IOTYPE_PNETCDF){ - // make sure we have room in the buffer ; - flush_output_buffer(file, false, tsize*(iodesc->maxiobuflen)); - } + if (file->iotype == PIO_IOTYPE_PNETCDF){ + // make sure we have room in the buffer ; + flush_output_buffer(file, false, tsize*(iodesc->maxiobuflen)); + } #endif - rrcnt=0; - for(regioncnt=0;regioncntmaxregions;regioncnt++){ - for(i=0;iloffset); - // this is a record based multidimensional array - if(vdesc->record >= 0){ - start[0] = vdesc->record; - for(i=1;istart[i-1]; - count[i] = region->count[i-1]; - } - if(count[1]>0) - count[0] = 1; - // Non-time dependent array - }else{ - for( i=0;istart[i]; - count[i] = region->count[i]; - } - } - } - - switch(file->iotype){ + rrcnt = 0; + for(regioncnt = 0;regioncntmaxregions;regioncnt++){ + for(i = 0;iloffset); + // this is a record based multidimensional array + if (vdesc->record >= 0){ + start[0] = vdesc->record; + for(i = 1;istart[i-1]; + count[i] = region->count[i-1]; + } + if (count[1]>0) + count[0] = 1; + // Non-time dependent array + } + else + { + for( i=0;istart[i]; + count[i] = region->count[i]; + } + } + } + + switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - ierr = nc_var_par_access(ncid, vid, NC_COLLECTIVE); - if(iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid,(size_t *) start,(size_t *) count, (const double *) bufptr); - } else if(iodesc->basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid, (size_t *) start, (size_t *) count, (const int *) bufptr); - }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid, vid, (size_t *) start, (size_t *) count, (const float *) bufptr); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) iodesc->basetype); - } - break; - case PIO_IOTYPE_NETCDF4C: + case PIO_IOTYPE_NETCDF4P: + ierr = nc_var_par_access(ncid, vid, NC_COLLECTIVE); + if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ + ierr = nc_put_vara_double (ncid, vid,(size_t *) start,(size_t *) count, (const double *) bufptr); + } else if (iodesc->basetype == MPI_INTEGER){ + ierr = nc_put_vara_int (ncid, vid, (size_t *) start, (size_t *) count, (const int *) bufptr); + }else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ + ierr = nc_put_vara_float (ncid, vid, (size_t *) start, (size_t *) count, (const float *) bufptr); + } + else + { + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) iodesc->basetype); + } + break; + case PIO_IOTYPE_NETCDF4C: #endif - case PIO_IOTYPE_NETCDF: - { - mpierr = MPI_Type_size(iodesc->basetype, &dsize); - size_t tstart[ndims], tcount[ndims]; - if(ios->io_rank==0){ - - for(i=0;inum_aiotasks;i++){ - if(i==0){ - buflen=1; - for(j=0;jio_comm); // handshake - tell the sending task I'm ready - mpierr = MPI_Recv( &buflen, 1, MPI_INT, i, 1, ios->io_comm, &status); - if(buflen>0){ - mpierr = MPI_Recv( tstart, ndims, MPI_OFFSET, i, ios->num_iotasks+i, ios->io_comm, &status); - mpierr = MPI_Recv( tcount, ndims, MPI_OFFSET, i,2*ios->num_iotasks+i, ios->io_comm, &status); - tmp_buf = malloc(buflen * dsize); - mpierr = MPI_Recv( tmp_buf, buflen, iodesc->basetype, i, i, ios->io_comm, &status); - } - } - - if(buflen>0){ - if(iodesc->basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid, tstart, tcount, (const int *) tmp_buf); - }else if(iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid, tstart, tcount, (const double *) tmp_buf); - }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid,vid, tstart, tcount, (const float *) tmp_buf); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) iodesc->basetype); - } - if(ierr == PIO_EEDGE){ - for(i=0;iio_rank < iodesc->num_aiotasks ){ - buflen=1; - for(i=0;iio_rank,tstart[0],tstart[1],tcount[0],tcount[1],buflen,ndims,fndims); - mpierr = MPI_Recv( &ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve - mpierr = MPI_Rsend( &buflen, 1, MPI_INT, 0, 1, ios->io_comm); - if(buflen>0) { - mpierr = MPI_Rsend( tstart, ndims, MPI_OFFSET, 0, ios->num_iotasks+ios->io_rank, ios->io_comm); - mpierr = MPI_Rsend( tcount, ndims, MPI_OFFSET, 0,2*ios->num_iotasks+ios->io_rank, ios->io_comm); - mpierr = MPI_Rsend( bufptr, buflen, iodesc->basetype, 0, ios->io_rank, ios->io_comm); - } - } - break; - } - break; - #endif - #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - for( i=0,dsize=1;ibasetype); - - /* if(regioncnt==0){ - for(i=0;imaxregions;i++){ - startlist[i] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - countlist[i] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - } - } - */ - if(dsize>0){ - // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); - startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - for( i=0; imaxregions-1){ - // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,ios->io_rank,iodesc->llen, tdsize); - // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, - // IOBUF, iodesc->llen, iodesc->basetype); - int reqn=0; - - - if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - - for(int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ - vdesc->request[i]=NC_REQ_NULL; - } - reqn = vdesc->nreqs; - }else{ - while(vdesc->request[reqn] != NC_REQ_NULL ){ - reqn++; - } - } - - ierr = ncmpi_bput_varn(ncid, vid, rrcnt, startlist, countlist, - IOBUF, iodesc->llen, iodesc->basetype, vdesc->request+reqn); - if(vdesc->request[reqn] == NC_REQ_NULL){ - vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync - } - vdesc->nreqs = reqn; - - // printf("%s %d %X %d\n",__FILE__,__LINE__,IOBUF,request); - for(i=0;ibasetype, &dsize); + size_t tstart[ndims], tcount[ndims]; + if (ios->io_rank==0){ + + for(i = 0;inum_aiotasks;i++){ + if (i==0){ + buflen = 1; + for(j = 0;jio_comm); // handshake - tell the sending task I'm ready + mpierr = MPI_Recv( &buflen, 1, MPI_INT, i, 1, ios->io_comm, &status); + if (buflen>0){ + mpierr = MPI_Recv( tstart, ndims, MPI_OFFSET, i, ios->num_iotasks+i, ios->io_comm, &status); + mpierr = MPI_Recv( tcount, ndims, MPI_OFFSET, i,2*ios->num_iotasks+i, ios->io_comm, &status); + tmp_buf = malloc(buflen * dsize); + mpierr = MPI_Recv( tmp_buf, buflen, iodesc->basetype, i, i, ios->io_comm, &status); + } + } + + if (buflen>0){ + if (iodesc->basetype == MPI_INTEGER){ + ierr = nc_put_vara_int (ncid, vid, tstart, tcount, (const int *) tmp_buf); + }else if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ + ierr = nc_put_vara_double (ncid, vid, tstart, tcount, (const double *) tmp_buf); + }else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ + ierr = nc_put_vara_float (ncid,vid, tstart, tcount, (const float *) tmp_buf); + } + else + { + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) iodesc->basetype); + } + if (ierr == PIO_EEDGE){ + for(i = 0;iio_rank < iodesc->num_aiotasks ){ + buflen = 1; + for(i = 0;iio_rank,tstart[0],tstart[1],tcount[0],tcount[1],buflen,ndims,fndims); + mpierr = MPI_Recv( &ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve + mpierr = MPI_Rsend( &buflen, 1, MPI_INT, 0, 1, ios->io_comm); + if (buflen>0) { + mpierr = MPI_Rsend( tstart, ndims, MPI_OFFSET, 0, ios->num_iotasks+ios->io_rank, ios->io_comm); + mpierr = MPI_Rsend( tcount, ndims, MPI_OFFSET, 0,2*ios->num_iotasks+ios->io_rank, ios->io_comm); + mpierr = MPI_Rsend( bufptr, buflen, iodesc->basetype, 0, ios->io_rank, ios->io_comm); + } + } + break; + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + for( i = 0,dsize = 1;ibasetype); + + /* if (regioncnt==0){ + for(i=0;imaxregions;i++){ + startlist[i] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); + countlist[i] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); + } + } + */ + if (dsize>0){ + // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); + startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); + countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); + for( i=0; imaxregions-1){ + // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,ios->io_rank,iodesc->llen, tdsize); + // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, + // IOBUF, iodesc->llen, iodesc->basetype); + int reqn=0; + + + if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + + for(int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ + vdesc->request[i]=NC_REQ_NULL; + } + reqn = vdesc->nreqs; + } + else + { + while(vdesc->request[reqn] != NC_REQ_NULL ){ + reqn++; + } + } + + ierr = ncmpi_bput_varn(ncid, vid, rrcnt, startlist, countlist, + IOBUF, iodesc->llen, iodesc->basetype, vdesc->request+reqn); + if (vdesc->request[reqn] == NC_REQ_NULL){ + vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync + } + vdesc->nreqs = reqn; + + // printf("%s %d %X %d\n",__FILE__,__LINE__,IOBUF,request); + for(i=0;iiotype,__FILE__,__LINE__); - } - if(region != NULL) - region = region->next; - } // for(regioncnt=0;regioncntmaxregions;regioncnt++){ - } // if(ios->ioproc) - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + if (region != NULL) + region = region->next; + } // for(regioncnt=0;regioncntmaxregions;regioncnt++){ + } // if (ios->ioproc) + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); #ifdef TIMING - GPTLstop("PIO:write_darray_nc"); + GPTLstop("PIO:write_darray_nc"); #endif - return ierr; - } + return ierr; +} -/** @brief Write a set of one or more aggregated arrays to output file - * @ingroup PIO_write_darray +/** Write a set of one or more aggregated arrays to output file + * @ingroup PIO_write_darray * - * This routine is used if aggregation is enabled, data is already on the - * io-tasks - * @param[in] file: a pointer to the open file descriptor for the file that will be written to - * @param[in] nvars: the number of variables to be written with this decomposition - * @param[in] vid: an array of the variable ids to be written - * @param[in] iodesc_ndims: the number of dimensions explicitly in the iodesc - * @param[in] basetype : the basic type of the minimal data unit - * @param[in] gsize : array of the global dimensions of the field to be written - * @param[in] maxregions : max number of blocks to be written from this iotask - * @param[in] firstregion : pointer to the first element of a linked list of region descriptions. - * @param[in] llen : length of the iobuffer on this task for a single field - * @param[in] maxiobuflen : maximum llen participating - * @param[in] num_aiotasks : actual number of iotasks participating - * @param[in] IOBUF: the buffer to be written from this mpi task - * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF + * This routine is used if aggregation is enabled, data is already on the + * io-tasks + * @param[in] file: a pointer to the open file descriptor for the file that will be written to + * @param[in] nvars: the number of variables to be written with this decomposition + * @param[in] vid: an array of the variable ids to be written + * @param[in] iodesc_ndims: the number of dimensions explicitly in the iodesc + * @param[in] basetype : the basic type of the minimal data unit + * @param[in] gsize : array of the global dimensions of the field to be written + * @param[in] maxregions : max number of blocks to be written from this iotask + * @param[in] firstregion : pointer to the first element of a linked list of region descriptions. + * @param[in] llen : length of the iobuffer on this task for a single field + * @param[in] maxiobuflen : maximum llen participating + * @param[in] num_aiotasks : actual number of iotasks participating + * @param[in] IOBUF: the buffer to be written from this mpi task + * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF */ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[], const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset gsize[], const int maxregions, io_region *firstregion, const PIO_Offset llen, const int maxiobuflen, const int num_aiotasks, void *IOBUF, const int frame[]) - { - iosystem_desc_t *ios; - var_desc_t *vdesc; - int ierr; - int i; - int msg; - int mpierr; - int dsize; - MPI_Status status; - PIO_Offset usage; - int fndims; - PIO_Offset tdsize; - int tsize; - int ncid; - tdsize=0; - ierr = PIO_NOERR; +{ + iosystem_desc_t *ios; + var_desc_t *vdesc; + int ierr; + int i; + int msg; + int mpierr; + int dsize; + MPI_Status status; + PIO_Offset usage; + int fndims; + PIO_Offset tdsize; + int tsize; + int ncid; + tdsize=0; + ierr = PIO_NOERR; #ifdef TIMING - GPTLstart("PIO:write_darray_multi_nc"); + GPTLstart("PIO:write_darray_multi_nc"); #endif - ios = file->iosystem; - if(ios == NULL){ - fprintf(stderr,"Failed to find iosystem handle \n"); - return PIO_EBADID; - } - vdesc = (file->varlist)+vid[0]; - ncid = file->fh; - - if(vdesc == NULL){ - fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); - return PIO_EBADID; - } - msg = 0; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - ierr = PIOc_inq_varndims(file->fh, vid[0], &fndims); - MPI_Type_size(basetype, &tsize); - - if(ios->ioproc){ - io_region *region; - int regioncnt; - int rrcnt; - void *bufptr; - int buflen, j; - size_t start[fndims]; - size_t count[fndims]; - int ndims = iodesc_ndims; - - PIO_Offset *startlist[maxregions]; - PIO_Offset *countlist[maxregions]; - - ncid = file->fh; - region = firstregion; - - - rrcnt=0; - for(regioncnt=0;regioncntstart[0],region->count[0],ndims,fndims,vdesc->record); - for(i=0;irecord >= 0){ - for(i=fndims-ndims;istart[i-(fndims-ndims)]; - count[i] = region->count[i-(fndims-ndims)]; - } - - if(fndims>1 && ndims0){ - count[0] = 1; - start[0] = frame[0]; - }else if(fndims==ndims){ - start[0]+=vdesc->record; - } - // Non-time dependent array - }else{ - for( i=0;istart[i]; - count[i] = region->count[i]; - } - } - } - - switch(file->iotype){ + ios = file->iosystem; + if (ios == NULL){ + fprintf(stderr,"Failed to find iosystem handle \n"); + return PIO_EBADID; + } + vdesc = (file->varlist)+vid[0]; + ncid = file->fh; + + if (vdesc == NULL){ + fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); + return PIO_EBADID; + } + msg = 0; + + if (ios->async_interface && ! ios->ioproc){ + if (ios->comp_rank==0) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + ierr = PIOc_inq_varndims(file->fh, vid[0], &fndims); + MPI_Type_size(basetype, &tsize); + + if (ios->ioproc){ + io_region *region; + int regioncnt; + int rrcnt; + void *bufptr; + int buflen, j; + size_t start[fndims]; + size_t count[fndims]; + int ndims = iodesc_ndims; + + PIO_Offset *startlist[maxregions]; + PIO_Offset *countlist[maxregions]; + + ncid = file->fh; + region = firstregion; + + rrcnt=0; + for(regioncnt=0;regioncntstart[0],region->count[0],ndims,fndims,vdesc->record); + for(i=0;irecord >= 0){ + for(i=fndims-ndims;istart[i-(fndims-ndims)]; + count[i] = region->count[i-(fndims-ndims)]; + } + + if (fndims>1 && ndims0){ + count[0] = 1; + start[0] = frame[0]; + }else if (fndims==ndims){ + start[0]+=vdesc->record; + } + // Non-time dependent array + } + else + { + for( i=0;istart[i]; + count[i] = region->count[i]; + } + } + } + + switch(file->iotype){ #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - for(int nv=0; nvrecord >= 0 && ndimsloffset)); - } - ierr = nc_var_par_access(ncid, vid[nv], NC_COLLECTIVE); - - if(basetype == MPI_DOUBLE ||basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid[nv],(size_t *) start,(size_t *) count, (const double *) bufptr); - } else if(basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid[nv], (size_t *) start, (size_t *) count, (const int *) bufptr); - }else if(basetype == MPI_FLOAT || basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid, vid[nv], (size_t *) start, (size_t *) count, (const float *) bufptr); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) basetype); - } - } - break; + case PIO_IOTYPE_NETCDF4P: + for(int nv=0; nvrecord >= 0 && ndimsloffset)); + } + ierr = nc_var_par_access(ncid, vid[nv], NC_COLLECTIVE); + + if (basetype == MPI_DOUBLE ||basetype == MPI_REAL8){ + ierr = nc_put_vara_double (ncid, vid[nv],(size_t *) start,(size_t *) count, (const double *) bufptr); + } else if (basetype == MPI_INTEGER){ + ierr = nc_put_vara_int (ncid, vid[nv], (size_t *) start, (size_t *) count, (const int *) bufptr); + }else if (basetype == MPI_FLOAT || basetype == MPI_REAL4){ + ierr = nc_put_vara_float (ncid, vid[nv], (size_t *) start, (size_t *) count, (const float *) bufptr); + } + else + { + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) basetype); + } + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - for( i=0,dsize=1;i0){ - // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); - startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - for( i=0; iio_rank,iodesc->llen, tdsize); - // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, - // IOBUF, iodesc->llen, iodesc->basetype); - - //printf("%s %d %ld \n",__FILE__,__LINE__,IOBUF); - for(int nv=0; nvvarlist)+vid[nv]; - if(vdesc->record >= 0 && ndimsnreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - - for(int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ - vdesc->request[i]=NC_REQ_NULL; - } - reqn = vdesc->nreqs; - }else{ - while(vdesc->request[reqn] != NC_REQ_NULL){ - reqn++; - } - } - ierr = ncmpi_iput_varn(ncid, vid[nv], rrcnt, startlist, countlist, - bufptr, llen, basetype, vdesc->request+reqn); - /* - ierr = ncmpi_bput_varn(ncid, vid[nv], rrcnt, startlist, countlist, - bufptr, llen, basetype, &(vdesc->request)); - */ - if(vdesc->request[reqn] == NC_REQ_NULL){ - vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync - } - vdesc->nreqs += reqn+1; - - // printf("%s %d %d %d\n",__FILE__,__LINE__,vdesc->nreqs,vdesc->request[reqn]); - } - for(i=0;i0){ + // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); + startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); + countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); + for( i=0; iio_rank,iodesc->llen, tdsize); + // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, + // IOBUF, iodesc->llen, iodesc->basetype); + + //printf("%s %d %ld \n",__FILE__,__LINE__,IOBUF); + for(int nv=0; nvvarlist)+vid[nv]; + if (vdesc->record >= 0 && ndimsnreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + + for(int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ + vdesc->request[i]=NC_REQ_NULL; + } + reqn = vdesc->nreqs; + } + else + { + while(vdesc->request[reqn] != NC_REQ_NULL){ + reqn++; + } + } + ierr = ncmpi_iput_varn(ncid, vid[nv], rrcnt, startlist, countlist, + bufptr, llen, basetype, vdesc->request+reqn); + /* + ierr = ncmpi_bput_varn(ncid, vid[nv], rrcnt, startlist, countlist, + bufptr, llen, basetype, &(vdesc->request)); + */ + if (vdesc->request[reqn] == NC_REQ_NULL){ + vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync + } + vdesc->nreqs += reqn+1; + + // printf("%s %d %d %d\n",__FILE__,__LINE__,vdesc->nreqs,vdesc->request[reqn]); + } + for(i=0;iiotype,__FILE__,__LINE__); - } - if(region != NULL) - region = region->next; - } // for(regioncnt=0;regioncntmaxregions;regioncnt++){ - } // if(ios->ioproc) - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + if (region != NULL) + region = region->next; + } // for(regioncnt=0;regioncntmaxregions;regioncnt++){ + } // if (ios->ioproc) + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); #ifdef TIMING - GPTLstop("PIO:write_darray_multi_nc"); + GPTLstop("PIO:write_darray_multi_nc"); #endif - return ierr; - } + return ierr; +} /** @brief Write a set of one or more aggregated arrays to output file * @ingroup PIO_write_darray @@ -573,1215 +602,1321 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF */ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const int vid[], - const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset gsize[], - const int maxregions, io_region *firstregion, const PIO_Offset llen, - const int maxiobuflen, const int num_aiotasks, - void *IOBUF, const int frame[]) - { - iosystem_desc_t *ios; - var_desc_t *vdesc; - int ierr; - int i; - int msg; - int mpierr; - int dsize; - MPI_Status status; - PIO_Offset usage; - int fndims; - PIO_Offset tdsize; - int tsize; - int ncid; - tdsize=0; - ierr = PIO_NOERR; + const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset gsize[], + const int maxregions, io_region *firstregion, const PIO_Offset llen, + const int maxiobuflen, const int num_aiotasks, + void *IOBUF, const int frame[]) +{ + iosystem_desc_t *ios; + var_desc_t *vdesc; + int ierr; + int i; + int msg; + int mpierr; + int dsize; + MPI_Status status; + PIO_Offset usage; + int fndims; + PIO_Offset tdsize; + int tsize; + int ncid; + tdsize=0; + ierr = PIO_NOERR; #ifdef TIMING - GPTLstart("PIO:write_darray_multi_nc_serial"); + GPTLstart("PIO:write_darray_multi_nc_serial"); #endif - ios = file->iosystem; - if(ios == NULL){ - fprintf(stderr,"Failed to find iosystem handle \n"); - return PIO_EBADID; - } - vdesc = (file->varlist)+vid[0]; - ncid = file->fh; - - if(vdesc == NULL){ - fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); - return PIO_EBADID; - } - msg = 0; - - if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); - } - - ierr = PIOc_inq_varndims(file->fh, vid[0], &fndims); - MPI_Type_size(basetype, &tsize); - - if(ios->ioproc){ - io_region *region; - int regioncnt; - int rrcnt; - void *bufptr; - int buflen, j; - size_t tmp_start[fndims*maxregions]; - size_t tmp_count[fndims*maxregions]; - - int ndims = iodesc_ndims; - - ncid = file->fh; - region = firstregion; - - - rrcnt=0; - for(regioncnt=0;regioncntrecord >= 0){ - for(i=fndims-ndims;istart[i-(fndims-ndims)]; - tmp_count[i+regioncnt*fndims] = region->count[i-(fndims-ndims)]; - } - // Non-time dependent array - }else{ - for( i=0;istart[i]; - tmp_count[i+regioncnt*fndims] = region->count[i]; - } - } - region = region->next; - } - } - if(ios->io_rank>0){ - mpierr = MPI_Recv( &ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve - MPI_Send( &llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); - if(llen>0){ - MPI_Send( &maxregions, 1, MPI_INT, 0, ios->io_rank+ios->num_iotasks, ios->io_comm); - MPI_Send( tmp_start, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+2*ios->num_iotasks, ios->io_comm); - MPI_Send( tmp_count, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+3*ios->num_iotasks, ios->io_comm); - // printf("%s %d %ld\n",__FILE__,__LINE__,nvars*llen); - MPI_Send( IOBUF, nvars*llen, basetype, 0, ios->io_rank+4*ios->num_iotasks, ios->io_comm); - } - }else{ - size_t rlen; - int rregions; - size_t start[fndims], count[fndims]; - size_t loffset; - mpierr = MPI_Type_size(basetype, &dsize); - - for(int rtask=0; rtasknum_iotasks; rtask++){ - if(rtask>0){ - mpierr = MPI_Send( &ierr, 1, MPI_INT, rtask, 0, ios->io_comm); // handshake - tell the sending task I'm ready - MPI_Recv( &rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); - if(rlen>0){ - MPI_Recv( &rregions, 1, MPI_INT, rtask, rtask+ios->num_iotasks, ios->io_comm, &status); - MPI_Recv( tmp_start, rregions*fndims, MPI_OFFSET, rtask, rtask+2*ios->num_iotasks, ios->io_comm, &status); - MPI_Recv( tmp_count, rregions*fndims, MPI_OFFSET, rtask, rtask+3*ios->num_iotasks, ios->io_comm, &status); - // printf("%s %d %d %ld\n",__FILE__,__LINE__,rtask,nvars*rlen); - MPI_Recv( IOBUF, nvars*rlen, basetype, rtask, rtask+4*ios->num_iotasks, ios->io_comm, &status); - } - }else{ - rlen = llen; - rregions = maxregions; - } - if(rlen>0){ - loffset = 0; - for(regioncnt=0;regioncntrecord>=0){ - if(fndims>1 && ndims0){ - count[0] = 1; - start[0] = frame[nv]; - }else if(fndims==ndims){ - start[0]+=vdesc->record; - } - } - - - - - if(basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid[nv], start, count, (const int *) bufptr); - }else if(basetype == MPI_DOUBLE || basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid[nv], start, count, (const double *) bufptr); - }else if(basetype == MPI_FLOAT || basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid,vid[nv], start, count, (const float *) bufptr); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) basetype); - } - - if(ierr != PIO_NOERR){ - for(i=0;imaxregions;regioncnt++){ - } // if(rlen>0) - } // for(int rtask=0; rtasknum_iotasks; rtask++){ - - } - } // if(ios->ioproc) - - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); -#ifdef TIMING - GPTLstop("PIO:write_darray_multi_nc_serial"); -#endif + ios = file->iosystem; + if (ios == NULL){ + fprintf(stderr,"Failed to find iosystem handle \n"); + return PIO_EBADID; + } + vdesc = (file->varlist)+vid[0]; + ncid = file->fh; + + if (vdesc == NULL){ + fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); + return PIO_EBADID; + } + msg = 0; - return ierr; - } + if (ios->async_interface && ! ios->ioproc){ + if (ios->comp_rank==0) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + ierr = PIOc_inq_varndims(file->fh, vid[0], &fndims); + MPI_Type_size(basetype, &tsize); + if (ios->ioproc){ + io_region *region; + int regioncnt; + int rrcnt; + void *bufptr; + int buflen, j; + size_t tmp_start[fndims*maxregions]; + size_t tmp_count[fndims*maxregions]; -/** @brief Write one or more arrays with the same IO decomposition to the file - * @ingroup PIO_write_darray + int ndims = iodesc_ndims; + + ncid = file->fh; + region = firstregion; + + + rrcnt=0; + for(regioncnt=0;regioncntrecord >= 0){ + for(i=fndims-ndims;istart[i-(fndims-ndims)]; + tmp_count[i+regioncnt*fndims] = region->count[i-(fndims-ndims)]; + } + // Non-time dependent array + } + else + { + for( i=0;istart[i]; + tmp_count[i+regioncnt*fndims] = region->count[i]; + } + } + region = region->next; + } + } + if (ios->io_rank>0){ + mpierr = MPI_Recv( &ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve + MPI_Send( &llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); + if (llen>0){ + MPI_Send( &maxregions, 1, MPI_INT, 0, ios->io_rank+ios->num_iotasks, ios->io_comm); + MPI_Send( tmp_start, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+2*ios->num_iotasks, ios->io_comm); + MPI_Send( tmp_count, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+3*ios->num_iotasks, ios->io_comm); + // printf("%s %d %ld\n",__FILE__,__LINE__,nvars*llen); + MPI_Send( IOBUF, nvars*llen, basetype, 0, ios->io_rank+4*ios->num_iotasks, ios->io_comm); + } + } + else + { + size_t rlen; + int rregions; + size_t start[fndims], count[fndims]; + size_t loffset; + mpierr = MPI_Type_size(basetype, &dsize); + + for(int rtask=0; rtasknum_iotasks; rtask++){ + if (rtask>0){ + mpierr = MPI_Send( &ierr, 1, MPI_INT, rtask, 0, ios->io_comm); // handshake - tell the sending task I'm ready + MPI_Recv( &rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); + if (rlen>0){ + MPI_Recv( &rregions, 1, MPI_INT, rtask, rtask+ios->num_iotasks, ios->io_comm, &status); + MPI_Recv( tmp_start, rregions*fndims, MPI_OFFSET, rtask, rtask+2*ios->num_iotasks, ios->io_comm, &status); + MPI_Recv( tmp_count, rregions*fndims, MPI_OFFSET, rtask, rtask+3*ios->num_iotasks, ios->io_comm, &status); + // printf("%s %d %d %ld\n",__FILE__,__LINE__,rtask,nvars*rlen); + MPI_Recv( IOBUF, nvars*rlen, basetype, rtask, rtask+4*ios->num_iotasks, ios->io_comm, &status); + } + } + else + { + rlen = llen; + rregions = maxregions; + } + if (rlen>0){ + loffset = 0; + for(regioncnt=0;regioncntrecord>=0){ + if (fndims>1 && ndims0){ + count[0] = 1; + start[0] = frame[nv]; + }else if (fndims==ndims){ + start[0]+=vdesc->record; + } + } + + + + + if (basetype == MPI_INTEGER){ + ierr = nc_put_vara_int (ncid, vid[nv], start, count, (const int *) bufptr); + }else if (basetype == MPI_DOUBLE || basetype == MPI_REAL8){ + ierr = nc_put_vara_double (ncid, vid[nv], start, count, (const double *) bufptr); + }else if (basetype == MPI_FLOAT || basetype == MPI_REAL4){ + ierr = nc_put_vara_float (ncid,vid[nv], start, count, (const float *) bufptr); + } + else + { + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) basetype); + } + + if (ierr != PIO_NOERR){ + for(i=0;imaxregions;regioncnt++){ + } // if (rlen>0) + } // for(int rtask=0; rtasknum_iotasks; rtask++){ + + } + } // if (ios->ioproc) + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +#ifdef TIMING + GPTLstop("PIO:write_darray_multi_nc_serial"); +#endif + + return ierr; +} + +/** Write one or more arrays with the same IO decomposition to the file + * @ingroup PIO_write_darray * + * @param ncid identifies the netCDF file + * @param vid + * @param ioid + * @param nvars number of variables + * @param arraylen + * @param array + * @param frame + * @param fillvalue + * @param flushtodisk */ -int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, const int nvars, const PIO_Offset arraylen, void *array, const int frame[], void *fillvalue[], bool flushtodisk) - { - iosystem_desc_t *ios; - file_desc_t *file; - io_desc_t *iodesc; - - int vsize, rlen; - int ierr; - var_desc_t *vdesc0; - - ierr = PIO_NOERR; - - file = pio_get_file_from_id(ncid); - if(file == NULL){ - fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); - return PIO_EBADID; - } - if(! (file->mode & PIO_WRITE)){ - fprintf(stderr,"ERROR: Attempt to write to read-only file\n"); - return PIO_EPERM; - } - - iodesc = pio_get_iodesc_from_id(ioid); - if(iodesc == NULL){ - // print_trace(NULL); - //fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); - return PIO_EBADID; - } - - vdesc0 = file->varlist+vid[0]; - - pioassert(nvars>0,"nvars <= 0",__FILE__,__LINE__); - - ios = file->iosystem; - // rlen = iodesc->llen*nvars; - rlen=0; - if(iodesc->llen>0){ - rlen = iodesc->maxiobuflen*nvars; - } - if(vdesc0->iobuf != NULL){ - piodie("Attempt to overwrite existing io buffer",__FILE__,__LINE__); - } - if(iodesc->rearranger>0){ - if(rlen>0){ - MPI_Type_size(iodesc->basetype, &vsize); - //printf("rlen*vsize = %ld\n",rlen*vsize); - - vdesc0->iobuf = bget((size_t) vsize* (size_t) rlen); - if(vdesc0->iobuf==NULL){ - printf("%s %d %d %ld\n",__FILE__,__LINE__,nvars,vsize*rlen); - piomemerror(*ios,(size_t) rlen*(size_t) vsize, __FILE__,__LINE__); - } - if(iodesc->needsfill && iodesc->rearranger==PIO_REARR_BOX){ - if(vsize==4){ - for(int nv=0;nv < nvars; nv++){ - for(int i=0;imaxiobuflen;i++){ - ((float *) vdesc0->iobuf)[i+nv*(iodesc->maxiobuflen)] = ((float *)fillvalue)[nv]; - } - } - }else if(vsize==8){ - for(int nv=0;nv < nvars; nv++){ - for(int i=0;imaxiobuflen;i++){ - ((double *)vdesc0->iobuf)[i+nv*(iodesc->maxiobuflen)] = ((double *)fillvalue)[nv]; - } - } - } - } - } - ierr = rearrange_comp2io(*ios, iodesc, array, vdesc0->iobuf, nvars); - }/* this is wrong, need to think about it - else{ - vdesc0->iobuf = array; - } */ - switch(file->iotype){ - case PIO_IOTYPE_NETCDF4P: - case PIO_IOTYPE_PNETCDF: - ierr = pio_write_darray_multi_nc(file, nvars, vid, - iodesc->ndims, iodesc->basetype, iodesc->gsize, - iodesc->maxregions, iodesc->firstregion, iodesc->llen, - iodesc->maxiobuflen, iodesc->num_aiotasks, - vdesc0->iobuf, frame); - break; - case PIO_IOTYPE_NETCDF4C: - case PIO_IOTYPE_NETCDF: - ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, - iodesc->ndims, iodesc->basetype, iodesc->gsize, - iodesc->maxregions, iodesc->firstregion, iodesc->llen, - iodesc->maxiobuflen, iodesc->num_aiotasks, - vdesc0->iobuf, frame); - if(vdesc0->iobuf != NULL){ - brel(vdesc0->iobuf); - vdesc0->iobuf = NULL; - } - break; - - } - - - - if(iodesc->rearranger == PIO_REARR_SUBSET && iodesc->needsfill && - iodesc->holegridsize>0){ - if(vdesc0->fillbuf != NULL){ - piodie("Attempt to overwrite existing buffer",__FILE__,__LINE__); - } - - vdesc0->fillbuf = bget(iodesc->holegridsize*vsize*nvars); - //printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); - if(vsize==4){ - for(int nv=0;nvholegridsize;i++){ - ((float *) vdesc0->fillbuf)[i+nv*iodesc->holegridsize] = ((float *) fillvalue)[nv]; - } - } - }else if(vsize==8){ - for(int nv=0;nvholegridsize;i++){ - ((double *) vdesc0->fillbuf)[i+nv*iodesc->holegridsize] = ((double *) fillvalue)[nv]; - } - } - } - switch(file->iotype){ - case PIO_IOTYPE_PNETCDF: - ierr = pio_write_darray_multi_nc(file, nvars, vid, - iodesc->ndims, iodesc->basetype, iodesc->gsize, - iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, - iodesc->holegridsize, iodesc->num_aiotasks, - vdesc0->fillbuf, frame); - break; - case PIO_IOTYPE_NETCDF4P: - case PIO_IOTYPE_NETCDF4C: - case PIO_IOTYPE_NETCDF: - /* ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, - iodesc->ndims, iodesc->basetype, iodesc->gsize, - iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, - iodesc->holegridsize, iodesc->num_aiotasks, - vdesc0->fillbuf, frame); - */ - /* if(vdesc0->fillbuf != NULL){ - printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); - brel(vdesc0->fillbuf); - vdesc0->fillbuf = NULL; - } - */ - break; - } - } - - flush_output_buffer(file, flushtodisk, 0); - - - return ierr; - - } +int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, + const int nvars, const PIO_Offset arraylen, + void *array, const int frame[], void *fillvalue[], + bool flushtodisk) +{ + iosystem_desc_t *ios; + file_desc_t *file; + io_desc_t *iodesc; -/** @brief Write a distributed array to the output file. - * @ingroup PIO_write_darray - * - * This routine aggregates output on the compute nodes and only sends - * it to the IO nodes when the compute buffer is full or when a flush - * is triggered. - * - * @param[in] ncid: the ncid of the open netCDF file. - * @param[in] vid: the variable ID returned by PIOc_def_var(). - * @param[in] ioid: the I/O description ID as passed back by - * PIOc_InitDecomp(). + int vsize, rlen; + int ierr; + var_desc_t *vdesc0; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if (file == NULL){ + fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); + return PIO_EBADID; + } + if (! (file->mode & PIO_WRITE)){ + fprintf(stderr,"ERROR: Attempt to write to read-only file\n"); + return PIO_EPERM; + } + + iodesc = pio_get_iodesc_from_id(ioid); + if (iodesc == NULL){ + // print_trace(NULL); + //fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); + return PIO_EBADID; + } + + vdesc0 = file->varlist+vid[0]; + + pioassert(nvars>0,"nvars <= 0",__FILE__,__LINE__); + + ios = file->iosystem; + // rlen = iodesc->llen*nvars; + rlen=0; + if (iodesc->llen>0){ + rlen = iodesc->maxiobuflen*nvars; + } + if (vdesc0->iobuf != NULL){ + piodie("Attempt to overwrite existing io buffer",__FILE__,__LINE__); + } + if (iodesc->rearranger>0){ + if (rlen>0){ + MPI_Type_size(iodesc->basetype, &vsize); + //printf("rlen*vsize = %ld\n",rlen*vsize); + + vdesc0->iobuf = bget((size_t) vsize* (size_t) rlen); + if (vdesc0->iobuf==NULL){ + printf("%s %d %d %ld\n",__FILE__,__LINE__,nvars,vsize*rlen); + piomemerror(*ios,(size_t) rlen*(size_t) vsize, __FILE__,__LINE__); + } + if (iodesc->needsfill && iodesc->rearranger==PIO_REARR_BOX){ + if (vsize==4){ + for(int nv=0;nv < nvars; nv++){ + for(int i=0;imaxiobuflen;i++){ + ((float *) vdesc0->iobuf)[i+nv*(iodesc->maxiobuflen)] = ((float *)fillvalue)[nv]; + } + } + }else if (vsize==8){ + for(int nv=0;nv < nvars; nv++){ + for(int i=0;imaxiobuflen;i++){ + ((double *)vdesc0->iobuf)[i+nv*(iodesc->maxiobuflen)] = ((double *)fillvalue)[nv]; + } + } + } + } + } + ierr = rearrange_comp2io(*ios, iodesc, array, vdesc0->iobuf, nvars); + }/* this is wrong, need to think about it + else{ + vdesc0->iobuf = array; + } */ + switch(file->iotype){ + case PIO_IOTYPE_NETCDF4P: + case PIO_IOTYPE_PNETCDF: + ierr = pio_write_darray_multi_nc(file, nvars, vid, + iodesc->ndims, iodesc->basetype, iodesc->gsize, + iodesc->maxregions, iodesc->firstregion, iodesc->llen, + iodesc->maxiobuflen, iodesc->num_aiotasks, + vdesc0->iobuf, frame); + break; + case PIO_IOTYPE_NETCDF4C: + case PIO_IOTYPE_NETCDF: + ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, + iodesc->ndims, iodesc->basetype, iodesc->gsize, + iodesc->maxregions, iodesc->firstregion, iodesc->llen, + iodesc->maxiobuflen, iodesc->num_aiotasks, + vdesc0->iobuf, frame); + if (vdesc0->iobuf != NULL){ + brel(vdesc0->iobuf); + vdesc0->iobuf = NULL; + } + break; - * @param[in] arraylen: the length of the array to be written. This - * is the length of the distrubited array. That is, the length of - * the portion of the data that is on the processor. + } - * @param[in] array: pointer to the data to be written. This is a - * pointer to the distributed portion of the array that is on this - * processor. + if (iodesc->rearranger == PIO_REARR_SUBSET && iodesc->needsfill && + iodesc->holegridsize>0){ + if (vdesc0->fillbuf != NULL){ + piodie("Attempt to overwrite existing buffer",__FILE__,__LINE__); + } - * @param[in] fillvalue: pointer to the fill value to be used for - * missing data. + vdesc0->fillbuf = bget(iodesc->holegridsize*vsize*nvars); + //printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); + if (vsize==4){ + for(int nv=0;nvholegridsize;i++){ + ((float *) vdesc0->fillbuf)[i+nv*iodesc->holegridsize] = ((float *) fillvalue)[nv]; + } + } + }else if (vsize==8){ + for(int nv=0;nvholegridsize;i++){ + ((double *) vdesc0->fillbuf)[i+nv*iodesc->holegridsize] = ((double *) fillvalue)[nv]; + } + } + } + switch(file->iotype){ + case PIO_IOTYPE_PNETCDF: + ierr = pio_write_darray_multi_nc(file, nvars, vid, + iodesc->ndims, iodesc->basetype, iodesc->gsize, + iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, + iodesc->holegridsize, iodesc->num_aiotasks, + vdesc0->fillbuf, frame); + break; + case PIO_IOTYPE_NETCDF4P: + case PIO_IOTYPE_NETCDF4C: + case PIO_IOTYPE_NETCDF: + /* ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, + iodesc->ndims, iodesc->basetype, iodesc->gsize, + iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, + iodesc->holegridsize, iodesc->num_aiotasks, + vdesc0->fillbuf, frame); + */ + /* if (vdesc0->fillbuf != NULL){ + printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); + brel(vdesc0->fillbuf); + vdesc0->fillbuf = NULL; + } + */ + break; + } + } + + flush_output_buffer(file, flushtodisk, 0); + + return ierr; +} + +/** @brief Write a distributed array to the output file. + * @ingroup PIO_write_darray + * + * This routine aggregates output on the compute nodes and only sends + * it to the IO nodes when the compute buffer is full or when a flush + * is triggered. + * + * @param[in] ncid: the ncid of the open netCDF file. + * @param[in] vid: the variable ID returned by PIOc_def_var(). + * @param[in] ioid: the I/O description ID as passed back by + * PIOc_InitDecomp(). + * @param[in] arraylen: the length of the array to be written. This + * is the length of the distrubited array. That is, the length of + * the portion of the data that is on the processor. + * @param[in] array: pointer to the data to be written. This is a + * pointer to the distributed portion of the array that is on this + * processor. + * @param[in] fillvalue: pointer to the fill value to be used for + * missing data. * * @returns 0 for success, non-zero error code for failure. */ #ifdef PIO_WRITE_BUFFERING - int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) - { - iosystem_desc_t *ios; - file_desc_t *file; - io_desc_t *iodesc; - var_desc_t *vdesc; - void *bufptr; - size_t rlen; - int ierr; - MPI_Datatype vtype; - wmulti_buffer *wmb; - int tsize; - int *tptr; - void *bptr; - void *fptr; - bool recordvar; - int needsflush; - bufsize totfree, maxfree; - - ierr = PIO_NOERR; - needsflush = 0; // false - file = pio_get_file_from_id(ncid); - if(file == NULL){ - fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); - return PIO_EBADID; - } - if(! (file->mode & PIO_WRITE)){ - fprintf(stderr,"ERROR: Attempt to write to read-only file\n"); - return PIO_EPERM; - } - - iodesc = pio_get_iodesc_from_id(ioid); - if(iodesc == NULL){ - fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); - return PIO_EBADID; - } - ios = file->iosystem; - - - vdesc = (file->varlist)+vid; - if(vdesc == NULL) - return PIO_EBADID; - - if(vdesc->record<0){ - recordvar=false; - }else{ - recordvar=true; - } - if(iodesc->ndof != arraylen){ - fprintf(stderr,"ndof=%ld, arraylen=%ld\n",iodesc->ndof,arraylen); - piodie("ndof != arraylen",__FILE__,__LINE__); - } - wmb = &(file->buffer); - if(wmb->ioid == -1){ - if(recordvar){ - wmb->ioid = ioid; - }else{ - wmb->ioid = -(ioid); - } - }else{ - // separate record and non-record variables - if(recordvar){ - while(wmb->next != NULL && wmb->ioid!=ioid){ - if(wmb->next!=NULL) - wmb = wmb->next; - } +int PIOc_write_darray(const int ncid, const int vid, const int ioid, + const PIO_Offset arraylen, void *array, void *fillvalue) +{ + iosystem_desc_t *ios; + file_desc_t *file; + io_desc_t *iodesc; + var_desc_t *vdesc; + void *bufptr; + size_t rlen; + int ierr; + MPI_Datatype vtype; + wmulti_buffer *wmb; + int tsize; + int *tptr; + void *bptr; + void *fptr; + bool recordvar; + int needsflush; + bufsize totfree, maxfree; + + ierr = PIO_NOERR; + needsflush = 0; // false + file = pio_get_file_from_id(ncid); + if (file == NULL){ + fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); + return PIO_EBADID; + } + if (! (file->mode & PIO_WRITE)){ + fprintf(stderr,"ERROR: Attempt to write to read-only file\n"); + return PIO_EPERM; + } + + iodesc = pio_get_iodesc_from_id(ioid); + if (iodesc == NULL){ + fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); + return PIO_EBADID; + } + ios = file->iosystem; + + vdesc = (file->varlist)+vid; + if (vdesc == NULL) + return PIO_EBADID; + + if (vdesc->record<0){ + recordvar=false; + } + else + { + recordvar=true; + } + if (iodesc->ndof != arraylen){ + fprintf(stderr,"ndof=%ld, arraylen=%ld\n",iodesc->ndof,arraylen); + piodie("ndof != arraylen",__FILE__,__LINE__); + } + wmb = &(file->buffer); + if (wmb->ioid == -1){ + if (recordvar){ + wmb->ioid = ioid; + } + else + { + wmb->ioid = -(ioid); + } + } + else + { + // separate record and non-record variables + if (recordvar){ + while(wmb->next != NULL && wmb->ioid!=ioid){ + if (wmb->next!=NULL) + wmb = wmb->next; + } #ifdef _PNETCDF - /* flush the previous record before starting a new one. this is collective */ - // if(vdesc->request != NULL && (vdesc->request[0] != NC_REQ_NULL) || - // (wmb->frame != NULL && vdesc->record != wmb->frame[0])){ - // needsflush = 2; // flush to disk - // } + /* flush the previous record before starting a new one. this is collective */ + // if (vdesc->request != NULL && (vdesc->request[0] != NC_REQ_NULL) || + // (wmb->frame != NULL && vdesc->record != wmb->frame[0])){ + // needsflush = 2; // flush to disk + // } #endif - }else{ - while(wmb->next != NULL && wmb->ioid!= -(ioid)){ - if(wmb->next!=NULL) - wmb = wmb->next; - } - } - } - if((recordvar && wmb->ioid != ioid) || (!recordvar && wmb->ioid != -(ioid))){ - wmb->next = (wmulti_buffer *) bget((bufsize) sizeof(wmulti_buffer)); - if(wmb->next == NULL){ - piomemerror(*ios,sizeof(wmulti_buffer), __FILE__,__LINE__); - } - wmb=wmb->next; - wmb->next=NULL; - if(recordvar){ - wmb->ioid = ioid; - }else{ - wmb->ioid = -(ioid); - } - wmb->validvars=0; - wmb->arraylen=arraylen; - wmb->vid=NULL; - wmb->data=NULL; - wmb->frame=NULL; - wmb->fillvalue=NULL; - } - + } + else + { + while(wmb->next != NULL && wmb->ioid!= -(ioid)){ + if (wmb->next!=NULL) + wmb = wmb->next; + } + } + } + if ((recordvar && wmb->ioid != ioid) || (!recordvar && wmb->ioid != -(ioid))){ + wmb->next = (wmulti_buffer *) bget((bufsize) sizeof(wmulti_buffer)); + if (wmb->next == NULL){ + piomemerror(*ios,sizeof(wmulti_buffer), __FILE__,__LINE__); + } + wmb=wmb->next; + wmb->next=NULL; + if (recordvar){ + wmb->ioid = ioid; + } + else + { + wmb->ioid = -(ioid); + } + wmb->validvars=0; + wmb->arraylen=arraylen; + wmb->vid=NULL; + wmb->data=NULL; + wmb->frame=NULL; + wmb->fillvalue=NULL; + } MPI_Type_size(iodesc->basetype, &tsize); - // At this point wmb should be pointing to a new or existing buffer - // so we can add the data + // At this point wmb should be pointing to a new or existing buffer + // so we can add the data // printf("%s %d %X %d %d %d\n",__FILE__,__LINE__,wmb->data,wmb->validvars,arraylen,tsize); // cn_buffer_report(*ios, true); bfreespace(&totfree, &maxfree); - if(needsflush==0){ - needsflush = (maxfree <= 1.1*(1+wmb->validvars)*arraylen*tsize ); + if (needsflush==0){ + needsflush = (maxfree <= 1.1*(1+wmb->validvars)*arraylen*tsize ); } MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1, MPI_INT, MPI_MAX, ios->comp_comm); - if(needsflush > 0 ){ - // need to flush first - // printf("%s %d %ld %d %ld %ld\n",__FILE__,__LINE__,maxfree, wmb->validvars, (1+wmb->validvars)*arraylen*tsize,totfree); - cn_buffer_report(*ios, true); + if (needsflush > 0 ){ + // need to flush first + // printf("%s %d %ld %d %ld %ld\n",__FILE__,__LINE__,maxfree, wmb->validvars, (1+wmb->validvars)*arraylen*tsize,totfree); + cn_buffer_report(*ios, true); - flush_buffer(ncid,wmb, needsflush==2); // if needsflush == 2 flush to disk otherwise just flush to io node + flush_buffer(ncid,wmb, needsflush==2); // if needsflush == 2 flush to disk otherwise just flush to io node } - if(arraylen > 0){ - wmb->data = bgetr( wmb->data, (1+wmb->validvars)*arraylen*tsize); - if(wmb->data == NULL){ - piomemerror(*ios, (1+wmb->validvars)*arraylen*tsize , __FILE__,__LINE__); - } + if (arraylen > 0){ + wmb->data = bgetr( wmb->data, (1+wmb->validvars)*arraylen*tsize); + if (wmb->data == NULL){ + piomemerror(*ios, (1+wmb->validvars)*arraylen*tsize , __FILE__,__LINE__); + } } wmb->vid = (int *) bgetr( wmb->vid,sizeof(int)*( 1+wmb->validvars)); - if(wmb->vid == NULL){ - piomemerror(*ios, (1+wmb->validvars)*sizeof(int) , __FILE__,__LINE__); - } - if(vdesc->record>=0){ - wmb->frame = (int *) bgetr( wmb->frame,sizeof(int)*( 1+wmb->validvars)); - if(wmb->frame == NULL){ + if (wmb->vid == NULL){ piomemerror(*ios, (1+wmb->validvars)*sizeof(int) , __FILE__,__LINE__); - } } - if(iodesc->needsfill){ - wmb->fillvalue = bgetr( wmb->fillvalue,tsize*( 1+wmb->validvars)); - if(wmb->fillvalue == NULL){ - piomemerror(*ios, (1+wmb->validvars)*tsize , __FILE__,__LINE__); - } + if (vdesc->record>=0){ + wmb->frame = (int *) bgetr( wmb->frame,sizeof(int)*( 1+wmb->validvars)); + if (wmb->frame == NULL){ + piomemerror(*ios, (1+wmb->validvars)*sizeof(int) , __FILE__,__LINE__); + } + } + if (iodesc->needsfill){ + wmb->fillvalue = bgetr( wmb->fillvalue,tsize*( 1+wmb->validvars)); + if (wmb->fillvalue == NULL){ + piomemerror(*ios, (1+wmb->validvars)*tsize , __FILE__,__LINE__); + } + } + + if (iodesc->needsfill){ + if (fillvalue != NULL){ + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars,fillvalue, tsize); + } + else + { + vtype = (MPI_Datatype) iodesc->basetype; + if (vtype == MPI_INTEGER){ + int fill = PIO_FILL_INT; + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); + }else if (vtype == MPI_FLOAT || vtype == MPI_REAL4){ + float fill = PIO_FILL_FLOAT; + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); + }else if (vtype == MPI_DOUBLE || vtype == MPI_REAL8){ + double fill = PIO_FILL_DOUBLE; + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); + }else if (vtype == MPI_CHARACTER){ + char fill = PIO_FILL_CHAR; + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); + } + else + { + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",vtype); + } + } + } + wmb->arraylen = arraylen; + wmb->vid[wmb->validvars]=vid; + bufptr = (void *)((char *) wmb->data + arraylen*tsize*wmb->validvars); + if (arraylen>0){ + memcpy(bufptr, array, arraylen*tsize); + } + /* + if (tsize==8){ + double asum=0.0; + printf("%s %d %d %d %d\n",__FILE__,__LINE__,vid,arraylen,iodesc->ndof); + for(int k=0;kneedsfill){ - if(fillvalue != NULL){ - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars,fillvalue, tsize); - }else{ - vtype = (MPI_Datatype) iodesc->basetype; - if(vtype == MPI_INTEGER){ - int fill = PIO_FILL_INT; - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); - }else if(vtype == MPI_FLOAT || vtype == MPI_REAL4){ - float fill = PIO_FILL_FLOAT; - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); - }else if(vtype == MPI_DOUBLE || vtype == MPI_REAL8){ - double fill = PIO_FILL_DOUBLE; - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); - }else if(vtype == MPI_CHARACTER){ - char fill = PIO_FILL_CHAR; - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",vtype); - } + // printf("%s %d %d %d %d %X\n",__FILE__,__LINE__,wmb->validvars,wmb->ioid,vid,bufptr); + + if (wmb->frame!=NULL){ + wmb->frame[wmb->validvars]=vdesc->record; + } + wmb->validvars++; + + // printf("%s %d %d %d %d %d\n",__FILE__,__LINE__,wmb->validvars,iodesc->maxbytes/tsize, iodesc->ndof, iodesc->llen); + if (wmb->validvars >= iodesc->maxbytes/tsize){ + PIOc_sync(ncid); } - } - - wmb->arraylen = arraylen; - wmb->vid[wmb->validvars]=vid; - bufptr = (void *)((char *) wmb->data + arraylen*tsize*wmb->validvars); - if(arraylen>0){ - memcpy(bufptr, array, arraylen*tsize); - } - /* - if(tsize==8){ - double asum=0.0; - printf("%s %d %d %d %d\n",__FILE__,__LINE__,vid,arraylen,iodesc->ndof); - for(int k=0;kvalidvars,wmb->ioid,vid,bufptr); - - if(wmb->frame!=NULL){ - wmb->frame[wmb->validvars]=vdesc->record; - } - wmb->validvars++; - - // printf("%s %d %d %d %d %d\n",__FILE__,__LINE__,wmb->validvars,iodesc->maxbytes/tsize, iodesc->ndof, iodesc->llen); - if(wmb->validvars >= iodesc->maxbytes/tsize){ - PIOc_sync(ncid); - } - - return ierr; - - } + return ierr; +} #else -/** @brief Write a distributed array to the output file +/** Write a distributed array to the output file. * @ingroup PIO_write_darray * - * This version of the routine does not buffer, all data is communicated to the io tasks - * before the routine returns - */ - int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) - { - iosystem_desc_t *ios; - file_desc_t *file; - io_desc_t *iodesc; - void *iobuf; - size_t rlen; - int tsize; - int ierr; - MPI_Datatype vtype; - - ierr = PIO_NOERR; - - - file = pio_get_file_from_id(ncid); - if(file == NULL){ - fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); - return PIO_EBADID; - } - iodesc = pio_get_iodesc_from_id(ioid); - if(iodesc == NULL){ - fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); - return PIO_EBADID; - } - iobuf = NULL; - - ios = file->iosystem; - - rlen = iodesc->llen; - if(iodesc->rearranger>0){ - if(rlen>0){ - MPI_Type_size(iodesc->basetype, &tsize); - // iobuf = bget(tsize*rlen); - iobuf = malloc((size_t) tsize*rlen); - if(iobuf==NULL){ - piomemerror(*ios,rlen*(size_t) tsize, __FILE__,__LINE__); - } - } - // printf(" rlen = %d %ld\n",rlen,iobuf); - - // } - - - ierr = rearrange_comp2io(*ios, iodesc, array, iobuf, 1); - - printf("%s %d ",__FILE__,__LINE__); - for(int n=0;n<4;n++) - printf(" %d ",((int *) iobuf)[n]); - printf("\n"); - - }else{ - iobuf = array; - } - switch(file->iotype){ - case PIO_IOTYPE_PNETCDF: - case PIO_IOTYPE_NETCDF: - case PIO_IOTYPE_NETCDF4P: - case PIO_IOTYPE_NETCDF4C: - ierr = pio_write_darray_nc(file, iodesc, vid, iobuf, fillvalue); - } - - if(iodesc->rearranger>0 && rlen>0) - free(iobuf); - - return ierr; - - } -#endif - -/** @brief Read an array of data from a file to the (parallel) IO library. - * @ingroup PIO_read_darray + * This version of the routine does not buffer, all data is + * communicated to the io tasks before the routine returns. + * + * @param ncid identifies the netCDF file + * @param vid + * @param ioid + * @param arraylen + * @param array + * @param fillvalue + * + * @return */ -int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF) +int PIOc_write_darray(const int ncid, const int vid, const int ioid, + const PIO_Offset arraylen, void *array, void *fillvalue) { - int ierr=PIO_NOERR; - iosystem_desc_t *ios; - var_desc_t *vdesc; - int ndims, fndims; - MPI_Status status; - int i; + iosystem_desc_t *ios; + file_desc_t *file; + io_desc_t *iodesc; + void *iobuf; + size_t rlen; + int tsize; + int ierr; + MPI_Datatype vtype; -#ifdef TIMING - GPTLstart("PIO:read_darray_nc"); -#endif - ios = file->iosystem; - if(ios == NULL) - return PIO_EBADID; + ierr = PIO_NOERR; - vdesc = (file->varlist)+vid; - if(vdesc == NULL) - return PIO_EBADID; + file = pio_get_file_from_id(ncid); + if (file == NULL){ + fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); + return PIO_EBADID; + } + iodesc = pio_get_iodesc_from_id(ioid); + if (iodesc == NULL){ + fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); + return PIO_EBADID; + } + iobuf = NULL; + + ios = file->iosystem; - ndims = iodesc->ndims; - ierr = PIOc_inq_varndims(file->fh, vid, &fndims); + rlen = iodesc->llen; + if (iodesc->rearranger>0){ + if (rlen>0){ + MPI_Type_size(iodesc->basetype, &tsize); + // iobuf = bget(tsize*rlen); + iobuf = malloc((size_t) tsize*rlen); + if (iobuf==NULL){ + piomemerror(*ios,rlen*(size_t) tsize, __FILE__,__LINE__); + } + } + // printf(" rlen = %d %ld\n",rlen,iobuf); - if(fndims==ndims) - vdesc->record=-1; + // } - if(ios->ioproc){ - io_region *region; - size_t start[fndims]; - size_t count[fndims]; - size_t tmp_start[fndims]; - size_t tmp_count[fndims]; - size_t tmp_bufsize=1; - int regioncnt; - void *bufptr; - int tsize; + ierr = rearrange_comp2io(*ios, iodesc, array, iobuf, 1); - int rrlen=0; - PIO_Offset *startlist[iodesc->maxregions]; - PIO_Offset *countlist[iodesc->maxregions]; + printf("%s %d ",__FILE__,__LINE__); + for(int n=0;n<4;n++) + printf(" %d ",((int *) iobuf)[n]); + printf("\n"); - // buffer is incremented by byte and loffset is in terms of the iodessc->basetype - // so we need to multiply by the size of the basetype - // We can potentially allow for one iodesc to have multiple datatypes by allowing the - // calling program to change the basetype. - region = iodesc->firstregion; - MPI_Type_size(iodesc->basetype, &tsize); - if(fndims>ndims){ - ndims++; - if(vdesc->record<0) - vdesc->record=0; } - for(regioncnt=0;regioncntmaxregions;regioncnt++){ - // printf("%s %d %d %ld %d %d\n",__FILE__,__LINE__,regioncnt,region,fndims,ndims); - tmp_bufsize=1; - if(region==NULL || iodesc->llen==0){ - for(i=0;iloffset); - - // printf("%s %d %d %d %d\n",__FILE__,__LINE__,iodesc->llen - region->loffset, iodesc->llen, region->loffset); - - if(vdesc->record >= 0 && fndims>1){ - start[0] = vdesc->record; - for(i=1;istart[i-1]; - count[i] = region->count[i-1]; - // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); - } - if(count[1]>0) - count[0] = 1; - }else{ - // Non-time dependent array - for(i=0;istart[i]; - count[i] = region->count[i]; - // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); - } + else + { + iobuf = array; + } + switch(file->iotype){ + case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_NETCDF: + case PIO_IOTYPE_NETCDF4P: + case PIO_IOTYPE_NETCDF4C: + ierr = pio_write_darray_nc(file, iodesc, vid, iobuf, fillvalue); + } + + if (iodesc->rearranger>0 && rlen>0) + free(iobuf); + + return ierr; +} +#endif + +/** Read an array of data from a file to the (parallel) IO library. + * @ingroup PIO_read_darray + * + * @param file + * @param iodesc + * @param vid + * @param IOBUF + */ +int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, + void *IOBUF) +{ + int ierr=PIO_NOERR; + iosystem_desc_t *ios; + var_desc_t *vdesc; + int ndims, fndims; + MPI_Status status; + int i; + +#ifdef TIMING + GPTLstart("PIO:read_darray_nc"); +#endif + ios = file->iosystem; + if (ios == NULL) + return PIO_EBADID; + + vdesc = (file->varlist)+vid; + + if (vdesc == NULL) + return PIO_EBADID; + + ndims = iodesc->ndims; + ierr = PIOc_inq_varndims(file->fh, vid, &fndims); + + if (fndims==ndims) + vdesc->record=-1; + + if (ios->ioproc){ + io_region *region; + size_t start[fndims]; + size_t count[fndims]; + size_t tmp_start[fndims]; + size_t tmp_count[fndims]; + size_t tmp_bufsize=1; + int regioncnt; + void *bufptr; + int tsize; + + int rrlen=0; + PIO_Offset *startlist[iodesc->maxregions]; + PIO_Offset *countlist[iodesc->maxregions]; + + // buffer is incremented by byte and loffset is in terms of the iodessc->basetype + // so we need to multiply by the size of the basetype + // We can potentially allow for one iodesc to have multiple datatypes by allowing the + // calling program to change the basetype. + region = iodesc->firstregion; + MPI_Type_size(iodesc->basetype, &tsize); + if (fndims>ndims){ + ndims++; + if (vdesc->record<0) + vdesc->record=0; } - } + for(regioncnt=0;regioncntmaxregions;regioncnt++){ + // printf("%s %d %d %ld %d %d\n",__FILE__,__LINE__,regioncnt,region,fndims,ndims); + tmp_bufsize=1; + if (region==NULL || iodesc->llen==0){ + for(i=0;iloffset); + + // printf("%s %d %d %d %d\n",__FILE__,__LINE__,iodesc->llen - region->loffset, iodesc->llen, region->loffset); + + if (vdesc->record >= 0 && fndims>1){ + start[0] = vdesc->record; + for(i=1;istart[i-1]; + count[i] = region->count[i-1]; + // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); + } + if (count[1]>0) + count[0] = 1; + } + else + { + // Non-time dependent array + for(i=0;istart[i]; + count[i] = region->count[i]; + // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); + } + } + } - switch(file->iotype){ + switch(file->iotype){ #ifdef _NETCDF4 - case PIO_IOTYPE_NETCDF4P: - if(iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_get_vara_double (file->fh, vid,start,count, bufptr); - } else if(iodesc->basetype == MPI_INTEGER){ - ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); - }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_read_darray\n",(int) iodesc->basetype); - } - break; + case PIO_IOTYPE_NETCDF4P: + if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ + ierr = nc_get_vara_double (file->fh, vid,start,count, bufptr); + } else if (iodesc->basetype == MPI_INTEGER){ + ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); + }else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ + ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); + } + else + { + fprintf(stderr,"Type not recognized %d in pioc_read_darray\n",(int) iodesc->basetype); + } + break; #endif #ifdef _PNETCDF - case PIO_IOTYPE_PNETCDF: - { - tmp_bufsize=1; - for(int j=0;j0){ - startlist[rrlen] = (PIO_Offset *) bget(fndims * sizeof(PIO_Offset)); - countlist[rrlen] = (PIO_Offset *) bget(fndims * sizeof(PIO_Offset)); - - for(int j=0;jmaxregions, j,start[j],count[j],tmp_bufsize); - } - rrlen++; - } - if(regioncnt==iodesc->maxregions-1){ - ierr = ncmpi_get_varn_all(file->fh, vid, rrlen, startlist, - countlist, IOBUF, iodesc->llen, iodesc->basetype); - for(i=0;i0){ + startlist[rrlen] = (PIO_Offset *) bget(fndims * sizeof(PIO_Offset)); + countlist[rrlen] = (PIO_Offset *) bget(fndims * sizeof(PIO_Offset)); + + for(int j=0;jmaxregions, j,start[j],count[j],tmp_bufsize); + } + rrlen++; + } + if (regioncnt==iodesc->maxregions-1){ + ierr = ncmpi_get_varn_all(file->fh, vid, rrlen, startlist, + countlist, IOBUF, iodesc->llen, iodesc->basetype); + for(i=0;iiotype,__FILE__,__LINE__); + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - if(region != NULL) - region = region->next; - } // for(regioncnt=0;...) - } + } + if (region != NULL) + region = region->next; + } // for(regioncnt=0;...) + } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); #ifdef TIMING - GPTLstop("PIO:read_darray_nc"); + GPTLstop("PIO:read_darray_nc"); #endif - return ierr; + return ierr; } - -/** @brief Read an array of data from a file to the (serial) IO library. +/** Read an array of data from a file to the (serial) IO library. * @ingroup PIO_read_darray + * + * @param file + * @param iodesc + * @param vid + * @param IOBUF + * + * @returns */ -int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF) +int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, + const int vid, void *IOBUF) { - int ierr=PIO_NOERR; - iosystem_desc_t *ios; - var_desc_t *vdesc; - int ndims, fndims; - MPI_Status status; - int i; + int ierr=PIO_NOERR; + iosystem_desc_t *ios; + var_desc_t *vdesc; + int ndims, fndims; + MPI_Status status; + int i; #ifdef TIMING - GPTLstart("PIO:read_darray_nc_serial"); + GPTLstart("PIO:read_darray_nc_serial"); #endif - ios = file->iosystem; - if(ios == NULL) - return PIO_EBADID; - - vdesc = (file->varlist)+vid; - - if(vdesc == NULL) - return PIO_EBADID; - - ndims = iodesc->ndims; - ierr = PIOc_inq_varndims(file->fh, vid, &fndims); - - if(fndims==ndims) - vdesc->record=-1; - - if(ios->ioproc){ - io_region *region; - size_t start[fndims]; - size_t count[fndims]; - size_t tmp_start[fndims*iodesc->maxregions]; - size_t tmp_count[fndims*iodesc->maxregions]; - size_t tmp_bufsize; - int regioncnt; - void *bufptr; - int tsize; - - int rrlen=0; - - // buffer is incremented by byte and loffset is in terms of the iodessc->basetype - // so we need to multiply by the size of the basetype - // We can potentially allow for one iodesc to have multiple datatypes by allowing the - // calling program to change the basetype. - region = iodesc->firstregion; - MPI_Type_size(iodesc->basetype, &tsize); - if(fndims>ndims){ - if(vdesc->record<0) - vdesc->record=0; - } - for(regioncnt=0;regioncntmaxregions;regioncnt++){ - if(region==NULL || iodesc->llen==0){ - for(i=0;irecord >= 0 && fndims>1){ - tmp_start[regioncnt*fndims] = vdesc->record; - for(i=1;istart[i-1]; - tmp_count[i+regioncnt*fndims] = region->count[i-1]; - } - if(tmp_count[1+regioncnt*fndims]>0) - tmp_count[regioncnt*fndims] = 1; - }else{ - // Non-time dependent array - for(i=0;istart[i]; - tmp_count[i+regioncnt*fndims] = region->count[i]; - } + ios = file->iosystem; + if (ios == NULL) + return PIO_EBADID; + + vdesc = (file->varlist)+vid; + + if (vdesc == NULL) + return PIO_EBADID; + + ndims = iodesc->ndims; + ierr = PIOc_inq_varndims(file->fh, vid, &fndims); + + if (fndims==ndims) + vdesc->record=-1; + + if (ios->ioproc){ + io_region *region; + size_t start[fndims]; + size_t count[fndims]; + size_t tmp_start[fndims*iodesc->maxregions]; + size_t tmp_count[fndims*iodesc->maxregions]; + size_t tmp_bufsize; + int regioncnt; + void *bufptr; + int tsize; + + int rrlen=0; + + // buffer is incremented by byte and loffset is in terms of the iodessc->basetype + // so we need to multiply by the size of the basetype + // We can potentially allow for one iodesc to have multiple datatypes by allowing the + // calling program to change the basetype. + region = iodesc->firstregion; + MPI_Type_size(iodesc->basetype, &tsize); + if (fndims>ndims){ + if (vdesc->record<0) + vdesc->record=0; } - /* for(i=0;imaxregions;regioncnt++){ + if (region==NULL || iodesc->llen==0){ + for(i=0;irecord >= 0 && fndims>1){ + tmp_start[regioncnt*fndims] = vdesc->record; + for(i=1;istart[i-1]; + tmp_count[i+regioncnt*fndims] = region->count[i-1]; + } + if (tmp_count[1+regioncnt*fndims]>0) + tmp_count[regioncnt*fndims] = 1; + } + else + { + // Non-time dependent array + for(i=0;istart[i]; + tmp_count[i+regioncnt*fndims] = region->count[i]; + } + } + /* for(i=0;inext; - } // for(regioncnt=0;...) - - if(ios->io_rank>0){ - MPI_Send( &(iodesc->llen), 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); - if(iodesc->llen > 0){ - MPI_Send( &(iodesc->maxregions), 1, MPI_INT, 0, ios->num_iotasks+ios->io_rank, ios->io_comm); - MPI_Send( tmp_count, iodesc->maxregions*fndims, MPI_OFFSET, 0, 2*ios->num_iotasks+ios->io_rank, ios->io_comm); - MPI_Send( tmp_start, iodesc->maxregions*fndims, MPI_OFFSET, 0, 3*ios->num_iotasks+ios->io_rank, ios->io_comm); - MPI_Recv(IOBUF, iodesc->llen, iodesc->basetype, 0, 4*ios->num_iotasks+ios->io_rank, ios->io_comm, &status); - } - }else if(ios->io_rank==0){ - int maxregions=0; - size_t loffset, regionsize; - size_t this_start[fndims*iodesc->maxregions]; - size_t this_count[fndims*iodesc->maxregions]; - // for( i=ios->num_iotasks-1; i>=0; i--){ - for(int rtask=1;rtask<=ios->num_iotasks;rtask++){ - if(rtasknum_iotasks){ - MPI_Recv(&tmp_bufsize, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); - if(tmp_bufsize>0){ - MPI_Recv(&maxregions, 1, MPI_INT, rtask, ios->num_iotasks+rtask, ios->io_comm, &status); - MPI_Recv(this_count, maxregions*fndims, MPI_OFFSET, rtask, 2*ios->num_iotasks+rtask, ios->io_comm, &status); - MPI_Recv(this_start, maxregions*fndims, MPI_OFFSET, rtask, 3*ios->num_iotasks+rtask, ios->io_comm, &status); - } - }else{ - maxregions=iodesc->maxregions; - tmp_bufsize=iodesc->llen; - } - loffset = 0; - for(regioncnt=0;regioncntnum_iotasks){ - for(int m=0; mnext; + } // for(regioncnt=0;...) + + if (ios->io_rank>0){ + MPI_Send( &(iodesc->llen), 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); + if (iodesc->llen > 0){ + MPI_Send( &(iodesc->maxregions), 1, MPI_INT, 0, ios->num_iotasks+ios->io_rank, ios->io_comm); + MPI_Send( tmp_count, iodesc->maxregions*fndims, MPI_OFFSET, 0, 2*ios->num_iotasks+ios->io_rank, ios->io_comm); + MPI_Send( tmp_start, iodesc->maxregions*fndims, MPI_OFFSET, 0, 3*ios->num_iotasks+ios->io_rank, ios->io_comm); + MPI_Recv(IOBUF, iodesc->llen, iodesc->basetype, 0, 4*ios->num_iotasks+ios->io_rank, ios->io_comm, &status); } - } - loffset+=regionsize; + }else if (ios->io_rank==0){ + int maxregions=0; + size_t loffset, regionsize; + size_t this_start[fndims*iodesc->maxregions]; + size_t this_count[fndims*iodesc->maxregions]; + // for( i=ios->num_iotasks-1; i>=0; i--){ + for(int rtask=1;rtask<=ios->num_iotasks;rtask++){ + if (rtasknum_iotasks){ + MPI_Recv(&tmp_bufsize, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); + if (tmp_bufsize>0){ + MPI_Recv(&maxregions, 1, MPI_INT, rtask, ios->num_iotasks+rtask, ios->io_comm, &status); + MPI_Recv(this_count, maxregions*fndims, MPI_OFFSET, rtask, 2*ios->num_iotasks+rtask, ios->io_comm, &status); + MPI_Recv(this_start, maxregions*fndims, MPI_OFFSET, rtask, 3*ios->num_iotasks+rtask, ios->io_comm, &status); + } + } + else + { + maxregions=iodesc->maxregions; + tmp_bufsize=iodesc->llen; + } + loffset = 0; + for(regioncnt=0;regioncntnum_iotasks){ + for(int m=0; mbasetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_get_vara_double (file->fh, vid,start, count, bufptr); - }else if(iodesc->basetype == MPI_INTEGER){ - ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); - }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray_nc_serial\n",(int) iodesc->basetype); - } - - if(ierr != PIO_NOERR){ - for(int i=0;ibasetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ + ierr = nc_get_vara_double (file->fh, vid,start, count, bufptr); + }else if (iodesc->basetype == MPI_INTEGER){ + ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); + }else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ + ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); + } + else + { + fprintf(stderr,"Type not recognized %d in pioc_write_darray_nc_serial\n",(int) iodesc->basetype); + } + + if (ierr != PIO_NOERR){ + for(int i=0;inum_iotasks){ + MPI_Send(IOBUF, tmp_bufsize, iodesc->basetype, rtask,4*ios->num_iotasks+rtask, ios->io_comm); + } + } } - if(rtasknum_iotasks){ - MPI_Send(IOBUF, tmp_bufsize, iodesc->basetype, rtask,4*ios->num_iotasks+rtask, ios->io_comm); - } - } } - } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); #ifdef TIMING - GPTLstop("PIO:read_darray_nc_serial"); + GPTLstop("PIO:read_darray_nc_serial"); #endif - return ierr; + return ierr; } - -/** @brief Read a field from a file to the IO library. - * @ingroup PIO_read_darray +/** Read a field from a file to the IO library. + * @ingroup PIO_read_darray + * + * @param ncid identifies the netCDF file + * @param vid + * @param ioid + * @param arraylen + * @param array * + * @return */ -int PIOc_read_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array) +int PIOc_read_darray(const int ncid, const int vid, const int ioid, + const PIO_Offset arraylen, void *array) { - iosystem_desc_t *ios; - file_desc_t *file; - io_desc_t *iodesc; - void *iobuf=NULL; - size_t rlen=0; - int ierr, tsize; - MPI_Datatype vtype; - - file = pio_get_file_from_id(ncid); - - if(file == NULL){ - fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); - return PIO_EBADID; - } - iodesc = pio_get_iodesc_from_id(ioid); - if(iodesc == NULL){ - fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); - return PIO_EBADID; - } - ios = file->iosystem; - if(ios->iomaster){ - rlen = iodesc->maxiobuflen; - }else{ - rlen = iodesc->llen; - } - - if(iodesc->rearranger > 0){ - if(ios->ioproc && rlen>0){ - MPI_Type_size(iodesc->basetype, &tsize); - iobuf = bget(((size_t) tsize)*rlen); - if(iobuf==NULL){ - piomemerror(*ios,rlen*((size_t) tsize), __FILE__,__LINE__); - } - } - }else{ - iobuf = array; - } - - switch(file->iotype){ - case PIO_IOTYPE_NETCDF: - case PIO_IOTYPE_NETCDF4C: - ierr = pio_read_darray_nc_serial(file, iodesc, vid, iobuf); - break; - case PIO_IOTYPE_PNETCDF: - case PIO_IOTYPE_NETCDF4P: - ierr = pio_read_darray_nc(file, iodesc, vid, iobuf); - break; - default: - ierr = iotype_error(file->iotype,__FILE__,__LINE__); - } - if(iodesc->rearranger > 0){ - ierr = rearrange_io2comp(*ios, iodesc, iobuf, array); - - if(rlen>0) - brel(iobuf); - } - - return ierr; + iosystem_desc_t *ios; + file_desc_t *file; + io_desc_t *iodesc; + void *iobuf=NULL; + size_t rlen=0; + int ierr, tsize; + MPI_Datatype vtype; + + file = pio_get_file_from_id(ncid); + + if (file == NULL){ + fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); + return PIO_EBADID; + } + iodesc = pio_get_iodesc_from_id(ioid); + if (iodesc == NULL){ + fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); + return PIO_EBADID; + } + ios = file->iosystem; + if (ios->iomaster){ + rlen = iodesc->maxiobuflen; + } + else + { + rlen = iodesc->llen; + } + + if (iodesc->rearranger > 0){ + if (ios->ioproc && rlen>0){ + MPI_Type_size(iodesc->basetype, &tsize); + iobuf = bget(((size_t) tsize)*rlen); + if (iobuf==NULL){ + piomemerror(*ios,rlen*((size_t) tsize), __FILE__,__LINE__); + } + } + } + else + { + iobuf = array; + } + + switch(file->iotype){ + case PIO_IOTYPE_NETCDF: + case PIO_IOTYPE_NETCDF4C: + ierr = pio_read_darray_nc_serial(file, iodesc, vid, iobuf); + break; + case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_NETCDF4P: + ierr = pio_read_darray_nc(file, iodesc, vid, iobuf); + break; + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + if (iodesc->rearranger > 0){ + ierr = rearrange_io2comp(*ios, iodesc, iobuf, array); + + if (rlen>0) + brel(iobuf); + } + + return ierr; } +/** Flush the output buffer. + * + * @param file + * @param force + * @param addsize + * + * @return + */ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) { - var_desc_t *vdesc; - int ierr=PIO_NOERR; + var_desc_t *vdesc; + int ierr=PIO_NOERR; #ifdef _PNETCDF - int *status; - PIO_Offset usage = 0; + int *status; + PIO_Offset usage = 0; #ifdef TIMING - GPTLstart("PIO:flush_output_buffer"); + GPTLstart("PIO:flush_output_buffer"); #endif - pioassert(file!=NULL,"file pointer not defined",__FILE__,__LINE__); - - - ierr = ncmpi_inq_buffer_usage(file->fh, &usage); - - if(!force && file->iosystem->io_comm != MPI_COMM_NULL){ - usage += addsize; - - MPI_Allreduce(MPI_IN_PLACE, &usage, 1, MPI_OFFSET, MPI_MAX, - file->iosystem->io_comm); - } - - if(usage > maxusage){ - maxusage = usage; - } - if(force || usage>=PIO_BUFFER_SIZE_LIMIT){ - int rcnt; - bool prev_dist=false; - int prev_record=-1; - int prev_type=0; - int maxreq; - int reqcnt; - maxreq = 0; - reqcnt=0; - rcnt=0; - for(int i=0; ivarlist+i; - reqcnt+=vdesc->nreqs; - if(vdesc->nreqs>0) maxreq = i; + pioassert(file!=NULL,"file pointer not defined",__FILE__,__LINE__); + + + ierr = ncmpi_inq_buffer_usage(file->fh, &usage); + + if (!force && file->iosystem->io_comm != MPI_COMM_NULL){ + usage += addsize; + + MPI_Allreduce(MPI_IN_PLACE, &usage, 1, MPI_OFFSET, MPI_MAX, + file->iosystem->io_comm); } - int request[reqcnt]; - int status[reqcnt]; - for(int i=0; i<=maxreq; i++){ - vdesc = file->varlist+i; -#ifdef MPIO_ONESIDED - /*onesided optimization requires that all of the requests in a wait_all call represent - a contiguous block of data in the file */ - if(rcnt>0 && (prev_record != vdesc->record || - vdesc->nreqs==0)){ - ierr = ncmpi_wait_all(file->fh, rcnt, request,status); + if (usage > maxusage){ + maxusage = usage; + } + if (force || usage>=PIO_BUFFER_SIZE_LIMIT){ + int rcnt; + bool prev_dist=false; + int prev_record=-1; + int prev_type=0; + int maxreq; + int reqcnt; + maxreq = 0; + reqcnt=0; rcnt=0; - } - prev_record = vdesc->record; + for(int i=0; ivarlist+i; + reqcnt+=vdesc->nreqs; + if (vdesc->nreqs>0) maxreq = i; + } + int request[reqcnt]; + int status[reqcnt]; + + for(int i=0; i<=maxreq; i++){ + vdesc = file->varlist+i; +#ifdef MPIO_ONESIDED + /*onesided optimization requires that all of the requests in a wait_all call represent + a contiguous block of data in the file */ + if (rcnt>0 && (prev_record != vdesc->record || + vdesc->nreqs==0)){ + ierr = ncmpi_wait_all(file->fh, rcnt, request,status); + rcnt=0; + } + prev_record = vdesc->record; #endif - // printf("%s %d %d %d %d \n",__FILE__,__LINE__,i,vdesc->nreqs,vdesc->request); - for(reqcnt=0;reqcntnreqs;reqcnt++){ - request[rcnt++] = max(vdesc->request[reqcnt],NC_REQ_NULL); - } - free(vdesc->request); - vdesc->request=NULL; - vdesc->nreqs=0; - // if(file->iosystem->io_rank < 2) printf("%s %d varid=%d\n",__FILE__,__LINE__,i); + // printf("%s %d %d %d %d \n",__FILE__,__LINE__,i,vdesc->nreqs,vdesc->request); + for(reqcnt=0;reqcntnreqs;reqcnt++){ + request[rcnt++] = max(vdesc->request[reqcnt],NC_REQ_NULL); + } + free(vdesc->request); + vdesc->request=NULL; + vdesc->nreqs=0; + // if (file->iosystem->io_rank < 2) printf("%s %d varid=%d\n",__FILE__,__LINE__,i); #ifdef FLUSH_EVERY_VAR - ierr = ncmpi_wait_all(file->fh, rcnt, request,status); - rcnt=0; + ierr = ncmpi_wait_all(file->fh, rcnt, request,status); + rcnt=0; #endif - } - // if(file->iosystem->io_rank==0){ - // printf("%s %d %d\n",__FILE__,__LINE__,rcnt); - // } - if(rcnt>0){ - /* - if(file->iosystem->io_rank==0){ - printf("%s %d %d ",__FILE__,__LINE__,rcnt); - for(int i=0; ifh, rcnt, request,status); - } - for(int i=0; ivarlist+i; - if(vdesc->iobuf != NULL){ - brel(vdesc->iobuf); - vdesc->iobuf=NULL; - } - if(vdesc->fillbuf != NULL){ - brel(vdesc->fillbuf); - vdesc->fillbuf=NULL; - } - } + // if (file->iosystem->io_rank==0){ + // printf("%s %d %d\n",__FILE__,__LINE__,rcnt); + // } + if (rcnt>0){ + /* + if (file->iosystem->io_rank==0){ + printf("%s %d %d ",__FILE__,__LINE__,rcnt); + for(int i=0; ifh, rcnt, request,status); + } + for(int i=0; ivarlist+i; + if (vdesc->iobuf != NULL){ + brel(vdesc->iobuf); + vdesc->iobuf=NULL; + } + if (vdesc->fillbuf != NULL){ + brel(vdesc->fillbuf); + vdesc->fillbuf=NULL; + } + } - } + } #ifdef TIMING - GPTLstop("PIO:flush_output_buffer"); + GPTLstop("PIO:flush_output_buffer"); #endif #endif - return ierr; + return ierr; } +/** Print out info about the buffer for debug purposes. + * + * @param ios the IO system structure + * @param collective true if collective report is desired + */ void cn_buffer_report(iosystem_desc_t ios, bool collective) { - if(CN_bpool != NULL){ - long bget_stats[5]; - long bget_mins[5]; - long bget_maxs[5]; - - bstats(bget_stats, bget_stats+1,bget_stats+2,bget_stats+3,bget_stats+4); - if(collective){ - MPI_Reduce(bget_stats, bget_maxs, 5, MPI_LONG, MPI_MAX, 0, ios.comp_comm); - MPI_Reduce(bget_stats, bget_mins, 5, MPI_LONG, MPI_MIN, 0, ios.comp_comm); - if(ios.compmaster){ - printf("PIO: Currently allocated buffer space %ld %ld\n",bget_mins[0],bget_maxs[0]); - printf("PIO: Currently available buffer space %ld %ld\n",bget_mins[1],bget_maxs[1]); - printf("PIO: Current largest free block %ld %ld\n",bget_mins[2],bget_maxs[2]); - printf("PIO: Number of successful bget calls %ld %ld\n",bget_mins[3],bget_maxs[3]); - printf("PIO: Number of successful brel calls %ld %ld\n",bget_mins[4],bget_maxs[4]); - // print_trace(stdout); - } - }else{ - printf("%d: PIO: Currently allocated buffer space %ld \n",ios.union_rank,bget_stats[0]) ; - printf("%d: PIO: Currently available buffer space %ld \n",ios.union_rank,bget_stats[1]); - printf("%d: PIO: Current largest free block %ld \n",ios.union_rank,bget_stats[2]); - printf("%d: PIO: Number of successful bget calls %ld \n",ios.union_rank,bget_stats[3]); - printf("%d: PIO: Number of successful brel calls %ld \n",ios.union_rank,bget_stats[4]); + if (CN_bpool != NULL){ + long bget_stats[5]; + long bget_mins[5]; + long bget_maxs[5]; + + bstats(bget_stats, bget_stats+1,bget_stats+2,bget_stats+3,bget_stats+4); + if (collective) + { + MPI_Reduce(bget_stats, bget_maxs, 5, MPI_LONG, MPI_MAX, 0, ios.comp_comm); + MPI_Reduce(bget_stats, bget_mins, 5, MPI_LONG, MPI_MIN, 0, ios.comp_comm); + if (ios.compmaster) + { + printf("PIO: Currently allocated buffer space %ld %ld\n",bget_mins[0],bget_maxs[0]); + printf("PIO: Currently available buffer space %ld %ld\n",bget_mins[1],bget_maxs[1]); + printf("PIO: Current largest free block %ld %ld\n",bget_mins[2],bget_maxs[2]); + printf("PIO: Number of successful bget calls %ld %ld\n",bget_mins[3],bget_maxs[3]); + printf("PIO: Number of successful brel calls %ld %ld\n",bget_mins[4],bget_maxs[4]); + // print_trace(stdout); + } + } + else + { + printf("%d: PIO: Currently allocated buffer space %ld \n",ios.union_rank,bget_stats[0]) ; + printf("%d: PIO: Currently available buffer space %ld \n",ios.union_rank,bget_stats[1]); + printf("%d: PIO: Current largest free block %ld \n",ios.union_rank,bget_stats[2]); + printf("%d: PIO: Number of successful bget calls %ld \n",ios.union_rank,bget_stats[3]); + printf("%d: PIO: Number of successful brel calls %ld \n",ios.union_rank,bget_stats[4]); + } } - } } +/** Free the buffer pool. + * + * @param ios + */ void free_cn_buffer_pool(iosystem_desc_t ios) { #ifndef PIO_USE_MALLOC - if(CN_bpool != NULL){ - cn_buffer_report(ios, true); - bpoolrelease(CN_bpool); - // free(CN_bpool); - CN_bpool=NULL; - } + if (CN_bpool != NULL){ + cn_buffer_report(ios, true); + bpoolrelease(CN_bpool); + // free(CN_bpool); + CN_bpool = NULL; + } #endif } +/** Flush the buffer. + * + * @param ncid + * @param wmb + * @param flushtodisk + */ void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) { - if(wmb->validvars>0){ - PIOc_write_darray_multi(ncid, wmb->vid, wmb->ioid, wmb->validvars, wmb->arraylen, wmb->data, wmb->frame, wmb->fillvalue, flushtodisk); - wmb->validvars=0; - brel(wmb->vid); - wmb->vid=NULL; - brel(wmb->data); - wmb->data=NULL; - if(wmb->fillvalue != NULL) - brel(wmb->fillvalue); - if(wmb->frame != NULL) - brel(wmb->frame); - wmb->fillvalue=NULL; - wmb->frame=NULL; - } + if (wmb->validvars > 0){ + PIOc_write_darray_multi(ncid, wmb->vid, wmb->ioid, wmb->validvars, + wmb->arraylen, wmb->data, wmb->frame, + wmb->fillvalue, flushtodisk); + wmb->validvars = 0; + brel(wmb->vid); + wmb->vid = NULL; + brel(wmb->data); + wmb->data = NULL; + if (wmb->fillvalue != NULL) + brel(wmb->fillvalue); + if (wmb->frame != NULL) + brel(wmb->frame); + wmb->fillvalue = NULL; + wmb->frame = NULL; + } } +/** Comput the maximum aggregate number of bytes. + * + * @param ios + * @param iodesc + */ void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc) { - int maxbytesoniotask=INT_MAX; - int maxbytesoncomputetask=INT_MAX; - int maxbytes; + int maxbytesoniotask = INT_MAX; + int maxbytesoncomputetask = INT_MAX; + int maxbytes; + + // printf("%s %d %d %d\n",__FILE__,__LINE__,iodesc->maxiobuflen, iodesc->ndof); + + if (ios.ioproc && iodesc->maxiobuflen > 0) + maxbytesoniotask = PIO_BUFFER_SIZE_LIMIT/ iodesc->maxiobuflen; - // printf("%s %d %d %d\n",__FILE__,__LINE__,iodesc->maxiobuflen, iodesc->ndof); + if (ios.comp_rank >= 0 && iodesc->ndof > 0) + maxbytesoncomputetask = PIO_CNBUFFER_LIMIT / iodesc->ndof; - if(ios.ioproc && iodesc->maxiobuflen>0){ - maxbytesoniotask = PIO_BUFFER_SIZE_LIMIT/ iodesc->maxiobuflen; - } - if(ios.comp_rank>=0 && iodesc->ndof>0){ - maxbytesoncomputetask = PIO_CNBUFFER_LIMIT/iodesc->ndof; - } - maxbytes = min(maxbytesoniotask,maxbytesoncomputetask); + maxbytes = min(maxbytesoniotask, maxbytesoncomputetask); - // printf("%s %d %d %d\n",__FILE__,__LINE__,maxbytesoniotask, maxbytesoncomputetask); + // printf("%s %d %d %d\n",__FILE__,__LINE__,maxbytesoniotask, maxbytesoncomputetask); - MPI_Allreduce(MPI_IN_PLACE, &maxbytes, 1, MPI_INT, MPI_MIN, ios.union_comm); - iodesc->maxbytes=maxbytes; - // printf("%s %d %d %d\n",__FILE__,__LINE__,iodesc->maxbytes,iodesc->maxiobuflen); + MPI_Allreduce(MPI_IN_PLACE, &maxbytes, 1, MPI_INT, MPI_MIN, ios.union_comm); + iodesc->maxbytes=maxbytes; + // printf("%s %d %d %d\n",__FILE__,__LINE__,iodesc->maxbytes,iodesc->maxiobuflen); } diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c index b748f7a420f..3d464bb660d 100644 --- a/tests/unit/test_nc4.c +++ b/tests/unit/test_nc4.c @@ -81,7 +81,7 @@ char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"}; int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; /** Length of chunksizes to use in netCDF-4 files. */ -size_t chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2}; +PIO_Offset chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2}; /** Run Tests for NetCDF-4 Functions. * @@ -147,7 +147,7 @@ main(int argc, char **argv) int storage; /** Chunksizes set in the file. */ - size_t my_chunksize[NDIM]; + PIO_Offset my_chunksize[NDIM]; /** The shuffle filter setting in the netCDF-4 test file. */ int shuffle; @@ -189,13 +189,13 @@ main(int argc, char **argv) int fmt, d, d1, i; /** For setting the chunk cache. */ - size_t chunk_cache_size = 1024*1024; - size_t chunk_cache_nelems = 1024; + PIO_Offset chunk_cache_size = 1024*1024; + PIO_Offset chunk_cache_nelems = 1024; float chunk_cache_preemption = 0.5; /* For reading the chunk cache. */ - size_t chunk_cache_size_in; - size_t chunk_cache_nelems_in; + PIO_Offset chunk_cache_size_in; + PIO_Offset chunk_cache_nelems_in; float chunk_cache_preemption_in; char varname[15]; From ede61821719bb8cd412ac1cf70246afbf99eaa70 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 15 Jun 2016 09:32:37 -0600 Subject: [PATCH 155/184] documentation fix --- doc/source/Error.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/source/Error.txt b/doc/source/Error.txt index 4664919cbc8..72c0da23e20 100644 --- a/doc/source/Error.txt +++ b/doc/source/Error.txt @@ -21,9 +21,6 @@ describing the error and then calling mpi_abort. Application developers can change this behaivior with a call to \ref PIO_seterrorhandling -For example, if a developer wanted -to see if an input netcdf file contained the variable 'U' they might do the following: - \verbinclude errorhandle \copydoc PIO_error_method From bce138fac70c6d5f737995a203b900ea50e058a2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 15 Jun 2016 09:58:17 -0600 Subject: [PATCH 156/184] starting to add darray test --- tests/unit/CMakeLists.txt | 7 + tests/unit/test_darray.c | 416 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 423 insertions(+) create mode 100644 tests/unit/test_darray.c diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index f4f49793221..0dbe18565c5 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -52,6 +52,8 @@ add_executable (test_names EXCLUDE_FROM_ALL test_names.c) target_link_libraries (test_names pioc) add_executable (test_nc4 EXCLUDE_FROM_ALL test_nc4.c) target_link_libraries (test_nc4 pioc) +add_executable (test_darray EXCLUDE_FROM_ALL test_darray.c) +target_link_libraries (test_darray pioc) if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" ) @@ -60,6 +62,7 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") endif () add_dependencies (tests test_names) +add_dependencies (tests test_darray) add_dependencies (tests test_nc4) add_dependencies (tests pio_unit_test) @@ -76,6 +79,10 @@ if (PIO_USE_MPISERIAL) set_tests_properties(pio_unit_test PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) else () + add_mpi_test(test_darray + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray + NUMPROCS 4 + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) if (PIO_ENABLE_ASYNC) add_mpi_test(test_intercomm EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_intercomm diff --git a/tests/unit/test_darray.c b/tests/unit/test_darray.c new file mode 100644 index 00000000000..9df6573f860 --- /dev/null +++ b/tests/unit/test_darray.c @@ -0,0 +1,416 @@ +/** + * @file + * Tests for darray functions. + * + */ +#include +#ifdef TIMING +#include +#endif + +#define NUM_NETCDF_FLAVORS 4 +#define NDIM 3 +#define X_DIM_LEN 400 +#define Y_DIM_LEN 400 +#define NUM_TIMESTEPS 6 +#define VAR_NAME "foo" +#define ATT_NAME "bar" +#define START_DATA_VAL 42 +#define ERR_AWFUL 1111 +#define VAR_CACHE_SIZE (1024 * 1024) +#define VAR_CACHE_NELEMS 10 +#define VAR_CACHE_PREEMPTION 0.5 + +/** Handle MPI errors. This should only be used with MPI library + * function calls. */ +#define MPIERR(e) do { \ + MPI_Error_string(e, err_buffer, &resultlen); \ + fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \ + MPI_Finalize(); \ + return ERR_AWFUL; \ + } while (0) + +/** Handle non-MPI errors by finalizing the MPI library and exiting + * with an exit code. */ +#define ERR(e) do { \ + fprintf(stderr, "Error %d in %s, line %d\n", e, __FILE__, __LINE__); \ + MPI_Finalize(); \ + return e; \ + } while (0) + +/** Global err buffer for MPI. */ +char err_buffer[MPI_MAX_ERROR_STRING]; +int resultlen; + +/** The dimension names. */ +char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"}; + +/** Length of the dimensions in the sample data. */ +int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; + +/** Length of chunksizes to use in netCDF-4 files. */ +size_t chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2}; + +/** Check the dimension names. + * + * @param my_rank rank of process + * @param ncid ncid of open netCDF file + * + * @returns 0 for success, error code otherwise. */ +int +check_dim_names(int my_rank, int ncid, int verbose) +{ + char dim_name[NC_MAX_NAME + 1]; + char zero_dim_name[NC_MAX_NAME + 1]; + int ret; + + for (int d = 0; d < NDIM; d++) + { + strcpy(dim_name, "11111111111111111111111111111111"); + if ((ret = PIOc_inq_dimname(ncid, d, dim_name))) + return ret; + if (verbose) + printf("my_rank %d dim %d name %s\n", my_rank, d, dim_name); + + /* Did other ranks get the same name? */ + if (!my_rank) + strcpy(zero_dim_name, dim_name); + /* if (verbose) */ + /* printf("rank %d dim_name %s zero_dim_name %s\n", my_rank, dim_name, zero_dim_name); */ + if ((ret = MPI_Bcast(&zero_dim_name, strlen(dim_name) + 1, MPI_CHAR, 0, + MPI_COMM_WORLD))) + MPIERR(ret); + if (strcmp(dim_name, zero_dim_name)) + return ERR_AWFUL; + } + return 0; +} + +/** Check the variable name. + * + * @param my_rank rank of process + * @param ncid ncid of open netCDF file + * + * @returns 0 for success, error code otherwise. */ +int +check_var_name(int my_rank, int ncid, int verbose) +{ + char var_name[NC_MAX_NAME + 1]; + char zero_var_name[NC_MAX_NAME + 1]; + int ret; + + strcpy(var_name, "11111111111111111111111111111111"); + if ((ret = PIOc_inq_varname(ncid, 0, var_name))) + return ret; + if (verbose) + printf("my_rank %d var name %s\n", my_rank, var_name); + + /* Did other ranks get the same name? */ + if (!my_rank) + strcpy(zero_var_name, var_name); + if ((ret = MPI_Bcast(&zero_var_name, strlen(var_name) + 1, MPI_CHAR, 0, + MPI_COMM_WORLD))) + MPIERR(ret); + if (strcmp(var_name, zero_var_name)) + return ERR_AWFUL; + return 0; +} + +/** Check the attribute name. + * + * @param my_rank rank of process + * @param ncid ncid of open netCDF file + * + * @returns 0 for success, error code otherwise. */ +int +check_att_name(int my_rank, int ncid, int verbose) +{ + char att_name[NC_MAX_NAME + 1]; + char zero_att_name[NC_MAX_NAME + 1]; + int ret; + + strcpy(att_name, "11111111111111111111111111111111"); + if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, att_name))) + return ret; + if (verbose) + printf("my_rank %d att name %s\n", my_rank, att_name); + + /* Did everyone ranks get the same length name? */ +/* if (strlen(att_name) != strlen(ATT_NAME)) + return ERR_AWFUL;*/ + if (!my_rank) + strcpy(zero_att_name, att_name); + if ((ret = MPI_Bcast(&zero_att_name, strlen(att_name) + 1, MPI_CHAR, 0, + MPI_COMM_WORLD))) + MPIERR(ret); + if (strcmp(att_name, zero_att_name)) + return ERR_AWFUL; + return 0; +} + +/** Run Tests for NetCDF-4 Functions. + * + * @param argc argument count + * @param argv array of arguments + */ +int +main(int argc, char **argv) +{ + int verbose = 1; + + /** Zero-based rank of processor. */ + int my_rank; + + /** Number of processors involved in current execution. */ + int ntasks; + + /** Specifies the flavor of netCDF output format. */ + int iotype; + + /** Different output flavors. */ + int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, + PIO_IOTYPE_NETCDF, + PIO_IOTYPE_NETCDF4C, + PIO_IOTYPE_NETCDF4P}; + + /** Names for the output files. */ + char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_names_pnetcdf.nc", + "test_names_classic.nc", + "test_names_serial4.nc", + "test_names_parallel4.nc"}; + + /** Number of processors that will do IO. In this test we + * will do IO from all processors. */ + int niotasks; + + /** Stride in the mpi rank between io tasks. Always 1 in this + * test. */ + int ioproc_stride = 1; + + /** Number of the aggregator? Always 0 in this test. */ + int numAggregator = 0; + + /** Zero based rank of first processor to be used for I/O. */ + int ioproc_start = 0; + + /** The dimension IDs. */ + int dimids[NDIM]; + + /** Array index per processing unit. */ + PIO_Offset elements_per_pe; + + /** The ID for the parallel I/O system. */ + int iosysid; + + /** The ncid of the netCDF file. */ + int ncid = 0; + + /** The ID of the netCDF varable. */ + int varid; + + /** Storage of netCDF-4 files (contiguous vs. chunked). */ + int storage; + + /** Chunksizes set in the file. */ + size_t my_chunksize[NDIM]; + + /** The shuffle filter setting in the netCDF-4 test file. */ + int shuffle; + + /** Non-zero if deflate set for the variable in the netCDF-4 test file. */ + int deflate; + + /** The deflate level set for the variable in the netCDF-4 test file. */ + int deflate_level; + + /** Non-zero if fletcher32 filter is used for variable. */ + int fletcher32; + + /** Endianness of variable. */ + int endianness; + + /* Size of the file chunk cache. */ + size_t chunk_cache_size; + + /* Number of elements in file cache. */ + size_t nelems; + + /* File cache preemption. */ + float preemption; + + /* Size of the var chunk cache. */ + size_t var_cache_size; + + /* Number of elements in var cache. */ + size_t var_cache_nelems; + + /* Var cache preemption. */ + float var_cache_preemption; + + /** The I/O description ID. */ + int ioid; + + /** A buffer for sample data. */ + float *buffer; + + /** A buffer for reading data back from the file. */ + int *read_buffer; + + /** The decomposition mapping. */ + PIO_Offset *compdof; + + /** Return code. */ + int ret; + + /** Index for loops. */ + int fmt, d, d1, i; + +#ifdef TIMING + /* Initialize the GPTL timing library. */ + if ((ret = GPTLinitialize ())) + return ret; +#endif + + /* Initialize MPI. */ + if ((ret = MPI_Init(&argc, &argv))) + MPIERR(ret); + + /* Learn my rank and the total number of processors. */ + if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) + MPIERR(ret); + if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) + MPIERR(ret); + + /* Check that a valid number of processors was specified. */ + if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || + ntasks == 8 || ntasks == 16)) + fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); + if (verbose) + printf("%d: ParallelIO Library example1 running on %d processors.\n", + my_rank, ntasks); + + /* keep things simple - 1 iotask per MPI process */ + niotasks = ntasks; + + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, + ioproc_start, PIO_REARR_SUBSET, &iosysid))) + ERR(ret); + + /* Describe the decomposition. This is a 1-based array, so add 1! */ + elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks; + if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + for (i = 0; i < elements_per_pe; i++) { + compdof[i] = my_rank * elements_per_pe + i + 1; + } + + /* Create the PIO decomposition for this test. */ + if (verbose) + printf("rank: %d Creating decomposition...\n", my_rank); + if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe, + compdof, &ioid, NULL, NULL, NULL))) + ERR(ret); + free(compdof); + + /* How many flavors will we be running for? */ + int num_flavors = 0; + int fmtidx = 0; +#ifdef _PNETCDF + num_flavors++; + format[fmtidx++] = PIO_IOTYPE_PNETCDF; +#endif +#ifdef _NETCDF + num_flavors++; + format[fmtidx++] = PIO_IOTYPE_NETCDF; +#endif +#ifdef _NETCDF4 + num_flavors += 2; + format[fmtidx++] = PIO_IOTYPE_NETCDF4C; + format[fmtidx] = PIO_IOTYPE_NETCDF4P; +#endif + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (fmt = 0; fmt < num_flavors; fmt++) + { + /* Create the netCDF output file. */ + if (verbose) + printf("rank: %d Creating sample file %s with format %d...\n", + my_rank, filename[fmt], format[fmt]); + if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt], + PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + if (verbose) + printf("rank: %d Defining netCDF metadata...\n", my_rank); + for (d = 0; d < NDIM; d++) { + if (verbose) + printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank, + dim_name[d], dim_len[d]); + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + } + + /* Check the dimension names. */ + if ((ret = check_dim_names(my_rank, ncid, verbose))) + ERR(ret); + + /* Define a global attribute. */ + int att_val = 42; + if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_val))) + ERR(ret); + + /* Check the attribute name. */ + if ((ret = check_att_name(my_rank, ncid, verbose))) + ERR(ret); + + /* Define a variable. */ + if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) + ERR(ret); + + /* Check the variable name. */ + if ((ret = check_var_name(my_rank, ncid, verbose))) + ERR(ret); + + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Close the netCDF file. */ + if (verbose) + printf("rank: %d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Put a barrier here to make verbose output look better. */ + if ((ret = MPI_Barrier(MPI_COMM_WORLD))) + MPIERR(ret); + + } + + /* Free the PIO decomposition. */ + if (verbose) + printf("rank: %d Freeing PIO decomposition...\n", my_rank); + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + /* Finalize the IO system. */ + if (verbose) + printf("rank: %d Freeing PIO resources...\n", my_rank); + if ((ret = PIOc_finalize(iosysid))) + ERR(ret); + + /* Finalize the MPI library. */ + MPI_Finalize(); + +#ifdef TIMING + /* Finalize the GPTL timing library. */ + if ((ret = GPTLfinalize ())) + return ret; +#endif + + + return 0; +} From 541a07037f88ffb1cc5ed801b8ba83809f33803b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 15 Jun 2016 10:04:26 -0600 Subject: [PATCH 157/184] cleanout of test_darray --- tests/unit/test_darray.c | 168 ++------------------------------------- 1 file changed, 6 insertions(+), 162 deletions(-) diff --git a/tests/unit/test_darray.c b/tests/unit/test_darray.c index 9df6573f860..abc1949633b 100644 --- a/tests/unit/test_darray.c +++ b/tests/unit/test_darray.c @@ -48,107 +48,7 @@ char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"}; /** Length of the dimensions in the sample data. */ int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; -/** Length of chunksizes to use in netCDF-4 files. */ -size_t chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2}; - -/** Check the dimension names. - * - * @param my_rank rank of process - * @param ncid ncid of open netCDF file - * - * @returns 0 for success, error code otherwise. */ -int -check_dim_names(int my_rank, int ncid, int verbose) -{ - char dim_name[NC_MAX_NAME + 1]; - char zero_dim_name[NC_MAX_NAME + 1]; - int ret; - - for (int d = 0; d < NDIM; d++) - { - strcpy(dim_name, "11111111111111111111111111111111"); - if ((ret = PIOc_inq_dimname(ncid, d, dim_name))) - return ret; - if (verbose) - printf("my_rank %d dim %d name %s\n", my_rank, d, dim_name); - - /* Did other ranks get the same name? */ - if (!my_rank) - strcpy(zero_dim_name, dim_name); - /* if (verbose) */ - /* printf("rank %d dim_name %s zero_dim_name %s\n", my_rank, dim_name, zero_dim_name); */ - if ((ret = MPI_Bcast(&zero_dim_name, strlen(dim_name) + 1, MPI_CHAR, 0, - MPI_COMM_WORLD))) - MPIERR(ret); - if (strcmp(dim_name, zero_dim_name)) - return ERR_AWFUL; - } - return 0; -} - -/** Check the variable name. - * - * @param my_rank rank of process - * @param ncid ncid of open netCDF file - * - * @returns 0 for success, error code otherwise. */ -int -check_var_name(int my_rank, int ncid, int verbose) -{ - char var_name[NC_MAX_NAME + 1]; - char zero_var_name[NC_MAX_NAME + 1]; - int ret; - - strcpy(var_name, "11111111111111111111111111111111"); - if ((ret = PIOc_inq_varname(ncid, 0, var_name))) - return ret; - if (verbose) - printf("my_rank %d var name %s\n", my_rank, var_name); - - /* Did other ranks get the same name? */ - if (!my_rank) - strcpy(zero_var_name, var_name); - if ((ret = MPI_Bcast(&zero_var_name, strlen(var_name) + 1, MPI_CHAR, 0, - MPI_COMM_WORLD))) - MPIERR(ret); - if (strcmp(var_name, zero_var_name)) - return ERR_AWFUL; - return 0; -} - -/** Check the attribute name. - * - * @param my_rank rank of process - * @param ncid ncid of open netCDF file - * - * @returns 0 for success, error code otherwise. */ -int -check_att_name(int my_rank, int ncid, int verbose) -{ - char att_name[NC_MAX_NAME + 1]; - char zero_att_name[NC_MAX_NAME + 1]; - int ret; - - strcpy(att_name, "11111111111111111111111111111111"); - if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, att_name))) - return ret; - if (verbose) - printf("my_rank %d att name %s\n", my_rank, att_name); - - /* Did everyone ranks get the same length name? */ -/* if (strlen(att_name) != strlen(ATT_NAME)) - return ERR_AWFUL;*/ - if (!my_rank) - strcpy(zero_att_name, att_name); - if ((ret = MPI_Bcast(&zero_att_name, strlen(att_name) + 1, MPI_CHAR, 0, - MPI_COMM_WORLD))) - MPIERR(ret); - if (strcmp(att_name, zero_att_name)) - return ERR_AWFUL; - return 0; -} - -/** Run Tests for NetCDF-4 Functions. +/** Run Tests for darray Functions. * * @param argc argument count * @param argv array of arguments @@ -174,10 +74,10 @@ main(int argc, char **argv) PIO_IOTYPE_NETCDF4P}; /** Names for the output files. */ - char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_names_pnetcdf.nc", - "test_names_classic.nc", - "test_names_serial4.nc", - "test_names_parallel4.nc"}; + char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_darray_pnetcdf.nc", + "test_darray_classic.nc", + "test_darray_serial4.nc", + "test_darray_parallel4.nc"}; /** Number of processors that will do IO. In this test we * will do IO from all processors. */ @@ -203,50 +103,11 @@ main(int argc, char **argv) int iosysid; /** The ncid of the netCDF file. */ - int ncid = 0; + int ncid; /** The ID of the netCDF varable. */ int varid; - /** Storage of netCDF-4 files (contiguous vs. chunked). */ - int storage; - - /** Chunksizes set in the file. */ - size_t my_chunksize[NDIM]; - - /** The shuffle filter setting in the netCDF-4 test file. */ - int shuffle; - - /** Non-zero if deflate set for the variable in the netCDF-4 test file. */ - int deflate; - - /** The deflate level set for the variable in the netCDF-4 test file. */ - int deflate_level; - - /** Non-zero if fletcher32 filter is used for variable. */ - int fletcher32; - - /** Endianness of variable. */ - int endianness; - - /* Size of the file chunk cache. */ - size_t chunk_cache_size; - - /* Number of elements in file cache. */ - size_t nelems; - - /* File cache preemption. */ - float preemption; - - /* Size of the var chunk cache. */ - size_t var_cache_size; - - /* Number of elements in var cache. */ - size_t var_cache_nelems; - - /* Var cache preemption. */ - float var_cache_preemption; - /** The I/O description ID. */ int ioid; @@ -354,27 +215,10 @@ main(int argc, char **argv) ERR(ret); } - /* Check the dimension names. */ - if ((ret = check_dim_names(my_rank, ncid, verbose))) - ERR(ret); - - /* Define a global attribute. */ - int att_val = 42; - if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_val))) - ERR(ret); - - /* Check the attribute name. */ - if ((ret = check_att_name(my_rank, ncid, verbose))) - ERR(ret); - /* Define a variable. */ if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) ERR(ret); - /* Check the variable name. */ - if ((ret = check_var_name(my_rank, ncid, verbose))) - ERR(ret); - if ((ret = PIOc_enddef(ncid))) ERR(ret); From 0f1fe882aff71b5c651065c329deaa7cd5fa7f35 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 15 Jun 2016 14:26:44 -0600 Subject: [PATCH 158/184] development of darray test --- src/clib/pio_darray.c | 296 ++++++++++++++++++++++------------------ src/clib/pioc_support.c | 6 +- 2 files changed, 167 insertions(+), 135 deletions(-) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index 11ecd8582c7..9ed3d453964 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -54,13 +54,15 @@ void compute_buffer_init(iosystem_desc_t ios) CN_bpool = malloc( PIO_CNBUFFER_LIMIT ); if (CN_bpool==NULL){ char errmsg[180]; - sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d: try reducing PIO_CNBUFFER_LIMIT\n",PIO_CNBUFFER_LIMIT,ios.comp_rank); + sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d:" + " try reducing PIO_CNBUFFER_LIMIT\n",PIO_CNBUFFER_LIMIT,ios.comp_rank); piodie(errmsg,__FILE__,__LINE__); } bpool( CN_bpool, PIO_CNBUFFER_LIMIT); if (CN_bpool==NULL){ char errmsg[180]; - sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d: try reducing PIO_CNBUFFER_LIMIT\n",PIO_CNBUFFER_LIMIT,ios.comp_rank); + sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d:" + " try reducing PIO_CNBUFFER_LIMIT\n",PIO_CNBUFFER_LIMIT,ios.comp_rank); piodie(errmsg,__FILE__,__LINE__); } bectl(NULL, malloc, free, PIO_CNBUFFER_LIMIT); @@ -91,45 +93,46 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, iosystem_desc_t *ios; var_desc_t *vdesc; int ndims; - int ierr; + int ierr = PIO_NOERR; /** Return code from function calls. */ int i; - int msg; - int mpierr; + int msg = 0; + int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ int dsize; MPI_Status status; PIO_Offset usage; int fndims; - PIO_Offset tdsize; + PIO_Offset tdsize = 0; #ifdef TIMING GPTLstart("PIO:write_darray_nc"); #endif - tdsize = 0; - ierr = PIO_NOERR; - - ios = file->iosystem; - if (ios == NULL){ - fprintf(stderr,"Failed to find iosystem handle \n"); + /* Get the IO system info. */ + if (!(ios = file->iosystem)) return PIO_EBADID; - } - vdesc = (file->varlist)+vid; - if (vdesc == NULL){ - fprintf(stderr,"Failed to find variable handle %d\n",vid); + vdesc = file->varlist + vid; + + if (!vdesc) return PIO_EBADID; - } + ndims = iodesc->ndims; - msg = 0; - if (ios->async_interface && ! ios->ioproc){ - if (ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + if (ios->async_interface) + { + if (!ios->ioproc) + { + if (ios->comp_rank==0) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + } } ierr = PIOc_inq_varndims(file->fh, vid, &fndims); - if (ios->ioproc){ + if (ios->ioproc) + { io_region *region; int ncid = file->fh; int regioncnt; @@ -147,37 +150,43 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, region = iodesc->firstregion; - if (vdesc->record >= 0 && ndimsrecord >= 0 && ndims < fndims) ndims++; + #ifdef _PNETCDF - if (file->iotype == PIO_IOTYPE_PNETCDF){ - // make sure we have room in the buffer ; + /* make sure we have room in the buffer. */ + if (file->iotype == PIO_IOTYPE_PNETCDF) flush_output_buffer(file, false, tsize*(iodesc->maxiobuflen)); - } #endif rrcnt = 0; - for(regioncnt = 0;regioncntmaxregions;regioncnt++){ - for(i = 0;imaxregions; regioncnt++) + { + for (i = 0; i < ndims; i++) + { start[i] = 0; count[i] = 0; } - if (region != NULL){ - bufptr = (void *)((char *) IOBUF+tsize*region->loffset); + if (region != NULL) + { + bufptr = (void *)((char *) IOBUF + tsize * region->loffset); // this is a record based multidimensional array - if (vdesc->record >= 0){ + if (vdesc->record >= 0) + { start[0] = vdesc->record; - for(i = 1;istart[i-1]; - count[i] = region->count[i-1]; + for (i = 1; i < ndims; i++) + { + start[i] = region->start[i - 1]; + count[i] = region->count[i - 1]; } - if (count[1]>0) + if (count[1] > 0) count[0] = 1; // Non-time dependent array } else { - for( i=0;istart[i]; count[i] = region->count[i]; } @@ -209,10 +218,10 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, size_t tstart[ndims], tcount[ndims]; if (ios->io_rank==0){ - for(i = 0;inum_aiotasks;i++){ + for (i = 0;inum_aiotasks;i++){ if (i==0){ buflen = 1; - for(j = 0;jbasetype); } if (ierr == PIO_EEDGE){ - for(i = 0;iio_rank < iodesc->num_aiotasks ){ buflen = 1; - for(i = 0;ibasetype); /* if (regioncnt==0){ - for(i=0;imaxregions;i++){ + for (i=0;imaxregions;i++){ startlist[i] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); countlist[i] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); } @@ -292,7 +301,7 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - for( i=0; irequest = realloc(vdesc->request, sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - for(int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ + for (int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ vdesc->request[i]=NC_REQ_NULL; } reqn = vdesc->nreqs; @@ -329,7 +338,7 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, vdesc->nreqs = reqn; // printf("%s %d %X %d\n",__FILE__,__LINE__,IOBUF,request); - for(i=0;inext; - } // for(regioncnt=0;regioncntmaxregions;regioncnt++){ + } // for (regioncnt=0;regioncntmaxregions;regioncnt++){ } // if (ios->ioproc) ierr = check_netcdf(file, ierr, __FILE__,__LINE__); @@ -357,6 +366,7 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, * * This routine is used if aggregation is enabled, data is already on the * io-tasks + * * @param[in] file: a pointer to the open file descriptor for the file that will be written to * @param[in] nvars: the number of variables to be written with this decomposition * @param[in] vid: an array of the variable ids to be written @@ -382,7 +392,7 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ int ierr; int i; int msg; - int mpierr; + int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ int dsize; MPI_Status status; PIO_Offset usage; @@ -410,16 +420,24 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ } msg = 0; - if (ios->async_interface && ! ios->ioproc){ - if (ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + /* If async is in use, send message to IO master task. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + if (ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } } ierr = PIOc_inq_varndims(file->fh, vid[0], &fndims); MPI_Type_size(basetype, &tsize); - if (ios->ioproc){ + if (ios->ioproc) + { io_region *region; int regioncnt; int rrcnt; @@ -436,16 +454,16 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ region = firstregion; rrcnt=0; - for(regioncnt=0;regioncntstart[0],region->count[0],ndims,fndims,vdesc->record); - for(i=0;irecord >= 0){ - for(i=fndims-ndims;istart[i-(fndims-ndims)]; count[i] = region->count[i-(fndims-ndims)]; } @@ -460,7 +478,7 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ } else { - for( i=0;istart[i]; count[i] = region->count[i]; } @@ -470,7 +488,7 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ switch(file->iotype){ #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - for(int nv=0; nvrecord >= 0 && ndimsio_rank,dsize); startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - for( i=0; illen, iodesc->basetype); //printf("%s %d %ld \n",__FILE__,__LINE__,IOBUF); - for(int nv=0; nvvarlist)+vid[nv]; if (vdesc->record >= 0 && ndimsrequest = realloc(vdesc->request, sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - for(int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ + for (int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ vdesc->request[i]=NC_REQ_NULL; } reqn = vdesc->nreqs; @@ -554,9 +572,9 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ // printf("%s %d %d %d\n",__FILE__,__LINE__,vdesc->nreqs,vdesc->request[reqn]); } - for(i=0;inext; - } // for(regioncnt=0;regioncntmaxregions;regioncnt++){ + } // for (regioncnt=0;regioncntmaxregions;regioncnt++){ } // if (ios->ioproc) ierr = check_netcdf(file, ierr, __FILE__,__LINE__); @@ -612,7 +630,7 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i int ierr; int i; int msg; - int mpierr; + int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ int dsize; MPI_Status status; PIO_Offset usage; @@ -640,10 +658,16 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i } msg = 0; - if (ios->async_interface && ! ios->ioproc){ - if (ios->comp_rank==0) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + if (ios->async_interface) + { + if (! ios->ioproc) + { + if (ios->comp_rank==0) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } } ierr = PIOc_inq_varndims(file->fh, vid[0], &fndims); @@ -665,15 +689,15 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i rrcnt=0; - for(regioncnt=0;regioncntrecord >= 0){ - for(i=fndims-ndims;istart[i-(fndims-ndims)]; tmp_count[i+regioncnt*fndims] = region->count[i-(fndims-ndims)]; } @@ -681,7 +705,7 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i } else { - for( i=0;istart[i]; tmp_count[i+regioncnt*fndims] = region->count[i]; } @@ -708,7 +732,7 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i size_t loffset; mpierr = MPI_Type_size(basetype, &dsize); - for(int rtask=0; rtasknum_iotasks; rtask++){ + for (int rtask=0; rtasknum_iotasks; rtask++){ if (rtask>0){ mpierr = MPI_Send( &ierr, 1, MPI_INT, rtask, 0, ios->io_comm); // handshake - tell the sending task I'm ready MPI_Recv( &rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); @@ -727,13 +751,13 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i } if (rlen>0){ loffset = 0; - for(regioncnt=0;regioncntrecord>=0){ @@ -761,19 +785,19 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i } if (ierr != PIO_NOERR){ - for(i=0;imaxregions;regioncnt++){ + }// for (regioncnt=0;regioncntmaxregions;regioncnt++){ } // if (rlen>0) - } // for(int rtask=0; rtasknum_iotasks; rtask++){ + } // for (int rtask=0; rtasknum_iotasks; rtask++){ } } // if (ios->ioproc) @@ -856,14 +880,14 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, } if (iodesc->needsfill && iodesc->rearranger==PIO_REARR_BOX){ if (vsize==4){ - for(int nv=0;nv < nvars; nv++){ - for(int i=0;imaxiobuflen;i++){ + for (int nv=0;nv < nvars; nv++){ + for (int i=0;imaxiobuflen;i++){ ((float *) vdesc0->iobuf)[i+nv*(iodesc->maxiobuflen)] = ((float *)fillvalue)[nv]; } } }else if (vsize==8){ - for(int nv=0;nv < nvars; nv++){ - for(int i=0;imaxiobuflen;i++){ + for (int nv=0;nv < nvars; nv++){ + for (int i=0;imaxiobuflen;i++){ ((double *)vdesc0->iobuf)[i+nv*(iodesc->maxiobuflen)] = ((double *)fillvalue)[nv]; } } @@ -908,14 +932,14 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, vdesc0->fillbuf = bget(iodesc->holegridsize*vsize*nvars); //printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); if (vsize==4){ - for(int nv=0;nvholegridsize;i++){ + for (int nv=0;nvholegridsize;i++){ ((float *) vdesc0->fillbuf)[i+nv*iodesc->holegridsize] = ((float *) fillvalue)[nv]; } } }else if (vsize==8){ - for(int nv=0;nvholegridsize;i++){ + for (int nv=0;nvholegridsize;i++){ ((double *) vdesc0->fillbuf)[i+nv*iodesc->holegridsize] = ((double *) fillvalue)[nv]; } } @@ -1165,7 +1189,7 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, if (tsize==8){ double asum=0.0; printf("%s %d %d %d %d\n",__FILE__,__LINE__,vid,arraylen,iodesc->ndof); - for(int k=0;krecord<0) vdesc->record=0; } - for(regioncnt=0;regioncntmaxregions;regioncnt++){ + for (regioncnt=0;regioncntmaxregions;regioncnt++){ // printf("%s %d %d %ld %d %d\n",__FILE__,__LINE__,regioncnt,region,fndims,ndims); tmp_bufsize=1; if (region==NULL || iodesc->llen==0){ - for(i=0;irecord >= 0 && fndims>1){ start[0] = vdesc->record; - for(i=1;istart[i-1]; count[i] = region->count[i-1]; // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); @@ -1367,7 +1391,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, else { // Non-time dependent array - for(i=0;istart[i]; count[i] = region->count[i]; // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); @@ -1395,7 +1419,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, case PIO_IOTYPE_PNETCDF: { tmp_bufsize=1; - for(int j=0;jmaxregions, j,start[j],count[j],tmp_bufsize); @@ -1413,7 +1437,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, if (regioncnt==iodesc->maxregions-1){ ierr = ncmpi_get_varn_all(file->fh, vid, rrlen, startlist, countlist, IOBUF, iodesc->llen, iodesc->basetype); - for(i=0;inext; - } // for(regioncnt=0;...) + } // for (regioncnt=0;...) } ierr = check_netcdf(file, ierr, __FILE__,__LINE__); @@ -1499,9 +1523,9 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, if (vdesc->record<0) vdesc->record=0; } - for(regioncnt=0;regioncntmaxregions;regioncnt++){ + for (regioncnt=0;regioncntmaxregions;regioncnt++){ if (region==NULL || iodesc->llen==0){ - for(i=0;irecord >= 0 && fndims>1){ tmp_start[regioncnt*fndims] = vdesc->record; - for(i=1;istart[i-1]; tmp_count[i+regioncnt*fndims] = region->count[i-1]; } @@ -1521,19 +1545,19 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, else { // Non-time dependent array - for(i=0;istart[i]; tmp_count[i+regioncnt*fndims] = region->count[i]; } } - /* for(i=0;inext; - } // for(regioncnt=0;...) + } // for (regioncnt=0;...) if (ios->io_rank>0){ MPI_Send( &(iodesc->llen), 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); @@ -1548,8 +1572,8 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, size_t loffset, regionsize; size_t this_start[fndims*iodesc->maxregions]; size_t this_count[fndims*iodesc->maxregions]; - // for( i=ios->num_iotasks-1; i>=0; i--){ - for(int rtask=1;rtask<=ios->num_iotasks;rtask++){ + // for ( i=ios->num_iotasks-1; i>=0; i--){ + for (int rtask=1;rtask<=ios->num_iotasks;rtask++){ if (rtasknum_iotasks){ MPI_Recv(&tmp_bufsize, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); if (tmp_bufsize>0){ @@ -1564,11 +1588,11 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, tmp_bufsize=iodesc->llen; } loffset = 0; - for(regioncnt=0;regioncntnum_iotasks){ - for(int m=0; mvarlist+i; - reqcnt+=vdesc->nreqs; - if (vdesc->nreqs>0) maxreq = i; + for (int i = 0; i < PIO_MAX_VARS; i++) + { + vdesc = file->varlist + i; + reqcnt += vdesc->nreqs; + if (vdesc->nreqs > 0) + maxreq = i; } int request[reqcnt]; int status[reqcnt]; - for(int i=0; i<=maxreq; i++){ - vdesc = file->varlist+i; + for (int i = 0; i <= maxreq; i++) + { + vdesc = file->varlist + i; #ifdef MPIO_ONESIDED /*onesided optimization requires that all of the requests in a wait_all call represent a contiguous block of data in the file */ - if (rcnt>0 && (prev_record != vdesc->record || - vdesc->nreqs==0)){ + if (rcnt>0 && (prev_record != vdesc->record || vdesc->nreqs==0)) + { ierr = ncmpi_wait_all(file->fh, rcnt, request,status); rcnt=0; } prev_record = vdesc->record; #endif // printf("%s %d %d %d %d \n",__FILE__,__LINE__,i,vdesc->nreqs,vdesc->request); - for(reqcnt=0;reqcntnreqs;reqcnt++){ + for (reqcnt=0;reqcntnreqs;reqcnt++) + { request[rcnt++] = max(vdesc->request[reqcnt],NC_REQ_NULL); } free(vdesc->request); - vdesc->request=NULL; - vdesc->nreqs=0; + vdesc->request = NULL; + vdesc->nreqs = 0; // if (file->iosystem->io_rank < 2) printf("%s %d varid=%d\n",__FILE__,__LINE__,i); #ifdef FLUSH_EVERY_VAR ierr = ncmpi_wait_all(file->fh, rcnt, request,status); - rcnt=0; + rcnt = 0; #endif } // if (file->iosystem->io_rank==0){ // printf("%s %d %d\n",__FILE__,__LINE__,rcnt); // } - if (rcnt>0){ + if (rcnt > 0) + { /* if (file->iosystem->io_rank==0){ printf("%s %d %d ",__FILE__,__LINE__,rcnt); - for(int i=0; ifh, rcnt, request,status); + ierr = ncmpi_wait_all(file->fh, rcnt, request, status); } - for(int i=0; ivarlist+i; - if (vdesc->iobuf != NULL){ + for (int i = 0; i < PIO_MAX_VARS; i++) + { + vdesc = file->varlist + i; + if (vdesc->iobuf != NULL) + { brel(vdesc->iobuf); vdesc->iobuf=NULL; } - if (vdesc->fillbuf != NULL){ + if (vdesc->fillbuf != NULL) + { brel(vdesc->fillbuf); vdesc->fillbuf=NULL; } diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index e08a019c517..3347d9ee69c 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -12,7 +12,7 @@ #include #define versno 2001 -#ifdef PIO_ENABLE_LOGGING +#if PIO_ENABLE_LOGGING int pio_log_level = 0; int my_rank; #endif /* PIO_ENABLE_LOGGING */ @@ -23,7 +23,7 @@ int my_rank; * logging, this function does nothing. */ int PIOc_set_log_level(int level) { -#ifdef PIO_ENABLE_LOGGING +#if PIO_ENABLE_LOGGING printf("setting log level to %d\n", level); pio_log_level = level; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); @@ -31,7 +31,7 @@ int PIOc_set_log_level(int level) #endif /* PIO_ENABLE_LOGGING */ } -#ifdef PIO_ENABLE_LOGGING +#if PIO_ENABLE_LOGGING /** This function prints out a message, if the severity of the message is lower than the global pio_log_level. To use it, do something like this: From 4717e6058e3c4f8e1ac5f1e8fa0727450d7a4d71 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 15 Jun 2016 15:21:34 -0600 Subject: [PATCH 159/184] more work on darray test --- src/clib/pio_darray.c | 593 +++++++++++++++++++++++++++--------------- 1 file changed, 378 insertions(+), 215 deletions(-) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index 9ed3d453964..a9d009f56c8 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -50,20 +50,23 @@ void compute_buffer_init(iosystem_desc_t ios) { #ifndef PIO_USE_MALLOC - if (CN_bpool == NULL){ - CN_bpool = malloc( PIO_CNBUFFER_LIMIT ); - if (CN_bpool==NULL){ + if (CN_bpool == NULL) + { + CN_bpool = malloc(PIO_CNBUFFER_LIMIT); + if (CN_bpool==NULL) + { char errmsg[180]; sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d:" - " try reducing PIO_CNBUFFER_LIMIT\n",PIO_CNBUFFER_LIMIT,ios.comp_rank); - piodie(errmsg,__FILE__,__LINE__); + " try reducing PIO_CNBUFFER_LIMIT\n", PIO_CNBUFFER_LIMIT, ios.comp_rank); + piodie(errmsg, __FILE__, __LINE__); } - bpool( CN_bpool, PIO_CNBUFFER_LIMIT); - if (CN_bpool==NULL){ + bpool(CN_bpool, PIO_CNBUFFER_LIMIT); + if (!CN_bpool) + { char errmsg[180]; sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d:" - " try reducing PIO_CNBUFFER_LIMIT\n",PIO_CNBUFFER_LIMIT,ios.comp_rank); - piodie(errmsg,__FILE__,__LINE__); + " try reducing PIO_CNBUFFER_LIMIT\n", PIO_CNBUFFER_LIMIT, ios.comp_rank); + piodie(errmsg, __FILE__, __LINE__); } bectl(NULL, malloc, free, PIO_CNBUFFER_LIMIT); } @@ -198,16 +201,22 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(ncid, vid, NC_COLLECTIVE); - if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid,(size_t *) start,(size_t *) count, (const double *) bufptr); - } else if (iodesc->basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid, (size_t *) start, (size_t *) count, (const int *) bufptr); - }else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid, vid, (size_t *) start, (size_t *) count, (const float *) bufptr); + if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) + { + ierr = nc_put_vara_double(ncid, vid,(size_t *)start,(size_t *)count, (const double *)bufptr); + } + else if (iodesc->basetype == MPI_INTEGER) + { + ierr = nc_put_vara_int(ncid, vid, (size_t *)start, (size_t *)count, (const int *)bufptr); + } + else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4) + { + ierr = nc_put_vara_float(ncid, vid, (size_t *)start, (size_t *)count, (const float *)bufptr); } else { - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) iodesc->basetype); + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n", + (int)iodesc->basetype); } break; case PIO_IOTYPE_NETCDF4C: @@ -216,12 +225,15 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, { mpierr = MPI_Type_size(iodesc->basetype, &dsize); size_t tstart[ndims], tcount[ndims]; - if (ios->io_rank==0){ - - for (i = 0;inum_aiotasks;i++){ - if (i==0){ + if (ios->io_rank == 0) + { + for (i = 0; i < iodesc->num_aiotasks; i++) + { + if (i == 0) + { buflen = 1; - for (j = 0;jio_comm); // handshake - tell the sending task I'm ready mpierr = MPI_Recv( &buflen, 1, MPI_INT, i, 1, ios->io_comm, &status); - if (buflen>0){ - mpierr = MPI_Recv( tstart, ndims, MPI_OFFSET, i, ios->num_iotasks+i, ios->io_comm, &status); - mpierr = MPI_Recv( tcount, ndims, MPI_OFFSET, i,2*ios->num_iotasks+i, ios->io_comm, &status); + if (buflen > 0) + { + mpierr = MPI_Recv(tstart, ndims, MPI_OFFSET, i, ios->num_iotasks+i, + ios->io_comm, &status); + mpierr = MPI_Recv(tcount, ndims, MPI_OFFSET, i, 2 * ios->num_iotasks + i, + ios->io_comm, &status); tmp_buf = malloc(buflen * dsize); - mpierr = MPI_Recv( tmp_buf, buflen, iodesc->basetype, i, i, ios->io_comm, &status); + mpierr = MPI_Recv(tmp_buf, buflen, iodesc->basetype, i, i, ios->io_comm, &status); } } - if (buflen>0){ - if (iodesc->basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid, tstart, tcount, (const int *) tmp_buf); - }else if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid, tstart, tcount, (const double *) tmp_buf); - }else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid,vid, tstart, tcount, (const float *) tmp_buf); + if (buflen>0) + { + if (iodesc->basetype == MPI_INTEGER) + { + ierr = nc_put_vara_int(ncid, vid, tstart, tcount, (const int *)tmp_buf); + } + else if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) + { + ierr = nc_put_vara_double(ncid, vid, tstart, tcount, (const double *)tmp_buf); + } + else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4) + { + ierr = nc_put_vara_float(ncid, vid, tstart, tcount, (const float *)tmp_buf); } else { - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) iodesc->basetype); + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n", + (int)iodesc->basetype); } - if (ierr == PIO_EEDGE){ - for (i = 0;iio_rank < iodesc->num_aiotasks ){ + }else if (ios->io_rank < iodesc->num_aiotasks ) + { buflen = 1; - for (i = 0;iio_comm, &status); // task0 is ready to recieve mpierr = MPI_Rsend( &buflen, 1, MPI_INT, 0, 1, ios->io_comm); if (buflen>0) { - mpierr = MPI_Rsend( tstart, ndims, MPI_OFFSET, 0, ios->num_iotasks+ios->io_rank, ios->io_comm); - mpierr = MPI_Rsend( tcount, ndims, MPI_OFFSET, 0,2*ios->num_iotasks+ios->io_rank, ios->io_comm); + mpierr = MPI_Rsend( tstart, ndims, MPI_OFFSET, 0, ios->num_iotasks+ios->io_rank, + ios->io_comm); + mpierr = MPI_Rsend( tcount, ndims, MPI_OFFSET, 0,2*ios->num_iotasks+ios->io_rank, + ios->io_comm); mpierr = MPI_Rsend( bufptr, buflen, iodesc->basetype, 0, ios->io_rank, ios->io_comm); } } @@ -283,7 +310,8 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - for ( i = 0,dsize = 1;i0){ + if (dsize > 0) + { // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - for ( i=0; imaxregions-1){ + if (regioncnt==iodesc->maxregions-1) + { // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,ios->io_rank,iodesc->llen, tdsize); // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, // IOBUF, iodesc->llen, iodesc->basetype); int reqn=0; - if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ) + { vdesc->request = realloc(vdesc->request, sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - for (int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ + for (int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++) + { vdesc->request[i]=NC_REQ_NULL; } reqn = vdesc->nreqs; } else { - while(vdesc->request[reqn] != NC_REQ_NULL ){ + while(vdesc->request[reqn] != NC_REQ_NULL ) + { reqn++; } } ierr = ncmpi_bput_varn(ncid, vid, rrcnt, startlist, countlist, IOBUF, iodesc->llen, iodesc->basetype, vdesc->request+reqn); - if (vdesc->request[reqn] == NC_REQ_NULL){ + if (vdesc->request[reqn] == NC_REQ_NULL) + { vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync } vdesc->nreqs = reqn; // printf("%s %d %X %d\n",__FILE__,__LINE__,IOBUF,request); - for (i=0;iiosystem; - if (ios == NULL){ + if (ios == NULL) + { fprintf(stderr,"Failed to find iosystem handle \n"); return PIO_EBADID; } vdesc = (file->varlist)+vid[0]; ncid = file->fh; - if (vdesc == NULL){ + if (vdesc == NULL) + { fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); return PIO_EBADID; } @@ -454,54 +492,72 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ region = firstregion; rrcnt=0; - for (regioncnt=0;regioncntstart[0],region->count[0],ndims,fndims,vdesc->record); - for (i=0;irecord >= 0){ - for (i=fndims-ndims;irecord >= 0) + { + for (i=fndims-ndims;istart[i-(fndims-ndims)]; count[i] = region->count[i-(fndims-ndims)]; } - if (fndims>1 && ndims0){ + if (fndims>1 && ndims0) + { count[0] = 1; start[0] = frame[0]; - }else if (fndims==ndims){ + } + else if (fndims==ndims) + { start[0]+=vdesc->record; } // Non-time dependent array } else { - for ( i=0;istart[i]; count[i] = region->count[i]; } } } - switch(file->iotype){ + switch(file->iotype) + { #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - for (int nv=0; nvrecord >= 0 && ndimsrecord >= 0 && ndimsloffset)); } ierr = nc_var_par_access(ncid, vid[nv], NC_COLLECTIVE); - if (basetype == MPI_DOUBLE ||basetype == MPI_REAL8){ + if (basetype == MPI_DOUBLE ||basetype == MPI_REAL8) + { ierr = nc_put_vara_double (ncid, vid[nv],(size_t *) start,(size_t *) count, (const double *) bufptr); - } else if (basetype == MPI_INTEGER){ + } + else if (basetype == MPI_INTEGER) + { ierr = nc_put_vara_int (ncid, vid[nv], (size_t *) start, (size_t *) count, (const int *) bufptr); - }else if (basetype == MPI_FLOAT || basetype == MPI_REAL4){ + } + else if (basetype == MPI_FLOAT || basetype == MPI_REAL4) + { ierr = nc_put_vara_float (ncid, vid[nv], (size_t *) start, (size_t *) count, (const float *) bufptr); } else @@ -513,49 +569,59 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - for ( i=0,dsize=1;i0){ + if (dsize>0) + { // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - for ( i=0; iio_rank,iodesc->llen, tdsize); // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, // IOBUF, iodesc->llen, iodesc->basetype); //printf("%s %d %ld \n",__FILE__,__LINE__,IOBUF); - for (int nv=0; nvvarlist)+vid[nv]; - if (vdesc->record >= 0 && ndimsrecord >= 0 && ndimsnreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ) + { vdesc->request = realloc(vdesc->request, sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - for (int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ + for (int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++) + { vdesc->request[i]=NC_REQ_NULL; } reqn = vdesc->nreqs; } else { - while(vdesc->request[reqn] != NC_REQ_NULL){ + while(vdesc->request[reqn] != NC_REQ_NULL) + { reqn++; } } @@ -565,16 +631,20 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ ierr = ncmpi_bput_varn(ncid, vid[nv], rrcnt, startlist, countlist, bufptr, llen, basetype, &(vdesc->request)); */ - if (vdesc->request[reqn] == NC_REQ_NULL){ + if (vdesc->request[reqn] == NC_REQ_NULL) + { vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync } vdesc->nreqs += reqn+1; // printf("%s %d %d %d\n",__FILE__,__LINE__,vdesc->nreqs,vdesc->request[reqn]); } - for (i=0;iiosystem; - if (ios == NULL){ + if (ios == NULL) + { fprintf(stderr,"Failed to find iosystem handle \n"); return PIO_EBADID; } vdesc = (file->varlist)+vid[0]; ncid = file->fh; - if (vdesc == NULL){ + if (vdesc == NULL) + { fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); return PIO_EBADID; } @@ -673,7 +745,8 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i ierr = PIOc_inq_varndims(file->fh, vid[0], &fndims); MPI_Type_size(basetype, &tsize); - if (ios->ioproc){ + if (ios->ioproc) + { io_region *region; int regioncnt; int rrcnt; @@ -689,15 +762,20 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i rrcnt=0; - for (regioncnt=0;regioncntrecord >= 0){ - for (i=fndims-ndims;irecord >= 0) + { + for (i=fndims-ndims;istart[i-(fndims-ndims)]; tmp_count[i+regioncnt*fndims] = region->count[i-(fndims-ndims)]; } @@ -705,7 +783,8 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i } else { - for ( i=0;istart[i]; tmp_count[i+regioncnt*fndims] = region->count[i]; } @@ -713,10 +792,12 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i region = region->next; } } - if (ios->io_rank>0){ + if (ios->io_rank>0) + { mpierr = MPI_Recv( &ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve MPI_Send( &llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); - if (llen>0){ + if (llen>0) + { MPI_Send( &maxregions, 1, MPI_INT, 0, ios->io_rank+ios->num_iotasks, ios->io_comm); MPI_Send( tmp_start, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+2*ios->num_iotasks, ios->io_comm); MPI_Send( tmp_count, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+3*ios->num_iotasks, ios->io_comm); @@ -732,8 +813,10 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i size_t loffset; mpierr = MPI_Type_size(basetype, &dsize); - for (int rtask=0; rtasknum_iotasks; rtask++){ - if (rtask>0){ + for (int rtask=0; rtasknum_iotasks; rtask++) + { + if (rtask>0) + { mpierr = MPI_Send( &ierr, 1, MPI_INT, rtask, 0, ios->io_comm); // handshake - tell the sending task I'm ready MPI_Recv( &rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); if (rlen>0){ @@ -749,34 +832,44 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i rlen = llen; rregions = maxregions; } - if (rlen>0){ + if (rlen>0) + { loffset = 0; - for (regioncnt=0;regioncntrecord>=0){ - if (fndims>1 && ndims0){ + if (vdesc->record>=0) + { + if (fndims>1 && ndims0) + { count[0] = 1; start[0] = frame[nv]; - }else if (fndims==ndims){ + } + else if (fndims==ndims) + { start[0]+=vdesc->record; } } - - - - if (basetype == MPI_INTEGER){ + if (basetype == MPI_INTEGER) + { ierr = nc_put_vara_int (ncid, vid[nv], start, count, (const int *) bufptr); - }else if (basetype == MPI_DOUBLE || basetype == MPI_REAL8){ + } + else if (basetype == MPI_DOUBLE || basetype == MPI_REAL8) + { ierr = nc_put_vara_double (ncid, vid[nv], start, count, (const double *) bufptr); - }else if (basetype == MPI_FLOAT || basetype == MPI_REAL4){ + } + else if (basetype == MPI_FLOAT || basetype == MPI_REAL4) + { ierr = nc_put_vara_float (ncid,vid[nv], start, count, (const float *) bufptr); } else @@ -791,7 +884,8 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i } size_t tsize; tsize = 1; - for (int i=0;inext != NULL && wmb->ioid!=ioid){ + if (recordvar) + { + while(wmb->next != NULL && wmb->ioid!=ioid) if (wmb->next!=NULL) wmb = wmb->next; - } #ifdef _PNETCDF /* flush the previous record before starting a new one. this is collective */ // if (vdesc->request != NULL && (vdesc->request[0] != NC_REQ_NULL) || @@ -1081,13 +1175,15 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, } else { - while(wmb->next != NULL && wmb->ioid!= -(ioid)){ + while(wmb->next != NULL && wmb->ioid!= -(ioid)) + { if (wmb->next!=NULL) wmb = wmb->next; } } } - if ((recordvar && wmb->ioid != ioid) || (!recordvar && wmb->ioid != -(ioid))){ + if ((recordvar && wmb->ioid != ioid) || (!recordvar && wmb->ioid != -(ioid))) + { wmb->next = (wmulti_buffer *) bget((bufsize) sizeof(wmulti_buffer)); if (wmb->next == NULL){ piomemerror(*ios,sizeof(wmulti_buffer), __FILE__,__LINE__); @@ -1115,44 +1211,55 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, // printf("%s %d %X %d %d %d\n",__FILE__,__LINE__,wmb->data,wmb->validvars,arraylen,tsize); // cn_buffer_report(*ios, true); bfreespace(&totfree, &maxfree); - if (needsflush==0){ + if (needsflush==0) + { needsflush = (maxfree <= 1.1*(1+wmb->validvars)*arraylen*tsize ); } MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1, MPI_INT, MPI_MAX, ios->comp_comm); - if (needsflush > 0 ){ + if (needsflush > 0 ) + { // need to flush first // printf("%s %d %ld %d %ld %ld\n",__FILE__,__LINE__,maxfree, wmb->validvars, (1+wmb->validvars)*arraylen*tsize,totfree); cn_buffer_report(*ios, true); flush_buffer(ncid,wmb, needsflush==2); // if needsflush == 2 flush to disk otherwise just flush to io node } - if (arraylen > 0){ + if (arraylen > 0) + { wmb->data = bgetr( wmb->data, (1+wmb->validvars)*arraylen*tsize); - if (wmb->data == NULL){ + if (wmb->data == NULL) + { piomemerror(*ios, (1+wmb->validvars)*arraylen*tsize , __FILE__,__LINE__); } } wmb->vid = (int *) bgetr( wmb->vid,sizeof(int)*( 1+wmb->validvars)); - if (wmb->vid == NULL){ + if (wmb->vid == NULL) + { piomemerror(*ios, (1+wmb->validvars)*sizeof(int) , __FILE__,__LINE__); } - if (vdesc->record>=0){ + if (vdesc->record>=0) + { wmb->frame = (int *) bgetr( wmb->frame,sizeof(int)*( 1+wmb->validvars)); - if (wmb->frame == NULL){ + if (wmb->frame == NULL) + { piomemerror(*ios, (1+wmb->validvars)*sizeof(int) , __FILE__,__LINE__); } } - if (iodesc->needsfill){ + if (iodesc->needsfill) + { wmb->fillvalue = bgetr( wmb->fillvalue,tsize*( 1+wmb->validvars)); - if (wmb->fillvalue == NULL){ + if (wmb->fillvalue == NULL) + { piomemerror(*ios, (1+wmb->validvars)*tsize , __FILE__,__LINE__); } } - if (iodesc->needsfill){ - if (fillvalue != NULL){ + if (iodesc->needsfill) + { + if (fillvalue != NULL) + { memcpy((char *) wmb->fillvalue+tsize*wmb->validvars,fillvalue, tsize); } else @@ -1182,9 +1289,8 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, wmb->arraylen = arraylen; wmb->vid[wmb->validvars]=vid; bufptr = (void *)((char *) wmb->data + arraylen*tsize*wmb->validvars); - if (arraylen>0){ + if (arraylen>0) memcpy(bufptr, array, arraylen*tsize); - } /* if (tsize==8){ double asum=0.0; @@ -1198,15 +1304,13 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, // printf("%s %d %d %d %d %X\n",__FILE__,__LINE__,wmb->validvars,wmb->ioid,vid,bufptr); - if (wmb->frame!=NULL){ + if (wmb->frame!=NULL) wmb->frame[wmb->validvars]=vdesc->record; - } wmb->validvars++; // printf("%s %d %d %d %d %d\n",__FILE__,__LINE__,wmb->validvars,iodesc->maxbytes/tsize, iodesc->ndof, iodesc->llen); - if (wmb->validvars >= iodesc->maxbytes/tsize){ + if (wmb->validvars >= iodesc->maxbytes/tsize) PIOc_sync(ncid); - } return ierr; } @@ -1243,12 +1347,14 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, file = pio_get_file_from_id(ncid); - if (file == NULL){ + if (file == NULL) + { fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); return PIO_EBADID; } iodesc = pio_get_iodesc_from_id(ioid); - if (iodesc == NULL){ + if (iodesc == NULL) + { fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); return PIO_EBADID; } @@ -1257,14 +1363,14 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, ios = file->iosystem; rlen = iodesc->llen; - if (iodesc->rearranger>0){ + if (iodesc->rearranger>0) + { if (rlen>0){ MPI_Type_size(iodesc->basetype, &tsize); // iobuf = bget(tsize*rlen); iobuf = malloc((size_t) tsize*rlen); - if (iobuf==NULL){ - piomemerror(*ios,rlen*(size_t) tsize, __FILE__,__LINE__); - } + if (!iobuf) + piomemerror(*ios, rlen * (size_t)tsize, __FILE__, __LINE__); } // printf(" rlen = %d %ld\n",rlen,iobuf); @@ -1282,7 +1388,8 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, { iobuf = array; } - switch(file->iotype){ + switch(file->iotype) + { case PIO_IOTYPE_PNETCDF: case PIO_IOTYPE_NETCDF: case PIO_IOTYPE_NETCDF4P: @@ -1333,7 +1440,8 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, if (fndims==ndims) vdesc->record=-1; - if (ios->ioproc){ + if (ios->ioproc) + { io_region *region; size_t start[fndims]; size_t count[fndims]; @@ -1354,16 +1462,20 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, // calling program to change the basetype. region = iodesc->firstregion; MPI_Type_size(iodesc->basetype, &tsize); - if (fndims>ndims){ + if (fndims>ndims) + { ndims++; if (vdesc->record<0) vdesc->record=0; } - for (regioncnt=0;regioncntmaxregions;regioncnt++){ + for (regioncnt=0;regioncntmaxregions;regioncnt++) + { // printf("%s %d %d %ld %d %d\n",__FILE__,__LINE__,regioncnt,region,fndims,ndims); tmp_bufsize=1; - if (region==NULL || iodesc->llen==0){ - for (i=0;illen==0) + { + for (i=0;illen - region->loffset, iodesc->llen, region->loffset); - if (vdesc->record >= 0 && fndims>1){ + if (vdesc->record >= 0 && fndims>1) + { start[0] = vdesc->record; - for (i=1;istart[i-1]; count[i] = region->count[i-1]; // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); } - if (count[1]>0) + if (count[1] > 0) count[0] = 1; } else { // Non-time dependent array - for (i=0;istart[i]; count[i] = region->count[i]; // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); @@ -1402,11 +1517,16 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, switch(file->iotype){ #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ + if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) + { ierr = nc_get_vara_double (file->fh, vid,start,count, bufptr); - } else if (iodesc->basetype == MPI_INTEGER){ + } + else if (iodesc->basetype == MPI_INTEGER) + { ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); - }else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ + } + else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4) + { ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); } else @@ -1419,25 +1539,29 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, case PIO_IOTYPE_PNETCDF: { tmp_bufsize=1; - for (int j=0;j0){ + if (tmp_bufsize > 0) + { startlist[rrlen] = (PIO_Offset *) bget(fndims * sizeof(PIO_Offset)); countlist[rrlen] = (PIO_Offset *) bget(fndims * sizeof(PIO_Offset)); - for (int j=0;jmaxregions, j,start[j],count[j],tmp_bufsize); + /* printf("%s %d %d %d %d %ld %ld %ld\n",__FILE__,__LINE__,realregioncnt, + iodesc->maxregions, j,start[j],count[j],tmp_bufsize);*/ } rrlen++; } - if (regioncnt==iodesc->maxregions-1){ + if (regioncnt==iodesc->maxregions-1) + { ierr = ncmpi_get_varn_all(file->fh, vid, rrlen, startlist, countlist, IOBUF, iodesc->llen, iodesc->basetype); - for (i=0;irecord=-1; - if (ios->ioproc){ + if (ios->ioproc) + { io_region *region; size_t start[fndims]; size_t count[fndims]; - size_t tmp_start[fndims*iodesc->maxregions]; - size_t tmp_count[fndims*iodesc->maxregions]; + size_t tmp_start[fndims * iodesc->maxregions]; + size_t tmp_count[fndims * iodesc->maxregions]; size_t tmp_bufsize; int regioncnt; void *bufptr; int tsize; - int rrlen=0; + int rrlen = 0; // buffer is incremented by byte and loffset is in terms of the iodessc->basetype // so we need to multiply by the size of the basetype @@ -1519,39 +1644,47 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, // calling program to change the basetype. region = iodesc->firstregion; MPI_Type_size(iodesc->basetype, &tsize); - if (fndims>ndims){ - if (vdesc->record<0) - vdesc->record=0; + if (fndims>ndims) + { + if (vdesc->record < 0) + vdesc->record = 0; } - for (regioncnt=0;regioncntmaxregions;regioncnt++){ - if (region==NULL || iodesc->llen==0){ - for (i=0;imaxregions;regioncnt++) + { + if (region==NULL || iodesc->llen==0) + { + for (i = 0; i < fndims; i++) + { + tmp_start[i + regioncnt * fndims] = 0; + tmp_count[i + regioncnt * fndims] = 0; } bufptr=NULL; } else { - if (vdesc->record >= 0 && fndims>1){ + if (vdesc->record >= 0 && fndims>1) + { tmp_start[regioncnt*fndims] = vdesc->record; - for (i=1;istart[i-1]; tmp_count[i+regioncnt*fndims] = region->count[i-1]; } - if (tmp_count[1+regioncnt*fndims]>0) - tmp_count[regioncnt*fndims] = 1; + if (tmp_count[1 + regioncnt * fndims] > 0) + tmp_count[regioncnt * fndims] = 1; } else { // Non-time dependent array - for (i=0;istart[i]; - tmp_count[i+regioncnt*fndims] = region->count[i]; + for (i = 0; i < fndims; i++) + { + tmp_start[i + regioncnt * fndims] = region->start[i]; + tmp_count[i + regioncnt * fndims] = region->count[i]; } } /* for (i=0;inext; } // for (regioncnt=0;...) - if (ios->io_rank>0){ + if (ios->io_rank>0) + { MPI_Send( &(iodesc->llen), 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); - if (iodesc->llen > 0){ - MPI_Send( &(iodesc->maxregions), 1, MPI_INT, 0, ios->num_iotasks+ios->io_rank, ios->io_comm); - MPI_Send( tmp_count, iodesc->maxregions*fndims, MPI_OFFSET, 0, 2*ios->num_iotasks+ios->io_rank, ios->io_comm); - MPI_Send( tmp_start, iodesc->maxregions*fndims, MPI_OFFSET, 0, 3*ios->num_iotasks+ios->io_rank, ios->io_comm); - MPI_Recv(IOBUF, iodesc->llen, iodesc->basetype, 0, 4*ios->num_iotasks+ios->io_rank, ios->io_comm, &status); + if (iodesc->llen > 0) + { + MPI_Send( &(iodesc->maxregions), 1, MPI_INT, 0, + ios->num_iotasks + ios->io_rank, ios->io_comm); + MPI_Send( tmp_count, iodesc->maxregions*fndims, MPI_OFFSET, 0, + 2 * ios->num_iotasks + ios->io_rank, ios->io_comm); + MPI_Send( tmp_start, iodesc->maxregions*fndims, MPI_OFFSET, 0, + 3 * ios->num_iotasks + ios->io_rank, ios->io_comm); + MPI_Recv(IOBUF, iodesc->llen, iodesc->basetype, 0, + 4 * ios->num_iotasks+ios->io_rank, ios->io_comm, &status); } - }else if (ios->io_rank==0){ + } + else if (ios->io_rank == 0) + { int maxregions=0; size_t loffset, regionsize; size_t this_start[fndims*iodesc->maxregions]; size_t this_count[fndims*iodesc->maxregions]; // for ( i=ios->num_iotasks-1; i>=0; i--){ - for (int rtask=1;rtask<=ios->num_iotasks;rtask++){ - if (rtasknum_iotasks){ + for (int rtask = 1; rtask <= ios->num_iotasks; rtask++) + { + if (rtasknum_iotasks) + { MPI_Recv(&tmp_bufsize, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); if (tmp_bufsize>0){ - MPI_Recv(&maxregions, 1, MPI_INT, rtask, ios->num_iotasks+rtask, ios->io_comm, &status); - MPI_Recv(this_count, maxregions*fndims, MPI_OFFSET, rtask, 2*ios->num_iotasks+rtask, ios->io_comm, &status); - MPI_Recv(this_start, maxregions*fndims, MPI_OFFSET, rtask, 3*ios->num_iotasks+rtask, ios->io_comm, &status); + MPI_Recv(&maxregions, 1, MPI_INT, rtask, ios->num_iotasks+rtask, + ios->io_comm, &status); + MPI_Recv(this_count, maxregions*fndims, MPI_OFFSET, rtask, + 2 * ios->num_iotasks + rtask, ios->io_comm, &status); + MPI_Recv(this_start, maxregions*fndims, MPI_OFFSET, rtask, + 3 * ios->num_iotasks + rtask, ios->io_comm, &status); } } else @@ -1588,11 +1734,13 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, tmp_bufsize=iodesc->llen; } loffset = 0; - for (regioncnt=0;regioncntnum_iotasks){ - for (int m=0; mbasetype); + fprintf(stderr,"Type not recognized %d in pioc_write_darray_nc_serial\n", + (int)iodesc->basetype); } - if (ierr != PIO_NOERR){ - for (int i=0;inum_iotasks){ - MPI_Send(IOBUF, tmp_bufsize, iodesc->basetype, rtask,4*ios->num_iotasks+rtask, ios->io_comm); - } + if (rtask < ios->num_iotasks) + MPI_Send(IOBUF, tmp_bufsize, iodesc->basetype, rtask, + 4 * ios->num_iotasks + rtask, ios->io_comm); } } } - ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + ierr = check_netcdf(file, ierr, __FILE__, __LINE__); #ifdef TIMING GPTLstop("PIO:read_darray_nc_serial"); #endif @@ -1800,7 +1952,7 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) vdesc->nreqs = 0; // if (file->iosystem->io_rank < 2) printf("%s %d varid=%d\n",__FILE__,__LINE__,i); #ifdef FLUSH_EVERY_VAR - ierr = ncmpi_wait_all(file->fh, rcnt, request,status); + ierr = ncmpi_wait_all(file->fh, rcnt, request, status); rcnt = 0; #endif } @@ -1863,21 +2015,31 @@ void cn_buffer_report(iosystem_desc_t ios, bool collective) MPI_Reduce(bget_stats, bget_mins, 5, MPI_LONG, MPI_MIN, 0, ios.comp_comm); if (ios.compmaster) { - printf("PIO: Currently allocated buffer space %ld %ld\n",bget_mins[0],bget_maxs[0]); - printf("PIO: Currently available buffer space %ld %ld\n",bget_mins[1],bget_maxs[1]); - printf("PIO: Current largest free block %ld %ld\n",bget_mins[2],bget_maxs[2]); - printf("PIO: Number of successful bget calls %ld %ld\n",bget_mins[3],bget_maxs[3]); - printf("PIO: Number of successful brel calls %ld %ld\n",bget_mins[4],bget_maxs[4]); + printf("PIO: Currently allocated buffer space %ld %ld\n", + bget_mins[0], bget_maxs[0]); + printf("PIO: Currently available buffer space %ld %ld\n", + bget_mins[1], bget_maxs[1]); + printf("PIO: Current largest free block %ld %ld\n", + bget_mins[2], bget_maxs[2]); + printf("PIO: Number of successful bget calls %ld %ld\n", + bget_mins[3], bget_maxs[3]); + printf("PIO: Number of successful brel calls %ld %ld\n", + bget_mins[4], bget_maxs[4]); // print_trace(stdout); } } else { - printf("%d: PIO: Currently allocated buffer space %ld \n",ios.union_rank,bget_stats[0]) ; - printf("%d: PIO: Currently available buffer space %ld \n",ios.union_rank,bget_stats[1]); - printf("%d: PIO: Current largest free block %ld \n",ios.union_rank,bget_stats[2]); - printf("%d: PIO: Number of successful bget calls %ld \n",ios.union_rank,bget_stats[3]); - printf("%d: PIO: Number of successful brel calls %ld \n",ios.union_rank,bget_stats[4]); + printf("%d: PIO: Currently allocated buffer space %ld \n", + ios.union_rank, bget_stats[0]) ; + printf("%d: PIO: Currently available buffer space %ld \n", + ios.union_rank, bget_stats[1]); + printf("%d: PIO: Current largest free block %ld \n", + ios.union_rank, bget_stats[2]); + printf("%d: PIO: Number of successful bget calls %ld \n", + ios.union_rank, bget_stats[3]); + printf("%d: PIO: Number of successful brel calls %ld \n", + ios.union_rank, bget_stats[4]); } } } @@ -1906,7 +2068,8 @@ void free_cn_buffer_pool(iosystem_desc_t ios) */ void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) { - if (wmb->validvars > 0){ + if (wmb->validvars > 0) + { PIOc_write_darray_multi(ncid, wmb->vid, wmb->ioid, wmb->validvars, wmb->arraylen, wmb->data, wmb->frame, wmb->fillvalue, flushtodisk); @@ -1938,7 +2101,7 @@ void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc) // printf("%s %d %d %d\n",__FILE__,__LINE__,iodesc->maxiobuflen, iodesc->ndof); if (ios.ioproc && iodesc->maxiobuflen > 0) - maxbytesoniotask = PIO_BUFFER_SIZE_LIMIT/ iodesc->maxiobuflen; + maxbytesoniotask = PIO_BUFFER_SIZE_LIMIT / iodesc->maxiobuflen; if (ios.comp_rank >= 0 && iodesc->ndof > 0) maxbytesoncomputetask = PIO_CNBUFFER_LIMIT / iodesc->ndof; @@ -1948,7 +2111,7 @@ void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc) // printf("%s %d %d %d\n",__FILE__,__LINE__,maxbytesoniotask, maxbytesoncomputetask); MPI_Allreduce(MPI_IN_PLACE, &maxbytes, 1, MPI_INT, MPI_MIN, ios.union_comm); - iodesc->maxbytes=maxbytes; + iodesc->maxbytes = maxbytes; // printf("%s %d %d %d\n",__FILE__,__LINE__,iodesc->maxbytes,iodesc->maxiobuflen); } From 79de1e3c37a2a3d4a71ecb21071c68987d848fa2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 15 Jun 2016 16:06:25 -0600 Subject: [PATCH 160/184] more work on darray test --- src/clib/pio_darray.c | 476 ++++++++++++++++++++++-------------------- 1 file changed, 254 insertions(+), 222 deletions(-) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index a9d009f56c8..96eb38240b7 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -50,16 +50,16 @@ void compute_buffer_init(iosystem_desc_t ios) { #ifndef PIO_USE_MALLOC - if (CN_bpool == NULL) + if (!CN_bpool) { - CN_bpool = malloc(PIO_CNBUFFER_LIMIT); - if (CN_bpool==NULL) + if (!(CN_bpool = malloc(PIO_CNBUFFER_LIMIT))) { char errmsg[180]; sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d:" " try reducing PIO_CNBUFFER_LIMIT\n", PIO_CNBUFFER_LIMIT, ios.comp_rank); piodie(errmsg, __FILE__, __LINE__); } + bpool(CN_bpool, PIO_CNBUFFER_LIMIT); if (!CN_bpool) { @@ -68,6 +68,7 @@ void compute_buffer_init(iosystem_desc_t ios) " try reducing PIO_CNBUFFER_LIMIT\n", PIO_CNBUFFER_LIMIT, ios.comp_rank); piodie(errmsg, __FILE__, __LINE__); } + bectl(NULL, malloc, free, PIO_CNBUFFER_LIMIT); } #endif @@ -93,18 +94,18 @@ void compute_buffer_init(iosystem_desc_t ios) int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF, void *fillvalue) { - iosystem_desc_t *ios; + iosystem_desc_t *ios; /** Pointer to io system information. */ var_desc_t *vdesc; int ndims; int ierr = PIO_NOERR; /** Return code from function calls. */ int i; - int msg = 0; int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ int dsize; MPI_Status status; PIO_Offset usage; int fndims; PIO_Offset tdsize = 0; + #ifdef TIMING GPTLstart("PIO:write_darray_nc"); #endif @@ -113,18 +114,20 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, if (!(ios = file->iosystem)) return PIO_EBADID; - vdesc = file->varlist + vid; - - if (!vdesc) + /* Get pointer to variable information. */ + if (!(vdesc = file->varlist + vid)) return PIO_EBADID; ndims = iodesc->ndims; + /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { if (!ios->ioproc) { - if (ios->comp_rank==0) + int msg = 0; + + if (ios->compmaster) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); if (!mpierr) @@ -159,7 +162,7 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, #ifdef _PNETCDF /* make sure we have room in the buffer. */ if (file->iotype == PIO_IOTYPE_PNETCDF) - flush_output_buffer(file, false, tsize*(iodesc->maxiobuflen)); + flush_output_buffer(file, false, tsize * (iodesc->maxiobuflen)); #endif rrcnt = 0; @@ -170,9 +173,9 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, start[i] = 0; count[i] = 0; } - if (region != NULL) + if (region) { - bufptr = (void *)((char *) IOBUF + tsize * region->loffset); + bufptr = (void *)((char *)IOBUF + tsize * region->loffset); // this is a record based multidimensional array if (vdesc->record >= 0) { @@ -196,28 +199,26 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, } } - switch(file->iotype){ + switch(file->iotype) + { #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: + + /* Use collective writes with this variable. */ ierr = nc_var_par_access(ncid, vid, NC_COLLECTIVE); if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) - { - ierr = nc_put_vara_double(ncid, vid,(size_t *)start,(size_t *)count, (const double *)bufptr); - } + ierr = nc_put_vara_double(ncid, vid, (size_t *)start, (size_t *)count, + (const double *)bufptr); else if (iodesc->basetype == MPI_INTEGER) - { - ierr = nc_put_vara_int(ncid, vid, (size_t *)start, (size_t *)count, (const int *)bufptr); - } + ierr = nc_put_vara_int(ncid, vid, (size_t *)start, (size_t *)count, + (const int *)bufptr); else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4) - { - ierr = nc_put_vara_float(ncid, vid, (size_t *)start, (size_t *)count, (const float *)bufptr); - } + ierr = nc_put_vara_float(ncid, vid, (size_t *)start, (size_t *)count, + (const float *)bufptr); else - { fprintf(stderr,"Type not recognized %d in pioc_write_darray\n", (int)iodesc->basetype); - } break; case PIO_IOTYPE_NETCDF4C: #endif @@ -242,8 +243,8 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, } else { - mpierr = MPI_Send( &ierr, 1, MPI_INT, i, 0, ios->io_comm); // handshake - tell the sending task I'm ready - mpierr = MPI_Recv( &buflen, 1, MPI_INT, i, 1, ios->io_comm, &status); + mpierr = MPI_Send(&ierr, 1, MPI_INT, i, 0, ios->io_comm); // handshake - tell the sending task I'm ready + mpierr = MPI_Recv(&buflen, 1, MPI_INT, i, 1, ios->io_comm, &status); if (buflen > 0) { mpierr = MPI_Recv(tstart, ndims, MPI_OFFSET, i, ios->num_iotasks+i, @@ -258,22 +259,15 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, if (buflen>0) { if (iodesc->basetype == MPI_INTEGER) - { ierr = nc_put_vara_int(ncid, vid, tstart, tcount, (const int *)tmp_buf); - } else if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) - { ierr = nc_put_vara_double(ncid, vid, tstart, tcount, (const double *)tmp_buf); - } else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4) - { ierr = nc_put_vara_float(ncid, vid, tstart, tcount, (const float *)tmp_buf); - } else - { fprintf(stderr,"Type not recognized %d in pioc_write_darray\n", (int)iodesc->basetype); - } + if (ierr == PIO_EEDGE) { for (i = 0; i < ndims; i++) @@ -294,14 +288,14 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, // printf("%s %d %d %d %d\n",__FILE__,__LINE__,i,tstart[i],tcount[i]); } // printf("%s %d %d %d %d %d %d %d %d %d\n",__FILE__,__LINE__,ios->io_rank,tstart[0],tstart[1],tcount[0],tcount[1],buflen,ndims,fndims); - mpierr = MPI_Recv( &ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve - mpierr = MPI_Rsend( &buflen, 1, MPI_INT, 0, 1, ios->io_comm); + mpierr = MPI_Recv(&ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve + mpierr = MPI_Rsend(&buflen, 1, MPI_INT, 0, 1, ios->io_comm); if (buflen>0) { - mpierr = MPI_Rsend( tstart, ndims, MPI_OFFSET, 0, ios->num_iotasks+ios->io_rank, + mpierr = MPI_Rsend(tstart, ndims, MPI_OFFSET, 0, ios->num_iotasks+ios->io_rank, ios->io_comm); - mpierr = MPI_Rsend( tcount, ndims, MPI_OFFSET, 0,2*ios->num_iotasks+ios->io_rank, + mpierr = MPI_Rsend(tcount, ndims, MPI_OFFSET, 0,2*ios->num_iotasks+ios->io_rank, ios->io_comm); - mpierr = MPI_Rsend( bufptr, buflen, iodesc->basetype, 0, ios->io_rank, ios->io_comm); + mpierr = MPI_Rsend(bufptr, buflen, iodesc->basetype, 0, ios->io_rank, ios->io_comm); } } break; @@ -310,7 +304,7 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - for ( i = 0,dsize = 1;iio_rank,dsize); startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - for ( i=0; iio_rank,iodesc->llen, tdsize); // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, // IOBUF, iodesc->llen, iodesc->basetype); - int reqn=0; + int reqn = 0; - if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ) + if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0 ) { vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + sizeof(int)*(vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)); - for (int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++) + for (int i=vdesc->nreqs;inreqs + PIO_REQUEST_ALLOC_CHUNK;i++) { - vdesc->request[i]=NC_REQ_NULL; + vdesc->request[i] = NC_REQ_NULL; } reqn = vdesc->nreqs; } @@ -384,7 +378,7 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, default: ierr = iotype_error(file->iotype,__FILE__,__LINE__); } - if (region != NULL) + if (region) region = region->next; } // for (regioncnt=0;regioncntmaxregions;regioncnt++){ } // if (ios->ioproc) @@ -423,11 +417,10 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ const int maxiobuflen, const int num_aiotasks, void *IOBUF, const int frame[]) { - iosystem_desc_t *ios; + iosystem_desc_t *ios; /** Pointer to io system information. */ var_desc_t *vdesc; int ierr; int i; - int msg; int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ int dsize; MPI_Status status; @@ -456,13 +449,13 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); return PIO_EBADID; } - msg = 0; /* If async is in use, send message to IO master task. */ if (ios->async_interface) { if (!ios->ioproc) { + int msg = 0; if (ios->compmaster) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); @@ -491,21 +484,21 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ ncid = file->fh; region = firstregion; - rrcnt=0; - for (regioncnt=0;regioncntstart[0],region->count[0],ndims,fndims,vdesc->record); - for (i=0;irecord >= 0) { - for (i=fndims-ndims;istart[i-(fndims-ndims)]; count[i] = region->count[i-(fndims-ndims)]; @@ -518,13 +511,13 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ } else if (fndims==ndims) { - start[0]+=vdesc->record; + start[0] += vdesc->record; } // Non-time dependent array } else { - for ( i=0;istart[i]; count[i] = region->count[i]; @@ -536,13 +529,13 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ { #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - for (int nv=0; nvrecord >= 0 && ndimsrecord >= 0 && ndims < fndims) { start[0] = frame[nv]; } - if (region != NULL) + if (region) { bufptr = (void *)((char *) IOBUF + tsize*(nv*llen + region->loffset)); } @@ -550,28 +543,32 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ if (basetype == MPI_DOUBLE ||basetype == MPI_REAL8) { - ierr = nc_put_vara_double (ncid, vid[nv],(size_t *) start,(size_t *) count, (const double *) bufptr); + ierr = nc_put_vara_double (ncid, vid[nv],(size_t *) start,(size_t *) count, + (const double *)bufptr); } else if (basetype == MPI_INTEGER) { - ierr = nc_put_vara_int (ncid, vid[nv], (size_t *) start, (size_t *) count, (const int *) bufptr); + ierr = nc_put_vara_int (ncid, vid[nv], (size_t *) start, (size_t *) count, + (const int *)bufptr); } else if (basetype == MPI_FLOAT || basetype == MPI_REAL4) { - ierr = nc_put_vara_float (ncid, vid[nv], (size_t *) start, (size_t *) count, (const float *) bufptr); + ierr = nc_put_vara_float (ncid, vid[nv], (size_t *) start, (size_t *) count, + (const float *)bufptr); } else { - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) basetype); + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n", + (int)basetype); } } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - for ( i=0,dsize=1;iio_rank,dsize); startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - for ( i=0; iiotype,__FILE__,__LINE__); } - if (region != NULL) + if (region) region = region->next; } // for (regioncnt=0;regioncntmaxregions;regioncnt++){ } // if (ios->ioproc) @@ -695,11 +692,10 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i const int maxiobuflen, const int num_aiotasks, void *IOBUF, const int frame[]) { - iosystem_desc_t *ios; + iosystem_desc_t *ios; /** Pointer to io system information. */ var_desc_t *vdesc; int ierr; int i; - int msg; int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ int dsize; MPI_Status status; @@ -714,26 +710,27 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i GPTLstart("PIO:write_darray_multi_nc_serial"); #endif - ios = file->iosystem; - if (ios == NULL) + if (!(ios = file->iosystem)) { fprintf(stderr,"Failed to find iosystem handle \n"); return PIO_EBADID; } - vdesc = (file->varlist)+vid[0]; + ncid = file->fh; - if (vdesc == NULL) + if (!(vdesc = (file->varlist) + vid[0])) { fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); return PIO_EBADID; } - msg = 0; + /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { if (! ios->ioproc) { + int msg = 0; + if (ios->comp_rank==0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); @@ -761,48 +758,48 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i region = firstregion; - rrcnt=0; - for (regioncnt=0;regioncntrecord >= 0) { - for (i=fndims-ndims;istart[i-(fndims-ndims)]; - tmp_count[i+regioncnt*fndims] = region->count[i-(fndims-ndims)]; + tmp_start[i + regioncnt * fndims] = region->start[i - (fndims - ndims)]; + tmp_count[i + regioncnt * fndims] = region->count[i - (fndims - ndims)]; } // Non-time dependent array } else { - for ( i=0;istart[i]; - tmp_count[i+regioncnt*fndims] = region->count[i]; + tmp_start[i + regioncnt * fndims] = region->start[i]; + tmp_count[i + regioncnt * fndims] = region->count[i]; } } region = region->next; } } - if (ios->io_rank>0) + if (ios->io_rank > 0) { - mpierr = MPI_Recv( &ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve - MPI_Send( &llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); + mpierr = MPI_Recv(&ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve + MPI_Send(&llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); if (llen>0) { - MPI_Send( &maxregions, 1, MPI_INT, 0, ios->io_rank+ios->num_iotasks, ios->io_comm); - MPI_Send( tmp_start, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+2*ios->num_iotasks, ios->io_comm); - MPI_Send( tmp_count, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+3*ios->num_iotasks, ios->io_comm); + MPI_Send(&maxregions, 1, MPI_INT, 0, ios->io_rank+ios->num_iotasks, ios->io_comm); + MPI_Send(tmp_start, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+2*ios->num_iotasks, ios->io_comm); + MPI_Send(tmp_count, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+3*ios->num_iotasks, ios->io_comm); // printf("%s %d %ld\n",__FILE__,__LINE__,nvars*llen); - MPI_Send( IOBUF, nvars*llen, basetype, 0, ios->io_rank+4*ios->num_iotasks, ios->io_comm); + MPI_Send(IOBUF, nvars*llen, basetype, 0, ios->io_rank+4*ios->num_iotasks, ios->io_comm); } } else @@ -817,14 +814,14 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i { if (rtask>0) { - mpierr = MPI_Send( &ierr, 1, MPI_INT, rtask, 0, ios->io_comm); // handshake - tell the sending task I'm ready - MPI_Recv( &rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); + mpierr = MPI_Send(&ierr, 1, MPI_INT, rtask, 0, ios->io_comm); // handshake - tell the sending task I'm ready + MPI_Recv(&rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); if (rlen>0){ - MPI_Recv( &rregions, 1, MPI_INT, rtask, rtask+ios->num_iotasks, ios->io_comm, &status); - MPI_Recv( tmp_start, rregions*fndims, MPI_OFFSET, rtask, rtask+2*ios->num_iotasks, ios->io_comm, &status); - MPI_Recv( tmp_count, rregions*fndims, MPI_OFFSET, rtask, rtask+3*ios->num_iotasks, ios->io_comm, &status); + MPI_Recv(&rregions, 1, MPI_INT, rtask, rtask+ios->num_iotasks, ios->io_comm, &status); + MPI_Recv(tmp_start, rregions*fndims, MPI_OFFSET, rtask, rtask+2*ios->num_iotasks, ios->io_comm, &status); + MPI_Recv(tmp_count, rregions*fndims, MPI_OFFSET, rtask, rtask+3*ios->num_iotasks, ios->io_comm, &status); // printf("%s %d %d %ld\n",__FILE__,__LINE__,rtask,nvars*rlen); - MPI_Recv( IOBUF, nvars*rlen, basetype, rtask, rtask+4*ios->num_iotasks, ios->io_comm, &status); + MPI_Recv(IOBUF, nvars*rlen, basetype, rtask, rtask+4*ios->num_iotasks, ios->io_comm, &status); } } else @@ -922,7 +919,7 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, void *array, const int frame[], void *fillvalue[], bool flushtodisk) { - iosystem_desc_t *ios; + iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; io_desc_t *iodesc; @@ -933,17 +930,20 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, ierr = PIO_NOERR; file = pio_get_file_from_id(ncid); - if (file == NULL){ + if (file == NULL) + { fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); return PIO_EBADID; } - if (! (file->mode & PIO_WRITE)){ + if (! (file->mode & PIO_WRITE)) + { fprintf(stderr,"ERROR: Attempt to write to read-only file\n"); return PIO_EPERM; } iodesc = pio_get_iodesc_from_id(ioid); - if (iodesc == NULL){ + if (iodesc == NULL) + { // print_trace(NULL); //fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); return PIO_EBADID; @@ -956,44 +956,59 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, ios = file->iosystem; // rlen = iodesc->llen*nvars; rlen=0; - if (iodesc->llen>0){ + if (iodesc->llen>0) + { rlen = iodesc->maxiobuflen*nvars; } - if (vdesc0->iobuf != NULL){ + if (vdesc0->iobuf) + { piodie("Attempt to overwrite existing io buffer",__FILE__,__LINE__); } - if (iodesc->rearranger>0){ - if (rlen>0){ + if (iodesc->rearranger>0) + { + if (rlen>0) + { MPI_Type_size(iodesc->basetype, &vsize); //printf("rlen*vsize = %ld\n",rlen*vsize); vdesc0->iobuf = bget((size_t) vsize* (size_t) rlen); - if (vdesc0->iobuf==NULL){ + if (vdesc0->iobuf==NULL) + { printf("%s %d %d %ld\n",__FILE__,__LINE__,nvars,vsize*rlen); piomemerror(*ios,(size_t) rlen*(size_t) vsize, __FILE__,__LINE__); } - if (iodesc->needsfill && iodesc->rearranger==PIO_REARR_BOX){ - if (vsize==4){ - for (int nv=0;nv < nvars; nv++){ - for (int i=0;imaxiobuflen;i++){ + if (iodesc->needsfill && iodesc->rearranger==PIO_REARR_BOX) + { + if (vsize==4) + { + for (int nv=0;nv < nvars; nv++) + { + for (int i=0;imaxiobuflen;i++) + { ((float *) vdesc0->iobuf)[i+nv*(iodesc->maxiobuflen)] = ((float *)fillvalue)[nv]; } } - }else if (vsize==8){ - for (int nv=0;nv < nvars; nv++){ - for (int i=0;imaxiobuflen;i++){ + } + else if (vsize==8) + { + for (int nv=0;nv < nvars; nv++) + { + for (int i=0;imaxiobuflen;i++) + { ((double *)vdesc0->iobuf)[i+nv*(iodesc->maxiobuflen)] = ((double *)fillvalue)[nv]; } } } } } + ierr = rearrange_comp2io(*ios, iodesc, array, vdesc0->iobuf, nvars); }/* this is wrong, need to think about it else{ vdesc0->iobuf = array; } */ - switch(file->iotype){ + switch(file->iotype) + { case PIO_IOTYPE_NETCDF4P: case PIO_IOTYPE_PNETCDF: ierr = pio_write_darray_multi_nc(file, nvars, vid, @@ -1009,7 +1024,8 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, iodesc->maxregions, iodesc->firstregion, iodesc->llen, iodesc->maxiobuflen, iodesc->num_aiotasks, vdesc0->iobuf, frame); - if (vdesc0->iobuf != NULL){ + if (vdesc0->iobuf) + { brel(vdesc0->iobuf); vdesc0->iobuf = NULL; } @@ -1018,27 +1034,37 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, } if (iodesc->rearranger == PIO_REARR_SUBSET && iodesc->needsfill && - iodesc->holegridsize>0){ - if (vdesc0->fillbuf != NULL){ + iodesc->holegridsize>0) + { + if (vdesc0->fillbuf) + { piodie("Attempt to overwrite existing buffer",__FILE__,__LINE__); } vdesc0->fillbuf = bget(iodesc->holegridsize*vsize*nvars); //printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); - if (vsize==4){ - for (int nv=0;nvholegridsize;i++){ + if (vsize==4) + { + for (int nv=0;nvholegridsize;i++) + { ((float *) vdesc0->fillbuf)[i+nv*iodesc->holegridsize] = ((float *) fillvalue)[nv]; } } - }else if (vsize==8){ - for (int nv=0;nvholegridsize;i++){ + } + else if (vsize==8) + { + for (int nv=0;nvholegridsize;i++) + { ((double *) vdesc0->fillbuf)[i+nv*iodesc->holegridsize] = ((double *) fillvalue)[nv]; } } } - switch(file->iotype){ + switch(file->iotype) + { case PIO_IOTYPE_PNETCDF: ierr = pio_write_darray_multi_nc(file, nvars, vid, iodesc->ndims, iodesc->basetype, iodesc->gsize, @@ -1096,7 +1122,7 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) { - iosystem_desc_t *ios; + iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; io_desc_t *iodesc; var_desc_t *vdesc; @@ -1116,17 +1142,20 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, ierr = PIO_NOERR; needsflush = 0; // false file = pio_get_file_from_id(ncid); - if (file == NULL){ + if (file == NULL) + { fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); return PIO_EBADID; } - if (! (file->mode & PIO_WRITE)){ + if (! (file->mode & PIO_WRITE)) + { fprintf(stderr,"ERROR: Attempt to write to read-only file\n"); return PIO_EPERM; } iodesc = pio_get_iodesc_from_id(ioid); - if (iodesc == NULL){ + if (iodesc == NULL) + { fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); return PIO_EBADID; } @@ -1136,33 +1165,28 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, if (vdesc == NULL) return PIO_EBADID; - if (vdesc->record<0){ - recordvar=false; - } - else + /* Is this a record variable? */ + recordvar = vdesc->record < 0 ? true : false; + + if (iodesc->ndof != arraylen) { - recordvar=true; - } - if (iodesc->ndof != arraylen){ fprintf(stderr,"ndof=%ld, arraylen=%ld\n",iodesc->ndof,arraylen); piodie("ndof != arraylen",__FILE__,__LINE__); } wmb = &(file->buffer); - if (wmb->ioid == -1){ - if (recordvar){ + if (wmb->ioid == -1) + { + if (recordvar) wmb->ioid = ioid; - } else - { wmb->ioid = -(ioid); - } } else { // separate record and non-record variables if (recordvar) { - while(wmb->next != NULL && wmb->ioid!=ioid) + while(wmb->next && wmb->ioid!=ioid) if (wmb->next!=NULL) wmb = wmb->next; #ifdef _PNETCDF @@ -1175,7 +1199,7 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, } else { - while(wmb->next != NULL && wmb->ioid!= -(ioid)) + while(wmb->next && wmb->ioid!= -(ioid)) { if (wmb->next!=NULL) wmb = wmb->next; @@ -1185,18 +1209,14 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, if ((recordvar && wmb->ioid != ioid) || (!recordvar && wmb->ioid != -(ioid))) { wmb->next = (wmulti_buffer *) bget((bufsize) sizeof(wmulti_buffer)); - if (wmb->next == NULL){ + if (wmb->next == NULL) piomemerror(*ios,sizeof(wmulti_buffer), __FILE__,__LINE__); - } wmb=wmb->next; wmb->next=NULL; - if (recordvar){ + if (recordvar) wmb->ioid = ioid; - } else - { wmb->ioid = -(ioid); - } wmb->validvars=0; wmb->arraylen=arraylen; wmb->vid=NULL; @@ -1211,70 +1231,60 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, // printf("%s %d %X %d %d %d\n",__FILE__,__LINE__,wmb->data,wmb->validvars,arraylen,tsize); // cn_buffer_report(*ios, true); bfreespace(&totfree, &maxfree); - if (needsflush==0) - { + if (needsflush == 0) needsflush = (maxfree <= 1.1*(1+wmb->validvars)*arraylen*tsize ); - } MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1, MPI_INT, MPI_MAX, ios->comp_comm); - if (needsflush > 0 ) { // need to flush first // printf("%s %d %ld %d %ld %ld\n",__FILE__,__LINE__,maxfree, wmb->validvars, (1+wmb->validvars)*arraylen*tsize,totfree); cn_buffer_report(*ios, true); - flush_buffer(ncid,wmb, needsflush==2); // if needsflush == 2 flush to disk otherwise just flush to io node + flush_buffer(ncid, wmb, needsflush == 2); // if needsflush == 2 flush to disk otherwise just flush to io node } + if (arraylen > 0) - { - wmb->data = bgetr( wmb->data, (1+wmb->validvars)*arraylen*tsize); - if (wmb->data == NULL) - { - piomemerror(*ios, (1+wmb->validvars)*arraylen*tsize , __FILE__,__LINE__); - } - } - wmb->vid = (int *) bgetr( wmb->vid,sizeof(int)*( 1+wmb->validvars)); - if (wmb->vid == NULL) - { - piomemerror(*ios, (1+wmb->validvars)*sizeof(int) , __FILE__,__LINE__); - } - if (vdesc->record>=0) - { - wmb->frame = (int *) bgetr( wmb->frame,sizeof(int)*( 1+wmb->validvars)); - if (wmb->frame == NULL) - { - piomemerror(*ios, (1+wmb->validvars)*sizeof(int) , __FILE__,__LINE__); - } - } + if (!(wmb->data = bgetr(wmb->data, (1+wmb->validvars)*arraylen*tsize))) + piomemerror(*ios, (1+wmb->validvars)*arraylen*tsize, __FILE__, __LINE__); + + if (!(wmb->vid = (int *) bgetr(wmb->vid,sizeof(int)*(1+wmb->validvars)))) + piomemerror(*ios, (1+wmb->validvars)*sizeof(int), __FILE__, __LINE__); + + if (vdesc->record >= 0) + if (!(wmb->frame = (int *)bgetr(wmb->frame, sizeof(int) * (1 + wmb->validvars)))) + piomemerror(*ios, (1+wmb->validvars)*sizeof(int), __FILE__, __LINE__); + if (iodesc->needsfill) - { - wmb->fillvalue = bgetr( wmb->fillvalue,tsize*( 1+wmb->validvars)); - if (wmb->fillvalue == NULL) - { + if (!(wmb->fillvalue = bgetr(wmb->fillvalue,tsize*(1+wmb->validvars)))) piomemerror(*ios, (1+wmb->validvars)*tsize , __FILE__,__LINE__); - } - } if (iodesc->needsfill) { - if (fillvalue != NULL) + if (fillvalue) { memcpy((char *) wmb->fillvalue+tsize*wmb->validvars,fillvalue, tsize); } else { vtype = (MPI_Datatype) iodesc->basetype; - if (vtype == MPI_INTEGER){ + if (vtype == MPI_INTEGER) + { int fill = PIO_FILL_INT; memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); - }else if (vtype == MPI_FLOAT || vtype == MPI_REAL4){ + } + else if (vtype == MPI_FLOAT || vtype == MPI_REAL4) + { float fill = PIO_FILL_FLOAT; memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); - }else if (vtype == MPI_DOUBLE || vtype == MPI_REAL8){ + } + else if (vtype == MPI_DOUBLE || vtype == MPI_REAL8) + { double fill = PIO_FILL_DOUBLE; memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); - }else if (vtype == MPI_CHARACTER){ + } + else if (vtype == MPI_CHARACTER) + { char fill = PIO_FILL_CHAR; memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); } @@ -1334,7 +1344,7 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) { - iosystem_desc_t *ios; + iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; io_desc_t *iodesc; void *iobuf; @@ -1365,7 +1375,8 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, rlen = iodesc->llen; if (iodesc->rearranger>0) { - if (rlen>0){ + if (rlen>0) + { MPI_Type_size(iodesc->basetype, &tsize); // iobuf = bget(tsize*rlen); iobuf = malloc((size_t) tsize*rlen); @@ -1416,7 +1427,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF) { int ierr=PIO_NOERR; - iosystem_desc_t *ios; + iosystem_desc_t *ios; /** Pointer to io system information. */ var_desc_t *vdesc; int ndims, fndims; MPI_Status status; @@ -1514,7 +1525,8 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, } } - switch(file->iotype){ + switch(file->iotype) + { #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) @@ -1573,7 +1585,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, ierr = iotype_error(file->iotype,__FILE__,__LINE__); } - if (region != NULL) + if (region) region = region->next; } // for (regioncnt=0;...) } @@ -1600,7 +1612,7 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF) { int ierr=PIO_NOERR; - iosystem_desc_t *ios; + iosystem_desc_t *ios; /** Pointer to io system information. */ var_desc_t *vdesc; int ndims, fndims; MPI_Status status; @@ -1688,20 +1700,20 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, }*/ } - if (region != NULL) + if (region) region = region->next; } // for (regioncnt=0;...) if (ios->io_rank>0) { - MPI_Send( &(iodesc->llen), 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); + MPI_Send(&(iodesc->llen), 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); if (iodesc->llen > 0) { - MPI_Send( &(iodesc->maxregions), 1, MPI_INT, 0, + MPI_Send(&(iodesc->maxregions), 1, MPI_INT, 0, ios->num_iotasks + ios->io_rank, ios->io_comm); - MPI_Send( tmp_count, iodesc->maxregions*fndims, MPI_OFFSET, 0, + MPI_Send(tmp_count, iodesc->maxregions*fndims, MPI_OFFSET, 0, 2 * ios->num_iotasks + ios->io_rank, ios->io_comm); - MPI_Send( tmp_start, iodesc->maxregions*fndims, MPI_OFFSET, 0, + MPI_Send(tmp_start, iodesc->maxregions*fndims, MPI_OFFSET, 0, 3 * ios->num_iotasks + ios->io_rank, ios->io_comm); MPI_Recv(IOBUF, iodesc->llen, iodesc->basetype, 0, 4 * ios->num_iotasks+ios->io_rank, ios->io_comm, &status); @@ -1713,13 +1725,14 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, size_t loffset, regionsize; size_t this_start[fndims*iodesc->maxregions]; size_t this_count[fndims*iodesc->maxregions]; - // for ( i=ios->num_iotasks-1; i>=0; i--){ + // for (i=ios->num_iotasks-1; i>=0; i--){ for (int rtask = 1; rtask <= ios->num_iotasks; rtask++) { if (rtasknum_iotasks) { MPI_Recv(&tmp_bufsize, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); - if (tmp_bufsize>0){ + if (tmp_bufsize>0) + { MPI_Recv(&maxregions, 1, MPI_INT, rtask, ios->num_iotasks+rtask, ios->io_comm, &status); MPI_Recv(this_count, maxregions*fndims, MPI_OFFSET, rtask, @@ -1738,7 +1751,8 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, { bufptr=(void *)((char *) IOBUF + tsize*loffset); regionsize=1; - if (rtasknum_iotasks){ + if (rtasknum_iotasks) + { for (int m=0; mbasetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ + if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) + { ierr = nc_get_vara_double (file->fh, vid,start, count, bufptr); - }else if (iodesc->basetype == MPI_INTEGER){ + } + else if (iodesc->basetype == MPI_INTEGER) + { ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); - }else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ + } + else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4) + { ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); } else @@ -1811,7 +1830,7 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int PIOc_read_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array) { - iosystem_desc_t *ios; + iosystem_desc_t *ios; /** Pointer to io system information. */ file_desc_t *file; io_desc_t *iodesc; void *iobuf=NULL; @@ -1821,17 +1840,20 @@ int PIOc_read_darray(const int ncid, const int vid, const int ioid, file = pio_get_file_from_id(ncid); - if (file == NULL){ + if (file == NULL) + { fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); return PIO_EBADID; } iodesc = pio_get_iodesc_from_id(ioid); - if (iodesc == NULL){ + if (iodesc == NULL) + { fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); return PIO_EBADID; } ios = file->iosystem; - if (ios->iomaster){ + if (ios->iomaster) + { rlen = iodesc->maxiobuflen; } else @@ -1839,11 +1861,14 @@ int PIOc_read_darray(const int ncid, const int vid, const int ioid, rlen = iodesc->llen; } - if (iodesc->rearranger > 0){ - if (ios->ioproc && rlen>0){ + if (iodesc->rearranger > 0) + { + if (ios->ioproc && rlen>0) + { MPI_Type_size(iodesc->basetype, &tsize); iobuf = bget(((size_t) tsize)*rlen); - if (iobuf==NULL){ + if (iobuf==NULL) + { piomemerror(*ios,rlen*((size_t) tsize), __FILE__,__LINE__); } } @@ -1853,7 +1878,8 @@ int PIOc_read_darray(const int ncid, const int vid, const int ioid, iobuf = array; } - switch(file->iotype){ + switch(file->iotype) + { case PIO_IOTYPE_NETCDF: case PIO_IOTYPE_NETCDF4C: ierr = pio_read_darray_nc_serial(file, iodesc, vid, iobuf); @@ -1865,7 +1891,8 @@ int PIOc_read_darray(const int ncid, const int vid, const int ioid, default: ierr = iotype_error(file->iotype,__FILE__,__LINE__); } - if (iodesc->rearranger > 0){ + if (iodesc->rearranger > 0) + { ierr = rearrange_io2comp(*ios, iodesc, iobuf, array); if (rlen>0) @@ -1899,17 +1926,20 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) ierr = ncmpi_inq_buffer_usage(file->fh, &usage); - if (!force && file->iosystem->io_comm != MPI_COMM_NULL){ + if (!force && file->iosystem->io_comm != MPI_COMM_NULL) + { usage += addsize; MPI_Allreduce(MPI_IN_PLACE, &usage, 1, MPI_OFFSET, MPI_MAX, file->iosystem->io_comm); } - if (usage > maxusage){ + if (usage > maxusage) + { maxusage = usage; } - if (force || usage>=PIO_BUFFER_SIZE_LIMIT){ + if (force || usage>=PIO_BUFFER_SIZE_LIMIT) + { int rcnt; bool prev_dist=false; int prev_record=-1; @@ -1974,12 +2004,12 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) for (int i = 0; i < PIO_MAX_VARS; i++) { vdesc = file->varlist + i; - if (vdesc->iobuf != NULL) + if (vdesc->iobuf) { brel(vdesc->iobuf); vdesc->iobuf=NULL; } - if (vdesc->fillbuf != NULL) + if (vdesc->fillbuf) { brel(vdesc->fillbuf); vdesc->fillbuf=NULL; @@ -2003,7 +2033,8 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) void cn_buffer_report(iosystem_desc_t ios, bool collective) { - if (CN_bpool != NULL){ + if (CN_bpool) + { long bget_stats[5]; long bget_mins[5]; long bget_maxs[5]; @@ -2051,7 +2082,8 @@ void cn_buffer_report(iosystem_desc_t ios, bool collective) void free_cn_buffer_pool(iosystem_desc_t ios) { #ifndef PIO_USE_MALLOC - if (CN_bpool != NULL){ + if (CN_bpool) + { cn_buffer_report(ios, true); bpoolrelease(CN_bpool); // free(CN_bpool); @@ -2078,9 +2110,9 @@ void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) wmb->vid = NULL; brel(wmb->data); wmb->data = NULL; - if (wmb->fillvalue != NULL) + if (wmb->fillvalue) brel(wmb->fillvalue); - if (wmb->frame != NULL) + if (wmb->frame) brel(wmb->frame); wmb->fillvalue = NULL; wmb->frame = NULL; From 77152b4c82227e0b462d0e67b9dc337edeeab4d3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 15 Jun 2016 16:17:52 -0600 Subject: [PATCH 161/184] more work on darray test --- src/clib/pio_darray.c | 50 ++++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index 96eb38240b7..abf6cf8ed0f 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -221,7 +221,7 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, (int)iodesc->basetype); break; case PIO_IOTYPE_NETCDF4C: -#endif +#endif /* _NETCDF4 */ case PIO_IOTYPE_NETCDF: { mpierr = MPI_Type_size(iodesc->basetype, &dsize); @@ -269,28 +269,30 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, (int)iodesc->basetype); if (ierr == PIO_EEDGE) - { for (i = 0; i < ndims; i++) fprintf(stderr,"dim %d start %ld count %ld\n", i, tstart[i], tcount[i]); - } + if (tmp_buf != bufptr) free(tmp_buf); } } - }else if (ios->io_rank < iodesc->num_aiotasks ) + } + else if (ios->io_rank < iodesc->num_aiotasks) { buflen = 1; - for (i = 0;iio_rank,tstart[0],tstart[1],tcount[0],tcount[1],buflen,ndims,fndims); + /* printf("%s %d %d %d %d %d %d %d %d %d\n",__FILE__,__LINE__,ios->io_rank,tstart[0], + tstart[1],tcount[0],tcount[1],buflen,ndims,fndims);*/ mpierr = MPI_Recv(&ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve mpierr = MPI_Rsend(&buflen, 1, MPI_INT, 0, 1, ios->io_comm); - if (buflen>0) { + if (buflen > 0) + { mpierr = MPI_Rsend(tstart, ndims, MPI_OFFSET, 0, ios->num_iotasks+ios->io_rank, ios->io_comm); mpierr = MPI_Rsend(tcount, ndims, MPI_OFFSET, 0,2*ios->num_iotasks+ios->io_rank, @@ -301,13 +303,12 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, break; } break; -#endif +#endif /* _NETCDF */ #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - for (i = 0,dsize = 1;ibasetype); @@ -324,46 +325,37 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - for (i=0; imaxregions-1) + if (regioncnt == iodesc->maxregions - 1) { // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,ios->io_rank,iodesc->llen, tdsize); // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, // IOBUF, iodesc->llen, iodesc->basetype); int reqn = 0; - if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0 ) { vdesc->request = realloc(vdesc->request, - sizeof(int)*(vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)); + sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)); - for (int i=vdesc->nreqs;inreqs + PIO_REQUEST_ALLOC_CHUNK;i++) - { + for (int i = vdesc->nreqs; i < vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK; i++) vdesc->request[i] = NC_REQ_NULL; - } reqn = vdesc->nreqs; } else - { - while(vdesc->request[reqn] != NC_REQ_NULL ) - { + while(vdesc->request[reqn] != NC_REQ_NULL) reqn++; - } - } ierr = ncmpi_bput_varn(ncid, vid, rrcnt, startlist, countlist, IOBUF, iodesc->llen, iodesc->basetype, vdesc->request+reqn); if (vdesc->request[reqn] == NC_REQ_NULL) - { vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync - } vdesc->nreqs = reqn; // printf("%s %d %X %d\n",__FILE__,__LINE__,IOBUF,request); @@ -374,7 +366,7 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, } } break; -#endif +#endif /* _PNETCDF */ default: ierr = iotype_error(file->iotype,__FILE__,__LINE__); } From e7dd43eccb898619d222a45d7034e0ad35c9df46 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 16 Jun 2016 04:53:59 -0600 Subject: [PATCH 162/184] fixing problems when built without logging, took out unneeded msgs --- src/clib/pio_internal.h | 138 ++-------------------------------------- src/clib/pioc_support.c | 4 +- 2 files changed, 9 insertions(+), 133 deletions(-) diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 412b3d99055..00d4817f6bf 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -17,7 +17,7 @@ #include #endif -#ifdef PIO_ENABLE_LOGGING +#if PIO_ENABLE_LOGGING void pio_log(int severity, const char *fmt, ...); #define LOG(e) pio_log e #else @@ -176,178 +176,54 @@ enum PIO_MSG{ PIO_MSG_INQ_VARID, PIO_MSG_DEF_VAR, PIO_MSG_INQ_VAR, - PIO_MSG_PUT_ATT_DOUBLE, - PIO_MSG_PUT_ATT_INT, PIO_MSG_RENAME_ATT, PIO_MSG_DEL_ATT, PIO_MSG_INQ, - PIO_MSG_GET_ATT_TEXT, - PIO_MSG_GET_ATT_SHORT, - PIO_MSG_PUT_ATT_LONG, PIO_MSG_REDEF, PIO_MSG_SET_FILL, PIO_MSG_ENDDEF, PIO_MSG_RENAME_VAR, - PIO_MSG_PUT_ATT_SHORT, - PIO_MSG_PUT_ATT_TEXT, PIO_MSG_INQ_ATTNAME, - PIO_MSG_GET_ATT_ULONGLONG, - PIO_MSG_GET_ATT_USHORT, - PIO_MSG_PUT_ATT_ULONGLONG, - PIO_MSG_GET_ATT_UINT, - PIO_MSG_GET_ATT_LONGLONG, - PIO_MSG_PUT_ATT_SCHAR, - PIO_MSG_PUT_ATT_FLOAT, PIO_MSG_RENAME_DIM, - PIO_MSG_GET_ATT_LONG, PIO_MSG_INQ_DIM, PIO_MSG_INQ_DIMID, - PIO_MSG_PUT_ATT_USHORT, - PIO_MSG_GET_ATT_FLOAT, PIO_MSG_SYNC, - PIO_MSG_PUT_ATT_LONGLONG, - PIO_MSG_PUT_ATT_UINT, - PIO_MSG_GET_ATT_SCHAR, PIO_MSG_INQ_ATTID, PIO_MSG_DEF_DIM, - PIO_MSG_GET_ATT_INT, - PIO_MSG_GET_ATT_DOUBLE, - PIO_MSG_PUT_ATT_UCHAR, - PIO_MSG_GET_ATT_UCHAR, - PIO_MSG_PUT_VARS_UCHAR, - PIO_MSG_GET_VAR1_SCHAR, - PIO_MSG_GET_VARS_ULONGLONG, PIO_MSG_GET_VARM_UCHAR, PIO_MSG_GET_VARM_SCHAR, - PIO_MSG_GET_VARS_SHORT, - PIO_MSG_GET_VAR_DOUBLE, - PIO_MSG_GET_VARA_DOUBLE, - PIO_MSG_GET_VAR_INT, - PIO_MSG_GET_VAR_USHORT, - PIO_MSG_PUT_VARS_USHORT, - PIO_MSG_GET_VARA_TEXT, - PIO_MSG_PUT_VARS_ULONGLONG, - PIO_MSG_GET_VARA_INT, PIO_MSG_PUT_VARM, - PIO_MSG_GET_VAR1_FLOAT, - PIO_MSG_GET_VAR1_SHORT, - PIO_MSG_GET_VARS_INT, - PIO_MSG_PUT_VARS_UINT, - PIO_MSG_GET_VAR_TEXT, PIO_MSG_GET_VARM_DOUBLE, PIO_MSG_PUT_VARM_UCHAR, - PIO_MSG_PUT_VAR_USHORT, - PIO_MSG_GET_VARS_SCHAR, - PIO_MSG_GET_VARA_USHORT, - PIO_MSG_PUT_VAR1_LONGLONG, - PIO_MSG_PUT_VARA_UCHAR, PIO_MSG_PUT_VARM_SHORT, - PIO_MSG_PUT_VAR1_LONG, - PIO_MSG_PUT_VARS_LONG, - PIO_MSG_GET_VAR1_USHORT, - PIO_MSG_PUT_VAR_SHORT, - PIO_MSG_PUT_VARA_INT, - PIO_MSG_GET_VAR_FLOAT, - PIO_MSG_PUT_VAR1_USHORT, - PIO_MSG_PUT_VARA_TEXT, PIO_MSG_PUT_VARM_TEXT, - PIO_MSG_GET_VARS_UCHAR, - PIO_MSG_GET_VAR, PIO_MSG_PUT_VARM_USHORT, - PIO_MSG_GET_VAR1_LONGLONG, - PIO_MSG_GET_VARS_USHORT, - PIO_MSG_GET_VAR_LONG, - PIO_MSG_GET_VAR1_DOUBLE, - PIO_MSG_PUT_VAR_ULONGLONG, - PIO_MSG_PUT_VAR_INT, - PIO_MSG_GET_VARA_UINT, - PIO_MSG_PUT_VAR_LONGLONG, - PIO_MSG_GET_VARS_LONGLONG, - PIO_MSG_PUT_VAR_SCHAR, - PIO_MSG_PUT_VAR_UINT, - PIO_MSG_PUT_VAR, - PIO_MSG_PUT_VARA_USHORT, - PIO_MSG_GET_VAR_LONGLONG, - PIO_MSG_GET_VARA_SHORT, - PIO_MSG_PUT_VARS_SHORT, - PIO_MSG_PUT_VARA_UINT, - PIO_MSG_PUT_VARA_SCHAR, PIO_MSG_PUT_VARM_ULONGLONG, - PIO_MSG_PUT_VAR1_UCHAR, PIO_MSG_PUT_VARM_INT, - PIO_MSG_PUT_VARS_SCHAR, - PIO_MSG_GET_VARA_LONG, - PIO_MSG_PUT_VAR1, - PIO_MSG_GET_VAR1_INT, - PIO_MSG_GET_VAR1_ULONGLONG, - PIO_MSG_GET_VAR_UCHAR, - PIO_MSG_PUT_VARA_FLOAT, - PIO_MSG_GET_VARA_UCHAR, - PIO_MSG_GET_VARS_FLOAT, - PIO_MSG_PUT_VAR1_FLOAT, PIO_MSG_PUT_VARM_FLOAT, - PIO_MSG_PUT_VAR1_TEXT, - PIO_MSG_PUT_VARS_TEXT, PIO_MSG_PUT_VARM_LONG, - PIO_MSG_GET_VARS_LONG, - PIO_MSG_PUT_VARS_DOUBLE, - PIO_MSG_GET_VAR1, - PIO_MSG_GET_VAR_UINT, - PIO_MSG_PUT_VARA_LONGLONG, - PIO_MSG_GET_VARA, - PIO_MSG_PUT_VAR_DOUBLE, - PIO_MSG_GET_VARA_SCHAR, - PIO_MSG_PUT_VAR_FLOAT, - PIO_MSG_GET_VAR1_UINT, - PIO_MSG_GET_VARS_UINT, - PIO_MSG_PUT_VAR1_ULONGLONG, PIO_MSG_PUT_VARM_UINT, - PIO_MSG_PUT_VAR1_UINT, - PIO_MSG_PUT_VAR1_INT, - PIO_MSG_GET_VARA_FLOAT, PIO_MSG_GET_VARM_TEXT, - PIO_MSG_PUT_VARS_FLOAT, - PIO_MSG_GET_VAR1_TEXT, - PIO_MSG_PUT_VARA_SHORT, - PIO_MSG_PUT_VAR1_SCHAR, - PIO_MSG_PUT_VARA_ULONGLONG, PIO_MSG_PUT_VARM_DOUBLE, PIO_MSG_GET_VARM_INT, - PIO_MSG_PUT_VARA, - PIO_MSG_PUT_VARA_LONG, PIO_MSG_GET_VARM_UINT, PIO_MSG_GET_VARM, - PIO_MSG_PUT_VAR1_DOUBLE, - PIO_MSG_GET_VARS_DOUBLE, - PIO_MSG_GET_VARA_LONGLONG, - PIO_MSG_GET_VAR_ULONGLONG, PIO_MSG_PUT_VARM_SCHAR, - PIO_MSG_GET_VARA_ULONGLONG, - PIO_MSG_GET_VAR_SHORT, PIO_MSG_GET_VARM_FLOAT, - PIO_MSG_PUT_VAR_TEXT, - PIO_MSG_PUT_VARS_INT, - PIO_MSG_GET_VAR1_LONG, PIO_MSG_GET_VARM_LONG, PIO_MSG_GET_VARM_USHORT, - PIO_MSG_PUT_VAR1_SHORT, - PIO_MSG_PUT_VARS_LONGLONG, PIO_MSG_GET_VARM_LONGLONG, - PIO_MSG_GET_VARS_TEXT, - PIO_MSG_PUT_VARA_DOUBLE, PIO_MSG_PUT_VARS, - PIO_MSG_PUT_VAR_UCHAR, - PIO_MSG_GET_VAR1_UCHAR, - PIO_MSG_PUT_VAR_LONG, PIO_MSG_GET_VARS, PIO_MSG_GET_VARM_SHORT, PIO_MSG_GET_VARM_ULONGLONG, PIO_MSG_PUT_VARM_LONGLONG, - PIO_MSG_GET_VAR_SCHAR, - PIO_MSG_GET_ATT_UBYTE, - PIO_MSG_PUT_ATT_STRING, - PIO_MSG_GET_ATT_STRING, - PIO_MSG_PUT_ATT_UBYTE, + PIO_MSG_PUT_VAR1, + PIO_MSG_PUT_VARA, + PIO_MSG_PUT_VAR, + PIO_MSG_GET_VAR1, + PIO_MSG_GET_VARA, + PIO_MSG_GET_VAR, PIO_MSG_INQ_VAR_FILL, PIO_MSG_DEF_VAR_FILL, PIO_MSG_DEF_VAR_DEFLATE, diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 3347d9ee69c..d4d7a292068 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -2,7 +2,7 @@ * Support functions. */ #include -#ifdef PIO_ENABLE_LOGGING +#if PIO_ENABLE_LOGGING #include #include #endif /* PIO_ENABLE_LOGGING */ @@ -27,8 +27,8 @@ int PIOc_set_log_level(int level) printf("setting log level to %d\n", level); pio_log_level = level; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - return PIO_NOERR; #endif /* PIO_ENABLE_LOGGING */ + return PIO_NOERR; } #if PIO_ENABLE_LOGGING From 58cb4dbb1154e4d83b995f33e2f0d6c8c315932a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 16 Jun 2016 05:09:41 -0600 Subject: [PATCH 163/184] put messages back in until async build is the only build --- src/clib/pio_internal.h | 138 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 131 insertions(+), 7 deletions(-) diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 00d4817f6bf..412b3d99055 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -17,7 +17,7 @@ #include #endif -#if PIO_ENABLE_LOGGING +#ifdef PIO_ENABLE_LOGGING void pio_log(int severity, const char *fmt, ...); #define LOG(e) pio_log e #else @@ -176,54 +176,178 @@ enum PIO_MSG{ PIO_MSG_INQ_VARID, PIO_MSG_DEF_VAR, PIO_MSG_INQ_VAR, + PIO_MSG_PUT_ATT_DOUBLE, + PIO_MSG_PUT_ATT_INT, PIO_MSG_RENAME_ATT, PIO_MSG_DEL_ATT, PIO_MSG_INQ, + PIO_MSG_GET_ATT_TEXT, + PIO_MSG_GET_ATT_SHORT, + PIO_MSG_PUT_ATT_LONG, PIO_MSG_REDEF, PIO_MSG_SET_FILL, PIO_MSG_ENDDEF, PIO_MSG_RENAME_VAR, + PIO_MSG_PUT_ATT_SHORT, + PIO_MSG_PUT_ATT_TEXT, PIO_MSG_INQ_ATTNAME, + PIO_MSG_GET_ATT_ULONGLONG, + PIO_MSG_GET_ATT_USHORT, + PIO_MSG_PUT_ATT_ULONGLONG, + PIO_MSG_GET_ATT_UINT, + PIO_MSG_GET_ATT_LONGLONG, + PIO_MSG_PUT_ATT_SCHAR, + PIO_MSG_PUT_ATT_FLOAT, PIO_MSG_RENAME_DIM, + PIO_MSG_GET_ATT_LONG, PIO_MSG_INQ_DIM, PIO_MSG_INQ_DIMID, + PIO_MSG_PUT_ATT_USHORT, + PIO_MSG_GET_ATT_FLOAT, PIO_MSG_SYNC, + PIO_MSG_PUT_ATT_LONGLONG, + PIO_MSG_PUT_ATT_UINT, + PIO_MSG_GET_ATT_SCHAR, PIO_MSG_INQ_ATTID, PIO_MSG_DEF_DIM, + PIO_MSG_GET_ATT_INT, + PIO_MSG_GET_ATT_DOUBLE, + PIO_MSG_PUT_ATT_UCHAR, + PIO_MSG_GET_ATT_UCHAR, + PIO_MSG_PUT_VARS_UCHAR, + PIO_MSG_GET_VAR1_SCHAR, + PIO_MSG_GET_VARS_ULONGLONG, PIO_MSG_GET_VARM_UCHAR, PIO_MSG_GET_VARM_SCHAR, + PIO_MSG_GET_VARS_SHORT, + PIO_MSG_GET_VAR_DOUBLE, + PIO_MSG_GET_VARA_DOUBLE, + PIO_MSG_GET_VAR_INT, + PIO_MSG_GET_VAR_USHORT, + PIO_MSG_PUT_VARS_USHORT, + PIO_MSG_GET_VARA_TEXT, + PIO_MSG_PUT_VARS_ULONGLONG, + PIO_MSG_GET_VARA_INT, PIO_MSG_PUT_VARM, + PIO_MSG_GET_VAR1_FLOAT, + PIO_MSG_GET_VAR1_SHORT, + PIO_MSG_GET_VARS_INT, + PIO_MSG_PUT_VARS_UINT, + PIO_MSG_GET_VAR_TEXT, PIO_MSG_GET_VARM_DOUBLE, PIO_MSG_PUT_VARM_UCHAR, + PIO_MSG_PUT_VAR_USHORT, + PIO_MSG_GET_VARS_SCHAR, + PIO_MSG_GET_VARA_USHORT, + PIO_MSG_PUT_VAR1_LONGLONG, + PIO_MSG_PUT_VARA_UCHAR, PIO_MSG_PUT_VARM_SHORT, + PIO_MSG_PUT_VAR1_LONG, + PIO_MSG_PUT_VARS_LONG, + PIO_MSG_GET_VAR1_USHORT, + PIO_MSG_PUT_VAR_SHORT, + PIO_MSG_PUT_VARA_INT, + PIO_MSG_GET_VAR_FLOAT, + PIO_MSG_PUT_VAR1_USHORT, + PIO_MSG_PUT_VARA_TEXT, PIO_MSG_PUT_VARM_TEXT, + PIO_MSG_GET_VARS_UCHAR, + PIO_MSG_GET_VAR, PIO_MSG_PUT_VARM_USHORT, + PIO_MSG_GET_VAR1_LONGLONG, + PIO_MSG_GET_VARS_USHORT, + PIO_MSG_GET_VAR_LONG, + PIO_MSG_GET_VAR1_DOUBLE, + PIO_MSG_PUT_VAR_ULONGLONG, + PIO_MSG_PUT_VAR_INT, + PIO_MSG_GET_VARA_UINT, + PIO_MSG_PUT_VAR_LONGLONG, + PIO_MSG_GET_VARS_LONGLONG, + PIO_MSG_PUT_VAR_SCHAR, + PIO_MSG_PUT_VAR_UINT, + PIO_MSG_PUT_VAR, + PIO_MSG_PUT_VARA_USHORT, + PIO_MSG_GET_VAR_LONGLONG, + PIO_MSG_GET_VARA_SHORT, + PIO_MSG_PUT_VARS_SHORT, + PIO_MSG_PUT_VARA_UINT, + PIO_MSG_PUT_VARA_SCHAR, PIO_MSG_PUT_VARM_ULONGLONG, + PIO_MSG_PUT_VAR1_UCHAR, PIO_MSG_PUT_VARM_INT, + PIO_MSG_PUT_VARS_SCHAR, + PIO_MSG_GET_VARA_LONG, + PIO_MSG_PUT_VAR1, + PIO_MSG_GET_VAR1_INT, + PIO_MSG_GET_VAR1_ULONGLONG, + PIO_MSG_GET_VAR_UCHAR, + PIO_MSG_PUT_VARA_FLOAT, + PIO_MSG_GET_VARA_UCHAR, + PIO_MSG_GET_VARS_FLOAT, + PIO_MSG_PUT_VAR1_FLOAT, PIO_MSG_PUT_VARM_FLOAT, + PIO_MSG_PUT_VAR1_TEXT, + PIO_MSG_PUT_VARS_TEXT, PIO_MSG_PUT_VARM_LONG, + PIO_MSG_GET_VARS_LONG, + PIO_MSG_PUT_VARS_DOUBLE, + PIO_MSG_GET_VAR1, + PIO_MSG_GET_VAR_UINT, + PIO_MSG_PUT_VARA_LONGLONG, + PIO_MSG_GET_VARA, + PIO_MSG_PUT_VAR_DOUBLE, + PIO_MSG_GET_VARA_SCHAR, + PIO_MSG_PUT_VAR_FLOAT, + PIO_MSG_GET_VAR1_UINT, + PIO_MSG_GET_VARS_UINT, + PIO_MSG_PUT_VAR1_ULONGLONG, PIO_MSG_PUT_VARM_UINT, + PIO_MSG_PUT_VAR1_UINT, + PIO_MSG_PUT_VAR1_INT, + PIO_MSG_GET_VARA_FLOAT, PIO_MSG_GET_VARM_TEXT, + PIO_MSG_PUT_VARS_FLOAT, + PIO_MSG_GET_VAR1_TEXT, + PIO_MSG_PUT_VARA_SHORT, + PIO_MSG_PUT_VAR1_SCHAR, + PIO_MSG_PUT_VARA_ULONGLONG, PIO_MSG_PUT_VARM_DOUBLE, PIO_MSG_GET_VARM_INT, + PIO_MSG_PUT_VARA, + PIO_MSG_PUT_VARA_LONG, PIO_MSG_GET_VARM_UINT, PIO_MSG_GET_VARM, + PIO_MSG_PUT_VAR1_DOUBLE, + PIO_MSG_GET_VARS_DOUBLE, + PIO_MSG_GET_VARA_LONGLONG, + PIO_MSG_GET_VAR_ULONGLONG, PIO_MSG_PUT_VARM_SCHAR, + PIO_MSG_GET_VARA_ULONGLONG, + PIO_MSG_GET_VAR_SHORT, PIO_MSG_GET_VARM_FLOAT, + PIO_MSG_PUT_VAR_TEXT, + PIO_MSG_PUT_VARS_INT, + PIO_MSG_GET_VAR1_LONG, PIO_MSG_GET_VARM_LONG, PIO_MSG_GET_VARM_USHORT, + PIO_MSG_PUT_VAR1_SHORT, + PIO_MSG_PUT_VARS_LONGLONG, PIO_MSG_GET_VARM_LONGLONG, + PIO_MSG_GET_VARS_TEXT, + PIO_MSG_PUT_VARA_DOUBLE, PIO_MSG_PUT_VARS, + PIO_MSG_PUT_VAR_UCHAR, + PIO_MSG_GET_VAR1_UCHAR, + PIO_MSG_PUT_VAR_LONG, PIO_MSG_GET_VARS, PIO_MSG_GET_VARM_SHORT, PIO_MSG_GET_VARM_ULONGLONG, PIO_MSG_PUT_VARM_LONGLONG, - PIO_MSG_PUT_VAR1, - PIO_MSG_PUT_VARA, - PIO_MSG_PUT_VAR, - PIO_MSG_GET_VAR1, - PIO_MSG_GET_VARA, - PIO_MSG_GET_VAR, + PIO_MSG_GET_VAR_SCHAR, + PIO_MSG_GET_ATT_UBYTE, + PIO_MSG_PUT_ATT_STRING, + PIO_MSG_GET_ATT_STRING, + PIO_MSG_PUT_ATT_UBYTE, PIO_MSG_INQ_VAR_FILL, PIO_MSG_DEF_VAR_FILL, PIO_MSG_DEF_VAR_DEFLATE, From b7db2ac96f1fb2c64dca69c90185a2c50eb52d44 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 16 Jun 2016 05:45:34 -0600 Subject: [PATCH 164/184] getting non-async build to work --- src/clib/pio_internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 412b3d99055..9b6dc4a09a1 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -17,7 +17,7 @@ #include #endif -#ifdef PIO_ENABLE_LOGGING +#if PIO_ENABLE_LOGGING void pio_log(int severity, const char *fmt, ...); #define LOG(e) pio_log e #else From d827fa650edac921003585fe5c3611ba7b93b84b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 20 Jun 2016 14:31:26 -0600 Subject: [PATCH 165/184] documentation and spacing changes --- src/clib/pio.h | 809 ++++++++++++++++++++-------------------- src/clib/pio_internal.h | 615 +++++++++++++++--------------- src/clib/pio_nc4.c | 56 +-- src/clib/pioc.c | 45 ++- 4 files changed, 772 insertions(+), 753 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index d7710f9f4ae..3b0fb191922 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -1,11 +1,8 @@ /** * @file + * Public headers for the PIO C interface. * @author Jim Edwards * @date 2014 - * @brief Public headers for the PIO C interface. - * - * - * * * @see http://code.google.com/p/parallelio/ */ @@ -43,7 +40,6 @@ /** The maximum number of variables allowed in a netCDF file. */ #define PIO_MAX_VARS NC_MAX_VARS - /** * @brief Variable description structure * @@ -54,16 +50,16 @@ * nreqs is the number of outstanding pnetcdf requests for this variable * fillbuf is a memory buffer to hold fill values for this variable (write only) * iobuf is a memory buffer to hold (write only) -*/ + */ typedef struct var_desc_t { - int record; - int ndims; + int record; + int ndims; - int *request; // used for pnetcdf iput calls - int nreqs; - void *fillbuf; - void *iobuf; + int *request; // used for pnetcdf iput calls + int nreqs; + void *fillbuf; + void *iobuf; } var_desc_t; @@ -74,13 +70,13 @@ typedef struct var_desc_t * arrays. Each IO task may in general have multiple io regions per variable. The * box rearranger will have at most one io region per variable. * -*/ + */ typedef struct io_region { - int loffset; - PIO_Offset *start; - PIO_Offset *count; - struct io_region *next; + int loffset; + PIO_Offset *start; + PIO_Offset *count; + struct io_region *next; } io_region; /** @@ -89,45 +85,44 @@ typedef struct io_region * This structure defines the mapping for a given variable between * compute and IO decomposition. * -*/ + */ typedef struct io_desc_t { - int ioid; - int async_id; - int nrecvs; - int ndof; - int ndims; - int num_aiotasks; - int rearranger; - int maxregions; - bool needsfill; // Does this decomp leave holes in the field (true) or write everywhere (false) - int maxbytes; // maximum number of bytes of this iodesc before flushing - MPI_Datatype basetype; - PIO_Offset llen; - int maxiobuflen; - PIO_Offset *gsize; - - int *rfrom; - int *rcount; - int *scount; - PIO_Offset *sindex; - PIO_Offset *rindex; - - MPI_Datatype *rtype; - MPI_Datatype *stype; - int num_stypes; - int holegridsize; - int maxfillregions; - io_region *firstregion; - io_region *fillregion; - - - bool handshake; - bool isend; - int max_requests; + int ioid; + int async_id; + int nrecvs; + int ndof; + int ndims; + int num_aiotasks; + int rearranger; + int maxregions; + bool needsfill; // Does this decomp leave holes in the field (true) or write everywhere (false) + int maxbytes; // maximum number of bytes of this iodesc before flushing + MPI_Datatype basetype; + PIO_Offset llen; + int maxiobuflen; + PIO_Offset *gsize; + + int *rfrom; + int *rcount; + int *scount; + PIO_Offset *sindex; + PIO_Offset *rindex; + + MPI_Datatype *rtype; + MPI_Datatype *stype; + int num_stypes; + int holegridsize; + int maxfillregions; + io_region *firstregion; + io_region *fillregion; + + bool handshake; + bool isend; + int max_requests; - MPI_Comm subset_comm; - struct io_desc_t *next; + MPI_Comm subset_comm; + struct io_desc_t *next; } io_desc_t; /** @@ -136,164 +131,160 @@ typedef struct io_desc_t * This structure contains the general IO subsystem data * and MPI structure * -*/ + */ typedef struct iosystem_desc_t { - /** The ID of this iosystem_desc_t. */ - int iosysid; + /** The ID of this iosystem_desc_t. */ + int iosysid; - /** This is an MPI intra communicator that includes all the tasks in - * both the IO and the computation communicators. */ - MPI_Comm union_comm; + /** This is an MPI intra communicator that includes all the tasks in + * both the IO and the computation communicators. */ + MPI_Comm union_comm; - /** This is an MPI intra communicator that includes all the tasks - * involved in IO. */ - MPI_Comm io_comm; + /** This is an MPI intra communicator that includes all the tasks + * involved in IO. */ + MPI_Comm io_comm; - /** This is an MPI intra communicator that includes all the tasks - * involved in computation. */ - MPI_Comm comp_comm; + /** This is an MPI intra communicator that includes all the tasks + * involved in computation. */ + MPI_Comm comp_comm; - /** This is an MPI inter communicator between IO communicator and - * computation communicator. */ - MPI_Comm intercomm; + /** This is an MPI inter communicator between IO communicator and + * computation communicator. */ + MPI_Comm intercomm; - /** This is a copy (but not an MPI copy) of either the comp (for - * non-async) or the union (for async) communicator. */ - MPI_Comm my_comm; + /** This is a copy (but not an MPI copy) of either the comp (for + * non-async) or the union (for async) communicator. */ + MPI_Comm my_comm; - /** This MPI group contains the processors involved in - * computation. */ - MPI_Group compgroup; + /** This MPI group contains the processors involved in + * computation. */ + MPI_Group compgroup; - /** This MPI group contains the processors involved in I/O. */ - MPI_Group iogroup; + /** This MPI group contains the processors involved in I/O. */ + MPI_Group iogroup; - /** The number of tasks in the IO communicator. */ - int num_iotasks; + /** The number of tasks in the IO communicator. */ + int num_iotasks; - /** The number of tasks in the computation communicator. */ - int num_comptasks; + /** The number of tasks in the computation communicator. */ + int num_comptasks; - /** Rank of this task in the union communicator. */ - int union_rank; + /** Rank of this task in the union communicator. */ + int union_rank; - /** The rank of this process in the computation communicator, or -1 - * if this process is not part of the computation communicator. */ - int comp_rank; + /** The rank of this process in the computation communicator, or -1 + * if this process is not part of the computation communicator. */ + int comp_rank; - /** The rank of this process in the IO communicator, or -1 if this - * process is not part of the IO communicator. */ - int io_rank; + /** The rank of this process in the IO communicator, or -1 if this + * process is not part of the IO communicator. */ + int io_rank; - /** Set to MPI_ROOT if this task is the master of IO communicator, 0 - * otherwise. */ - int iomaster; + /** Set to MPI_ROOT if this task is the master of IO communicator, 0 + * otherwise. */ + int iomaster; - /** Set to MPI_ROOT if this task is the master of comp communicator, 0 - * otherwise. */ - int compmaster; + /** Set to MPI_ROOT if this task is the master of comp communicator, 0 + * otherwise. */ + int compmaster; - /** Rank of IO root task (which is rank 0 in io_comm) in the union - * communicator. */ - int ioroot; + /** Rank of IO root task (which is rank 0 in io_comm) in the union + * communicator. */ + int ioroot; - /** Rank of computation root task (which is rank 0 in - * comm_comms[cmp]) in the union communicator. */ - int comproot; + /** Rank of computation root task (which is rank 0 in + * comm_comms[cmp]) in the union communicator. */ + int comproot; - /** An array of the ranks of all IO tasks within the union - * communicator. */ - int *ioranks; + /** An array of the ranks of all IO tasks within the union + * communicator. */ + int *ioranks; - /** Controls handling errors. */ - int error_handler; + /** Controls handling errors. */ + int error_handler; - /** ??? */ - int default_rearranger; + /** ??? */ + int default_rearranger; - /** True if asynchronous interface is in use. */ - bool async_interface; + /** True if asynchronous interface is in use. */ + bool async_interface; - /** True if this task is a member of the IO communicator. */ - bool ioproc; + /** True if this task is a member of the IO communicator. */ + bool ioproc; - /** MPI Info object. */ - MPI_Info info; + /** MPI Info object. */ + MPI_Info info; - /** Pointer to the next iosystem_desc_t in the list. */ - struct iosystem_desc_t *next; + /** Pointer to the next iosystem_desc_t in the list. */ + struct iosystem_desc_t *next; } iosystem_desc_t; /** * @brief multi buffer * -*/ + */ typedef struct wmulti_buffer { - int ioid; - int validvars; - int arraylen; - int *vid; - int *frame; - void *fillvalue; - void *data; - struct wmulti_buffer *next; + int ioid; + int validvars; + int arraylen; + int *vid; + int *frame; + void *fillvalue; + void *data; + struct wmulti_buffer *next; } wmulti_buffer; - - /** * @brief io system descriptor structure * * This structure holds information associated with each open file * -*/ + */ typedef struct file_desc_t { - iosystem_desc_t *iosystem; - PIO_Offset buffsize; - int fh; - int iotype; - struct var_desc_t varlist[PIO_MAX_VARS]; - int mode; - struct wmulti_buffer buffer; - struct file_desc_t *next; - - /** True if this task should participate in IO (only true for one - * task with netcdf serial files. */ - int do_io; + iosystem_desc_t *iosystem; + PIO_Offset buffsize; + int fh; + int iotype; + struct var_desc_t varlist[PIO_MAX_VARS]; + int mode; + struct wmulti_buffer buffer; + struct file_desc_t *next; + + /** True if this task should participate in IO (only true for one + * task with netcdf serial files. */ + int do_io; } file_desc_t; /** * @brief These are the supported output formats - * -*/ - + */ enum PIO_IOTYPE{ - PIO_IOTYPE_PNETCDF=1, //< Parallel Netcdf (parallel) - PIO_IOTYPE_NETCDF=2, //< Netcdf3 Classic format (serial) - PIO_IOTYPE_NETCDF4C=3, //< NetCDF4 (HDF5) compressed format (serial) - PIO_IOTYPE_NETCDF4P=4 //< NetCDF4 (HDF5) parallel + PIO_IOTYPE_PNETCDF=1, //< Parallel Netcdf (parallel) + PIO_IOTYPE_NETCDF=2, //< Netcdf3 Classic format (serial) + PIO_IOTYPE_NETCDF4C=3, //< NetCDF4 (HDF5) compressed format (serial) + PIO_IOTYPE_NETCDF4P=4 //< NetCDF4 (HDF5) parallel }; /** * @brief These are the supported output data rearrangement methods * -*/ + */ enum PIO_REARRANGERS{ - PIO_REARR_BOX = 1, - PIO_REARR_SUBSET = 2 + PIO_REARR_BOX = 1, + PIO_REARR_SUBSET = 2 }; /** * @brief These are the supported error handlers * -*/ + */ enum PIO_ERROR_HANDLERS{ - PIO_INTERNAL_ERROR=(-51), //< Errors cause abort - PIO_BCAST_ERROR=(-52), //< Error codes are broadcast to all tasks - PIO_RETURN_ERROR=(-53) //< Errors are returned to caller with no internal action + PIO_INTERNAL_ERROR=(-51), //< Errors cause abort + PIO_BCAST_ERROR=(-52), //< Error codes are broadcast to all tasks + PIO_RETURN_ERROR=(-53) //< Errors are returned to caller with no internal action }; #if defined( _PNETCDF) || defined(_NETCDF) @@ -400,263 +391,267 @@ enum PIO_ERROR_HANDLERS{ #else #define PIO_EINDEP (-203) #endif + +#define PIO_EBADIOTYPE -255 +#define PIO_REQ_NULL (NC_REQ_NULL-1) + #if defined(__cplusplus) extern "C" { #endif -#define PIO_EBADIOTYPE -255 -#define PIO_REQ_NULL (NC_REQ_NULL-1) -int PIOc_freedecomp(int iosysid, int ioid); -int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp); -int PIOc_inq_format (int ncid, int *formatp); -int PIOc_inq_varid (int ncid, const char *name, int *varidp); -int PIOc_inq_varnatts (int ncid, int varid, int *nattsp); -int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp); -int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, - int deflate_level); -int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep, - int *deflate_levelp); -int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp); -int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp); -int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp); -int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value); -int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep); -int PIOc_def_var_endian(int ncid, int varid, int endian); -int PIOc_inq_var_endian(int ncid, int varid, int *endianp); -int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems, float preemption); -int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nelemsp, float *preemptionp); -int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems, - float preemption); -int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp, - float *preemptionp); -int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp, int *nattsp); -int PIOc_inq_varname (int ncid, int varid, char *name); -int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op); -int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op); -int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname); -int PIOc_del_att (int ncid, int varid, const char *name); -int PIOc_inq_natts (int ncid, int *ngattsp); -int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp); -int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip); -int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip); -int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op); -int PIOc_redef (int ncid); -int PIOc_set_fill (int ncid, int fillmode, int *old_modep); -int PIOc_enddef (int ncid); -int PIOc_rename_var (int ncid, int varid, const char *name); -int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op); -int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, const char *op); -int PIOc_inq_attname (int ncid, int varid, int attnum, char *name); -int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip); -int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip); -int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op); -int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp); -int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip); -int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip); -int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op); -int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const float *op); -int PIOc_inq_nvars (int ncid, int *nvarsp); -int PIOc_rename_dim (int ncid, int dimid, const char *name); -int PIOc_inq_varndims (int ncid, int varid, int *ndimsp); -int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip); -int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp); -int PIOc_inq_dimid (int ncid, const char *name, int *idp); -int PIOc_inq_unlimdim (int ncid, int *unlimdimidp); -int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp); -int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp); -int PIOc_inq_dimname (int ncid, int dimid, char *name); -int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op); -int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip); -int PIOc_sync (int ncid); -int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long long *op); -int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned int *op); -int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip); -int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp); -int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp); -int PIOc_inq_ndims (int ncid, int *ndimsp); -int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep); -int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip); -int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip); -int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep); -int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op); -int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip); -int PIOc_InitDecomp(const int iosysid, const int basetype,const int ndims, const int dims[], - const int maplen, const PIO_Offset *compmap, int *ioidp, const int *rearr, - const PIO_Offset *iostart,const PIO_Offset *iocount); -int PIOc_Init_Intracomm(const MPI_Comm comp_comm, - const int num_iotasks, const int stride, - const int base, const int rearr, int *iosysidp); -int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, MPI_Comm *comp_comms, - MPI_Comm io_comm, int *iosysidp); -int PIOc_closefile(int ncid); -int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, - const char *fname, const int mode); -int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, - const char *fname, const int mode); -int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue); - int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, const int nvars, const PIO_Offset arraylen, void *array, const int frame[], void *fillvalue[], bool flushtodisk); - -int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip); -int PIOc_put_att_ubyte (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) ; -int PIOc_set_blocksize(const int newblocksize); - int PIOc_readmap(const char file[], int *ndims, int *gdims[], PIO_Offset *fmaplen, PIO_Offset *map[], const MPI_Comm comm); - int PIOc_readmap_from_f90(const char file[],int *ndims, int *gdims[], PIO_Offset *maplen, PIO_Offset *map[], const int f90_comm); - int PIOc_writemap(const char file[], const int ndims, const int gdims[], PIO_Offset maplen, PIO_Offset map[], const MPI_Comm comm); - int PIOc_writemap_from_f90(const char file[], const int ndims, const int gdims[], const PIO_Offset maplen, const PIO_Offset map[], const int f90_comm); - int PIOc_deletefile(const int iosysid, const char filename[]); - int PIOc_File_is_Open(int ncid); - int PIOc_Set_File_Error_Handling(int ncid, int method); - int PIOc_advanceframe(int ncid, int varid); - int PIOc_setframe(const int ncid, const int varid,const int frame); - int PIOc_get_numiotasks(int iosysid, int *numiotasks); - int PIOc_get_iorank(int iosysid, int *iorank); - int PIOc_get_local_array_size(int ioid); - int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method); - int PIOc_set_hint(const int iosysid, char hint[], const char hintval[]); - int PIOc_Init_Intracomm(const MPI_Comm comp_comm, - const int num_iotasks, const int stride, - const int base,const int rearr, int *iosysidp); - int PIOc_finalize(const int iosysid); - int PIOc_iam_iotask(const int iosysid, bool *ioproc); - int PIOc_iotask_rank(const int iosysid, int *iorank); - int PIOc_iosystem_is_active(const int iosysid, bool *active); - int PIOc_put_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned char *op) ; - int PIOc_get_var1_schar (int ncid, int varid, const PIO_Offset index[], signed char *buf) ; - int PIOc_put_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned short *op) ; - int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF); - int PIOc_put_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned long long *op) ; - int PIOc_get_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned long long *buf) ; - int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) ; - int PIOc_read_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array); - int PIOc_put_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned int *op) ; - int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], signed char *buf) ; - int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned char *op) ; - int PIOc_put_var_ushort (int ncid, int varid, const unsigned short *op) ; - int PIOc_get_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], short *buf) ; - int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) ; - int PIOc_get_var_double (int ncid, int varid, double *buf) ; - int PIOc_put_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned char *op) ; - int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const short *op) ; - int PIOc_get_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], double *buf) ; - int PIOc_put_var1_long (int ncid, int varid, const PIO_Offset index[], const long *ip) ; - int PIOc_get_var_int (int ncid, int varid, int *buf) ; - int PIOc_put_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long *op) ; - int PIOc_put_var_short (int ncid, int varid, const short *op) ; - int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], char *buf) ; - int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op) ; - - int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op); - int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op); - int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const char *op); - int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned short *op); - int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op); - int PIOc_put_var_int (int ncid, int varid, const int *op); - int PIOc_put_var_longlong (int ncid, int varid, const long long *op); - int PIOc_put_var_schar (int ncid, int varid, const signed char *op); - int PIOc_put_var_uint (int ncid, int varid, const unsigned int *op); - int PIOc_put_var (int ncid, int varid, const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_put_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned short *op); - int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const short *op); - int PIOc_put_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned int *op); - int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const signed char *op); - int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned long long *op); - int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], const unsigned char *op); - int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const int *op); - int PIOc_put_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const signed char *op); - int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_put_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const float *op); - int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], const float *op); - int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const float *op); - int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op); - int PIOc_put_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const char *op); - int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long *op); - int PIOc_put_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const double *op); - int PIOc_put_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long long *op); - int PIOc_put_var_double (int ncid, int varid, const double *op); - int PIOc_put_var_float (int ncid, int varid, const float *op); - int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op); - int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned int *op); - int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const unsigned int *op); - int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int *op); - int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const float *op); - int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const short *op); - int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const signed char *op); - int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned long long *op); - int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const double *op); - int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long *op); - int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const double *op); - int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const signed char *op); - int PIOc_put_var_text (int ncid, int varid, const char *op); - int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const int *op); - int PIOc_put_var1_short (int ncid, int varid, const PIO_Offset index[], const short *op); - int PIOc_put_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long long *op); - int PIOc_put_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const double *op); - int PIOc_put_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], nc_type xtype, const void *buf); - int PIOc_put_var_uchar (int ncid, int varid, const unsigned char *op); - int PIOc_put_var_long (int ncid, int varid, const long *op); - int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op); - int PIOc_get_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], int *buf); - int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset index[], float *buf); - int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset index[], short *buf); - int PIOc_get_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], int *buf); - int PIOc_get_var_text (int ncid, int varid, char *buf); - int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], double *buf); - int PIOc_get_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], signed char *buf); - int PIOc_get_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned short *buf); - int PIOc_get_var1_ushort (int ncid, int varid, const PIO_Offset index[], unsigned short *buf); - int PIOc_get_var_float (int ncid, int varid, float *buf); - int PIOc_get_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned char *buf); - int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset index[], long long *buf); - int PIOc_get_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned short *buf); - int PIOc_get_var_long (int ncid, int varid, long *buf); - int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset index[], double *buf); - int PIOc_get_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned int *buf); - int PIOc_get_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long long *buf); - int PIOc_get_var_longlong (int ncid, int varid, long long *buf); - int PIOc_get_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], short *buf); - int PIOc_get_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long *buf); - int PIOc_get_var1_int (int ncid, int varid, const PIO_Offset index[], int *buf); - int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], unsigned long long *buf); - int PIOc_get_var_uchar (int ncid, int varid, unsigned char *buf); - int PIOc_get_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned char *buf); - int PIOc_get_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], float *buf); - int PIOc_get_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long *buf); - int PIOc_get_var1 (int ncid, int varid, const PIO_Offset index[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf); - int PIOc_get_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], signed char *buf); - int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset index[], unsigned int *buf); - int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned int *buf); - int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], float *buf); - int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], char *buf); - int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset index[], char *buf); - int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], int *buf); - int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned int *buf); - int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], double *buf); - int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long long *buf); - int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf); - int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned long long *buf); - int PIOc_get_var_short (int ncid, int varid, short *buf); - int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], float *buf); - int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset index[], long *buf); - int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long *buf); - int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned short *buf); - int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long long *buf); - int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], char *buf); - int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset index[], unsigned char *buf); - int PIOc_get_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); - int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], nc_type xtype, void *buf); - int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf); - int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf); - int PIOc_get_var_schar (int ncid, int varid, signed char *buf); - int PIOc_iotype_available(const int iotype); - int PIOc_set_log_level(int level); - int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const void *op); - int PIOc_get_att(int ncid, int varid, const char *name, void *ip); - int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep); + int PIOc_freedecomp(int iosysid, int ioid); + int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp); + int PIOc_inq_format (int ncid, int *formatp); + int PIOc_inq_varid (int ncid, const char *name, int *varidp); + int PIOc_inq_varnatts (int ncid, int varid, int *nattsp); + int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp); + int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, + int deflate_level); + int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep, + int *deflate_levelp); + int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp); + int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp); + int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp); + int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value); + int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep); + int PIOc_def_var_endian(int ncid, int varid, int endian); + int PIOc_inq_var_endian(int ncid, int varid, int *endianp); + int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems, float preemption); + int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nelemsp, float *preemptionp); + int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems, + float preemption); + int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp, + float *preemptionp); + int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp, int *nattsp); + int PIOc_inq_varname (int ncid, int varid, char *name); + int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const double *op); + int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op); + int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname); + int PIOc_del_att (int ncid, int varid, const char *name); + int PIOc_inq_natts (int ncid, int *ngattsp); + int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp); + int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip); + int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip); + int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op); + int PIOc_redef (int ncid); + int PIOc_set_fill (int ncid, int fillmode, int *old_modep); + int PIOc_enddef (int ncid); + int PIOc_rename_var (int ncid, int varid, const char *name); + int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op); + int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, const char *op); + int PIOc_inq_attname (int ncid, int varid, int attnum, char *name); + int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip); + int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip); + int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op); + int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp); + int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip); + int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip); + int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op); + int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const float *op); + int PIOc_inq_nvars (int ncid, int *nvarsp); + int PIOc_rename_dim (int ncid, int dimid, const char *name); + int PIOc_inq_varndims (int ncid, int varid, int *ndimsp); + int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip); + int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp); + int PIOc_inq_dimid (int ncid, const char *name, int *idp); + int PIOc_inq_unlimdim (int ncid, int *unlimdimidp); + int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp); + int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp); + int PIOc_inq_dimname (int ncid, int dimid, char *name); + int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op); + int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip); + int PIOc_sync (int ncid); + int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long long *op); + int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned int *op); + int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip); + int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp); + int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp); + int PIOc_inq_ndims (int ncid, int *ndimsp); + int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep); + int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip); + int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip); + int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep); + int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op); + int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip); + int PIOc_InitDecomp(const int iosysid, const int basetype,const int ndims, const int dims[], + const int maplen, const PIO_Offset *compmap, int *ioidp, const int *rearr, + const PIO_Offset *iostart,const PIO_Offset *iocount); + int PIOc_Init_Intracomm(const MPI_Comm comp_comm, + const int num_iotasks, const int stride, + const int base, const int rearr, int *iosysidp); + int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, MPI_Comm *comp_comms, + MPI_Comm io_comm, int *iosysidp); + int PIOc_closefile(int ncid); + int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, + const char *fname, const int mode); + int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, + const char *fname, const int mode); + int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, + void *array, void *fillvalue); + int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, const int nvars, const PIO_Offset arraylen, + void *array, const int frame[], void *fillvalue[], bool flushtodisk); + + int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip); + int PIOc_put_att_ubyte (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) ; + int PIOc_set_blocksize(const int newblocksize); + int PIOc_readmap(const char file[], int *ndims, int *gdims[], PIO_Offset *fmaplen, PIO_Offset *map[], const MPI_Comm comm); + int PIOc_readmap_from_f90(const char file[],int *ndims, int *gdims[], PIO_Offset *maplen, PIO_Offset *map[], const int f90_comm); + int PIOc_writemap(const char file[], const int ndims, const int gdims[], PIO_Offset maplen, PIO_Offset map[], const MPI_Comm comm); + int PIOc_writemap_from_f90(const char file[], const int ndims, const int gdims[], const PIO_Offset maplen, const PIO_Offset map[], const int f90_comm); + int PIOc_deletefile(const int iosysid, const char filename[]); + int PIOc_File_is_Open(int ncid); + int PIOc_Set_File_Error_Handling(int ncid, int method); + int PIOc_advanceframe(int ncid, int varid); + int PIOc_setframe(const int ncid, const int varid,const int frame); + int PIOc_get_numiotasks(int iosysid, int *numiotasks); + int PIOc_get_iorank(int iosysid, int *iorank); + int PIOc_get_local_array_size(int ioid); + int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method); + int PIOc_set_hint(const int iosysid, char hint[], const char hintval[]); + int PIOc_Init_Intracomm(const MPI_Comm comp_comm, + const int num_iotasks, const int stride, + const int base,const int rearr, int *iosysidp); + int PIOc_finalize(const int iosysid); + int PIOc_iam_iotask(const int iosysid, bool *ioproc); + int PIOc_iotask_rank(const int iosysid, int *iorank); + int PIOc_iosystem_is_active(const int iosysid, bool *active); + int PIOc_put_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned char *op) ; + int PIOc_get_var1_schar (int ncid, int varid, const PIO_Offset index[], signed char *buf) ; + int PIOc_put_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned short *op) ; + int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF); + int PIOc_put_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned long long *op) ; + int PIOc_get_vars_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned long long *buf) ; + int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype) ; + int PIOc_read_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array); + int PIOc_put_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const unsigned int *op) ; + int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], signed char *buf) ; + int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned char *op) ; + int PIOc_put_var_ushort (int ncid, int varid, const unsigned short *op) ; + int PIOc_get_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], short *buf) ; + int PIOc_put_var1_longlong (int ncid, int varid, const PIO_Offset index[], const long long *op) ; + int PIOc_get_var_double (int ncid, int varid, double *buf) ; + int PIOc_put_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned char *op) ; + int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const short *op) ; + int PIOc_get_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], double *buf) ; + int PIOc_put_var1_long (int ncid, int varid, const PIO_Offset index[], const long *ip) ; + int PIOc_get_var_int (int ncid, int varid, int *buf) ; + int PIOc_put_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long *op) ; + int PIOc_put_var_short (int ncid, int varid, const short *op) ; + int PIOc_get_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], char *buf) ; + int PIOc_put_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const int *op) ; + + int PIOc_put_var1_ushort (int ncid, int varid, const PIO_Offset index[], const unsigned short *op); + int PIOc_put_vara_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const char *op); + int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const char *op); + int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned short *op); + int PIOc_put_var_ulonglong (int ncid, int varid, const unsigned long long *op); + int PIOc_put_var_int (int ncid, int varid, const int *op); + int PIOc_put_var_longlong (int ncid, int varid, const long long *op); + int PIOc_put_var_schar (int ncid, int varid, const signed char *op); + int PIOc_put_var_uint (int ncid, int varid, const unsigned int *op); + int PIOc_put_var (int ncid, int varid, const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); + int PIOc_put_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned short *op); + int PIOc_put_vars_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const short *op); + int PIOc_put_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned int *op); + int PIOc_put_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const signed char *op); + int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned long long *op); + int PIOc_put_var1_uchar (int ncid, int varid, const PIO_Offset index[], const unsigned char *op); + int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const int *op); + int PIOc_put_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const signed char *op); + int PIOc_put_var1 (int ncid, int varid, const PIO_Offset index[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); + int PIOc_put_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const float *op); + int PIOc_put_var1_float (int ncid, int varid, const PIO_Offset index[], const float *op); + int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const float *op); + int PIOc_put_var1_text (int ncid, int varid, const PIO_Offset index[], const char *op); + int PIOc_put_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const char *op); + int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long *op); + int PIOc_put_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const double *op); + int PIOc_put_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long long *op); + int PIOc_put_var_double (int ncid, int varid, const double *op); + int PIOc_put_var_float (int ncid, int varid, const float *op); + int PIOc_put_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], const unsigned long long *op); + int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned int *op); + int PIOc_put_var1_uint (int ncid, int varid, const PIO_Offset index[], const unsigned int *op); + int PIOc_put_var1_int (int ncid, int varid, const PIO_Offset index[], const int *op); + int PIOc_put_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const float *op); + int PIOc_put_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const short *op); + int PIOc_put_var1_schar (int ncid, int varid, const PIO_Offset index[], const signed char *op); + int PIOc_put_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const unsigned long long *op); + int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const double *op); + int PIOc_put_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); + int PIOc_put_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const long *op); + int PIOc_put_var1_double (int ncid, int varid, const PIO_Offset index[], const double *op); + int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const signed char *op); + int PIOc_put_var_text (int ncid, int varid, const char *op); + int PIOc_put_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const int *op); + int PIOc_put_var1_short (int ncid, int varid, const PIO_Offset index[], const short *op); + int PIOc_put_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const long long *op); + int PIOc_put_vara_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const double *op); + int PIOc_put_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype); + int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], nc_type xtype, const void *buf); + int PIOc_put_var_uchar (int ncid, int varid, const unsigned char *op); + int PIOc_put_var_long (int ncid, int varid, const long *op); + int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op); + int PIOc_get_vara_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], int *buf); + int PIOc_get_var1_float (int ncid, int varid, const PIO_Offset index[], float *buf); + int PIOc_get_var1_short (int ncid, int varid, const PIO_Offset index[], short *buf); + int PIOc_get_vars_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], int *buf); + int PIOc_get_var_text (int ncid, int varid, char *buf); + int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], double *buf); + int PIOc_get_vars_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], signed char *buf); + int PIOc_get_vara_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned short *buf); + int PIOc_get_var1_ushort (int ncid, int varid, const PIO_Offset index[], unsigned short *buf); + int PIOc_get_var_float (int ncid, int varid, float *buf); + int PIOc_get_vars_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned char *buf); + int PIOc_get_var (int ncid, int varid, void *buf, PIO_Offset bufcount, MPI_Datatype buftype); + int PIOc_get_var1_longlong (int ncid, int varid, const PIO_Offset index[], long long *buf); + int PIOc_get_vars_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned short *buf); + int PIOc_get_var_long (int ncid, int varid, long *buf); + int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset index[], double *buf); + int PIOc_get_vara_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned int *buf); + int PIOc_get_vars_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long long *buf); + int PIOc_get_var_longlong (int ncid, int varid, long long *buf); + int PIOc_get_vara_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], short *buf); + int PIOc_get_vara_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long *buf); + int PIOc_get_var1_int (int ncid, int varid, const PIO_Offset index[], int *buf); + int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset index[], unsigned long long *buf); + int PIOc_get_var_uchar (int ncid, int varid, unsigned char *buf); + int PIOc_get_vara_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned char *buf); + int PIOc_get_vars_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], float *buf); + int PIOc_get_vars_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], long *buf); + int PIOc_get_var1 (int ncid, int varid, const PIO_Offset index[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); + int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf); + int PIOc_get_vara (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); + int PIOc_get_vara_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], signed char *buf); + int PIOc_get_var1_uint (int ncid, int varid, const PIO_Offset index[], unsigned int *buf); + int PIOc_get_vars_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], unsigned int *buf); + int PIOc_get_vara_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], float *buf); + int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], char *buf); + int PIOc_get_var1_text (int ncid, int varid, const PIO_Offset index[], char *buf); + int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], int *buf); + int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned int *buf); + int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); + int PIOc_get_vars_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], double *buf); + int PIOc_get_vara_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], long long *buf); + int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf); + int PIOc_get_vara_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], unsigned long long *buf); + int PIOc_get_var_short (int ncid, int varid, short *buf); + int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], float *buf); + int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset index[], long *buf); + int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long *buf); + int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned short *buf); + int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long long *buf); + int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], char *buf); + int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset index[], unsigned char *buf); + int PIOc_get_vars (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype); + int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], nc_type xtype, void *buf); + int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf); + int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf); + int PIOc_get_var_schar (int ncid, int varid, signed char *buf); + int PIOc_iotype_available(const int iotype); + int PIOc_set_log_level(int level); + int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const void *op); + int PIOc_get_att(int ncid, int varid, const char *name, void *ip); + int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep); #if defined(__cplusplus) } #endif diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 9b6dc4a09a1..c2e22a57a9c 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -1,6 +1,17 @@ +/** + * @file + * Private headers and defines for the PIO C interface. + * @author Jim Edwards + * @date 2014 + * + * @see http://code.google.com/p/parallelio/ + */ + #ifndef __PIO_INTERNAL__ #define __PIO_INTERNAL__ + #include + // It seems that some versions of openmpi fail to define MPI_OFFSET #ifdef OMPI_OFFSET_DATATYPE #ifndef MPI_OFFSET @@ -24,15 +35,15 @@ void pio_log(int severity, const char *fmt, ...); #define LOG(e) #endif /* PIO_ENABLE_LOGGING */ -#define max(a,b) \ - ({ __typeof__ (a) _a = (a); \ - __typeof__ (b) _b = (b); \ - _a > _b ? _a : _b; }) +#define max(a,b) \ + ({ __typeof__ (a) _a = (a); \ + __typeof__ (b) _b = (b); \ + _a > _b ? _a : _b; }) -#define min(a,b) \ - ({ __typeof__ (a) _a = (a); \ - __typeof__ (b) _b = (b); \ - _a < _b ? _a : _b; }) +#define min(a,b) \ + ({ __typeof__ (a) _a = (a); \ + __typeof__ (b) _b = (b); \ + _a < _b ? _a : _b; }) #define MAX_GATHER_BLOCK_SIZE 0 @@ -42,123 +53,123 @@ void pio_log(int severity, const char *fmt, ...); extern "C" { #endif -extern PIO_Offset PIO_BUFFER_SIZE_LIMIT; -extern bool PIO_Save_Decomps; + extern PIO_Offset PIO_BUFFER_SIZE_LIMIT; + extern bool PIO_Save_Decomps; /** ** @brief Used to sort map points in the subset rearranger -*/ -typedef struct mapsort -{ - int rfrom; - PIO_Offset soffset; - PIO_Offset iomap; -} mapsort; + */ + typedef struct mapsort + { + int rfrom; + PIO_Offset soffset; + PIO_Offset iomap; + } mapsort; /** * @brief swapm defaults. * -*/ -typedef struct pio_swapm_defaults -{ - int nreqs; - bool handshake; - bool isend; -} pio_swapm_defaults; + */ + typedef struct pio_swapm_defaults + { + int nreqs; + bool handshake; + bool isend; + } pio_swapm_defaults; - void pio_get_env(void); - int pio_add_to_iodesc_list(io_desc_t *iodesc); - io_desc_t *pio_get_iodesc_from_id(int ioid); - int pio_delete_iodesc_from_list(int ioid); + void pio_get_env(void); + int pio_add_to_iodesc_list(io_desc_t *iodesc); + io_desc_t *pio_get_iodesc_from_id(int ioid); + int pio_delete_iodesc_from_list(int ioid); - file_desc_t *pio_get_file_from_id(int ncid); - int pio_delete_file_from_list(int ncid); - void pio_add_to_file_list(file_desc_t *file); - void pio_push_request(file_desc_t *file, int request); + file_desc_t *pio_get_file_from_id(int ncid); + int pio_delete_file_from_list(int ncid); + void pio_add_to_file_list(file_desc_t *file); + void pio_push_request(file_desc_t *file, int request); - iosystem_desc_t *pio_get_iosystem_from_id(int iosysid); - int pio_add_to_iosystem_list(iosystem_desc_t *ios); + iosystem_desc_t *pio_get_iosystem_from_id(int iosysid); + int pio_add_to_iosystem_list(iosystem_desc_t *ios); - int check_netcdf(file_desc_t *file,const int status, const char *fname, const int line); - int iotype_error(const int iotype, const char *fname, const int line); - void piodie(const char *msg,const char *fname, const int line); - void pioassert(bool exp, const char *msg,const char *fname, const int line); - int CalcStartandCount(const int basetype, const int ndims, const int *gdims, const int num_io_procs, - const int myiorank, PIO_Offset *start, PIO_Offset *kount); - void CheckMPIReturn(const int ierr,const char file[],const int line); - int pio_fc_gather( void *sendbuf, const int sendcnt, const MPI_Datatype sendtype, - void *recvbuf, const int recvcnt, const MPI_Datatype recvtype, const int root, - MPI_Comm comm, const int flow_cntl); - int pio_fc_gatherv( void *sendbuf, const int sendcnt, const MPI_Datatype sendtype, - void *recvbuf, const int recvcnts[], const int recvdispl[], const MPI_Datatype recvtype, const int root, - MPI_Comm comm, const int flow_cntl); + int check_netcdf(file_desc_t *file,const int status, const char *fname, const int line); + int iotype_error(const int iotype, const char *fname, const int line); + void piodie(const char *msg,const char *fname, const int line); + void pioassert(bool exp, const char *msg,const char *fname, const int line); + int CalcStartandCount(const int basetype, const int ndims, const int *gdims, const int num_io_procs, + const int myiorank, PIO_Offset *start, PIO_Offset *kount); + void CheckMPIReturn(const int ierr,const char file[],const int line); + int pio_fc_gather( void *sendbuf, const int sendcnt, const MPI_Datatype sendtype, + void *recvbuf, const int recvcnt, const MPI_Datatype recvtype, const int root, + MPI_Comm comm, const int flow_cntl); + int pio_fc_gatherv( void *sendbuf, const int sendcnt, const MPI_Datatype sendtype, + void *recvbuf, const int recvcnts[], const int recvdispl[], const MPI_Datatype recvtype, const int root, + MPI_Comm comm, const int flow_cntl); - int pio_fc_gatherv( void *sendbuf, const int sendcnt, const MPI_Datatype sendtype, - void *recvbuf, const int recvcnts[], const int rdispls[], const MPI_Datatype recvtype, const int root, - MPI_Comm comm, const int flow_cntl); + int pio_fc_gatherv( void *sendbuf, const int sendcnt, const MPI_Datatype sendtype, + void *recvbuf, const int recvcnts[], const int rdispls[], const MPI_Datatype recvtype, const int root, + MPI_Comm comm, const int flow_cntl); - int pio_swapm(void *sndbuf, int sndlths[], int sdispls[], MPI_Datatype stypes[], - void *rcvbuf, int rcvlths[], int rdispls[], MPI_Datatype rtypes[], - MPI_Comm comm, const bool handshake, bool isend, const int max_requests); + int pio_swapm(void *sndbuf, int sndlths[], int sdispls[], MPI_Datatype stypes[], + void *rcvbuf, int rcvlths[], int rdispls[], MPI_Datatype rtypes[], + MPI_Comm comm, const bool handshake, bool isend, const int max_requests); - long long lgcd_array(int nain, long long*ain); + long long lgcd_array(int nain, long long*ain); - void PIO_Offset_size(MPI_Datatype *dtype, int *tsize); - PIO_Offset GCDblocksize(const int arrlen, const PIO_Offset arr_in[]); + void PIO_Offset_size(MPI_Datatype *dtype, int *tsize); + PIO_Offset GCDblocksize(const int arrlen, const PIO_Offset arr_in[]); - int subset_rearrange_create(const iosystem_desc_t ios,const int maplen, PIO_Offset compmap[], const int gsize[], - const int ndim, io_desc_t *iodesc); + int subset_rearrange_create(const iosystem_desc_t ios,const int maplen, PIO_Offset compmap[], const int gsize[], + const int ndim, io_desc_t *iodesc); - int box_rearrange_create(const iosystem_desc_t ios,const int maplen, const PIO_Offset compmap[], const int gsize[], - const int ndim, io_desc_t *iodesc); + int box_rearrange_create(const iosystem_desc_t ios,const int maplen, const PIO_Offset compmap[], const int gsize[], + const int ndim, io_desc_t *iodesc); - int rearrange_io2comp(const iosystem_desc_t ios, io_desc_t *iodesc, void *sbuf, - void *rbuf); - int rearrange_comp2io(const iosystem_desc_t ios, io_desc_t *iodesc, void *sbuf, - void *rbuf, const int nvars); - int calcdisplace(const int bsize, const int numblocks,const PIO_Offset map[],int displace[]); - io_desc_t *malloc_iodesc(const int piotype, const int ndims); - void performance_tune_rearranger(iosystem_desc_t ios, io_desc_t *iodesc); + int rearrange_io2comp(const iosystem_desc_t ios, io_desc_t *iodesc, void *sbuf, + void *rbuf); + int rearrange_comp2io(const iosystem_desc_t ios, io_desc_t *iodesc, void *sbuf, + void *rbuf, const int nvars); + int calcdisplace(const int bsize, const int numblocks,const PIO_Offset map[],int displace[]); + io_desc_t *malloc_iodesc(const int piotype, const int ndims); + void performance_tune_rearranger(iosystem_desc_t ios, io_desc_t *iodesc); - int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize); - void compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc); - io_region *alloc_region(const int ndims); - int pio_delete_iosystem_from_list(int piosysid); - int gcd(int a, int b); - long long lgcd (long long a,long long b ); - int gcd_array(int nain, int *ain); - void free_region_list(io_region *top); - void gindex_to_coord(const int ndims, const PIO_Offset gindex, const PIO_Offset gstride[], PIO_Offset *gcoord); - PIO_Offset coord_to_lindex(const int ndims, const PIO_Offset lcoord[], const PIO_Offset count[]); + int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize); + void compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc); + io_region *alloc_region(const int ndims); + int pio_delete_iosystem_from_list(int piosysid); + int gcd(int a, int b); + long long lgcd (long long a,long long b ); + int gcd_array(int nain, int *ain); + void free_region_list(io_region *top); + void gindex_to_coord(const int ndims, const PIO_Offset gindex, const PIO_Offset gstride[], PIO_Offset *gcoord); + PIO_Offset coord_to_lindex(const int ndims, const PIO_Offset lcoord[], const PIO_Offset count[]); - int ceil2(const int i); - int pair(const int np, const int p, const int k); - int define_iodesc_datatypes(const iosystem_desc_t ios, io_desc_t *iodesc); + int ceil2(const int i); + int pair(const int np, const int p, const int k); + int define_iodesc_datatypes(const iosystem_desc_t ios, io_desc_t *iodesc); - int create_mpi_datatypes(const MPI_Datatype basetype,const int msgcnt,const PIO_Offset dlen, - const PIO_Offset mindex[],const int mcount[],int *mfrom, MPI_Datatype mtype[]); - int compare_offsets(const void *a,const void *b) ; + int create_mpi_datatypes(const MPI_Datatype basetype,const int msgcnt,const PIO_Offset dlen, + const PIO_Offset mindex[],const int mcount[],int *mfrom, MPI_Datatype mtype[]); + int compare_offsets(const void *a,const void *b) ; - int subset_rearrange_create(const iosystem_desc_t ios, int maplen, PIO_Offset compmap[], - const int gsize[], const int ndims, io_desc_t *iodesc); - void print_trace (FILE *fp); - void cn_buffer_report(iosystem_desc_t ios, bool collective); - void compute_buffer_init(iosystem_desc_t ios); - void free_cn_buffer_pool(iosystem_desc_t ios); -void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk); - void piomemerror(iosystem_desc_t ios, size_t req, char *fname, const int line); - void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc); - int check_mpi(file_desc_t *file, const int mpierr, const char *filename, - const int line); + int subset_rearrange_create(const iosystem_desc_t ios, int maplen, PIO_Offset compmap[], + const int gsize[], const int ndims, io_desc_t *iodesc); + void print_trace (FILE *fp); + void cn_buffer_report(iosystem_desc_t ios, bool collective); + void compute_buffer_init(iosystem_desc_t ios); + void free_cn_buffer_pool(iosystem_desc_t ios); + void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk); + void piomemerror(iosystem_desc_t ios, size_t req, char *fname, const int line); + void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc); + int check_mpi(file_desc_t *file, const int mpierr, const char *filename, + const int line); #ifdef BGQ - void identity(MPI_Comm comm, int *iotask); - void determineiotasks(const MPI_Comm comm, int *numiotasks,int *base, int *stride, int *rearr, - bool *iamIOtask); + void identity(MPI_Comm comm, int *iotask); + void determineiotasks(const MPI_Comm comm, int *numiotasks,int *base, int *stride, int *rearr, + bool *iamIOtask); #endif @@ -169,211 +180,211 @@ void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk); /** These are the messages that can be sent over the intercomm when * async is being used. */ enum PIO_MSG{ - PIO_MSG_OPEN_FILE, - PIO_MSG_CREATE_FILE, - PIO_MSG_INQ_ATT, - PIO_MSG_INQ_FORMAT, - PIO_MSG_INQ_VARID, - PIO_MSG_DEF_VAR, - PIO_MSG_INQ_VAR, - PIO_MSG_PUT_ATT_DOUBLE, - PIO_MSG_PUT_ATT_INT, - PIO_MSG_RENAME_ATT, - PIO_MSG_DEL_ATT, - PIO_MSG_INQ, - PIO_MSG_GET_ATT_TEXT, - PIO_MSG_GET_ATT_SHORT, - PIO_MSG_PUT_ATT_LONG, - PIO_MSG_REDEF, - PIO_MSG_SET_FILL, - PIO_MSG_ENDDEF, - PIO_MSG_RENAME_VAR, - PIO_MSG_PUT_ATT_SHORT, - PIO_MSG_PUT_ATT_TEXT, - PIO_MSG_INQ_ATTNAME, - PIO_MSG_GET_ATT_ULONGLONG, - PIO_MSG_GET_ATT_USHORT, - PIO_MSG_PUT_ATT_ULONGLONG, - PIO_MSG_GET_ATT_UINT, - PIO_MSG_GET_ATT_LONGLONG, - PIO_MSG_PUT_ATT_SCHAR, - PIO_MSG_PUT_ATT_FLOAT, - PIO_MSG_RENAME_DIM, - PIO_MSG_GET_ATT_LONG, - PIO_MSG_INQ_DIM, - PIO_MSG_INQ_DIMID, - PIO_MSG_PUT_ATT_USHORT, - PIO_MSG_GET_ATT_FLOAT, - PIO_MSG_SYNC, - PIO_MSG_PUT_ATT_LONGLONG, - PIO_MSG_PUT_ATT_UINT, - PIO_MSG_GET_ATT_SCHAR, - PIO_MSG_INQ_ATTID, - PIO_MSG_DEF_DIM, - PIO_MSG_GET_ATT_INT, - PIO_MSG_GET_ATT_DOUBLE, - PIO_MSG_PUT_ATT_UCHAR, - PIO_MSG_GET_ATT_UCHAR, - PIO_MSG_PUT_VARS_UCHAR, - PIO_MSG_GET_VAR1_SCHAR, - PIO_MSG_GET_VARS_ULONGLONG, - PIO_MSG_GET_VARM_UCHAR, - PIO_MSG_GET_VARM_SCHAR, - PIO_MSG_GET_VARS_SHORT, - PIO_MSG_GET_VAR_DOUBLE, - PIO_MSG_GET_VARA_DOUBLE, - PIO_MSG_GET_VAR_INT, - PIO_MSG_GET_VAR_USHORT, - PIO_MSG_PUT_VARS_USHORT, - PIO_MSG_GET_VARA_TEXT, - PIO_MSG_PUT_VARS_ULONGLONG, - PIO_MSG_GET_VARA_INT, - PIO_MSG_PUT_VARM, - PIO_MSG_GET_VAR1_FLOAT, - PIO_MSG_GET_VAR1_SHORT, - PIO_MSG_GET_VARS_INT, - PIO_MSG_PUT_VARS_UINT, - PIO_MSG_GET_VAR_TEXT, - PIO_MSG_GET_VARM_DOUBLE, - PIO_MSG_PUT_VARM_UCHAR, - PIO_MSG_PUT_VAR_USHORT, - PIO_MSG_GET_VARS_SCHAR, - PIO_MSG_GET_VARA_USHORT, - PIO_MSG_PUT_VAR1_LONGLONG, - PIO_MSG_PUT_VARA_UCHAR, - PIO_MSG_PUT_VARM_SHORT, - PIO_MSG_PUT_VAR1_LONG, - PIO_MSG_PUT_VARS_LONG, - PIO_MSG_GET_VAR1_USHORT, - PIO_MSG_PUT_VAR_SHORT, - PIO_MSG_PUT_VARA_INT, - PIO_MSG_GET_VAR_FLOAT, - PIO_MSG_PUT_VAR1_USHORT, - PIO_MSG_PUT_VARA_TEXT, - PIO_MSG_PUT_VARM_TEXT, - PIO_MSG_GET_VARS_UCHAR, - PIO_MSG_GET_VAR, - PIO_MSG_PUT_VARM_USHORT, - PIO_MSG_GET_VAR1_LONGLONG, - PIO_MSG_GET_VARS_USHORT, - PIO_MSG_GET_VAR_LONG, - PIO_MSG_GET_VAR1_DOUBLE, - PIO_MSG_PUT_VAR_ULONGLONG, - PIO_MSG_PUT_VAR_INT, - PIO_MSG_GET_VARA_UINT, - PIO_MSG_PUT_VAR_LONGLONG, - PIO_MSG_GET_VARS_LONGLONG, - PIO_MSG_PUT_VAR_SCHAR, - PIO_MSG_PUT_VAR_UINT, - PIO_MSG_PUT_VAR, - PIO_MSG_PUT_VARA_USHORT, - PIO_MSG_GET_VAR_LONGLONG, - PIO_MSG_GET_VARA_SHORT, - PIO_MSG_PUT_VARS_SHORT, - PIO_MSG_PUT_VARA_UINT, - PIO_MSG_PUT_VARA_SCHAR, - PIO_MSG_PUT_VARM_ULONGLONG, - PIO_MSG_PUT_VAR1_UCHAR, - PIO_MSG_PUT_VARM_INT, - PIO_MSG_PUT_VARS_SCHAR, - PIO_MSG_GET_VARA_LONG, - PIO_MSG_PUT_VAR1, - PIO_MSG_GET_VAR1_INT, - PIO_MSG_GET_VAR1_ULONGLONG, - PIO_MSG_GET_VAR_UCHAR, - PIO_MSG_PUT_VARA_FLOAT, - PIO_MSG_GET_VARA_UCHAR, - PIO_MSG_GET_VARS_FLOAT, - PIO_MSG_PUT_VAR1_FLOAT, - PIO_MSG_PUT_VARM_FLOAT, - PIO_MSG_PUT_VAR1_TEXT, - PIO_MSG_PUT_VARS_TEXT, - PIO_MSG_PUT_VARM_LONG, - PIO_MSG_GET_VARS_LONG, - PIO_MSG_PUT_VARS_DOUBLE, - PIO_MSG_GET_VAR1, - PIO_MSG_GET_VAR_UINT, - PIO_MSG_PUT_VARA_LONGLONG, - PIO_MSG_GET_VARA, - PIO_MSG_PUT_VAR_DOUBLE, - PIO_MSG_GET_VARA_SCHAR, - PIO_MSG_PUT_VAR_FLOAT, - PIO_MSG_GET_VAR1_UINT, - PIO_MSG_GET_VARS_UINT, - PIO_MSG_PUT_VAR1_ULONGLONG, - PIO_MSG_PUT_VARM_UINT, - PIO_MSG_PUT_VAR1_UINT, - PIO_MSG_PUT_VAR1_INT, - PIO_MSG_GET_VARA_FLOAT, - PIO_MSG_GET_VARM_TEXT, - PIO_MSG_PUT_VARS_FLOAT, - PIO_MSG_GET_VAR1_TEXT, - PIO_MSG_PUT_VARA_SHORT, - PIO_MSG_PUT_VAR1_SCHAR, - PIO_MSG_PUT_VARA_ULONGLONG, - PIO_MSG_PUT_VARM_DOUBLE, - PIO_MSG_GET_VARM_INT, - PIO_MSG_PUT_VARA, - PIO_MSG_PUT_VARA_LONG, - PIO_MSG_GET_VARM_UINT, - PIO_MSG_GET_VARM, - PIO_MSG_PUT_VAR1_DOUBLE, - PIO_MSG_GET_VARS_DOUBLE, - PIO_MSG_GET_VARA_LONGLONG, - PIO_MSG_GET_VAR_ULONGLONG, - PIO_MSG_PUT_VARM_SCHAR, - PIO_MSG_GET_VARA_ULONGLONG, - PIO_MSG_GET_VAR_SHORT, - PIO_MSG_GET_VARM_FLOAT, - PIO_MSG_PUT_VAR_TEXT, - PIO_MSG_PUT_VARS_INT, - PIO_MSG_GET_VAR1_LONG, - PIO_MSG_GET_VARM_LONG, - PIO_MSG_GET_VARM_USHORT, - PIO_MSG_PUT_VAR1_SHORT, - PIO_MSG_PUT_VARS_LONGLONG, - PIO_MSG_GET_VARM_LONGLONG, - PIO_MSG_GET_VARS_TEXT, - PIO_MSG_PUT_VARA_DOUBLE, - PIO_MSG_PUT_VARS, - PIO_MSG_PUT_VAR_UCHAR, - PIO_MSG_GET_VAR1_UCHAR, - PIO_MSG_PUT_VAR_LONG, - PIO_MSG_GET_VARS, - PIO_MSG_GET_VARM_SHORT, - PIO_MSG_GET_VARM_ULONGLONG, - PIO_MSG_PUT_VARM_LONGLONG, - PIO_MSG_GET_VAR_SCHAR, - PIO_MSG_GET_ATT_UBYTE, - PIO_MSG_PUT_ATT_STRING, - PIO_MSG_GET_ATT_STRING, - PIO_MSG_PUT_ATT_UBYTE, - PIO_MSG_INQ_VAR_FILL, - PIO_MSG_DEF_VAR_FILL, - PIO_MSG_DEF_VAR_DEFLATE, - PIO_MSG_INQ_VAR_DEFLATE, - PIO_MSG_INQ_VAR_SZIP, - PIO_MSG_DEF_VAR_FLETCHER32, - PIO_MSG_INQ_VAR_FLETCHER32, - PIO_MSG_DEF_VAR_CHUNKING, - PIO_MSG_INQ_VAR_CHUNKING, - PIO_MSG_DEF_VAR_ENDIAN, - PIO_MSG_INQ_VAR_ENDIAN, - PIO_MSG_SET_CHUNK_CACHE, - PIO_MSG_GET_CHUNK_CACHE, - PIO_MSG_SET_VAR_CHUNK_CACHE, - PIO_MSG_GET_VAR_CHUNK_CACHE, - PIO_MSG_INITDECOMP_DOF, - PIO_MSG_WRITEDARRAY, - PIO_MSG_READDARRAY, - PIO_MSG_SETERRORHANDLING, - PIO_MSG_FREEDECOMP, - PIO_MSG_CLOSE_FILE, - PIO_MSG_DELETE_FILE, - PIO_MSG_EXIT, - PIO_MSG_GET_ATT, - PIO_MSG_PUT_ATT, - PIO_MSG_INQ_TYPE + PIO_MSG_OPEN_FILE, + PIO_MSG_CREATE_FILE, + PIO_MSG_INQ_ATT, + PIO_MSG_INQ_FORMAT, + PIO_MSG_INQ_VARID, + PIO_MSG_DEF_VAR, + PIO_MSG_INQ_VAR, + PIO_MSG_PUT_ATT_DOUBLE, + PIO_MSG_PUT_ATT_INT, + PIO_MSG_RENAME_ATT, + PIO_MSG_DEL_ATT, + PIO_MSG_INQ, + PIO_MSG_GET_ATT_TEXT, + PIO_MSG_GET_ATT_SHORT, + PIO_MSG_PUT_ATT_LONG, + PIO_MSG_REDEF, + PIO_MSG_SET_FILL, + PIO_MSG_ENDDEF, + PIO_MSG_RENAME_VAR, + PIO_MSG_PUT_ATT_SHORT, + PIO_MSG_PUT_ATT_TEXT, + PIO_MSG_INQ_ATTNAME, + PIO_MSG_GET_ATT_ULONGLONG, + PIO_MSG_GET_ATT_USHORT, + PIO_MSG_PUT_ATT_ULONGLONG, + PIO_MSG_GET_ATT_UINT, + PIO_MSG_GET_ATT_LONGLONG, + PIO_MSG_PUT_ATT_SCHAR, + PIO_MSG_PUT_ATT_FLOAT, + PIO_MSG_RENAME_DIM, + PIO_MSG_GET_ATT_LONG, + PIO_MSG_INQ_DIM, + PIO_MSG_INQ_DIMID, + PIO_MSG_PUT_ATT_USHORT, + PIO_MSG_GET_ATT_FLOAT, + PIO_MSG_SYNC, + PIO_MSG_PUT_ATT_LONGLONG, + PIO_MSG_PUT_ATT_UINT, + PIO_MSG_GET_ATT_SCHAR, + PIO_MSG_INQ_ATTID, + PIO_MSG_DEF_DIM, + PIO_MSG_GET_ATT_INT, + PIO_MSG_GET_ATT_DOUBLE, + PIO_MSG_PUT_ATT_UCHAR, + PIO_MSG_GET_ATT_UCHAR, + PIO_MSG_PUT_VARS_UCHAR, + PIO_MSG_GET_VAR1_SCHAR, + PIO_MSG_GET_VARS_ULONGLONG, + PIO_MSG_GET_VARM_UCHAR, + PIO_MSG_GET_VARM_SCHAR, + PIO_MSG_GET_VARS_SHORT, + PIO_MSG_GET_VAR_DOUBLE, + PIO_MSG_GET_VARA_DOUBLE, + PIO_MSG_GET_VAR_INT, + PIO_MSG_GET_VAR_USHORT, + PIO_MSG_PUT_VARS_USHORT, + PIO_MSG_GET_VARA_TEXT, + PIO_MSG_PUT_VARS_ULONGLONG, + PIO_MSG_GET_VARA_INT, + PIO_MSG_PUT_VARM, + PIO_MSG_GET_VAR1_FLOAT, + PIO_MSG_GET_VAR1_SHORT, + PIO_MSG_GET_VARS_INT, + PIO_MSG_PUT_VARS_UINT, + PIO_MSG_GET_VAR_TEXT, + PIO_MSG_GET_VARM_DOUBLE, + PIO_MSG_PUT_VARM_UCHAR, + PIO_MSG_PUT_VAR_USHORT, + PIO_MSG_GET_VARS_SCHAR, + PIO_MSG_GET_VARA_USHORT, + PIO_MSG_PUT_VAR1_LONGLONG, + PIO_MSG_PUT_VARA_UCHAR, + PIO_MSG_PUT_VARM_SHORT, + PIO_MSG_PUT_VAR1_LONG, + PIO_MSG_PUT_VARS_LONG, + PIO_MSG_GET_VAR1_USHORT, + PIO_MSG_PUT_VAR_SHORT, + PIO_MSG_PUT_VARA_INT, + PIO_MSG_GET_VAR_FLOAT, + PIO_MSG_PUT_VAR1_USHORT, + PIO_MSG_PUT_VARA_TEXT, + PIO_MSG_PUT_VARM_TEXT, + PIO_MSG_GET_VARS_UCHAR, + PIO_MSG_GET_VAR, + PIO_MSG_PUT_VARM_USHORT, + PIO_MSG_GET_VAR1_LONGLONG, + PIO_MSG_GET_VARS_USHORT, + PIO_MSG_GET_VAR_LONG, + PIO_MSG_GET_VAR1_DOUBLE, + PIO_MSG_PUT_VAR_ULONGLONG, + PIO_MSG_PUT_VAR_INT, + PIO_MSG_GET_VARA_UINT, + PIO_MSG_PUT_VAR_LONGLONG, + PIO_MSG_GET_VARS_LONGLONG, + PIO_MSG_PUT_VAR_SCHAR, + PIO_MSG_PUT_VAR_UINT, + PIO_MSG_PUT_VAR, + PIO_MSG_PUT_VARA_USHORT, + PIO_MSG_GET_VAR_LONGLONG, + PIO_MSG_GET_VARA_SHORT, + PIO_MSG_PUT_VARS_SHORT, + PIO_MSG_PUT_VARA_UINT, + PIO_MSG_PUT_VARA_SCHAR, + PIO_MSG_PUT_VARM_ULONGLONG, + PIO_MSG_PUT_VAR1_UCHAR, + PIO_MSG_PUT_VARM_INT, + PIO_MSG_PUT_VARS_SCHAR, + PIO_MSG_GET_VARA_LONG, + PIO_MSG_PUT_VAR1, + PIO_MSG_GET_VAR1_INT, + PIO_MSG_GET_VAR1_ULONGLONG, + PIO_MSG_GET_VAR_UCHAR, + PIO_MSG_PUT_VARA_FLOAT, + PIO_MSG_GET_VARA_UCHAR, + PIO_MSG_GET_VARS_FLOAT, + PIO_MSG_PUT_VAR1_FLOAT, + PIO_MSG_PUT_VARM_FLOAT, + PIO_MSG_PUT_VAR1_TEXT, + PIO_MSG_PUT_VARS_TEXT, + PIO_MSG_PUT_VARM_LONG, + PIO_MSG_GET_VARS_LONG, + PIO_MSG_PUT_VARS_DOUBLE, + PIO_MSG_GET_VAR1, + PIO_MSG_GET_VAR_UINT, + PIO_MSG_PUT_VARA_LONGLONG, + PIO_MSG_GET_VARA, + PIO_MSG_PUT_VAR_DOUBLE, + PIO_MSG_GET_VARA_SCHAR, + PIO_MSG_PUT_VAR_FLOAT, + PIO_MSG_GET_VAR1_UINT, + PIO_MSG_GET_VARS_UINT, + PIO_MSG_PUT_VAR1_ULONGLONG, + PIO_MSG_PUT_VARM_UINT, + PIO_MSG_PUT_VAR1_UINT, + PIO_MSG_PUT_VAR1_INT, + PIO_MSG_GET_VARA_FLOAT, + PIO_MSG_GET_VARM_TEXT, + PIO_MSG_PUT_VARS_FLOAT, + PIO_MSG_GET_VAR1_TEXT, + PIO_MSG_PUT_VARA_SHORT, + PIO_MSG_PUT_VAR1_SCHAR, + PIO_MSG_PUT_VARA_ULONGLONG, + PIO_MSG_PUT_VARM_DOUBLE, + PIO_MSG_GET_VARM_INT, + PIO_MSG_PUT_VARA, + PIO_MSG_PUT_VARA_LONG, + PIO_MSG_GET_VARM_UINT, + PIO_MSG_GET_VARM, + PIO_MSG_PUT_VAR1_DOUBLE, + PIO_MSG_GET_VARS_DOUBLE, + PIO_MSG_GET_VARA_LONGLONG, + PIO_MSG_GET_VAR_ULONGLONG, + PIO_MSG_PUT_VARM_SCHAR, + PIO_MSG_GET_VARA_ULONGLONG, + PIO_MSG_GET_VAR_SHORT, + PIO_MSG_GET_VARM_FLOAT, + PIO_MSG_PUT_VAR_TEXT, + PIO_MSG_PUT_VARS_INT, + PIO_MSG_GET_VAR1_LONG, + PIO_MSG_GET_VARM_LONG, + PIO_MSG_GET_VARM_USHORT, + PIO_MSG_PUT_VAR1_SHORT, + PIO_MSG_PUT_VARS_LONGLONG, + PIO_MSG_GET_VARM_LONGLONG, + PIO_MSG_GET_VARS_TEXT, + PIO_MSG_PUT_VARA_DOUBLE, + PIO_MSG_PUT_VARS, + PIO_MSG_PUT_VAR_UCHAR, + PIO_MSG_GET_VAR1_UCHAR, + PIO_MSG_PUT_VAR_LONG, + PIO_MSG_GET_VARS, + PIO_MSG_GET_VARM_SHORT, + PIO_MSG_GET_VARM_ULONGLONG, + PIO_MSG_PUT_VARM_LONGLONG, + PIO_MSG_GET_VAR_SCHAR, + PIO_MSG_GET_ATT_UBYTE, + PIO_MSG_PUT_ATT_STRING, + PIO_MSG_GET_ATT_STRING, + PIO_MSG_PUT_ATT_UBYTE, + PIO_MSG_INQ_VAR_FILL, + PIO_MSG_DEF_VAR_FILL, + PIO_MSG_DEF_VAR_DEFLATE, + PIO_MSG_INQ_VAR_DEFLATE, + PIO_MSG_INQ_VAR_SZIP, + PIO_MSG_DEF_VAR_FLETCHER32, + PIO_MSG_INQ_VAR_FLETCHER32, + PIO_MSG_DEF_VAR_CHUNKING, + PIO_MSG_INQ_VAR_CHUNKING, + PIO_MSG_DEF_VAR_ENDIAN, + PIO_MSG_INQ_VAR_ENDIAN, + PIO_MSG_SET_CHUNK_CACHE, + PIO_MSG_GET_CHUNK_CACHE, + PIO_MSG_SET_VAR_CHUNK_CACHE, + PIO_MSG_GET_VAR_CHUNK_CACHE, + PIO_MSG_INITDECOMP_DOF, + PIO_MSG_WRITEDARRAY, + PIO_MSG_READDARRAY, + PIO_MSG_SETERRORHANDLING, + PIO_MSG_FREEDECOMP, + PIO_MSG_CLOSE_FILE, + PIO_MSG_DELETE_FILE, + PIO_MSG_EXIT, + PIO_MSG_GET_ATT, + PIO_MSG_PUT_ATT, + PIO_MSG_INQ_TYPE }; #endif diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c index 735e3755a99..a311ed003f4 100644 --- a/src/clib/pio_nc4.c +++ b/src/clib/pio_nc4.c @@ -235,28 +235,40 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp) { - int ierr; - int msg; - int mpierr; - iosystem_desc_t *ios; - file_desc_t *file; + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; /** Pointer to file information. */ + int ierr = PIO_NOERR; /** Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ char *errstr; - errstr = NULL; - ierr = PIO_NOERR; + /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_DEF_VAR_CHUNKING; - if (ios->async_interface && ! ios->ioproc) + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) { - if (ios->compmaster) - mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); - mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + if (!ios->ioproc) + { + int msg = PIO_MSG_DEF_VAR_CHUNKING; + + if (ios->compmaster) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); + } + + /* Handle MPI errors. */ + if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) + return check_mpi(file, mpierr2, __FILE__, __LINE__); + if (mpierr) + return check_mpi(file, mpierr, __FILE__, __LINE__); } + /* If this is an IO task, then call the netCDF function. */ if (ios->ioproc) { switch (file->iotype) @@ -287,20 +299,12 @@ int PIOc_def_var_chunking(int ncid, int varid, int storage, } } - /* Allocate an error string if needed. */ - if (ierr != PIO_NOERR) - { - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); - } - - /* Check for netCDF error. */ - ierr = check_netcdf(file, ierr, errstr,__LINE__); - - /* Free the error string if it was allocated. */ - if (errstr != NULL) - free(errstr); - + /* Broadcast and check the return code. */ + if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm))) + return check_mpi(file, mpierr, __FILE__, __LINE__); + if (ierr) + return check_netcdf(file, ierr, __FILE__, __LINE__); + return ierr; } diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 612d99bde7c..97afd948977 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -7,11 +7,10 @@ * @see http://code.google.com/p/parallelio/ */ - +#include #include #include - static int counter=0; /** @@ -176,8 +175,6 @@ int PIOc_get_local_array_size(int ioid) ** @param iostart An optional array of start values for block cyclic decompositions (optional input) ** @param iocount An optional array of count values for block cyclic decompositions (optional input) */ - - int PIOc_InitDecomp(const int iosysid, const int basetype,const int ndims, const int dims[], const int maplen, const PIO_Offset *compmap, int *ioidp,const int *rearranger, const PIO_Offset *iostart,const PIO_Offset *iocount) @@ -189,8 +186,6 @@ int PIOc_InitDecomp(const int iosysid, const int basetype,const int ndims, const int iosize; int ndisp; - - for(int i=0;inum_comptasks,ndims,counter); } - PIOc_writemap(filename,ndims,dims,maplen,compmap,ios->comp_comm); + PIOc_writemap(filename,ndims,dims,maplen, (PIO_Offset *)compmap,ios->comp_comm); counter++; } @@ -281,8 +276,6 @@ int PIOc_InitDecomp(const int iosysid, const int basetype,const int ndims, const ** expressed in terms of start and count on the file. ** in this case we compute the compdof and use the subset rearranger */ - - int PIOc_InitDecomp_bc(const int iosysid, const int basetype,const int ndims, const int dims[], const long int start[], const long int count[], int *ioidp) @@ -340,15 +333,31 @@ int PIOc_InitDecomp_bc(const int iosysid, const int basetype,const int ndims, co return PIO_NOERR; } -/** - ** @ingroup PIO_init - ** @brief library initialization used when IO tasks are a subset of compute tasks - ** @param comp_comm the MPI_Comm of the compute tasks - ** @param num_iotasks the number of io tasks to use - ** @param stride the offset between io tasks in the comp_comm - ** @param base the comp_comm index of the first io task - ** @param rearr the rearranger to use by default, this may be overriden in the @ref PIO_initdecomp - ** @param iosysidp index of the defined system descriptor +/* @ingroup PIO_init + * + * Library initialization used when IO tasks are a subset of compute + * tasks. + * + * This function creates an MPI intracommunicator between a set of IO + * tasks and one or more sets of computational tasks. + * + * The caller must create all comp_comm and the io_comm MPI + * communicators before calling this function. + * + * @param comp_comm the MPI_Comm of the compute tasks + * + * @param num_iotasks the number of io tasks to use + * + * @param stride the offset between io tasks in the comp_comm + * + * @param base the comp_comm index of the first io task + * + * @param rearr the rearranger to use by default, this may be + * overriden in the @ref PIO_initdecomp + * + * @param iosysidp index of the defined system descriptor + * + * @return 0 on success, otherwise a PIO error code. */ int PIOc_Init_Intracomm(const MPI_Comm comp_comm, const int num_iotasks, const int stride, const int base, const int rearr, From 5068c6069e9200186b582230234863f82e9be843 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 20 Jun 2016 15:54:28 -0600 Subject: [PATCH 166/184] cleanup and documentation --- src/clib/pio.h | 124 +++++++++++++++++--------- src/clib/pio_darray.c | 12 +-- src/clib/pio_file.c | 60 ++++++------- src/clib/pio_get_nc_async.c | 76 +++++++++++++--- src/clib/pio_internal.h | 20 ++--- src/clib/pio_nc_async.c | 169 ++++++++++++++++++------------------ src/clib/pio_put_nc_async.c | 48 ++++++---- 7 files changed, 303 insertions(+), 206 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index 3b0fb191922..8a75bfce292 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -41,33 +41,35 @@ #define PIO_MAX_VARS NC_MAX_VARS /** - * @brief Variable description structure - * - * The variable record is the index into the unlimited dimension in the netcdf file - * typically this is the time dimension. - * ndims is the number of dimensions on the file for this variable - * request is the id of each outstanding pnetcdf request for this variable - * nreqs is the number of outstanding pnetcdf requests for this variable - * fillbuf is a memory buffer to hold fill values for this variable (write only) - * iobuf is a memory buffer to hold (write only) + * Variable description structure. */ typedef struct var_desc_t { - int record; + /** The unlimited dimension in the netCDF file (typically the time + * dimension). */ + int record; + + /** Number of dimensions for this variable. */ int ndims; - int *request; // used for pnetcdf iput calls + /** ID of each outstanding pnetcdf request for this variable. */ + int *request; + + /** Number of requests bending with pnetcdf. */ int nreqs; + + /** Buffer that contains the fill value for this variable. */ void *fillbuf; - void *iobuf; + /** ??? */ + void *iobuf; } var_desc_t; /** - * @brief io region structure + * IO region structure. * * Each IO region is a unit of data which can be described using start and count - * arrays. Each IO task may in general have multiple io regions per variable. The + * arrays. Each IO task may in general have multiple io regions per variable. The * box rearranger will have at most one io region per variable. * */ @@ -80,14 +82,14 @@ typedef struct io_region } io_region; /** - * @brief io descriptor structure + * IO descriptor structure. * * This structure defines the mapping for a given variable between * compute and IO decomposition. - * */ typedef struct io_desc_t { + /** The ID of this io_desc_t. */ int ioid; int async_id; int nrecvs; @@ -97,7 +99,9 @@ typedef struct io_desc_t int rearranger; int maxregions; bool needsfill; // Does this decomp leave holes in the field (true) or write everywhere (false) - int maxbytes; // maximum number of bytes of this iodesc before flushing + + /** The maximum number of bytes of this iodesc before flushing. */ + int maxbytes; MPI_Datatype basetype; PIO_Offset llen; int maxiobuflen; @@ -122,19 +126,21 @@ typedef struct io_desc_t int max_requests; MPI_Comm subset_comm; + + /** Pointer to the next io_desc_t in the list. */ struct io_desc_t *next; } io_desc_t; /** - * @brief io system descriptor structure + * IO system descriptor structure. * - * This structure contains the general IO subsystem data - * and MPI structure - * + * This structure contains the general IO subsystem data and MPI + * structure */ typedef struct iosystem_desc_t { - /** The ID of this iosystem_desc_t. */ + /** The ID of this iosystem_desc_t. This will be obtained by + * calling PIOc_Init_Intercomm() or PIOc_Init_Intracomm(). */ int iosysid; /** This is an MPI intra communicator that includes all the tasks in @@ -204,7 +210,8 @@ typedef struct iosystem_desc_t /** Controls handling errors. */ int error_handler; - /** ??? */ + /** The rearranger decides which parts of a distributed array are + * handled by which IO tasks. */ int default_rearranger; /** True if asynchronous interface is in use. */ @@ -221,8 +228,7 @@ typedef struct iosystem_desc_t } iosystem_desc_t; /** - * @brief multi buffer - * + * multi buffer. */ typedef struct wmulti_buffer { @@ -237,20 +243,35 @@ typedef struct wmulti_buffer } wmulti_buffer; /** - * @brief io system descriptor structure + * File descriptor structure. * * This structure holds information associated with each open file - * */ typedef struct file_desc_t { + /** The IO system ID used to open this file. */ iosystem_desc_t *iosystem; - PIO_Offset buffsize; + + /** The buffersize does not seem to be used anywhere. */ + /* PIO_Offset buffsize;*/ + + /** The ncid returned for this file by the underlying library + * (netcdf or pnetcdf). */ int fh; + + /** The PIO_TYPE value that was used to open this file. */ int iotype; + + /** List of variables in this file. */ struct var_desc_t varlist[PIO_MAX_VARS]; + + /** ??? */ int mode; + + /** ??? */ struct wmulti_buffer buffer; + + /** Pointer to the next file_desc_t in the list of open files. */ struct file_desc_t *next; /** True if this task should participate in IO (only true for one @@ -259,32 +280,49 @@ typedef struct file_desc_t } file_desc_t; /** - * @brief These are the supported output formats + * These are the supported methods of reading/writing netCDF + * files. (Not all methods can be used with all netCDF files.) */ -enum PIO_IOTYPE{ - PIO_IOTYPE_PNETCDF=1, //< Parallel Netcdf (parallel) - PIO_IOTYPE_NETCDF=2, //< Netcdf3 Classic format (serial) - PIO_IOTYPE_NETCDF4C=3, //< NetCDF4 (HDF5) compressed format (serial) - PIO_IOTYPE_NETCDF4P=4 //< NetCDF4 (HDF5) parallel +enum PIO_IOTYPE +{ + /** Parallel Netcdf (parallel) */ + PIO_IOTYPE_PNETCDF = 1, + + /** Netcdf3 Classic format (serial) */ + PIO_IOTYPE_NETCDF = 2, + + /** NetCDF4 (HDF5) compressed format (serial) */ + PIO_IOTYPE_NETCDF4C = 3, + + /** NetCDF4 (HDF5) parallel */ + PIO_IOTYPE_NETCDF4P = 4 }; /** - * @brief These are the supported output data rearrangement methods - * + * These are the supported output data rearrangement methods. */ -enum PIO_REARRANGERS{ +enum PIO_REARRANGERS +{ + /** Box rearranger. */ PIO_REARR_BOX = 1, + + /** Subset rearranger. */ PIO_REARR_SUBSET = 2 }; /** - * @brief These are the supported error handlers - * + * These are the supported error handlers. */ -enum PIO_ERROR_HANDLERS{ - PIO_INTERNAL_ERROR=(-51), //< Errors cause abort - PIO_BCAST_ERROR=(-52), //< Error codes are broadcast to all tasks - PIO_RETURN_ERROR=(-53) //< Errors are returned to caller with no internal action +enum PIO_ERROR_HANDLERS +{ + /** Errors cause abort. */ + PIO_INTERNAL_ERROR = (-51), + + /** Error codes are broadcast to all tasks. */ + PIO_BCAST_ERROR = (-52), + + /** Errors are returned to caller with no internal action. */ + PIO_RETURN_ERROR = (-53) }; #if defined( _PNETCDF) || defined(_NETCDF) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index abf6cf8ed0f..6ab9a872671 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -74,9 +74,7 @@ void compute_buffer_init(iosystem_desc_t ios) #endif } -/** @ingroup PIO_write_darray - * - * Write a single distributed field to output. This routine is only +/** Write a single distributed field to output. This routine is only * used if aggregation is off. * * @param[in] file: a pointer to the open file descriptor for the file @@ -90,6 +88,10 @@ void compute_buffer_init(iosystem_desc_t ios) * * @param[in] fillvalue: the optional fillvalue to be used for missing * data in this buffer + * + * @return 0 for success, error code otherwise. + * + * @ingroup PIO_write_darray */ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF, void *fillvalue) @@ -1088,8 +1090,7 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, return ierr; } -/** @brief Write a distributed array to the output file. - * @ingroup PIO_write_darray +/** Write a distributed array to the output file. * * This routine aggregates output on the compute nodes and only sends * it to the IO nodes when the compute buffer is full or when a flush @@ -1109,6 +1110,7 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, * missing data. * * @returns 0 for success, non-zero error code for failure. + * @ingroup PIO_write_darray */ #ifdef PIO_WRITE_BUFFERING int PIOc_write_darray(const int ncid, const int vid, const int ioid, diff --git a/src/clib/pio_file.c b/src/clib/pio_file.c index 44bd07888f3..5003288caa0 100644 --- a/src/clib/pio_file.c +++ b/src/clib/pio_file.c @@ -2,24 +2,26 @@ #include #include -/* Open an existing file using pio - * @public - * @ingroup PIO_openfile +/** Open an existing file using PIO library. * - * @details Input parameters are read on comp task 0 and ignored elsewhere. + * Input parameters are read on comp task 0 and ignored elsewhere. + * * @param iosysid : A defined pio system descriptor (input) * @param ncidp : A pio file descriptor (output) * @param iotype : A pio output format (input) * @param filename : The filename to open * @param mode : The netcdf mode for the open operation + * + * @return 0 for success, error code otherwise. + * @ingroup PIO_openfile */ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, const char *filename, const int mode) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ LOG((1, "PIOc_openfile iosysid = %d", iosysid)); @@ -64,7 +66,7 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, file->buffer.frame = NULL; file->buffer.fillvalue = NULL; - /** Set to true if this task should participate in IO (only true for + /* Set to true if this task should participate in IO (only true for * one task with netcdf serial files. */ if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF || ios->io_rank == 0) @@ -193,7 +195,7 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, return ierr; } -/* Open a new file using pio. Input parameters are read on comp task +/** Open a new file using pio. Input parameters are read on comp task * 0 and ignored elsewhere. * * @public @@ -209,10 +211,10 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, const char filename[], const int mode) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ /* User must provide valid input for these parameters. */ if (!ncidp || !iotype || !filename || strlen(filename) > NC_MAX_NAME) @@ -253,7 +255,7 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, file->mode = mode; - /** Set to true if this task should participate in IO (only true for + /* Set to true if this task should participate in IO (only true for * one task with netcdf serial files. */ if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF || ios->io_rank == 0) @@ -358,17 +360,17 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, return ierr; } -/* Close a file previously opened with PIO. +/** Close a file previously opened with PIO. * @ingroup PIO_closefile * * @param ncid: the file pointer */ int PIOc_closefile(int ncid) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ /* Find the info about this file. */ if (!(file = pio_get_file_from_id(ncid))) @@ -446,7 +448,7 @@ int PIOc_closefile(int ncid) return ierr; } -/* Delete a file. +/** Delete a file. * @ingroup PIO_deletefile * * @param iosysid : a pio system handle @@ -454,10 +456,10 @@ int PIOc_closefile(int ncid) */ int PIOc_deletefile(const int iosysid, const char filename[]) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ int msg = PIO_MSG_DELETE_FILE; size_t len; @@ -513,10 +515,10 @@ int PIOc_deletefile(const int iosysid, const char filename[]) */ int PIOc_sync(int ncid) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ wmulti_buffer *wmb, *twmb; /* Get the file info from the ncid. */ diff --git a/src/clib/pio_get_nc_async.c b/src/clib/pio_get_nc_async.c index 7636dd6251e..7f6aeb79ce3 100644 --- a/src/clib/pio_get_nc_async.c +++ b/src/clib/pio_get_nc_async.c @@ -1,18 +1,66 @@ +/** + * @file + * PIO functions to get data (excluding varm functions). + * + * @author Ed Hartnett + * @date 2016 + * + * @see http://code.google.com/p/parallelio/ + */ + #include #include #include +/** + * Internal PIO function which provides a type-neutral interface to + * nc_get_vars. + * + * Users should not call this function directly. Instead, call one of + * the derived functions, depending on the type of data you are + * reading: PIOc_get_vars_text(), PIOc_get_vars_uchar(), + * PIOc_get_vars_schar(), PIOc_get_vars_ushort(), + * PIOc_get_vars_short(), PIOc_get_vars_uint(), PIOc_get_vars_int(), + * PIOc_get_vars_long(), PIOc_get_vars_float(), + * PIOc_get_vars_double(), PIOc_get_vars_ulonglong(), + * PIOc_get_vars_longlong() + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param start an array of start indicies (must have same number of + * entries as variable has dimensions). If NULL, indices of 0 will be + * used. + * + * @param count an array of counts (must have same number of entries + * as variable has dimensions). If NULL, counts matching the size of + * the variable will be used. + * + * @param stride an array of strides (must have same number of + * entries as variable has dimensions). If NULL, strides of 1 will be + * used. + * + * @param xtype the netCDF type of the data being passed in buf. Data + * will be automatically covnerted from the type of the variable being + * read from to this type. + * + * @param buf pointer to the data to be written. + * + * @return PIO_NOERR on success, error code otherwise. + */ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, nc_type xtype, void *buf) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - int ndims; /** The number of dimensions in the variable. */ - int *dimids; /** The IDs of the dimensions for this variable. */ - PIO_Offset typelen; /** Size (in bytes) of the data type of data in buf. */ - PIO_Offset num_elem = 1; /** Number of data elements in the buffer. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + int ndims; /* The number of dimensions in the variable. */ + int *dimids; /* The IDs of the dimensions for this variable. */ + PIO_Offset typelen; /* Size (in bytes) of the data type of data in buf. */ + PIO_Offset num_elem = 1; /* Number of data elements in the buffer. */ int bcast = false; LOG((1, "PIOc_get_vars_tc ncid = %d varid = %d start = %d count = %d " @@ -283,7 +331,7 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off return ierr; } -int PIOc_get_vars_text (int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, +int PIOc_get_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, char *buf) { return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_CHAR, buf); @@ -295,7 +343,7 @@ int PIOc_get_vars_uchar(int ncid, int varid, const PIO_Offset *start, return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_UBYTE, buf); } -int PIOc_get_vars_schar (int ncid, int varid, const PIO_Offset *start, +int PIOc_get_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, signed char *buf) { return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_BYTE, buf); @@ -350,7 +398,7 @@ int PIOc_get_vars_ulonglong(int ncid, int varid, const PIO_Offset *start, return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_UINT64, buf); } -int PIOc_get_vars_longlong (int ncid, int varid, const PIO_Offset *start, +int PIOc_get_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, long long *buf) { return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_UINT64, buf); @@ -453,7 +501,7 @@ int PIOc_get_var_short(int ncid, int varid, short *buf) return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_SHORT, buf); } -int PIOc_get_var_uint (int ncid, int varid, unsigned int *buf) +int PIOc_get_var_uint(int ncid, int varid, unsigned int *buf) { return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UINT, buf); } @@ -478,12 +526,12 @@ int PIOc_get_var_double(int ncid, int varid, double *buf) return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_DOUBLE, buf); } -int PIOc_get_var_ulonglong (int ncid, int varid, unsigned long long *buf) +int PIOc_get_var_ulonglong(int ncid, int varid, unsigned long long *buf) { return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_UINT64, buf); } -int PIOc_get_var_longlong (int ncid, int varid, long long *buf) +int PIOc_get_var_longlong(int ncid, int varid, long long *buf) { return PIOc_get_vars_tc(ncid, varid, NULL, NULL, NULL, NC_INT64, buf); } diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index c2e22a57a9c..40d7e12f450 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -12,7 +12,8 @@ #include -// It seems that some versions of openmpi fail to define MPI_OFFSET +/* It seems that some versions of openmpi fail to define + * MPI_OFFSET. */ #ifdef OMPI_OFFSET_DATATYPE #ifndef MPI_OFFSET #define MPI_OFFSET OMPI_OFFSET_DATATYPE @@ -45,7 +46,6 @@ void pio_log(int severity, const char *fmt, ...); __typeof__ (b) _b = (b); \ _a < _b ? _a : _b; }) - #define MAX_GATHER_BLOCK_SIZE 0 #define PIO_REQUEST_ALLOC_CHUNK 16 @@ -56,10 +56,7 @@ extern "C" { extern PIO_Offset PIO_BUFFER_SIZE_LIMIT; extern bool PIO_Save_Decomps; - -/** - ** @brief Used to sort map points in the subset rearranger - */ + /** Used to sort map points in the subset rearranger. */ typedef struct mapsort { int rfrom; @@ -67,10 +64,7 @@ extern "C" { PIO_Offset iomap; } mapsort; -/** - * @brief swapm defaults. - * - */ + /** swapm defaults. */ typedef struct pio_swapm_defaults { int nreqs; @@ -78,7 +72,6 @@ extern "C" { bool isend; } pio_swapm_defaults; - void pio_get_env(void); int pio_add_to_iodesc_list(io_desc_t *iodesc); io_desc_t *pio_get_iodesc_from_id(int ioid); @@ -179,7 +172,8 @@ extern "C" { /** These are the messages that can be sent over the intercomm when * async is being used. */ -enum PIO_MSG{ +enum PIO_MSG +{ PIO_MSG_OPEN_FILE, PIO_MSG_CREATE_FILE, PIO_MSG_INQ_ATT, @@ -387,4 +381,4 @@ enum PIO_MSG{ PIO_MSG_INQ_TYPE }; -#endif +#endif /* __PIO_INTERNAL__ */ diff --git a/src/clib/pio_nc_async.c b/src/clib/pio_nc_async.c index 73322b86d18..147fc548d7a 100644 --- a/src/clib/pio_nc_async.c +++ b/src/clib/pio_nc_async.c @@ -35,13 +35,12 @@ * @return PIO_NOERR for success, error code otherwise. See * PIOc_Set_File_Error_Handling */ -int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, - int *unlimdimidp) +int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ LOG((1, "PIOc_inq ncid = %d", ncid)); @@ -55,7 +54,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, { if (!ios->ioproc) { - int msg = PIO_MSG_INQ; /** Message for async notification. */ + int msg = PIO_MSG_INQ; /* Message for async notification. */ char ndims_present = ndimsp ? true : false; char nvars_present = nvarsp ? true : false; char ngatts_present = ngattsp ? true : false; @@ -243,10 +242,10 @@ int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name, */ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ int typelen; LOG((1, "PIOc_inq_type ncid = %d xtype = %d", ncid, xtype)); @@ -261,7 +260,7 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) { if (!ios->ioproc) { - int msg = PIO_MSG_INQ_TYPE; /** Message for async notification. */ + int msg = PIO_MSG_INQ_TYPE; /* Message for async notification. */ char name_present = name ? true : false; char size_present = sizep ? true : false; @@ -332,10 +331,10 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep) */ int PIOc_inq_format (int ncid, int *formatp) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ LOG((1, "PIOc_inq ncid = %d", ncid)); @@ -414,10 +413,10 @@ int PIOc_inq_format (int ncid, int *formatp) */ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ LOG((1, "PIOc_inq_dim")); @@ -534,7 +533,7 @@ int PIOc_inq_dimid(int ncid, const char *name, int *idp) iosystem_desc_t *ios; file_desc_t *file; int ierr = PIO_NOERR; - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ /* Name must be provided. */ if (!name) @@ -624,9 +623,9 @@ int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, { iosystem_desc_t *ios; file_desc_t *file; - int ndims; /** The number of dimensions for this variable. */ + int ndims; /* The number of dimensions for this variable. */ int ierr = PIO_NOERR; - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ LOG((1, "PIOc_inq_var ncid = %d varid = %d", ncid, varid)); @@ -809,10 +808,10 @@ int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) */ int PIOc_inq_varid (int ncid, const char *name, int *varidp) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ /* Caller must provide name. */ if (!name || strlen(name) > NC_MAX_NAME) @@ -902,7 +901,7 @@ int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep, int msg = PIO_MSG_INQ_ATT; iosystem_desc_t *ios; file_desc_t *file; - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ int ierr = PIO_NOERR; /* Caller must provide a name. */ @@ -1021,10 +1020,10 @@ int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep) */ int PIOc_inq_attname(int ncid, int varid, int attnum, char *name) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ LOG((1, "PIOc_inq_attname ncid = %d varid = %d attnum = %d", ncid, varid, attnum)); @@ -1116,10 +1115,10 @@ int PIOc_inq_attname(int ncid, int varid, int attnum, char *name) */ int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ /* User must provide name shorter than NC_MAX_NAME +1. */ if (!name || strlen(name) > NC_MAX_NAME) @@ -1211,10 +1210,10 @@ int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp) */ int PIOc_rename_dim(int ncid, int dimid, const char *name) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ /* User must provide name of correct length. */ if (!name || strlen(name) > NC_MAX_NAME) @@ -1232,7 +1231,7 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name) { if (!ios->ioproc) { - int msg = PIO_MSG_RENAME_DIM; /** Message for async notification. */ + int msg = PIO_MSG_RENAME_DIM; /* Message for async notification. */ int namelen = strlen(name); if(ios->compmaster) @@ -1299,10 +1298,10 @@ int PIOc_rename_dim(int ncid, int dimid, const char *name) */ int PIOc_rename_var(int ncid, int varid, const char *name) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ /* User must provide name of correct length. */ if (!name || strlen(name) > NC_MAX_NAME) @@ -1320,7 +1319,7 @@ int PIOc_rename_var(int ncid, int varid, const char *name) { if (!ios->ioproc) { - int msg = PIO_MSG_RENAME_VAR; /** Message for async notification. */ + int msg = PIO_MSG_RENAME_VAR; /* Message for async notification. */ int namelen = strlen(name); if(ios->compmaster) @@ -1389,10 +1388,10 @@ int PIOc_rename_var(int ncid, int varid, const char *name) int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI functions. */ /* User must provide names of correct length. */ if (!name || strlen(name) > NC_MAX_NAME || @@ -1412,7 +1411,7 @@ int PIOc_rename_att (int ncid, int varid, const char *name, { if (!ios->ioproc) { - int msg = PIO_MSG_RENAME_ATT; /** Message for async notification. */ + int msg = PIO_MSG_RENAME_ATT; /* Message for async notification. */ int namelen = strlen(name); int newnamelen = strlen(newname); @@ -1481,10 +1480,10 @@ int PIOc_rename_att (int ncid, int varid, const char *name, */ int PIOc_del_att(int ncid, int varid, const char *name) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI functions. */ /* User must provide name of correct length. */ if (!name || strlen(name) > NC_MAX_NAME) @@ -1503,7 +1502,7 @@ int PIOc_del_att(int ncid, int varid, const char *name) if (!ios->ioproc) { int msg = PIO_MSG_DEL_ATT; - int namelen = strlen(name); /** Length of name string. */ + int namelen = strlen(name); /* Length of name string. */ if(ios->compmaster) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); @@ -1565,10 +1564,10 @@ int PIOc_del_att(int ncid, int varid, const char *name) */ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI functions. */ LOG((1, "PIOc_set_fill ncid = %d fillmode = %d old_modep = %d", ncid, fillmode, old_modep)); @@ -1632,10 +1631,10 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) * @returns PIO_NOERR on success, error code on failure. */ int pioc_change_def(int ncid, int is_enddef) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI functions. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI functions. */ LOG((1, "pioc_change_def ncid = %d is_enddef = %d", ncid, is_enddef)); @@ -1753,10 +1752,10 @@ int PIOc_redef(int ncid) */ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ /* User must provide name. */ if (!name || strlen(name) > NC_MAX_NAME) @@ -1848,10 +1847,10 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ /* User must provide name and storage for varid. */ if (!name || !varidp || strlen(name) > NC_MAX_NAME) @@ -1955,10 +1954,10 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, */ int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ LOG((1, "PIOc_inq ncid = %d", ncid)); @@ -2035,10 +2034,10 @@ int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep) */ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ PIO_Offset attlen, typelen; nc_type atttype; @@ -2170,11 +2169,11 @@ int PIOc_get_att(int ncid, int varid, const char *name, void *ip) int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const void *op) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - PIO_Offset typelen; /** Length (in bytes) of the type. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + PIO_Offset typelen; /* Length (in bytes) of the type. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ LOG((1, "PIOc_put_att ncid = %d varid = %d name = %s", ncid, varid, name)); diff --git a/src/clib/pio_put_nc_async.c b/src/clib/pio_put_nc_async.c index 055acc247ff..9ef0d63da12 100644 --- a/src/clib/pio_put_nc_async.c +++ b/src/clib/pio_put_nc_async.c @@ -1,48 +1,62 @@ +/** + * @file + * PIO functions to write data. + * + * @author Ed Hartnett + * @date 2016 + * @see http://code.google.com/p/parallelio/ + */ + #include #include #include /** * Internal PIO function which provides a type-neutral interface to - * nc_put_vars + * nc_put_vars. + * + * Users should not call this function directly. Instead, call one of + * the derived functions, depending on the type of data you are + * writing: PIOc_put_vars_text(), PIOc_put_vars_uchar(), + * PIOc_put_vars_schar(), PIOc_put_vars_ushort(), + * PIOc_put_vars_short(), PIOc_put_vars_uint(), PIOc_put_vars_int(), + * PIOc_put_vars_long(), PIOc_put_vars_float(), + * PIOc_put_vars_longlong(), PIOc_put_vars_double(), + * PIOc_put_vars_ulonglong(). * * This routine is called collectively by all tasks in the * communicator ios.union_comm. * * @param ncid identifies the netCDF file * @param varid the variable ID number - - * @param *start an array of start indicies (must have same number of + * @param start an array of start indicies (must have same number of * entries as variable has dimensions). If NULL, indices of 0 will be * used. - * - * @param *count an array of counts (must have same number of entries + * @param count an array of counts (must have same number of entries * as variable has dimensions). If NULL, counts matching the size of * the variable will be used. - * - * @param *stride an array of strides (must have same number of + * @param stride an array of strides (must have same number of * entries as variable has dimensions). If NULL, strides of 1 will be * used. - * * @param xtype the netCDF type of the data being passed in buf. Data * will be automatically covnerted from this type to the type of the * variable being written to. - * * @param buf pointer to the data to be written. * * @return PIO_NOERR on success, error code otherwise. + * @private */ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, const PIO_Offset *stride, nc_type xtype, const void *buf) { - iosystem_desc_t *ios; /** Pointer to io system information. */ - file_desc_t *file; /** Pointer to file information. */ - int ierr = PIO_NOERR; /** Return code from function calls. */ - int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */ - int ndims; /** The number of dimensions in the variable. */ - int *dimids; /** The IDs of the dimensions for this variable. */ - PIO_Offset typelen; /** Size (in bytes) of the data type of data in buf. */ - PIO_Offset num_elem = 1; /** Number of data elements in the buffer. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ + int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ + int ndims; /* The number of dimensions in the variable. */ + int *dimids; /* The IDs of the dimensions for this variable. */ + PIO_Offset typelen; /* Size (in bytes) of the data type of data in buf. */ + PIO_Offset num_elem = 1; /* Number of data elements in the buffer. */ char start_present = start ? true : false; /* Is start non-NULL? */ char count_present = count ? true : false; /* Is count non-NULL? */ char stride_present = stride ? true : false; /* Is stride non-NULL? */ From 1e317dc4ddacdb2b9d837be9d0f2483b7b64c3c6 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 21 Jun 2016 07:55:05 -0600 Subject: [PATCH 167/184] split off pio_darray_async.c --- src/clib/CMakeLists.txt | 6 +- src/clib/pio_darray_async.c | 2143 +++++++++++++++++++++++++++++++++++ 2 files changed, 2146 insertions(+), 3 deletions(-) create mode 100644 src/clib/pio_darray_async.c diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt index 4344261aa28..8c01336969c 100644 --- a/src/clib/CMakeLists.txt +++ b/src/clib/CMakeLists.txt @@ -14,7 +14,6 @@ set (PIO_C_SRCS topology.c pioc_sc.c pio_spmd.c pio_rearrange.c - pio_darray.c pio_nc4.c bget.c) @@ -23,9 +22,10 @@ set (PIO_GENNC_SRCS ${CMAKE_CURRENT_BINARY_DIR}/pio_put_nc.c ${CMAKE_CURRENT_BINARY_DIR}/pio_nc.c) if (PIO_ENABLE_ASYNC) - set (PIO_ADDL_SRCS pio_nc_async.c pio_put_nc_async.c pio_get_nc_async.c pio_msg.c pio_varm.c) + set (PIO_ADDL_SRCS pio_nc_async.c pio_put_nc_async.c pio_get_nc_async.c + pio_msg.c pio_varm.c pio_darray_async.c) else () - set (PIO_ADDL_SRCS ${PIO_GENNC_SRCS}) + set (PIO_ADDL_SRCS pio_darray.c ${PIO_GENNC_SRCS}) endif () add_library (pioc ${PIO_C_SRCS} ${PIO_ADDL_SRCS}) diff --git a/src/clib/pio_darray_async.c b/src/clib/pio_darray_async.c new file mode 100644 index 00000000000..6ab9a872671 --- /dev/null +++ b/src/clib/pio_darray_async.c @@ -0,0 +1,2143 @@ +/** @file + * + * @brief This file contains the routines that read and write + * distributed arrays in PIO. + * + * When arrays are distributed, each processor holds some of the + * array. Only by combining the distributed arrays from all processor + * can the full array be obtained. + * + * @author Jim Edwards + * @bug No Known bugs + */ + +#include +#include + +#define PIO_WRITE_BUFFERING 1 + +PIO_Offset PIO_BUFFER_SIZE_LIMIT = 10485760; // 10MB default limit +bufsize PIO_CNBUFFER_LIMIT = 33554432; +static void *CN_bpool = NULL; +static PIO_Offset maxusage = 0; + +/** Set the pio buffer size limit. This is the size of the data buffer + * on the IO nodes. + * + * The pio_buffer_size_limit will only apply to files opened after + * the setting is changed. + * + * @param limit the size of the buffer on the IO nodes + * @return The previous limit setting. + */ +PIO_Offset PIOc_set_buffer_size_limit(const PIO_Offset limit) +{ + PIO_Offset oldsize; + oldsize = PIO_BUFFER_SIZE_LIMIT; + if (limit>0) + PIO_BUFFER_SIZE_LIMIT = limit; + return(oldsize); +} + +/** Initialize the compute buffer to size PIO_CNBUFFER_LIMIT. + * + * This routine initializes the compute buffer pool if the bget memory + * management is used. + * + * @param ios the iosystem descriptor which will use the new buffer + */ +void compute_buffer_init(iosystem_desc_t ios) +{ +#ifndef PIO_USE_MALLOC + + if (!CN_bpool) + { + if (!(CN_bpool = malloc(PIO_CNBUFFER_LIMIT))) + { + char errmsg[180]; + sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d:" + " try reducing PIO_CNBUFFER_LIMIT\n", PIO_CNBUFFER_LIMIT, ios.comp_rank); + piodie(errmsg, __FILE__, __LINE__); + } + + bpool(CN_bpool, PIO_CNBUFFER_LIMIT); + if (!CN_bpool) + { + char errmsg[180]; + sprintf(errmsg,"Unable to allocate a buffer pool of size %d on task %d:" + " try reducing PIO_CNBUFFER_LIMIT\n", PIO_CNBUFFER_LIMIT, ios.comp_rank); + piodie(errmsg, __FILE__, __LINE__); + } + + bectl(NULL, malloc, free, PIO_CNBUFFER_LIMIT); + } +#endif +} + +/** Write a single distributed field to output. This routine is only + * used if aggregation is off. + * + * @param[in] file: a pointer to the open file descriptor for the file + * that will be written to + * + * @param[in] iodesc: a pointer to the defined iodescriptor for the buffer + * + * @param[in] vid: the variable id to be written + * + * @param[in] IOBUF: the buffer to be written from this mpi task + * + * @param[in] fillvalue: the optional fillvalue to be used for missing + * data in this buffer + * + * @return 0 for success, error code otherwise. + * + * @ingroup PIO_write_darray + */ +int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, + void *IOBUF, void *fillvalue) +{ + iosystem_desc_t *ios; /** Pointer to io system information. */ + var_desc_t *vdesc; + int ndims; + int ierr = PIO_NOERR; /** Return code from function calls. */ + int i; + int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ + int dsize; + MPI_Status status; + PIO_Offset usage; + int fndims; + PIO_Offset tdsize = 0; + +#ifdef TIMING + GPTLstart("PIO:write_darray_nc"); +#endif + + /* Get the IO system info. */ + if (!(ios = file->iosystem)) + return PIO_EBADID; + + /* Get pointer to variable information. */ + if (!(vdesc = file->varlist + vid)) + return PIO_EBADID; + + ndims = iodesc->ndims; + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = 0; + + if (ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->compmaster, ios->intercomm); + } + } + + ierr = PIOc_inq_varndims(file->fh, vid, &fndims); + + if (ios->ioproc) + { + io_region *region; + int ncid = file->fh; + int regioncnt; + int rrcnt; + void *bufptr; + void *tmp_buf = NULL; + int tsize; + size_t start[fndims]; + size_t count[fndims]; + int buflen, j; + + PIO_Offset *startlist[iodesc->maxregions]; + PIO_Offset *countlist[iodesc->maxregions]; + MPI_Type_size(iodesc->basetype, &tsize); + + region = iodesc->firstregion; + + if (vdesc->record >= 0 && ndims < fndims) + ndims++; + +#ifdef _PNETCDF + /* make sure we have room in the buffer. */ + if (file->iotype == PIO_IOTYPE_PNETCDF) + flush_output_buffer(file, false, tsize * (iodesc->maxiobuflen)); +#endif + + rrcnt = 0; + for (regioncnt = 0; regioncnt < iodesc->maxregions; regioncnt++) + { + for (i = 0; i < ndims; i++) + { + start[i] = 0; + count[i] = 0; + } + if (region) + { + bufptr = (void *)((char *)IOBUF + tsize * region->loffset); + // this is a record based multidimensional array + if (vdesc->record >= 0) + { + start[0] = vdesc->record; + for (i = 1; i < ndims; i++) + { + start[i] = region->start[i - 1]; + count[i] = region->count[i - 1]; + } + if (count[1] > 0) + count[0] = 1; + // Non-time dependent array + } + else + { + for (i = 0; i < ndims; i++) + { + start[i] = region->start[i]; + count[i] = region->count[i]; + } + } + } + + switch(file->iotype) + { +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + + /* Use collective writes with this variable. */ + ierr = nc_var_par_access(ncid, vid, NC_COLLECTIVE); + if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) + ierr = nc_put_vara_double(ncid, vid, (size_t *)start, (size_t *)count, + (const double *)bufptr); + else if (iodesc->basetype == MPI_INTEGER) + ierr = nc_put_vara_int(ncid, vid, (size_t *)start, (size_t *)count, + (const int *)bufptr); + else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4) + ierr = nc_put_vara_float(ncid, vid, (size_t *)start, (size_t *)count, + (const float *)bufptr); + else + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n", + (int)iodesc->basetype); + break; + case PIO_IOTYPE_NETCDF4C: +#endif /* _NETCDF4 */ + case PIO_IOTYPE_NETCDF: + { + mpierr = MPI_Type_size(iodesc->basetype, &dsize); + size_t tstart[ndims], tcount[ndims]; + if (ios->io_rank == 0) + { + for (i = 0; i < iodesc->num_aiotasks; i++) + { + if (i == 0) + { + buflen = 1; + for (j = 0; j < ndims; j++) + { + tstart[j] = start[j]; + tcount[j] = count[j]; + buflen *= tcount[j]; + tmp_buf = bufptr; + } + } + else + { + mpierr = MPI_Send(&ierr, 1, MPI_INT, i, 0, ios->io_comm); // handshake - tell the sending task I'm ready + mpierr = MPI_Recv(&buflen, 1, MPI_INT, i, 1, ios->io_comm, &status); + if (buflen > 0) + { + mpierr = MPI_Recv(tstart, ndims, MPI_OFFSET, i, ios->num_iotasks+i, + ios->io_comm, &status); + mpierr = MPI_Recv(tcount, ndims, MPI_OFFSET, i, 2 * ios->num_iotasks + i, + ios->io_comm, &status); + tmp_buf = malloc(buflen * dsize); + mpierr = MPI_Recv(tmp_buf, buflen, iodesc->basetype, i, i, ios->io_comm, &status); + } + } + + if (buflen>0) + { + if (iodesc->basetype == MPI_INTEGER) + ierr = nc_put_vara_int(ncid, vid, tstart, tcount, (const int *)tmp_buf); + else if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) + ierr = nc_put_vara_double(ncid, vid, tstart, tcount, (const double *)tmp_buf); + else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4) + ierr = nc_put_vara_float(ncid, vid, tstart, tcount, (const float *)tmp_buf); + else + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n", + (int)iodesc->basetype); + + if (ierr == PIO_EEDGE) + for (i = 0; i < ndims; i++) + fprintf(stderr,"dim %d start %ld count %ld\n", i, tstart[i], tcount[i]); + + if (tmp_buf != bufptr) + free(tmp_buf); + } + } + } + else if (ios->io_rank < iodesc->num_aiotasks) + { + buflen = 1; + for (i = 0; i < ndims; i++) + { + tstart[i] = (size_t) start[i]; + tcount[i] = (size_t) count[i]; + buflen *= tcount[i]; + // printf("%s %d %d %d %d\n",__FILE__,__LINE__,i,tstart[i],tcount[i]); + } + /* printf("%s %d %d %d %d %d %d %d %d %d\n",__FILE__,__LINE__,ios->io_rank,tstart[0], + tstart[1],tcount[0],tcount[1],buflen,ndims,fndims);*/ + mpierr = MPI_Recv(&ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve + mpierr = MPI_Rsend(&buflen, 1, MPI_INT, 0, 1, ios->io_comm); + if (buflen > 0) + { + mpierr = MPI_Rsend(tstart, ndims, MPI_OFFSET, 0, ios->num_iotasks+ios->io_rank, + ios->io_comm); + mpierr = MPI_Rsend(tcount, ndims, MPI_OFFSET, 0,2*ios->num_iotasks+ios->io_rank, + ios->io_comm); + mpierr = MPI_Rsend(bufptr, buflen, iodesc->basetype, 0, ios->io_rank, ios->io_comm); + } + } + break; + } + break; +#endif /* _NETCDF */ +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + for (i = 0, dsize = 1; i < ndims; i++) + dsize *= count[i]; + + tdsize += dsize; + // if (dsize==1 && ndims==2) + // printf("%s %d %d\n",__FILE__,__LINE__,iodesc->basetype); + + /* if (regioncnt==0){ + for (i=0;imaxregions;i++){ + startlist[i] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); + countlist[i] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); + } + } + */ + if (dsize > 0) + { + // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); + startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); + countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); + for (i = 0; i < fndims; i++) + { + startlist[rrcnt][i] = start[i]; + countlist[rrcnt][i] = count[i]; + } + rrcnt++; + } + if (regioncnt == iodesc->maxregions - 1) + { + // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,ios->io_rank,iodesc->llen, tdsize); + // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, + // IOBUF, iodesc->llen, iodesc->basetype); + int reqn = 0; + + if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0 ) + { + vdesc->request = realloc(vdesc->request, + sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK)); + + for (int i = vdesc->nreqs; i < vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK; i++) + vdesc->request[i] = NC_REQ_NULL; + reqn = vdesc->nreqs; + } + else + while(vdesc->request[reqn] != NC_REQ_NULL) + reqn++; + + ierr = ncmpi_bput_varn(ncid, vid, rrcnt, startlist, countlist, + IOBUF, iodesc->llen, iodesc->basetype, vdesc->request+reqn); + if (vdesc->request[reqn] == NC_REQ_NULL) + vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync + vdesc->nreqs = reqn; + + // printf("%s %d %X %d\n",__FILE__,__LINE__,IOBUF,request); + for (i=0;iiotype,__FILE__,__LINE__); + } + if (region) + region = region->next; + } // for (regioncnt=0;regioncntmaxregions;regioncnt++){ + } // if (ios->ioproc) + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +#ifdef TIMING + GPTLstop("PIO:write_darray_nc"); +#endif + + return ierr; +} + +/** Write a set of one or more aggregated arrays to output file + * @ingroup PIO_write_darray + * + * This routine is used if aggregation is enabled, data is already on the + * io-tasks + * + * @param[in] file: a pointer to the open file descriptor for the file that will be written to + * @param[in] nvars: the number of variables to be written with this decomposition + * @param[in] vid: an array of the variable ids to be written + * @param[in] iodesc_ndims: the number of dimensions explicitly in the iodesc + * @param[in] basetype : the basic type of the minimal data unit + * @param[in] gsize : array of the global dimensions of the field to be written + * @param[in] maxregions : max number of blocks to be written from this iotask + * @param[in] firstregion : pointer to the first element of a linked list of region descriptions. + * @param[in] llen : length of the iobuffer on this task for a single field + * @param[in] maxiobuflen : maximum llen participating + * @param[in] num_aiotasks : actual number of iotasks participating + * @param[in] IOBUF: the buffer to be written from this mpi task + * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF + */ +int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[], + const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset gsize[], + const int maxregions, io_region *firstregion, const PIO_Offset llen, + const int maxiobuflen, const int num_aiotasks, + void *IOBUF, const int frame[]) +{ + iosystem_desc_t *ios; /** Pointer to io system information. */ + var_desc_t *vdesc; + int ierr; + int i; + int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ + int dsize; + MPI_Status status; + PIO_Offset usage; + int fndims; + PIO_Offset tdsize; + int tsize; + int ncid; + tdsize=0; + ierr = PIO_NOERR; +#ifdef TIMING + GPTLstart("PIO:write_darray_multi_nc"); +#endif + + ios = file->iosystem; + if (ios == NULL) + { + fprintf(stderr,"Failed to find iosystem handle \n"); + return PIO_EBADID; + } + vdesc = (file->varlist)+vid[0]; + ncid = file->fh; + + if (vdesc == NULL) + { + fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); + return PIO_EBADID; + } + + /* If async is in use, send message to IO master task. */ + if (ios->async_interface) + { + if (!ios->ioproc) + { + int msg = 0; + if (ios->compmaster) + mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + } + + ierr = PIOc_inq_varndims(file->fh, vid[0], &fndims); + MPI_Type_size(basetype, &tsize); + + if (ios->ioproc) + { + io_region *region; + int regioncnt; + int rrcnt; + void *bufptr; + int buflen, j; + size_t start[fndims]; + size_t count[fndims]; + int ndims = iodesc_ndims; + + PIO_Offset *startlist[maxregions]; + PIO_Offset *countlist[maxregions]; + + ncid = file->fh; + region = firstregion; + + rrcnt = 0; + for (regioncnt = 0; regioncnt < maxregions; regioncnt++) + { + // printf("%s %d %d %d %d %d %d\n",__FILE__,__LINE__,region->start[0],region->count[0],ndims,fndims,vdesc->record); + for (i = 0; i < fndims; i++) + { + start[i] = 0; + count[i] = 0; + } + if (region) + { + // this is a record based multidimensional array + if (vdesc->record >= 0) + { + for (i = fndims - ndims; i < fndims; i++) + { + start[i] = region->start[i-(fndims-ndims)]; + count[i] = region->count[i-(fndims-ndims)]; + } + + if (fndims>1 && ndims0) + { + count[0] = 1; + start[0] = frame[0]; + } + else if (fndims==ndims) + { + start[0] += vdesc->record; + } + // Non-time dependent array + } + else + { + for (i = 0; i < ndims; i++) + { + start[i] = region->start[i]; + count[i] = region->count[i]; + } + } + } + + switch(file->iotype) + { +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + for (int nv = 0; nv < nvars; nv++) + { + if (vdesc->record >= 0 && ndims < fndims) + { + start[0] = frame[nv]; + } + if (region) + { + bufptr = (void *)((char *) IOBUF + tsize*(nv*llen + region->loffset)); + } + ierr = nc_var_par_access(ncid, vid[nv], NC_COLLECTIVE); + + if (basetype == MPI_DOUBLE ||basetype == MPI_REAL8) + { + ierr = nc_put_vara_double (ncid, vid[nv],(size_t *) start,(size_t *) count, + (const double *)bufptr); + } + else if (basetype == MPI_INTEGER) + { + ierr = nc_put_vara_int (ncid, vid[nv], (size_t *) start, (size_t *) count, + (const int *)bufptr); + } + else if (basetype == MPI_FLOAT || basetype == MPI_REAL4) + { + ierr = nc_put_vara_float (ncid, vid[nv], (size_t *) start, (size_t *) count, + (const float *)bufptr); + } + else + { + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n", + (int)basetype); + } + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + for (i = 0, dsize = 1; i < fndims; i++) + { + dsize *= count[i]; + } + tdsize += dsize; + + if (dsize>0) + { + // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); + startlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); + countlist[rrcnt] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); + for (i = 0; i < fndims; i++) + { + startlist[rrcnt][i]=start[i]; + countlist[rrcnt][i]=count[i]; + } + rrcnt++; + } + if (regioncnt==maxregions-1) + { + //printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,ios->io_rank,iodesc->llen, tdsize); + // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, + // IOBUF, iodesc->llen, iodesc->basetype); + + //printf("%s %d %ld \n",__FILE__,__LINE__,IOBUF); + for (int nv=0; nvvarlist)+vid[nv]; + if (vdesc->record >= 0 && ndimsnreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ) + { + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + + for (int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++) + { + vdesc->request[i]=NC_REQ_NULL; + } + reqn = vdesc->nreqs; + } + else + { + while(vdesc->request[reqn] != NC_REQ_NULL) + { + reqn++; + } + } + ierr = ncmpi_iput_varn(ncid, vid[nv], rrcnt, startlist, countlist, + bufptr, llen, basetype, vdesc->request+reqn); + /* + ierr = ncmpi_bput_varn(ncid, vid[nv], rrcnt, startlist, countlist, + bufptr, llen, basetype, &(vdesc->request)); + */ + if (vdesc->request[reqn] == NC_REQ_NULL) + { + vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync + } + vdesc->nreqs += reqn+1; + + // printf("%s %d %d %d\n",__FILE__,__LINE__,vdesc->nreqs,vdesc->request[reqn]); + } + for (i=0;iiotype,__FILE__,__LINE__); + } + if (region) + region = region->next; + } // for (regioncnt=0;regioncntmaxregions;regioncnt++){ + } // if (ios->ioproc) + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +#ifdef TIMING + GPTLstop("PIO:write_darray_multi_nc"); +#endif + + return ierr; +} + +/** @brief Write a set of one or more aggregated arrays to output file + * @ingroup PIO_write_darray + * + * This routine is used if aggregation is enabled, data is already on the + * io-tasks + * @param[in] file: a pointer to the open file descriptor for the file that will be written to + * @param[in] nvars: the number of variables to be written with this decomposition + * @param[in] vid: an array of the variable ids to be written + * @param[in] iodesc_ndims: the number of dimensions explicitly in the iodesc + * @param[in] basetype : the basic type of the minimal data unit + * @param[in] gsize : array of the global dimensions of the field to be written + * @param[in] maxregions : max number of blocks to be written from this iotask + * @param[in] firstregion : pointer to the first element of a linked list of region descriptions. + * @param[in] llen : length of the iobuffer on this task for a single field + * @param[in] maxiobuflen : maximum llen participating + * @param[in] num_aiotasks : actual number of iotasks participating + * @param[in] IOBUF: the buffer to be written from this mpi task + * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF + */ +int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const int vid[], + const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset gsize[], + const int maxregions, io_region *firstregion, const PIO_Offset llen, + const int maxiobuflen, const int num_aiotasks, + void *IOBUF, const int frame[]) +{ + iosystem_desc_t *ios; /** Pointer to io system information. */ + var_desc_t *vdesc; + int ierr; + int i; + int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ + int dsize; + MPI_Status status; + PIO_Offset usage; + int fndims; + PIO_Offset tdsize; + int tsize; + int ncid; + tdsize=0; + ierr = PIO_NOERR; +#ifdef TIMING + GPTLstart("PIO:write_darray_multi_nc_serial"); +#endif + + if (!(ios = file->iosystem)) + { + fprintf(stderr,"Failed to find iosystem handle \n"); + return PIO_EBADID; + } + + ncid = file->fh; + + if (!(vdesc = (file->varlist) + vid[0])) + { + fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); + return PIO_EBADID; + } + + /* If async is in use, and this is not an IO task, bcast the parameters. */ + if (ios->async_interface) + { + if (! ios->ioproc) + { + int msg = 0; + + if (ios->comp_rank==0) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + + if (!mpierr) + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + } + + ierr = PIOc_inq_varndims(file->fh, vid[0], &fndims); + MPI_Type_size(basetype, &tsize); + + if (ios->ioproc) + { + io_region *region; + int regioncnt; + int rrcnt; + void *bufptr; + int buflen, j; + size_t tmp_start[fndims*maxregions]; + size_t tmp_count[fndims*maxregions]; + + int ndims = iodesc_ndims; + + ncid = file->fh; + region = firstregion; + + + rrcnt = 0; + for (regioncnt = 0; regioncnt < maxregions; regioncnt++) + { + for (i = 0; i < fndims; i++) + { + tmp_start[i + regioncnt * fndims] = 0; + tmp_count[i + regioncnt * fndims] = 0; + } + if (region) + { + // this is a record based multidimensional array + if (vdesc->record >= 0) + { + for (i = fndims - ndims; i < fndims; i++) + { + tmp_start[i + regioncnt * fndims] = region->start[i - (fndims - ndims)]; + tmp_count[i + regioncnt * fndims] = region->count[i - (fndims - ndims)]; + } + // Non-time dependent array + } + else + { + for (i = 0; i < ndims; i++) + { + tmp_start[i + regioncnt * fndims] = region->start[i]; + tmp_count[i + regioncnt * fndims] = region->count[i]; + } + } + region = region->next; + } + } + if (ios->io_rank > 0) + { + mpierr = MPI_Recv(&ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve + MPI_Send(&llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); + if (llen>0) + { + MPI_Send(&maxregions, 1, MPI_INT, 0, ios->io_rank+ios->num_iotasks, ios->io_comm); + MPI_Send(tmp_start, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+2*ios->num_iotasks, ios->io_comm); + MPI_Send(tmp_count, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+3*ios->num_iotasks, ios->io_comm); + // printf("%s %d %ld\n",__FILE__,__LINE__,nvars*llen); + MPI_Send(IOBUF, nvars*llen, basetype, 0, ios->io_rank+4*ios->num_iotasks, ios->io_comm); + } + } + else + { + size_t rlen; + int rregions; + size_t start[fndims], count[fndims]; + size_t loffset; + mpierr = MPI_Type_size(basetype, &dsize); + + for (int rtask=0; rtasknum_iotasks; rtask++) + { + if (rtask>0) + { + mpierr = MPI_Send(&ierr, 1, MPI_INT, rtask, 0, ios->io_comm); // handshake - tell the sending task I'm ready + MPI_Recv(&rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); + if (rlen>0){ + MPI_Recv(&rregions, 1, MPI_INT, rtask, rtask+ios->num_iotasks, ios->io_comm, &status); + MPI_Recv(tmp_start, rregions*fndims, MPI_OFFSET, rtask, rtask+2*ios->num_iotasks, ios->io_comm, &status); + MPI_Recv(tmp_count, rregions*fndims, MPI_OFFSET, rtask, rtask+3*ios->num_iotasks, ios->io_comm, &status); + // printf("%s %d %d %ld\n",__FILE__,__LINE__,rtask,nvars*rlen); + MPI_Recv(IOBUF, nvars*rlen, basetype, rtask, rtask+4*ios->num_iotasks, ios->io_comm, &status); + } + } + else + { + rlen = llen; + rregions = maxregions; + } + if (rlen>0) + { + loffset = 0; + for (regioncnt=0;regioncntrecord>=0) + { + if (fndims>1 && ndims0) + { + count[0] = 1; + start[0] = frame[nv]; + } + else if (fndims==ndims) + { + start[0]+=vdesc->record; + } + } + + if (basetype == MPI_INTEGER) + { + ierr = nc_put_vara_int (ncid, vid[nv], start, count, (const int *) bufptr); + } + else if (basetype == MPI_DOUBLE || basetype == MPI_REAL8) + { + ierr = nc_put_vara_double (ncid, vid[nv], start, count, (const double *) bufptr); + } + else if (basetype == MPI_FLOAT || basetype == MPI_REAL4) + { + ierr = nc_put_vara_float (ncid,vid[nv], start, count, (const float *) bufptr); + } + else + { + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) basetype); + } + + if (ierr != PIO_NOERR){ + for (i=0;imaxregions;regioncnt++){ + } // if (rlen>0) + } // for (int rtask=0; rtasknum_iotasks; rtask++){ + + } + } // if (ios->ioproc) + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +#ifdef TIMING + GPTLstop("PIO:write_darray_multi_nc_serial"); +#endif + + return ierr; +} + +/** Write one or more arrays with the same IO decomposition to the file + * @ingroup PIO_write_darray + * + * @param ncid identifies the netCDF file + * @param vid + * @param ioid + * @param nvars number of variables + * @param arraylen + * @param array + * @param frame + * @param fillvalue + * @param flushtodisk + */ +int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, + const int nvars, const PIO_Offset arraylen, + void *array, const int frame[], void *fillvalue[], + bool flushtodisk) +{ + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; + io_desc_t *iodesc; + + int vsize, rlen; + int ierr; + var_desc_t *vdesc0; + + ierr = PIO_NOERR; + + file = pio_get_file_from_id(ncid); + if (file == NULL) + { + fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); + return PIO_EBADID; + } + if (! (file->mode & PIO_WRITE)) + { + fprintf(stderr,"ERROR: Attempt to write to read-only file\n"); + return PIO_EPERM; + } + + iodesc = pio_get_iodesc_from_id(ioid); + if (iodesc == NULL) + { + // print_trace(NULL); + //fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); + return PIO_EBADID; + } + + vdesc0 = file->varlist+vid[0]; + + pioassert(nvars>0,"nvars <= 0",__FILE__,__LINE__); + + ios = file->iosystem; + // rlen = iodesc->llen*nvars; + rlen=0; + if (iodesc->llen>0) + { + rlen = iodesc->maxiobuflen*nvars; + } + if (vdesc0->iobuf) + { + piodie("Attempt to overwrite existing io buffer",__FILE__,__LINE__); + } + if (iodesc->rearranger>0) + { + if (rlen>0) + { + MPI_Type_size(iodesc->basetype, &vsize); + //printf("rlen*vsize = %ld\n",rlen*vsize); + + vdesc0->iobuf = bget((size_t) vsize* (size_t) rlen); + if (vdesc0->iobuf==NULL) + { + printf("%s %d %d %ld\n",__FILE__,__LINE__,nvars,vsize*rlen); + piomemerror(*ios,(size_t) rlen*(size_t) vsize, __FILE__,__LINE__); + } + if (iodesc->needsfill && iodesc->rearranger==PIO_REARR_BOX) + { + if (vsize==4) + { + for (int nv=0;nv < nvars; nv++) + { + for (int i=0;imaxiobuflen;i++) + { + ((float *) vdesc0->iobuf)[i+nv*(iodesc->maxiobuflen)] = ((float *)fillvalue)[nv]; + } + } + } + else if (vsize==8) + { + for (int nv=0;nv < nvars; nv++) + { + for (int i=0;imaxiobuflen;i++) + { + ((double *)vdesc0->iobuf)[i+nv*(iodesc->maxiobuflen)] = ((double *)fillvalue)[nv]; + } + } + } + } + } + + ierr = rearrange_comp2io(*ios, iodesc, array, vdesc0->iobuf, nvars); + }/* this is wrong, need to think about it + else{ + vdesc0->iobuf = array; + } */ + switch(file->iotype) + { + case PIO_IOTYPE_NETCDF4P: + case PIO_IOTYPE_PNETCDF: + ierr = pio_write_darray_multi_nc(file, nvars, vid, + iodesc->ndims, iodesc->basetype, iodesc->gsize, + iodesc->maxregions, iodesc->firstregion, iodesc->llen, + iodesc->maxiobuflen, iodesc->num_aiotasks, + vdesc0->iobuf, frame); + break; + case PIO_IOTYPE_NETCDF4C: + case PIO_IOTYPE_NETCDF: + ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, + iodesc->ndims, iodesc->basetype, iodesc->gsize, + iodesc->maxregions, iodesc->firstregion, iodesc->llen, + iodesc->maxiobuflen, iodesc->num_aiotasks, + vdesc0->iobuf, frame); + if (vdesc0->iobuf) + { + brel(vdesc0->iobuf); + vdesc0->iobuf = NULL; + } + break; + + } + + if (iodesc->rearranger == PIO_REARR_SUBSET && iodesc->needsfill && + iodesc->holegridsize>0) + { + if (vdesc0->fillbuf) + { + piodie("Attempt to overwrite existing buffer",__FILE__,__LINE__); + } + + vdesc0->fillbuf = bget(iodesc->holegridsize*vsize*nvars); + //printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); + if (vsize==4) + { + for (int nv=0;nvholegridsize;i++) + { + ((float *) vdesc0->fillbuf)[i+nv*iodesc->holegridsize] = ((float *) fillvalue)[nv]; + } + } + } + else if (vsize==8) + { + for (int nv=0;nvholegridsize;i++) + { + ((double *) vdesc0->fillbuf)[i+nv*iodesc->holegridsize] = ((double *) fillvalue)[nv]; + } + } + } + switch(file->iotype) + { + case PIO_IOTYPE_PNETCDF: + ierr = pio_write_darray_multi_nc(file, nvars, vid, + iodesc->ndims, iodesc->basetype, iodesc->gsize, + iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, + iodesc->holegridsize, iodesc->num_aiotasks, + vdesc0->fillbuf, frame); + break; + case PIO_IOTYPE_NETCDF4P: + case PIO_IOTYPE_NETCDF4C: + case PIO_IOTYPE_NETCDF: + /* ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, + iodesc->ndims, iodesc->basetype, iodesc->gsize, + iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, + iodesc->holegridsize, iodesc->num_aiotasks, + vdesc0->fillbuf, frame); + */ + /* if (vdesc0->fillbuf != NULL){ + printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); + brel(vdesc0->fillbuf); + vdesc0->fillbuf = NULL; + } + */ + break; + } + } + + flush_output_buffer(file, flushtodisk, 0); + + return ierr; +} + +/** Write a distributed array to the output file. + * + * This routine aggregates output on the compute nodes and only sends + * it to the IO nodes when the compute buffer is full or when a flush + * is triggered. + * + * @param[in] ncid: the ncid of the open netCDF file. + * @param[in] vid: the variable ID returned by PIOc_def_var(). + * @param[in] ioid: the I/O description ID as passed back by + * PIOc_InitDecomp(). + * @param[in] arraylen: the length of the array to be written. This + * is the length of the distrubited array. That is, the length of + * the portion of the data that is on the processor. + * @param[in] array: pointer to the data to be written. This is a + * pointer to the distributed portion of the array that is on this + * processor. + * @param[in] fillvalue: pointer to the fill value to be used for + * missing data. + * + * @returns 0 for success, non-zero error code for failure. + * @ingroup PIO_write_darray + */ +#ifdef PIO_WRITE_BUFFERING +int PIOc_write_darray(const int ncid, const int vid, const int ioid, + const PIO_Offset arraylen, void *array, void *fillvalue) +{ + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; + io_desc_t *iodesc; + var_desc_t *vdesc; + void *bufptr; + size_t rlen; + int ierr; + MPI_Datatype vtype; + wmulti_buffer *wmb; + int tsize; + int *tptr; + void *bptr; + void *fptr; + bool recordvar; + int needsflush; + bufsize totfree, maxfree; + + ierr = PIO_NOERR; + needsflush = 0; // false + file = pio_get_file_from_id(ncid); + if (file == NULL) + { + fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); + return PIO_EBADID; + } + if (! (file->mode & PIO_WRITE)) + { + fprintf(stderr,"ERROR: Attempt to write to read-only file\n"); + return PIO_EPERM; + } + + iodesc = pio_get_iodesc_from_id(ioid); + if (iodesc == NULL) + { + fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); + return PIO_EBADID; + } + ios = file->iosystem; + + vdesc = (file->varlist)+vid; + if (vdesc == NULL) + return PIO_EBADID; + + /* Is this a record variable? */ + recordvar = vdesc->record < 0 ? true : false; + + if (iodesc->ndof != arraylen) + { + fprintf(stderr,"ndof=%ld, arraylen=%ld\n",iodesc->ndof,arraylen); + piodie("ndof != arraylen",__FILE__,__LINE__); + } + wmb = &(file->buffer); + if (wmb->ioid == -1) + { + if (recordvar) + wmb->ioid = ioid; + else + wmb->ioid = -(ioid); + } + else + { + // separate record and non-record variables + if (recordvar) + { + while(wmb->next && wmb->ioid!=ioid) + if (wmb->next!=NULL) + wmb = wmb->next; +#ifdef _PNETCDF + /* flush the previous record before starting a new one. this is collective */ + // if (vdesc->request != NULL && (vdesc->request[0] != NC_REQ_NULL) || + // (wmb->frame != NULL && vdesc->record != wmb->frame[0])){ + // needsflush = 2; // flush to disk + // } +#endif + } + else + { + while(wmb->next && wmb->ioid!= -(ioid)) + { + if (wmb->next!=NULL) + wmb = wmb->next; + } + } + } + if ((recordvar && wmb->ioid != ioid) || (!recordvar && wmb->ioid != -(ioid))) + { + wmb->next = (wmulti_buffer *) bget((bufsize) sizeof(wmulti_buffer)); + if (wmb->next == NULL) + piomemerror(*ios,sizeof(wmulti_buffer), __FILE__,__LINE__); + wmb=wmb->next; + wmb->next=NULL; + if (recordvar) + wmb->ioid = ioid; + else + wmb->ioid = -(ioid); + wmb->validvars=0; + wmb->arraylen=arraylen; + wmb->vid=NULL; + wmb->data=NULL; + wmb->frame=NULL; + wmb->fillvalue=NULL; + } + + MPI_Type_size(iodesc->basetype, &tsize); + // At this point wmb should be pointing to a new or existing buffer + // so we can add the data + // printf("%s %d %X %d %d %d\n",__FILE__,__LINE__,wmb->data,wmb->validvars,arraylen,tsize); + // cn_buffer_report(*ios, true); + bfreespace(&totfree, &maxfree); + if (needsflush == 0) + needsflush = (maxfree <= 1.1*(1+wmb->validvars)*arraylen*tsize ); + MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1, MPI_INT, MPI_MAX, ios->comp_comm); + + if (needsflush > 0 ) + { + // need to flush first + // printf("%s %d %ld %d %ld %ld\n",__FILE__,__LINE__,maxfree, wmb->validvars, (1+wmb->validvars)*arraylen*tsize,totfree); + cn_buffer_report(*ios, true); + + flush_buffer(ncid, wmb, needsflush == 2); // if needsflush == 2 flush to disk otherwise just flush to io node + } + + if (arraylen > 0) + if (!(wmb->data = bgetr(wmb->data, (1+wmb->validvars)*arraylen*tsize))) + piomemerror(*ios, (1+wmb->validvars)*arraylen*tsize, __FILE__, __LINE__); + + if (!(wmb->vid = (int *) bgetr(wmb->vid,sizeof(int)*(1+wmb->validvars)))) + piomemerror(*ios, (1+wmb->validvars)*sizeof(int), __FILE__, __LINE__); + + if (vdesc->record >= 0) + if (!(wmb->frame = (int *)bgetr(wmb->frame, sizeof(int) * (1 + wmb->validvars)))) + piomemerror(*ios, (1+wmb->validvars)*sizeof(int), __FILE__, __LINE__); + + if (iodesc->needsfill) + if (!(wmb->fillvalue = bgetr(wmb->fillvalue,tsize*(1+wmb->validvars)))) + piomemerror(*ios, (1+wmb->validvars)*tsize , __FILE__,__LINE__); + + if (iodesc->needsfill) + { + if (fillvalue) + { + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars,fillvalue, tsize); + } + else + { + vtype = (MPI_Datatype) iodesc->basetype; + if (vtype == MPI_INTEGER) + { + int fill = PIO_FILL_INT; + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); + } + else if (vtype == MPI_FLOAT || vtype == MPI_REAL4) + { + float fill = PIO_FILL_FLOAT; + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); + } + else if (vtype == MPI_DOUBLE || vtype == MPI_REAL8) + { + double fill = PIO_FILL_DOUBLE; + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); + } + else if (vtype == MPI_CHARACTER) + { + char fill = PIO_FILL_CHAR; + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); + } + else + { + fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",vtype); + } + } + + } + + wmb->arraylen = arraylen; + wmb->vid[wmb->validvars]=vid; + bufptr = (void *)((char *) wmb->data + arraylen*tsize*wmb->validvars); + if (arraylen>0) + memcpy(bufptr, array, arraylen*tsize); + /* + if (tsize==8){ + double asum=0.0; + printf("%s %d %d %d %d\n",__FILE__,__LINE__,vid,arraylen,iodesc->ndof); + for (int k=0;kvalidvars,wmb->ioid,vid,bufptr); + + if (wmb->frame!=NULL) + wmb->frame[wmb->validvars]=vdesc->record; + wmb->validvars++; + + // printf("%s %d %d %d %d %d\n",__FILE__,__LINE__,wmb->validvars,iodesc->maxbytes/tsize, iodesc->ndof, iodesc->llen); + if (wmb->validvars >= iodesc->maxbytes/tsize) + PIOc_sync(ncid); + + return ierr; +} +#else + +/** Write a distributed array to the output file. + * @ingroup PIO_write_darray + * + * This version of the routine does not buffer, all data is + * communicated to the io tasks before the routine returns. + * + * @param ncid identifies the netCDF file + * @param vid + * @param ioid + * @param arraylen + * @param array + * @param fillvalue + * + * @return + */ +int PIOc_write_darray(const int ncid, const int vid, const int ioid, + const PIO_Offset arraylen, void *array, void *fillvalue) +{ + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; + io_desc_t *iodesc; + void *iobuf; + size_t rlen; + int tsize; + int ierr; + MPI_Datatype vtype; + + ierr = PIO_NOERR; + + + file = pio_get_file_from_id(ncid); + if (file == NULL) + { + fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); + return PIO_EBADID; + } + iodesc = pio_get_iodesc_from_id(ioid); + if (iodesc == NULL) + { + fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); + return PIO_EBADID; + } + iobuf = NULL; + + ios = file->iosystem; + + rlen = iodesc->llen; + if (iodesc->rearranger>0) + { + if (rlen>0) + { + MPI_Type_size(iodesc->basetype, &tsize); + // iobuf = bget(tsize*rlen); + iobuf = malloc((size_t) tsize*rlen); + if (!iobuf) + piomemerror(*ios, rlen * (size_t)tsize, __FILE__, __LINE__); + } + // printf(" rlen = %d %ld\n",rlen,iobuf); + + // } + + ierr = rearrange_comp2io(*ios, iodesc, array, iobuf, 1); + + printf("%s %d ",__FILE__,__LINE__); + for (int n=0;n<4;n++) + printf(" %d ",((int *) iobuf)[n]); + printf("\n"); + + } + else + { + iobuf = array; + } + switch(file->iotype) + { + case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_NETCDF: + case PIO_IOTYPE_NETCDF4P: + case PIO_IOTYPE_NETCDF4C: + ierr = pio_write_darray_nc(file, iodesc, vid, iobuf, fillvalue); + } + + if (iodesc->rearranger>0 && rlen>0) + free(iobuf); + + return ierr; +} +#endif + +/** Read an array of data from a file to the (parallel) IO library. + * @ingroup PIO_read_darray + * + * @param file + * @param iodesc + * @param vid + * @param IOBUF + */ +int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, + void *IOBUF) +{ + int ierr=PIO_NOERR; + iosystem_desc_t *ios; /** Pointer to io system information. */ + var_desc_t *vdesc; + int ndims, fndims; + MPI_Status status; + int i; + +#ifdef TIMING + GPTLstart("PIO:read_darray_nc"); +#endif + ios = file->iosystem; + if (ios == NULL) + return PIO_EBADID; + + vdesc = (file->varlist)+vid; + + if (vdesc == NULL) + return PIO_EBADID; + + ndims = iodesc->ndims; + ierr = PIOc_inq_varndims(file->fh, vid, &fndims); + + if (fndims==ndims) + vdesc->record=-1; + + if (ios->ioproc) + { + io_region *region; + size_t start[fndims]; + size_t count[fndims]; + size_t tmp_start[fndims]; + size_t tmp_count[fndims]; + size_t tmp_bufsize=1; + int regioncnt; + void *bufptr; + int tsize; + + int rrlen=0; + PIO_Offset *startlist[iodesc->maxregions]; + PIO_Offset *countlist[iodesc->maxregions]; + + // buffer is incremented by byte and loffset is in terms of the iodessc->basetype + // so we need to multiply by the size of the basetype + // We can potentially allow for one iodesc to have multiple datatypes by allowing the + // calling program to change the basetype. + region = iodesc->firstregion; + MPI_Type_size(iodesc->basetype, &tsize); + if (fndims>ndims) + { + ndims++; + if (vdesc->record<0) + vdesc->record=0; + } + for (regioncnt=0;regioncntmaxregions;regioncnt++) + { + // printf("%s %d %d %ld %d %d\n",__FILE__,__LINE__,regioncnt,region,fndims,ndims); + tmp_bufsize=1; + if (region==NULL || iodesc->llen==0) + { + for (i=0;iloffset); + + // printf("%s %d %d %d %d\n",__FILE__,__LINE__,iodesc->llen - region->loffset, iodesc->llen, region->loffset); + + if (vdesc->record >= 0 && fndims>1) + { + start[0] = vdesc->record; + for (i=1;istart[i-1]; + count[i] = region->count[i-1]; + // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); + } + if (count[1] > 0) + count[0] = 1; + } + else + { + // Non-time dependent array + for (i=0;istart[i]; + count[i] = region->count[i]; + // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); + } + } + } + + switch(file->iotype) + { +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) + { + ierr = nc_get_vara_double (file->fh, vid,start,count, bufptr); + } + else if (iodesc->basetype == MPI_INTEGER) + { + ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); + } + else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4) + { + ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); + } + else + { + fprintf(stderr,"Type not recognized %d in pioc_read_darray\n",(int) iodesc->basetype); + } + break; +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + { + tmp_bufsize=1; + for (int j = 0; j < fndims; j++) + tmp_bufsize *= count[j]; + + if (tmp_bufsize > 0) + { + startlist[rrlen] = (PIO_Offset *) bget(fndims * sizeof(PIO_Offset)); + countlist[rrlen] = (PIO_Offset *) bget(fndims * sizeof(PIO_Offset)); + + for (int j = 0; j < fndims; j++) + { + startlist[rrlen][j] = start[j]; + countlist[rrlen][j] = count[j]; + /* printf("%s %d %d %d %d %ld %ld %ld\n",__FILE__,__LINE__,realregioncnt, + iodesc->maxregions, j,start[j],count[j],tmp_bufsize);*/ + } + rrlen++; + } + if (regioncnt==iodesc->maxregions-1) + { + ierr = ncmpi_get_varn_all(file->fh, vid, rrlen, startlist, + countlist, IOBUF, iodesc->llen, iodesc->basetype); + for (i=0;iiotype,__FILE__,__LINE__); + + } + if (region) + region = region->next; + } // for (regioncnt=0;...) + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +#ifdef TIMING + GPTLstop("PIO:read_darray_nc"); +#endif + + return ierr; +} + +/** Read an array of data from a file to the (serial) IO library. + * @ingroup PIO_read_darray + * + * @param file + * @param iodesc + * @param vid + * @param IOBUF + * + * @returns + */ +int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, + const int vid, void *IOBUF) +{ + int ierr=PIO_NOERR; + iosystem_desc_t *ios; /** Pointer to io system information. */ + var_desc_t *vdesc; + int ndims, fndims; + MPI_Status status; + int i; + +#ifdef TIMING + GPTLstart("PIO:read_darray_nc_serial"); +#endif + ios = file->iosystem; + if (ios == NULL) + return PIO_EBADID; + + vdesc = (file->varlist)+vid; + + if (vdesc == NULL) + return PIO_EBADID; + + ndims = iodesc->ndims; + ierr = PIOc_inq_varndims(file->fh, vid, &fndims); + + if (fndims==ndims) + vdesc->record=-1; + + if (ios->ioproc) + { + io_region *region; + size_t start[fndims]; + size_t count[fndims]; + size_t tmp_start[fndims * iodesc->maxregions]; + size_t tmp_count[fndims * iodesc->maxregions]; + size_t tmp_bufsize; + int regioncnt; + void *bufptr; + int tsize; + + int rrlen = 0; + + // buffer is incremented by byte and loffset is in terms of the iodessc->basetype + // so we need to multiply by the size of the basetype + // We can potentially allow for one iodesc to have multiple datatypes by allowing the + // calling program to change the basetype. + region = iodesc->firstregion; + MPI_Type_size(iodesc->basetype, &tsize); + if (fndims>ndims) + { + if (vdesc->record < 0) + vdesc->record = 0; + } + for (regioncnt=0;regioncntmaxregions;regioncnt++) + { + if (region==NULL || iodesc->llen==0) + { + for (i = 0; i < fndims; i++) + { + tmp_start[i + regioncnt * fndims] = 0; + tmp_count[i + regioncnt * fndims] = 0; + } + bufptr=NULL; + } + else + { + if (vdesc->record >= 0 && fndims>1) + { + tmp_start[regioncnt*fndims] = vdesc->record; + for (i=1;istart[i-1]; + tmp_count[i+regioncnt*fndims] = region->count[i-1]; + } + if (tmp_count[1 + regioncnt * fndims] > 0) + tmp_count[regioncnt * fndims] = 1; + } + else + { + // Non-time dependent array + for (i = 0; i < fndims; i++) + { + tmp_start[i + regioncnt * fndims] = region->start[i]; + tmp_count[i + regioncnt * fndims] = region->count[i]; + } + } + /* for (i=0;inext; + } // for (regioncnt=0;...) + + if (ios->io_rank>0) + { + MPI_Send(&(iodesc->llen), 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); + if (iodesc->llen > 0) + { + MPI_Send(&(iodesc->maxregions), 1, MPI_INT, 0, + ios->num_iotasks + ios->io_rank, ios->io_comm); + MPI_Send(tmp_count, iodesc->maxregions*fndims, MPI_OFFSET, 0, + 2 * ios->num_iotasks + ios->io_rank, ios->io_comm); + MPI_Send(tmp_start, iodesc->maxregions*fndims, MPI_OFFSET, 0, + 3 * ios->num_iotasks + ios->io_rank, ios->io_comm); + MPI_Recv(IOBUF, iodesc->llen, iodesc->basetype, 0, + 4 * ios->num_iotasks+ios->io_rank, ios->io_comm, &status); + } + } + else if (ios->io_rank == 0) + { + int maxregions=0; + size_t loffset, regionsize; + size_t this_start[fndims*iodesc->maxregions]; + size_t this_count[fndims*iodesc->maxregions]; + // for (i=ios->num_iotasks-1; i>=0; i--){ + for (int rtask = 1; rtask <= ios->num_iotasks; rtask++) + { + if (rtasknum_iotasks) + { + MPI_Recv(&tmp_bufsize, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); + if (tmp_bufsize>0) + { + MPI_Recv(&maxregions, 1, MPI_INT, rtask, ios->num_iotasks+rtask, + ios->io_comm, &status); + MPI_Recv(this_count, maxregions*fndims, MPI_OFFSET, rtask, + 2 * ios->num_iotasks + rtask, ios->io_comm, &status); + MPI_Recv(this_start, maxregions*fndims, MPI_OFFSET, rtask, + 3 * ios->num_iotasks + rtask, ios->io_comm, &status); + } + } + else + { + maxregions=iodesc->maxregions; + tmp_bufsize=iodesc->llen; + } + loffset = 0; + for (regioncnt=0;regioncntnum_iotasks) + { + for (int m=0; mbasetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) + { + ierr = nc_get_vara_double (file->fh, vid,start, count, bufptr); + } + else if (iodesc->basetype == MPI_INTEGER) + { + ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); + } + else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4) + { + ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); + } + else + { + fprintf(stderr,"Type not recognized %d in pioc_write_darray_nc_serial\n", + (int)iodesc->basetype); + } + + if (ierr != PIO_NOERR) + { + for (int i = 0; i < fndims; i++) + fprintf(stderr,"vid %d dim %d start %ld count %ld err %d\n", + vid, i, start[i], count[i], ierr); + + } + +#endif + } + if (rtask < ios->num_iotasks) + MPI_Send(IOBUF, tmp_bufsize, iodesc->basetype, rtask, + 4 * ios->num_iotasks + rtask, ios->io_comm); + } + } + } + + ierr = check_netcdf(file, ierr, __FILE__, __LINE__); +#ifdef TIMING + GPTLstop("PIO:read_darray_nc_serial"); +#endif + + return ierr; +} + +/** Read a field from a file to the IO library. + * @ingroup PIO_read_darray + * + * @param ncid identifies the netCDF file + * @param vid + * @param ioid + * @param arraylen + * @param array + * + * @return + */ +int PIOc_read_darray(const int ncid, const int vid, const int ioid, + const PIO_Offset arraylen, void *array) +{ + iosystem_desc_t *ios; /** Pointer to io system information. */ + file_desc_t *file; + io_desc_t *iodesc; + void *iobuf=NULL; + size_t rlen=0; + int ierr, tsize; + MPI_Datatype vtype; + + file = pio_get_file_from_id(ncid); + + if (file == NULL) + { + fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); + return PIO_EBADID; + } + iodesc = pio_get_iodesc_from_id(ioid); + if (iodesc == NULL) + { + fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); + return PIO_EBADID; + } + ios = file->iosystem; + if (ios->iomaster) + { + rlen = iodesc->maxiobuflen; + } + else + { + rlen = iodesc->llen; + } + + if (iodesc->rearranger > 0) + { + if (ios->ioproc && rlen>0) + { + MPI_Type_size(iodesc->basetype, &tsize); + iobuf = bget(((size_t) tsize)*rlen); + if (iobuf==NULL) + { + piomemerror(*ios,rlen*((size_t) tsize), __FILE__,__LINE__); + } + } + } + else + { + iobuf = array; + } + + switch(file->iotype) + { + case PIO_IOTYPE_NETCDF: + case PIO_IOTYPE_NETCDF4C: + ierr = pio_read_darray_nc_serial(file, iodesc, vid, iobuf); + break; + case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_NETCDF4P: + ierr = pio_read_darray_nc(file, iodesc, vid, iobuf); + break; + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); + } + if (iodesc->rearranger > 0) + { + ierr = rearrange_io2comp(*ios, iodesc, iobuf, array); + + if (rlen>0) + brel(iobuf); + } + + return ierr; + +} + +/** Flush the output buffer. + * + * @param file + * @param force + * @param addsize + * + * @return + */ +int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) +{ + var_desc_t *vdesc; + int ierr=PIO_NOERR; +#ifdef _PNETCDF + int *status; + PIO_Offset usage = 0; +#ifdef TIMING + GPTLstart("PIO:flush_output_buffer"); +#endif + pioassert(file!=NULL,"file pointer not defined",__FILE__,__LINE__); + + + ierr = ncmpi_inq_buffer_usage(file->fh, &usage); + + if (!force && file->iosystem->io_comm != MPI_COMM_NULL) + { + usage += addsize; + + MPI_Allreduce(MPI_IN_PLACE, &usage, 1, MPI_OFFSET, MPI_MAX, + file->iosystem->io_comm); + } + + if (usage > maxusage) + { + maxusage = usage; + } + if (force || usage>=PIO_BUFFER_SIZE_LIMIT) + { + int rcnt; + bool prev_dist=false; + int prev_record=-1; + int prev_type=0; + int maxreq; + int reqcnt; + maxreq = 0; + reqcnt=0; + rcnt=0; + for (int i = 0; i < PIO_MAX_VARS; i++) + { + vdesc = file->varlist + i; + reqcnt += vdesc->nreqs; + if (vdesc->nreqs > 0) + maxreq = i; + } + int request[reqcnt]; + int status[reqcnt]; + + for (int i = 0; i <= maxreq; i++) + { + vdesc = file->varlist + i; +#ifdef MPIO_ONESIDED + /*onesided optimization requires that all of the requests in a wait_all call represent + a contiguous block of data in the file */ + if (rcnt>0 && (prev_record != vdesc->record || vdesc->nreqs==0)) + { + ierr = ncmpi_wait_all(file->fh, rcnt, request,status); + rcnt=0; + } + prev_record = vdesc->record; +#endif + // printf("%s %d %d %d %d \n",__FILE__,__LINE__,i,vdesc->nreqs,vdesc->request); + for (reqcnt=0;reqcntnreqs;reqcnt++) + { + request[rcnt++] = max(vdesc->request[reqcnt],NC_REQ_NULL); + } + free(vdesc->request); + vdesc->request = NULL; + vdesc->nreqs = 0; + // if (file->iosystem->io_rank < 2) printf("%s %d varid=%d\n",__FILE__,__LINE__,i); +#ifdef FLUSH_EVERY_VAR + ierr = ncmpi_wait_all(file->fh, rcnt, request, status); + rcnt = 0; +#endif + } + // if (file->iosystem->io_rank==0){ + // printf("%s %d %d\n",__FILE__,__LINE__,rcnt); + // } + if (rcnt > 0) + { + /* + if (file->iosystem->io_rank==0){ + printf("%s %d %d ",__FILE__,__LINE__,rcnt); + for (int i=0; ifh, rcnt, request, status); + } + for (int i = 0; i < PIO_MAX_VARS; i++) + { + vdesc = file->varlist + i; + if (vdesc->iobuf) + { + brel(vdesc->iobuf); + vdesc->iobuf=NULL; + } + if (vdesc->fillbuf) + { + brel(vdesc->fillbuf); + vdesc->fillbuf=NULL; + } + } + + } +#ifdef TIMING + GPTLstop("PIO:flush_output_buffer"); +#endif + +#endif + return ierr; +} + +/** Print out info about the buffer for debug purposes. + * + * @param ios the IO system structure + * @param collective true if collective report is desired + */ +void cn_buffer_report(iosystem_desc_t ios, bool collective) +{ + + if (CN_bpool) + { + long bget_stats[5]; + long bget_mins[5]; + long bget_maxs[5]; + + bstats(bget_stats, bget_stats+1,bget_stats+2,bget_stats+3,bget_stats+4); + if (collective) + { + MPI_Reduce(bget_stats, bget_maxs, 5, MPI_LONG, MPI_MAX, 0, ios.comp_comm); + MPI_Reduce(bget_stats, bget_mins, 5, MPI_LONG, MPI_MIN, 0, ios.comp_comm); + if (ios.compmaster) + { + printf("PIO: Currently allocated buffer space %ld %ld\n", + bget_mins[0], bget_maxs[0]); + printf("PIO: Currently available buffer space %ld %ld\n", + bget_mins[1], bget_maxs[1]); + printf("PIO: Current largest free block %ld %ld\n", + bget_mins[2], bget_maxs[2]); + printf("PIO: Number of successful bget calls %ld %ld\n", + bget_mins[3], bget_maxs[3]); + printf("PIO: Number of successful brel calls %ld %ld\n", + bget_mins[4], bget_maxs[4]); + // print_trace(stdout); + } + } + else + { + printf("%d: PIO: Currently allocated buffer space %ld \n", + ios.union_rank, bget_stats[0]) ; + printf("%d: PIO: Currently available buffer space %ld \n", + ios.union_rank, bget_stats[1]); + printf("%d: PIO: Current largest free block %ld \n", + ios.union_rank, bget_stats[2]); + printf("%d: PIO: Number of successful bget calls %ld \n", + ios.union_rank, bget_stats[3]); + printf("%d: PIO: Number of successful brel calls %ld \n", + ios.union_rank, bget_stats[4]); + } + } +} + +/** Free the buffer pool. + * + * @param ios + */ +void free_cn_buffer_pool(iosystem_desc_t ios) +{ +#ifndef PIO_USE_MALLOC + if (CN_bpool) + { + cn_buffer_report(ios, true); + bpoolrelease(CN_bpool); + // free(CN_bpool); + CN_bpool = NULL; + } +#endif +} + +/** Flush the buffer. + * + * @param ncid + * @param wmb + * @param flushtodisk + */ +void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) +{ + if (wmb->validvars > 0) + { + PIOc_write_darray_multi(ncid, wmb->vid, wmb->ioid, wmb->validvars, + wmb->arraylen, wmb->data, wmb->frame, + wmb->fillvalue, flushtodisk); + wmb->validvars = 0; + brel(wmb->vid); + wmb->vid = NULL; + brel(wmb->data); + wmb->data = NULL; + if (wmb->fillvalue) + brel(wmb->fillvalue); + if (wmb->frame) + brel(wmb->frame); + wmb->fillvalue = NULL; + wmb->frame = NULL; + } +} + +/** Comput the maximum aggregate number of bytes. + * + * @param ios + * @param iodesc + */ +void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc) +{ + int maxbytesoniotask = INT_MAX; + int maxbytesoncomputetask = INT_MAX; + int maxbytes; + + // printf("%s %d %d %d\n",__FILE__,__LINE__,iodesc->maxiobuflen, iodesc->ndof); + + if (ios.ioproc && iodesc->maxiobuflen > 0) + maxbytesoniotask = PIO_BUFFER_SIZE_LIMIT / iodesc->maxiobuflen; + + if (ios.comp_rank >= 0 && iodesc->ndof > 0) + maxbytesoncomputetask = PIO_CNBUFFER_LIMIT / iodesc->ndof; + + maxbytes = min(maxbytesoniotask, maxbytesoncomputetask); + + // printf("%s %d %d %d\n",__FILE__,__LINE__,maxbytesoniotask, maxbytesoncomputetask); + + MPI_Allreduce(MPI_IN_PLACE, &maxbytes, 1, MPI_INT, MPI_MIN, ios.union_comm); + iodesc->maxbytes = maxbytes; + // printf("%s %d %d %d\n",__FILE__,__LINE__,iodesc->maxbytes,iodesc->maxiobuflen); + +} From 80dfb558660ceb72f09f8c9d7cef7bbf57af0f28 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 21 Jun 2016 07:58:30 -0600 Subject: [PATCH 168/184] added test_darray_async.c for async darray testing --- tests/unit/CMakeLists.txt | 3 + tests/unit/test_darray_async.c | 261 +++++++++++++++++++++++++++++++++ 2 files changed, 264 insertions(+) create mode 100644 tests/unit/test_darray_async.c diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 0dbe18565c5..12999f2518e 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -47,6 +47,9 @@ if (PIO_ENABLE_ASYNC AND NOT PIO_USE_MPI_SERIAL) add_executable (test_intercomm EXCLUDE_FROM_ALL test_intercomm.c) target_link_libraries (test_intercomm pioc) add_dependencies (tests test_intercomm) + add_executable (test_darray_async EXCLUDE_FROM_ALL test_darray_async.c) + target_link_libraries (test_darray_async pioc) + add_dependencies (tests test_darray_async) endif () add_executable (test_names EXCLUDE_FROM_ALL test_names.c) target_link_libraries (test_names pioc) diff --git a/tests/unit/test_darray_async.c b/tests/unit/test_darray_async.c new file mode 100644 index 00000000000..f589f01e3a5 --- /dev/null +++ b/tests/unit/test_darray_async.c @@ -0,0 +1,261 @@ +/** + * @file + * Tests for darray functions. + * @author Ed Hartnett + * + */ +#include +#ifdef TIMING +#include +#endif + +#define NUM_NETCDF_FLAVORS 4 +#define NDIM 3 +#define X_DIM_LEN 400 +#define Y_DIM_LEN 400 +#define NUM_TIMESTEPS 6 +#define VAR_NAME "foo" +#define ATT_NAME "bar" +#define START_DATA_VAL 42 +#define ERR_AWFUL 1111 +#define VAR_CACHE_SIZE (1024 * 1024) +#define VAR_CACHE_NELEMS 10 +#define VAR_CACHE_PREEMPTION 0.5 + +/** Handle MPI errors. This should only be used with MPI library + * function calls. */ +#define MPIERR(e) do { \ + MPI_Error_string(e, err_buffer, &resultlen); \ + fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \ + MPI_Finalize(); \ + return ERR_AWFUL; \ + } while (0) + +/** Handle non-MPI errors by finalizing the MPI library and exiting + * with an exit code. */ +#define ERR(e) do { \ + fprintf(stderr, "Error %d in %s, line %d\n", e, __FILE__, __LINE__); \ + MPI_Finalize(); \ + return e; \ + } while (0) + +/** Global err buffer for MPI. */ +char err_buffer[MPI_MAX_ERROR_STRING]; +int resultlen; + +/** The dimension names. */ +char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"}; + +/** Length of the dimensions in the sample data. */ +int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; + +/** Run Tests for darray Functions. + * + * @param argc argument count + * @param argv array of arguments + */ +int +main(int argc, char **argv) +{ + int verbose = 1; + + /** Zero-based rank of processor. */ + int my_rank; + + /** Number of processors involved in current execution. */ + int ntasks; + + /** Specifies the flavor of netCDF output format. */ + int iotype; + + /** Different output flavors. */ + int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, + PIO_IOTYPE_NETCDF, + PIO_IOTYPE_NETCDF4C, + PIO_IOTYPE_NETCDF4P}; + + /** Names for the output files. */ + char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_darray_pnetcdf.nc", + "test_darray_classic.nc", + "test_darray_serial4.nc", + "test_darray_parallel4.nc"}; + + /** Number of processors that will do IO. In this test we + * will do IO from all processors. */ + int niotasks; + + /** Stride in the mpi rank between io tasks. Always 1 in this + * test. */ + int ioproc_stride = 1; + + /** Number of the aggregator? Always 0 in this test. */ + int numAggregator = 0; + + /** Zero based rank of first processor to be used for I/O. */ + int ioproc_start = 0; + + /** The dimension IDs. */ + int dimids[NDIM]; + + /** Array index per processing unit. */ + PIO_Offset elements_per_pe; + + /** The ID for the parallel I/O system. */ + int iosysid; + + /** The ncid of the netCDF file. */ + int ncid; + + /** The ID of the netCDF varable. */ + int varid; + + /** The I/O description ID. */ + int ioid; + + /** A buffer for sample data. */ + float *buffer; + + /** A buffer for reading data back from the file. */ + int *read_buffer; + + /** The decomposition mapping. */ + PIO_Offset *compdof; + + /** Return code. */ + int ret; + + /** Index for loops. */ + int fmt, d, d1, i; + +#ifdef TIMING + /* Initialize the GPTL timing library. */ + if ((ret = GPTLinitialize ())) + return ret; +#endif + + /* Initialize MPI. */ + if ((ret = MPI_Init(&argc, &argv))) + MPIERR(ret); + + /* Learn my rank and the total number of processors. */ + if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) + MPIERR(ret); + if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) + MPIERR(ret); + + /* Check that a valid number of processors was specified. */ + if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || + ntasks == 8 || ntasks == 16)) + fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); + if (verbose) + printf("%d: ParallelIO Library example1 running on %d processors.\n", + my_rank, ntasks); + + /* keep things simple - 1 iotask per MPI process */ + niotasks = ntasks; + + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, + ioproc_start, PIO_REARR_SUBSET, &iosysid))) + ERR(ret); + + /* Describe the decomposition. This is a 1-based array, so add 1! */ + elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks; + if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + for (i = 0; i < elements_per_pe; i++) { + compdof[i] = my_rank * elements_per_pe + i + 1; + } + + /* Create the PIO decomposition for this test. */ + if (verbose) + printf("rank: %d Creating decomposition...\n", my_rank); + if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe, + compdof, &ioid, NULL, NULL, NULL))) + ERR(ret); + free(compdof); + + /* How many flavors will we be running for? */ + int num_flavors = 0; + int fmtidx = 0; +#ifdef _PNETCDF + num_flavors++; + format[fmtidx++] = PIO_IOTYPE_PNETCDF; +#endif +#ifdef _NETCDF + num_flavors++; + format[fmtidx++] = PIO_IOTYPE_NETCDF; +#endif +#ifdef _NETCDF4 + num_flavors += 2; + format[fmtidx++] = PIO_IOTYPE_NETCDF4C; + format[fmtidx] = PIO_IOTYPE_NETCDF4P; +#endif + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (fmt = 0; fmt < num_flavors; fmt++) + { + /* Create the netCDF output file. */ + if (verbose) + printf("rank: %d Creating sample file %s with format %d...\n", + my_rank, filename[fmt], format[fmt]); + if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt], + PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + if (verbose) + printf("rank: %d Defining netCDF metadata...\n", my_rank); + for (d = 0; d < NDIM; d++) { + if (verbose) + printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank, + dim_name[d], dim_len[d]); + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + } + + /* Define a variable. */ + if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) + ERR(ret); + + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Close the netCDF file. */ + if (verbose) + printf("rank: %d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Put a barrier here to make verbose output look better. */ + if ((ret = MPI_Barrier(MPI_COMM_WORLD))) + MPIERR(ret); + + } + + /* Free the PIO decomposition. */ + if (verbose) + printf("rank: %d Freeing PIO decomposition...\n", my_rank); + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + /* Finalize the IO system. */ + if (verbose) + printf("rank: %d Freeing PIO resources...\n", my_rank); + if ((ret = PIOc_finalize(iosysid))) + ERR(ret); + + /* Finalize the MPI library. */ + MPI_Finalize(); + +#ifdef TIMING + /* Finalize the GPTL timing library. */ + if ((ret = GPTLfinalize ())) + return ret; +#endif + + + return 0; +} From 7cd49c190154509acbae1767f4c92b4e537cab50 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 21 Jun 2016 09:52:39 -0600 Subject: [PATCH 169/184] added config.h include --- src/clib/bget.c | 1 - src/clib/pio_darray_async.c | 38 ++++++++++++++++++------------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/clib/bget.c b/src/clib/bget.c index 752ac219e74..020fb724b76 100644 --- a/src/clib/bget.c +++ b/src/clib/bget.c @@ -397,7 +397,6 @@ BGET CONFIGURATION ================== */ -//#define PIO_USE_MALLOC 1 #ifdef PIO_USE_MALLOC #include #endif diff --git a/src/clib/pio_darray_async.c b/src/clib/pio_darray_async.c index 6ab9a872671..6c9bb5b657a 100644 --- a/src/clib/pio_darray_async.c +++ b/src/clib/pio_darray_async.c @@ -7,10 +7,10 @@ * array. Only by combining the distributed arrays from all processor * can the full array be obtained. * - * @author Jim Edwards - * @bug No Known bugs + * @author Jim Edwards, Ed Hartnett */ +#include #include #include @@ -48,7 +48,7 @@ PIO_Offset PIOc_set_buffer_size_limit(const PIO_Offset limit) */ void compute_buffer_init(iosystem_desc_t ios) { -#ifndef PIO_USE_MALLOC +#if !PIO_USE_MALLOC if (!CN_bpool) { @@ -96,12 +96,12 @@ void compute_buffer_init(iosystem_desc_t ios) int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF, void *fillvalue) { - iosystem_desc_t *ios; /** Pointer to io system information. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ var_desc_t *vdesc; int ndims; - int ierr = PIO_NOERR; /** Return code from function calls. */ + int ierr = PIO_NOERR; /* Return code from function calls. */ int i; - int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ + int mpierr = MPI_SUCCESS; /* Return code from MPI function codes. */ int dsize; MPI_Status status; PIO_Offset usage; @@ -411,11 +411,11 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ const int maxiobuflen, const int num_aiotasks, void *IOBUF, const int frame[]) { - iosystem_desc_t *ios; /** Pointer to io system information. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ var_desc_t *vdesc; int ierr; int i; - int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ + int mpierr = MPI_SUCCESS; /* Return code from MPI function codes. */ int dsize; MPI_Status status; PIO_Offset usage; @@ -686,11 +686,11 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i const int maxiobuflen, const int num_aiotasks, void *IOBUF, const int frame[]) { - iosystem_desc_t *ios; /** Pointer to io system information. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ var_desc_t *vdesc; int ierr; int i; - int mpierr = MPI_SUCCESS; /** Return code from MPI function codes. */ + int mpierr = MPI_SUCCESS; /* Return code from MPI function codes. */ int dsize; MPI_Status status; PIO_Offset usage; @@ -913,7 +913,7 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, void *array, const int frame[], void *fillvalue[], bool flushtodisk) { - iosystem_desc_t *ios; /** Pointer to io system information. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ file_desc_t *file; io_desc_t *iodesc; @@ -1116,7 +1116,7 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) { - iosystem_desc_t *ios; /** Pointer to io system information. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ file_desc_t *file; io_desc_t *iodesc; var_desc_t *vdesc; @@ -1338,7 +1338,7 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) { - iosystem_desc_t *ios; /** Pointer to io system information. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ file_desc_t *file; io_desc_t *iodesc; void *iobuf; @@ -1421,7 +1421,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF) { int ierr=PIO_NOERR; - iosystem_desc_t *ios; /** Pointer to io system information. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ var_desc_t *vdesc; int ndims, fndims; MPI_Status status; @@ -1606,7 +1606,7 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF) { int ierr=PIO_NOERR; - iosystem_desc_t *ios; /** Pointer to io system information. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ var_desc_t *vdesc; int ndims, fndims; MPI_Status status; @@ -1824,7 +1824,7 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int PIOc_read_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array) { - iosystem_desc_t *ios; /** Pointer to io system information. */ + iosystem_desc_t *ios; /* Pointer to io system information. */ file_desc_t *file; io_desc_t *iodesc; void *iobuf=NULL; @@ -2075,7 +2075,7 @@ void cn_buffer_report(iosystem_desc_t ios, bool collective) */ void free_cn_buffer_pool(iosystem_desc_t ios) { -#ifndef PIO_USE_MALLOC +#if !PIO_USE_MALLOC if (CN_bpool) { cn_buffer_report(ios, true); @@ -2083,7 +2083,7 @@ void free_cn_buffer_pool(iosystem_desc_t ios) // free(CN_bpool); CN_bpool = NULL; } -#endif +#endif /* !PIO_USE_MALLOC */ } /** Flush the buffer. @@ -2113,7 +2113,7 @@ void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) } } -/** Comput the maximum aggregate number of bytes. +/** Compute the maximum aggregate number of bytes. * * @param ios * @param iodesc From 23b06b479ae539985f4b27d17bf8038092f5a0be Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 21 Jun 2016 10:09:19 -0600 Subject: [PATCH 170/184] added some comments --- src/clib/pio_darray_async.c | 79 ++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/src/clib/pio_darray_async.c b/src/clib/pio_darray_async.c index 6c9bb5b657a..119f95a3d2f 100644 --- a/src/clib/pio_darray_async.c +++ b/src/clib/pio_darray_async.c @@ -34,9 +34,9 @@ PIO_Offset PIOc_set_buffer_size_limit(const PIO_Offset limit) { PIO_Offset oldsize; oldsize = PIO_BUFFER_SIZE_LIMIT; - if (limit>0) + if (limit > 0) PIO_BUFFER_SIZE_LIMIT = limit; - return(oldsize); + return oldsize; } /** Initialize the compute buffer to size PIO_CNBUFFER_LIMIT. @@ -98,17 +98,18 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, { iosystem_desc_t *ios; /* Pointer to io system information. */ var_desc_t *vdesc; - int ndims; + int ndims; /* Number of dimensions according to iodesc. */ int ierr = PIO_NOERR; /* Return code from function calls. */ - int i; + int i; /* Loop counter. */ int mpierr = MPI_SUCCESS; /* Return code from MPI function codes. */ - int dsize; - MPI_Status status; - PIO_Offset usage; - int fndims; - PIO_Offset tdsize = 0; + int dsize; /* Size of the type. */ + MPI_Status status; /* Status from MPI_Recv calls. */ + PIO_Offset usage; /* Size of current buffer. */ + int fndims; /* Number of dims for variable according to netCDF. */ + PIO_Offset tdsize = 0; /* Total size. */ #ifdef TIMING + /* Start timing this function. */ GPTLstart("PIO:write_darray_nc"); #endif @@ -122,6 +123,9 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, ndims = iodesc->ndims; + /* Get the number of dims for this var from netcdf. */ + ierr = PIOc_inq_varndims(file->fh, vid, &fndims); + /* If async is in use, and this is not an IO task, bcast the parameters. */ if (ios->async_interface) { @@ -137,8 +141,7 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, } } - ierr = PIOc_inq_varndims(file->fh, vid, &fndims); - + /* If this is an IO task, write the data. */ if (ios->ioproc) { io_region *region; @@ -147,52 +150,67 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, int rrcnt; void *bufptr; void *tmp_buf = NULL; - int tsize; - size_t start[fndims]; - size_t count[fndims]; - int buflen, j; + int tsize; /* Type size. */ + size_t start[fndims]; /* Local start array for this task. */ + size_t count[fndims]; /* Local count array for this task. */ + int buflen; + int j; /* Loop counter. */ PIO_Offset *startlist[iodesc->maxregions]; PIO_Offset *countlist[iodesc->maxregions]; + + /* Get the type size (again?) */ MPI_Type_size(iodesc->basetype, &tsize); region = iodesc->firstregion; + /* If this is a var with an unlimited dimension, and the + * iodesc ndims doesn't contain it, then add it to ndims. */ if (vdesc->record >= 0 && ndims < fndims) ndims++; #ifdef _PNETCDF - /* make sure we have room in the buffer. */ + /* Make sure we have room in the buffer. */ if (file->iotype == PIO_IOTYPE_PNETCDF) flush_output_buffer(file, false, tsize * (iodesc->maxiobuflen)); #endif rrcnt = 0; + /* For each region, figure out start/count arrays. */ for (regioncnt = 0; regioncnt < iodesc->maxregions; regioncnt++) { + /* Init arrays to zeros. */ for (i = 0; i < ndims; i++) { start[i] = 0; count[i] = 0; } + if (region) { bufptr = (void *)((char *)IOBUF + tsize * region->loffset); - // this is a record based multidimensional array if (vdesc->record >= 0) { + /* This is a record based multidimensional array. */ + + /* This does not look correct, but will work if + * unlimited dim is dim 0. */ start[0] = vdesc->record; + + /* Set the local start and count arrays. */ for (i = 1; i < ndims; i++) { start[i] = region->start[i - 1]; count[i] = region->count[i - 1]; } + + /* If there is data to be written, write one timestep. */ if (count[1] > 0) count[0] = 1; - // Non-time dependent array } else { + /* Array without unlimited dimension. */ for (i = 0; i < ndims; i++) { start[i] = region->start[i]; @@ -209,6 +227,8 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, /* Use collective writes with this variable. */ ierr = nc_var_par_access(ncid, vid, NC_COLLECTIVE); + + /* Write the data. */ if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) ierr = nc_put_vara_double(ncid, vid, (size_t *)start, (size_t *)count, (const double *)bufptr); @@ -226,8 +246,13 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, #endif /* _NETCDF4 */ case PIO_IOTYPE_NETCDF: { + /* Find the type size (again?) */ mpierr = MPI_Type_size(iodesc->basetype, &dsize); + size_t tstart[ndims], tcount[ndims]; + + /* The IO master task does all the data writes, but + * sends the data to the other IO tasks (why?). */ if (ios->io_rank == 0) { for (i = 0; i < iodesc->num_aiotasks; i++) @@ -245,7 +270,8 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, } else { - mpierr = MPI_Send(&ierr, 1, MPI_INT, i, 0, ios->io_comm); // handshake - tell the sending task I'm ready + /* Handshake - tell the sending task I'm ready. */ + mpierr = MPI_Send(&ierr, 1, MPI_INT, i, 0, ios->io_comm); mpierr = MPI_Recv(&buflen, 1, MPI_INT, i, 1, ios->io_comm, &status); if (buflen > 0) { @@ -258,8 +284,9 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, } } - if (buflen>0) + if (buflen > 0) { + /* Write the data. */ if (iodesc->basetype == MPI_INTEGER) ierr = nc_put_vara_int(ncid, vid, tstart, tcount, (const int *)tmp_buf); else if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) @@ -270,10 +297,12 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, fprintf(stderr,"Type not recognized %d in pioc_write_darray\n", (int)iodesc->basetype); + /* Was there an error from netCDF? */ if (ierr == PIO_EEDGE) for (i = 0; i < ndims; i++) fprintf(stderr,"dim %d start %ld count %ld\n", i, tstart[i], tcount[i]); + /* Free the temporary buffer, if we don't need it any more. */ if (tmp_buf != bufptr) free(tmp_buf); } @@ -315,13 +344,6 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, // if (dsize==1 && ndims==2) // printf("%s %d %d\n",__FILE__,__LINE__,iodesc->basetype); - /* if (regioncnt==0){ - for (i=0;imaxregions;i++){ - startlist[i] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - countlist[i] = (PIO_Offset *) calloc(fndims, sizeof(PIO_Offset)); - } - } - */ if (dsize > 0) { // printf("%s %d %d %d\n",__FILE__,__LINE__,ios->io_rank,dsize); @@ -377,8 +399,11 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, } // for (regioncnt=0;regioncntmaxregions;regioncnt++){ } // if (ios->ioproc) + /* Check the error code returned by netCDF. */ ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + #ifdef TIMING + /* Stop timing this function. */ GPTLstop("PIO:write_darray_nc"); #endif From ecbc513f805b90d0e28388980e2ac6abd5843566 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 21 Jun 2016 10:31:50 -0600 Subject: [PATCH 171/184] more documentation changes --- src/clib/pio_darray_async.c | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/clib/pio_darray_async.c b/src/clib/pio_darray_async.c index 119f95a3d2f..57f30c94d27 100644 --- a/src/clib/pio_darray_async.c +++ b/src/clib/pio_darray_async.c @@ -145,7 +145,6 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, if (ios->ioproc) { io_region *region; - int ncid = file->fh; int regioncnt; int rrcnt; void *bufptr; @@ -226,17 +225,17 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, case PIO_IOTYPE_NETCDF4P: /* Use collective writes with this variable. */ - ierr = nc_var_par_access(ncid, vid, NC_COLLECTIVE); + ierr = nc_var_par_access(file->fh, vid, NC_COLLECTIVE); /* Write the data. */ if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) - ierr = nc_put_vara_double(ncid, vid, (size_t *)start, (size_t *)count, + ierr = nc_put_vara_double(file->fh, vid, (size_t *)start, (size_t *)count, (const double *)bufptr); else if (iodesc->basetype == MPI_INTEGER) - ierr = nc_put_vara_int(ncid, vid, (size_t *)start, (size_t *)count, + ierr = nc_put_vara_int(file->fh, vid, (size_t *)start, (size_t *)count, (const int *)bufptr); else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4) - ierr = nc_put_vara_float(ncid, vid, (size_t *)start, (size_t *)count, + ierr = nc_put_vara_float(file->fh, vid, (size_t *)start, (size_t *)count, (const float *)bufptr); else fprintf(stderr,"Type not recognized %d in pioc_write_darray\n", @@ -288,11 +287,11 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, { /* Write the data. */ if (iodesc->basetype == MPI_INTEGER) - ierr = nc_put_vara_int(ncid, vid, tstart, tcount, (const int *)tmp_buf); + ierr = nc_put_vara_int(file->fh, vid, tstart, tcount, (const int *)tmp_buf); else if (iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8) - ierr = nc_put_vara_double(ncid, vid, tstart, tcount, (const double *)tmp_buf); + ierr = nc_put_vara_double(file->fh, vid, tstart, tcount, (const double *)tmp_buf); else if (iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4) - ierr = nc_put_vara_float(ncid, vid, tstart, tcount, (const float *)tmp_buf); + ierr = nc_put_vara_float(file->fh, vid, tstart, tcount, (const float *)tmp_buf); else fprintf(stderr,"Type not recognized %d in pioc_write_darray\n", (int)iodesc->basetype); @@ -359,7 +358,7 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, if (regioncnt == iodesc->maxregions - 1) { // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,ios->io_rank,iodesc->llen, tdsize); - // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, + // ierr = ncmpi_put_varn_all(file->fh, vid, iodesc->maxregions, startlist, countlist, // IOBUF, iodesc->llen, iodesc->basetype); int reqn = 0; @@ -376,7 +375,7 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, while(vdesc->request[reqn] != NC_REQ_NULL) reqn++; - ierr = ncmpi_bput_varn(ncid, vid, rrcnt, startlist, countlist, + ierr = ncmpi_bput_varn(file->fh, vid, rrcnt, startlist, countlist, IOBUF, iodesc->llen, iodesc->basetype, vdesc->request+reqn); if (vdesc->request[reqn] == NC_REQ_NULL) vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync @@ -394,6 +393,8 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, default: ierr = iotype_error(file->iotype,__FILE__,__LINE__); } + + /* Move to the next region. */ if (region) region = region->next; } // for (regioncnt=0;regioncntmaxregions;regioncnt++){ @@ -1346,19 +1347,19 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, #else /** Write a distributed array to the output file. - * @ingroup PIO_write_darray * - * This version of the routine does not buffer, all data is - * communicated to the io tasks before the routine returns. + * This version of the routine does not buffer, all data is + * communicated to the io tasks before the routine returns. * * @param ncid identifies the netCDF file - * @param vid + * @param vid variable ID * @param ioid - * @param arraylen - * @param array - * @param fillvalue + * @param arraylen total length of array + * @param array pointer to the data to be written + * @param fillvalue fill value to be used when writing data * - * @return + * @return 0 for success, error code otherwise. + * @ingroup PIO_write_darray */ int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) From 31147026271e09366b9e90a37f88bf9f3deddc88 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 21 Jun 2016 10:41:58 -0600 Subject: [PATCH 172/184] more comments, some code cleanup --- src/clib/pio_darray_async.c | 66 ++++++++++++++----------------------- 1 file changed, 25 insertions(+), 41 deletions(-) diff --git a/src/clib/pio_darray_async.c b/src/clib/pio_darray_async.c index 57f30c94d27..276e727458c 100644 --- a/src/clib/pio_darray_async.c +++ b/src/clib/pio_darray_async.c @@ -1365,70 +1365,54 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) { iosystem_desc_t *ios; /* Pointer to io system information. */ - file_desc_t *file; - io_desc_t *iodesc; - void *iobuf; - size_t rlen; - int tsize; - int ierr; + file_desc_t *file; /* Info about the open file. */ + io_desc_t *iodesc; /* Infor about the IO desc? */ + void *iobuf = NULL; /* The data buffer. */ + size_t rlen; + int tsize; /* Size of base MPI type. */ + int ierr = PIO_NOERR; /* Return code from netCDF calls. */ MPI_Datatype vtype; - ierr = PIO_NOERR; - + LOG((1, "PIOc_write_darray ncid = %d vid = %d ioid = %d arraylen = %d", + ncid, vid, ioid, arraylen)); - file = pio_get_file_from_id(ncid); - if (file == NULL) - { - fprintf(stderr,"File handle not found %d %d\n",ncid,__LINE__); + /* Get the file info for this ncid. */ + if (!(file = pio_get_file_from_id(ncid))) return PIO_EBADID; - } - iodesc = pio_get_iodesc_from_id(ioid); - if (iodesc == NULL) + ios = file->iosystem; + + /* Get the iodesc info. */ + if (!(iodesc = pio_get_iodesc_from_id(ioid))) { fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); return PIO_EBADID; } - iobuf = NULL; - - ios = file->iosystem; rlen = iodesc->llen; - if (iodesc->rearranger>0) + if (iodesc->rearranger > 0) { - if (rlen>0) + if (rlen > 0) { + /* Get the size of the base type. */ MPI_Type_size(iodesc->basetype, &tsize); - // iobuf = bget(tsize*rlen); - iobuf = malloc((size_t) tsize*rlen); - if (!iobuf) + + /* Allocate the data buffer. */ + if (!(iobuf = malloc((size_t)tsize * rlen))) piomemerror(*ios, rlen * (size_t)tsize, __FILE__, __LINE__); } - // printf(" rlen = %d %ld\n",rlen,iobuf); - - // } ierr = rearrange_comp2io(*ios, iodesc, array, iobuf, 1); - - printf("%s %d ",__FILE__,__LINE__); - for (int n=0;n<4;n++) - printf(" %d ",((int *) iobuf)[n]); - printf("\n"); - } else { iobuf = array; } - switch(file->iotype) - { - case PIO_IOTYPE_PNETCDF: - case PIO_IOTYPE_NETCDF: - case PIO_IOTYPE_NETCDF4P: - case PIO_IOTYPE_NETCDF4C: - ierr = pio_write_darray_nc(file, iodesc, vid, iobuf, fillvalue); - } - if (iodesc->rearranger>0 && rlen>0) + /* Call the darray_nc function to do the writes. */ + ierr = pio_write_darray_nc(file, iodesc, vid, iobuf, fillvalue); + + /* Free the buffer if necessary. */ + if (iodesc->rearranger > 0 && rlen > 0) free(iobuf); return ierr; From d6bd625db7db25e332b7de9773c88f1cc73c2579 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 21 Jun 2016 11:02:48 -0600 Subject: [PATCH 173/184] removed some dead code, improved comments --- src/clib/pio_darray_async.c | 88 +++++-------------------------------- 1 file changed, 10 insertions(+), 78 deletions(-) diff --git a/src/clib/pio_darray_async.c b/src/clib/pio_darray_async.c index 276e727458c..8dfa074c051 100644 --- a/src/clib/pio_darray_async.c +++ b/src/clib/pio_darray_async.c @@ -14,11 +14,16 @@ #include #include -#define PIO_WRITE_BUFFERING 1 +/* 10MB default limit. */ +PIO_Offset PIO_BUFFER_SIZE_LIMIT = 10485760; -PIO_Offset PIO_BUFFER_SIZE_LIMIT = 10485760; // 10MB default limit +/* Initial size of compute buffer. */ bufsize PIO_CNBUFFER_LIMIT = 33554432; + +/* Global buffer pool pointer. */ static void *CN_bpool = NULL; + +/* Maximum buffer usage. */ static PIO_Offset maxusage = 0; /** Set the pio buffer size limit. This is the size of the data buffer @@ -1138,7 +1143,6 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, * @returns 0 for success, non-zero error code for failure. * @ingroup PIO_write_darray */ -#ifdef PIO_WRITE_BUFFERING int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) { @@ -1344,88 +1348,16 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, return ierr; } -#else - -/** Write a distributed array to the output file. - * - * This version of the routine does not buffer, all data is - * communicated to the io tasks before the routine returns. - * - * @param ncid identifies the netCDF file - * @param vid variable ID - * @param ioid - * @param arraylen total length of array - * @param array pointer to the data to be written - * @param fillvalue fill value to be used when writing data - * - * @return 0 for success, error code otherwise. - * @ingroup PIO_write_darray - */ -int PIOc_write_darray(const int ncid, const int vid, const int ioid, - const PIO_Offset arraylen, void *array, void *fillvalue) -{ - iosystem_desc_t *ios; /* Pointer to io system information. */ - file_desc_t *file; /* Info about the open file. */ - io_desc_t *iodesc; /* Infor about the IO desc? */ - void *iobuf = NULL; /* The data buffer. */ - size_t rlen; - int tsize; /* Size of base MPI type. */ - int ierr = PIO_NOERR; /* Return code from netCDF calls. */ - MPI_Datatype vtype; - - LOG((1, "PIOc_write_darray ncid = %d vid = %d ioid = %d arraylen = %d", - ncid, vid, ioid, arraylen)); - - /* Get the file info for this ncid. */ - if (!(file = pio_get_file_from_id(ncid))) - return PIO_EBADID; - ios = file->iosystem; - - /* Get the iodesc info. */ - if (!(iodesc = pio_get_iodesc_from_id(ioid))) - { - fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); - return PIO_EBADID; - } - - rlen = iodesc->llen; - if (iodesc->rearranger > 0) - { - if (rlen > 0) - { - /* Get the size of the base type. */ - MPI_Type_size(iodesc->basetype, &tsize); - - /* Allocate the data buffer. */ - if (!(iobuf = malloc((size_t)tsize * rlen))) - piomemerror(*ios, rlen * (size_t)tsize, __FILE__, __LINE__); - } - - ierr = rearrange_comp2io(*ios, iodesc, array, iobuf, 1); - } - else - { - iobuf = array; - } - - /* Call the darray_nc function to do the writes. */ - ierr = pio_write_darray_nc(file, iodesc, vid, iobuf, fillvalue); - - /* Free the buffer if necessary. */ - if (iodesc->rearranger > 0 && rlen > 0) - free(iobuf); - - return ierr; -} -#endif /** Read an array of data from a file to the (parallel) IO library. - * @ingroup PIO_read_darray * * @param file * @param iodesc * @param vid * @param IOBUF + * + * @return 0 on success, error code otherwise. + * @ingroup PIO_read_darray */ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF) From ed31fb01305ff4a959a3dabeff6cdcffa810971e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 21 Jun 2016 12:10:33 -0600 Subject: [PATCH 174/184] minor cleanup --- src/clib/pio_darray_async.c | 229 ++++++++++++++++++++++-------------- 1 file changed, 140 insertions(+), 89 deletions(-) diff --git a/src/clib/pio_darray_async.c b/src/clib/pio_darray_async.c index 8dfa074c051..fdb91b8971a 100644 --- a/src/clib/pio_darray_async.c +++ b/src/clib/pio_darray_async.c @@ -1,6 +1,6 @@ /** @file * - * @brief This file contains the routines that read and write + * This file contains the routines that read and write * distributed arrays in PIO. * * When arrays are distributed, each processor holds some of the @@ -47,7 +47,8 @@ PIO_Offset PIOc_set_buffer_size_limit(const PIO_Offset limit) /** Initialize the compute buffer to size PIO_CNBUFFER_LIMIT. * * This routine initializes the compute buffer pool if the bget memory - * management is used. + * management is used. If malloc is used (that is, PIO_USE_MALLOC is + * non zero), this function does nothing. * * @param ios the iosystem descriptor which will use the new buffer */ @@ -82,20 +83,15 @@ void compute_buffer_init(iosystem_desc_t ios) /** Write a single distributed field to output. This routine is only * used if aggregation is off. * - * @param[in] file: a pointer to the open file descriptor for the file + * @param file a pointer to the open file descriptor for the file * that will be written to - * - * @param[in] iodesc: a pointer to the defined iodescriptor for the buffer - * - * @param[in] vid: the variable id to be written - * - * @param[in] IOBUF: the buffer to be written from this mpi task - * - * @param[in] fillvalue: the optional fillvalue to be used for missing + * @param iodesc a pointer to the defined iodescriptor for the buffer + * @param vid the variable id to be written + * @param IOBUF the buffer to be written from this mpi task + * @param fillvalue the optional fillvalue to be used for missing * data in this buffer * * @return 0 for success, error code otherwise. - * * @ingroup PIO_write_darray */ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, @@ -416,31 +412,41 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, return ierr; } -/** Write a set of one or more aggregated arrays to output file - * @ingroup PIO_write_darray +/** Write a set of one or more aggregated arrays to output file. * - * This routine is used if aggregation is enabled, data is already on the - * io-tasks + * This routine is used if aggregation is enabled, data is already on + * the io-tasks + * + * @param file a pointer to the open file descriptor for the file + * that will be written to + * @param nvars the number of variables to be written with this + * decomposition + * @param vid: an array of the variable ids to be written + * @param iodesc_ndims: the number of dimensions explicitly in the + * iodesc + * @param basetype the basic type of the minimal data unit + * @param gsize array of the global dimensions of the field to + * be written + * @param maxregions max number of blocks to be written from + * this iotask + * @param firstregion pointer to the first element of a linked + * list of region descriptions. + * @param llen length of the iobuffer on this task for a single + * field + * @param maxiobuflen maximum llen participating + * @param num_aiotasks actual number of iotasks participating + * @param IOBUF the buffer to be written from this mpi task + * @param frame the frame or record dimension for each of the nvars + * variables in IOBUF * - * @param[in] file: a pointer to the open file descriptor for the file that will be written to - * @param[in] nvars: the number of variables to be written with this decomposition - * @param[in] vid: an array of the variable ids to be written - * @param[in] iodesc_ndims: the number of dimensions explicitly in the iodesc - * @param[in] basetype : the basic type of the minimal data unit - * @param[in] gsize : array of the global dimensions of the field to be written - * @param[in] maxregions : max number of blocks to be written from this iotask - * @param[in] firstregion : pointer to the first element of a linked list of region descriptions. - * @param[in] llen : length of the iobuffer on this task for a single field - * @param[in] maxiobuflen : maximum llen participating - * @param[in] num_aiotasks : actual number of iotasks participating - * @param[in] IOBUF: the buffer to be written from this mpi task - * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF + * @return 0 for success, error code otherwise. + * @ingroup PIO_write_darray */ -int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[], - const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset gsize[], +int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int *vid, + const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset *gsize, const int maxregions, io_region *firstregion, const PIO_Offset llen, const int maxiobuflen, const int num_aiotasks, - void *IOBUF, const int frame[]) + void *IOBUF, const int *frame) { iosystem_desc_t *ios; /* Pointer to io system information. */ var_desc_t *vdesc; @@ -692,30 +698,42 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ return ierr; } -/** @brief Write a set of one or more aggregated arrays to output file - * @ingroup PIO_write_darray +/** Write a set of one or more aggregated arrays to output file in + * serial mode. * - * This routine is used if aggregation is enabled, data is already on the - * io-tasks - * @param[in] file: a pointer to the open file descriptor for the file that will be written to - * @param[in] nvars: the number of variables to be written with this decomposition - * @param[in] vid: an array of the variable ids to be written - * @param[in] iodesc_ndims: the number of dimensions explicitly in the iodesc - * @param[in] basetype : the basic type of the minimal data unit - * @param[in] gsize : array of the global dimensions of the field to be written - * @param[in] maxregions : max number of blocks to be written from this iotask - * @param[in] firstregion : pointer to the first element of a linked list of region descriptions. - * @param[in] llen : length of the iobuffer on this task for a single field - * @param[in] maxiobuflen : maximum llen participating - * @param[in] num_aiotasks : actual number of iotasks participating - * @param[in] IOBUF: the buffer to be written from this mpi task - * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF + * This routine is used if aggregation is enabled, data is already on the + * io-tasks + * + * @param file: a pointer to the open file descriptor for the file + * that will be written to + * @param nvars: the number of variables to be written with this + * decomposition + * @param vid: an array of the variable ids to be written + * @param iodesc_ndims: the number of dimensions explicitly in the + * iodesc + * @param basetype : the basic type of the minimal data unit + * @param gsize : array of the global dimensions of the field to be + * written + * @param maxregions : max number of blocks to be written from this + * iotask + * @param firstregion : pointer to the first element of a linked + * list of region descriptions. + * @param llen : length of the iobuffer on this task for a single + * field + * @param maxiobuflen : maximum llen participating + * @param num_aiotasks : actual number of iotasks participating + * @param IOBUF: the buffer to be written from this mpi task + * @param frame : the frame or record dimension for each of the + * nvars variables in IOBUF + * + * @return 0 for success, error code otherwise. + * @ingroup PIO_write_darray */ -int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const int vid[], - const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset gsize[], +int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const int *vid, + const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset *gsize, const int maxregions, io_region *firstregion, const PIO_Offset llen, const int maxiobuflen, const int num_aiotasks, - void *IOBUF, const int frame[]) + void *IOBUF, const int *frame) { iosystem_desc_t *ios; /* Pointer to io system information. */ var_desc_t *vdesc; @@ -926,22 +944,32 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i return ierr; } -/** Write one or more arrays with the same IO decomposition to the file - * @ingroup PIO_write_darray +/** Write one or more arrays with the same IO decomposition to the file. * * @param ncid identifies the netCDF file - * @param vid - * @param ioid - * @param nvars number of variables - * @param arraylen - * @param array - * @param frame - * @param fillvalue + * @param vid: an array of the variable ids to be written + * @param ioid: the I/O description ID as passed back by + * PIOc_InitDecomp(). + * @param nvars the number of variables to be written with this + * decomposition + * @param arraylen: the length of the array to be written. This + * is the length of the distrubited array. That is, the length of + * the portion of the data that is on the processor. + * @param array: pointer to the data to be written. This is a + * pointer to the distributed portion of the array that is on this + * processor. + * @param frame the frame or record dimension for each of the nvars + * variables in IOBUF + * @param fillvalue: pointer to the fill value to be used for + * missing data. * @param flushtodisk + * + * @return 0 for success, error code otherwise. + * @ingroup PIO_write_darray */ -int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, +int PIOc_write_darray_multi(const int ncid, const int *vid, const int ioid, const int nvars, const PIO_Offset arraylen, - void *array, const int frame[], void *fillvalue[], + void *array, const int *frame, void **fillvalue, bool flushtodisk) { iosystem_desc_t *ios; /* Pointer to io system information. */ @@ -1127,17 +1155,17 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, * it to the IO nodes when the compute buffer is full or when a flush * is triggered. * - * @param[in] ncid: the ncid of the open netCDF file. - * @param[in] vid: the variable ID returned by PIOc_def_var(). - * @param[in] ioid: the I/O description ID as passed back by + * @param ncid: the ncid of the open netCDF file. + * @param vid: the variable ID returned by PIOc_def_var(). + * @param ioid: the I/O description ID as passed back by * PIOc_InitDecomp(). - * @param[in] arraylen: the length of the array to be written. This + * @param arraylen: the length of the array to be written. This * is the length of the distrubited array. That is, the length of * the portion of the data that is on the processor. - * @param[in] array: pointer to the data to be written. This is a + * @param array: pointer to the data to be written. This is a * pointer to the distributed portion of the array that is on this * processor. - * @param[in] fillvalue: pointer to the fill value to be used for + * @param fillvalue: pointer to the fill value to be used for * missing data. * * @returns 0 for success, non-zero error code for failure. @@ -1351,10 +1379,11 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid, /** Read an array of data from a file to the (parallel) IO library. * - * @param file - * @param iodesc - * @param vid - * @param IOBUF + * @param file a pointer to the open file descriptor for the file + * that will be written to + * @param iodesc a pointer to the defined iodescriptor for the buffer + * @param vid the variable id to be read + * @param IOBUF the buffer to be read into from this mpi task * * @return 0 on success, error code otherwise. * @ingroup PIO_read_darray @@ -1537,9 +1566,10 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, /** Read an array of data from a file to the (serial) IO library. * @ingroup PIO_read_darray * - * @param file - * @param iodesc - * @param vid + * @param file a pointer to the open file descriptor for the file + * that will be written to + * @param iodesc a pointer to the defined iodescriptor for the buffer + * @param vid the variable id to be read. * @param IOBUF * * @returns @@ -1756,12 +1786,18 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, * @ingroup PIO_read_darray * * @param ncid identifies the netCDF file - * @param vid - * @param ioid - * @param arraylen - * @param array + * @param vid the variable ID to be read + * @param ioid: the I/O description ID as passed back by + * PIOc_InitDecomp(). + * @param arraylen: the length of the array to be read. This + * is the length of the distrubited array. That is, the length of + * the portion of the data that is on the processor. + * @param array: pointer to the data to be read. This is a + * pointer to the distributed portion of the array that is on this + * processor. * - * @return + * @return 0 for success, error code otherwise. + * @ingroup PIO_write_darray */ int PIOc_read_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array) @@ -1841,11 +1877,14 @@ int PIOc_read_darray(const int ncid, const int vid, const int ioid, /** Flush the output buffer. * - * @param file - * @param force - * @param addsize + * @param file a pointer to the open file descriptor for the file + * that will be written to + * @param force true to force the flushing of the buffer + * @param addsize additional size to add to buffer (in bytes) * - * @return + * @return 0 for success, error code otherwise. + * @private + * @ingroup PIO_write_darray */ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) { @@ -1965,6 +2004,9 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) * * @param ios the IO system structure * @param collective true if collective report is desired + * + * @private + * @ingroup PIO_write_darray */ void cn_buffer_report(iosystem_desc_t ios, bool collective) { @@ -2013,7 +2055,10 @@ void cn_buffer_report(iosystem_desc_t ios, bool collective) /** Free the buffer pool. * - * @param ios + * @param ios the IO system structure + * + * @private + * @ingroup PIO_write_darray */ void free_cn_buffer_pool(iosystem_desc_t ios) { @@ -2030,9 +2075,12 @@ void free_cn_buffer_pool(iosystem_desc_t ios) /** Flush the buffer. * - * @param ncid + * @param ncid identifies the netCDF file * @param wmb * @param flushtodisk + * + * @private + * @ingroup PIO_write_darray */ void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) { @@ -2057,8 +2105,11 @@ void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) /** Compute the maximum aggregate number of bytes. * - * @param ios - * @param iodesc + * @param ios the IO system structure + * @param iodesc a pointer to the defined iodescriptor for the buffer + * + * @private + * @ingroup PIO_write_darray */ void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc) { From 5831e9fdf0451220cb509dffe62773c776056e5d Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 21 Jun 2016 12:36:53 -0600 Subject: [PATCH 175/184] more comments --- src/clib/pio.h | 2 +- src/clib/pio_darray_async.c | 48 +++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index 8a75bfce292..c6c0a1332fa 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -46,7 +46,7 @@ typedef struct var_desc_t { /** The unlimited dimension in the netCDF file (typically the time - * dimension). */ + * dimension). -1 if there is no unlimited dimension. */ int record; /** Number of dimensions for this variable. */ diff --git a/src/clib/pio_darray_async.c b/src/clib/pio_darray_async.c index fdb91b8971a..cf66c643332 100644 --- a/src/clib/pio_darray_async.c +++ b/src/clib/pio_darray_async.c @@ -33,6 +33,7 @@ static PIO_Offset maxusage = 0; * the setting is changed. * * @param limit the size of the buffer on the IO nodes + * * @return The previous limit setting. */ PIO_Offset PIOc_set_buffer_size_limit(const PIO_Offset limit) @@ -109,6 +110,8 @@ int pio_write_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, int fndims; /* Number of dims for variable according to netCDF. */ PIO_Offset tdsize = 0; /* Total size. */ + LOG((1, "pio_write_array_nc vid = %d", vid)); + #ifdef TIMING /* Start timing this function. */ GPTLstart("PIO:write_darray_nc"); @@ -462,7 +465,9 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int *vid int ncid; tdsize=0; ierr = PIO_NOERR; + #ifdef TIMING + /* Start timing this function. */ GPTLstart("PIO:write_darray_multi_nc"); #endif @@ -691,7 +696,9 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int *vid } // if (ios->ioproc) ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + #ifdef TIMING + /* Stop timing this function. */ GPTLstop("PIO:write_darray_multi_nc"); #endif @@ -750,6 +757,7 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i tdsize=0; ierr = PIO_NOERR; #ifdef TIMING + /* Start timing this function. */ GPTLstart("PIO:write_darray_multi_nc_serial"); #endif @@ -937,7 +945,9 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i } // if (ios->ioproc) ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + #ifdef TIMING + /* Stop timing this function. */ GPTLstop("PIO:write_darray_multi_nc_serial"); #endif @@ -1399,6 +1409,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, int i; #ifdef TIMING + /* Start timing this function. */ GPTLstart("PIO:read_darray_nc"); #endif ios = file->iosystem; @@ -1556,7 +1567,9 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, } ierr = check_netcdf(file, ierr, __FILE__,__LINE__); + #ifdef TIMING + /* Stop timing this function. */ GPTLstop("PIO:read_darray_nc"); #endif @@ -1585,6 +1598,7 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int i; #ifdef TIMING + /* Start timing this function. */ GPTLstart("PIO:read_darray_nc_serial"); #endif ios = file->iosystem; @@ -1775,7 +1789,9 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, } ierr = check_netcdf(file, ierr, __FILE__, __LINE__); + #ifdef TIMING + /* Stop timing this function. */ GPTLstop("PIO:read_darray_nc_serial"); #endif @@ -1875,7 +1891,8 @@ int PIOc_read_darray(const int ncid, const int vid, const int ioid, } -/** Flush the output buffer. +/** Flush the output buffer. This is only relevant for files opened + * with pnetcdf. * * @param file a pointer to the open file descriptor for the file * that will be written to @@ -1888,32 +1905,40 @@ int PIOc_read_darray(const int ncid, const int vid, const int ioid, */ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) { - var_desc_t *vdesc; - int ierr=PIO_NOERR; + int ierr = PIO_NOERR; + #ifdef _PNETCDF + var_desc_t *vdesc; int *status; PIO_Offset usage = 0; + #ifdef TIMING + /* Start timing this function. */ GPTLstart("PIO:flush_output_buffer"); #endif - pioassert(file!=NULL,"file pointer not defined",__FILE__,__LINE__); + pioassert(file != NULL, "file pointer not defined", __FILE__, + __LINE__); + /* Find out the buffer usage. */ ierr = ncmpi_inq_buffer_usage(file->fh, &usage); + /* If we are not forcing a flush, spread the usage to all IO + * tasks. */ if (!force && file->iosystem->io_comm != MPI_COMM_NULL) { usage += addsize; - MPI_Allreduce(MPI_IN_PLACE, &usage, 1, MPI_OFFSET, MPI_MAX, file->iosystem->io_comm); } + /* Keep track of the maximum usage. */ if (usage > maxusage) - { maxusage = usage; - } - if (force || usage>=PIO_BUFFER_SIZE_LIMIT) + + /* If the user forces it, or the buffer has exceeded the size + * limit, then flush to disk. */ + if (force || usage >= PIO_BUFFER_SIZE_LIMIT) { int rcnt; bool prev_dist=false; @@ -1992,11 +2017,13 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) } } + #ifdef TIMING + /* Stop timing this function. */ GPTLstop("PIO:flush_output_buffer"); #endif -#endif +#endif /* _PNETCDF */ return ierr; } @@ -2053,7 +2080,8 @@ void cn_buffer_report(iosystem_desc_t ios, bool collective) } } -/** Free the buffer pool. +/** Free the buffer pool. If malloc is used (that is, PIO_USE_MALLOC is + * non zero), this function does nothing. * * @param ios the IO system structure * From 004aeb906a374fe909ab79ea617681b7d74e0387 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 1 Aug 2016 14:02:10 -0600 Subject: [PATCH 176/184] Fixed doc build for async vs. non-async builds --- doc/CMakeLists.txt | 9 ++++++++- doc/Doxyfile.in | 8 ++++++-- src/clib/pio_darray_async.c | 6 +++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index e8d97644217..c2ccb2749a8 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -5,7 +5,14 @@ #============================================================================== find_package(Doxygen) -if(DOXYGEN_FOUND) + if (PIO_ENABLE_ASYNC) + SET(C_SRC_FILES "@CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_sc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_darray_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_get_nc_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_internal.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc4.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_put_nc_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_spmd.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_support.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_lists.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_varm.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/dtypes.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_file.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_msg.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_rearrange.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/topology.c") + else () + SET(C_SRC_FILES "@CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_sc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_internal.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc4.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_spmd.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_support.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_darray.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_get_nc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_lists.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_put_nc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_varm.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/dtypes.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_file.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_msg.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_rearrange.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/topology.c") + endif () + + if(DOXYGEN_FOUND) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY) add_custom_target(doc diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 31dc3ffd113..c39e69b13d2 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -769,11 +769,15 @@ WARN_LOGFILE = # Note: If this tag is empty the current directory is searched. INPUT = @CMAKE_CURRENT_SOURCE_DIR@/source \ - @CMAKE_CURRENT_SOURCE_DIR@/../src/clib \ @CMAKE_CURRENT_SOURCE_DIR@/../src/flib \ @CMAKE_CURRENT_SOURCE_DIR@/../examples/c \ @CMAKE_CURRENT_SOURCE_DIR@/../examples/f03 \ - @CMAKE_BINARY_DIR@/src/flib + @CMAKE_BINARY_DIR@/src/clib \ + @CMAKE_BINARY_DIR@/src/flib \ + @C_SRC_FILES@ + +# Uncomment this after the async code is fully merged into PIO. +# @CMAKE_CURRENT_SOURCE_DIR@/../src/clib # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/src/clib/pio_darray_async.c b/src/clib/pio_darray_async.c index cf66c643332..d9e41a8340e 100644 --- a/src/clib/pio_darray_async.c +++ b/src/clib/pio_darray_async.c @@ -1577,15 +1577,15 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, } /** Read an array of data from a file to the (serial) IO library. - * @ingroup PIO_read_darray * * @param file a pointer to the open file descriptor for the file * that will be written to * @param iodesc a pointer to the defined iodescriptor for the buffer * @param vid the variable id to be read. - * @param IOBUF + * @param IOBUF the buffer to be read into from this mpi task * * @returns + * @ingroup PIO_read_darray */ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF) @@ -1813,7 +1813,7 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, * processor. * * @return 0 for success, error code otherwise. - * @ingroup PIO_write_darray + * @ingroup PIO_read_darray */ int PIOc_read_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array) From e51ccebefd524b96f1f7e22b6ff3f551b0dd639e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 1 Aug 2016 14:10:23 -0600 Subject: [PATCH 177/184] Comment and clean up --- doc/CMakeLists.txt | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index c2ccb2749a8..0294c950a40 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -5,24 +5,30 @@ #============================================================================== find_package(Doxygen) - if (PIO_ENABLE_ASYNC) - SET(C_SRC_FILES "@CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_sc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_darray_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_get_nc_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_internal.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc4.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_put_nc_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_spmd.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_support.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_lists.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_varm.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/dtypes.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_file.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_msg.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_rearrange.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/topology.c") - else () - SET(C_SRC_FILES "@CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_sc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_internal.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc4.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_spmd.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_support.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_darray.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_get_nc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_lists.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_put_nc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_varm.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/dtypes.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_file.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_msg.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_rearrange.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/topology.c") - endif () - if(DOXYGEN_FOUND) +# This supports the build with/witout async code. Once async code is +# fully merged, remove the definition of C_SRC_FILES and its mention +# in Doxyfile.in for simplicity. +if (PIO_ENABLE_ASYNC) + SET(C_SRC_FILES "@CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_sc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_darray_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_get_nc_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_internal.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc4.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_put_nc_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_spmd.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_support.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_lists.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_varm.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/dtypes.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_file.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_msg.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_rearrange.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/topology.c") +else () + SET(C_SRC_FILES "@CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_sc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_internal.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc4.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_spmd.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_support.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_darray.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_get_nc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_lists.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_put_nc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_varm.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/dtypes.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_file.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_msg.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_rearrange.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/topology.c") +endif () - configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in - ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY) - add_custom_target(doc - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/customdoxygen.css - ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/DoxygenLayout.xml - ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/doxygen.sty - ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Generating API documentation with Doxygen" VERBATIM) +if(DOXYGEN_FOUND) + # Process the Doxyfile using options set during configure. + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in + ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY) + + # Copy necessary files. + add_custom_target(doc + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/customdoxygen.css + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/DoxygenLayout.xml + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/doxygen.sty + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating API documentation with Doxygen" VERBATIM) endif(DOXYGEN_FOUND) From 1c5f4315922734bf1501fd9be0833e064f28a7c3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 3 Aug 2016 07:10:32 -0600 Subject: [PATCH 178/184] added PIOc_strerror() and test for it --- src/clib/pio.h | 13 +++++++---- src/clib/pioc_support.c | 43 ++++++++++++++++++++++++++++++++++ tests/unit/test_names.c | 52 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 102 insertions(+), 6 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index c6c0a1332fa..2dd570831c5 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -325,6 +325,7 @@ enum PIO_ERROR_HANDLERS PIO_RETURN_ERROR = (-53) }; +/** Define the netCDF-based error codes. */ #if defined( _PNETCDF) || defined(_NETCDF) #define PIO_GLOBAL NC_GLOBAL #define PIO_UNLIMITED NC_UNLIMITED @@ -417,25 +418,29 @@ enum PIO_ERROR_HANDLERS #define PIO_EBADCHUNK NC_EBADCHUNK #define PIO_ENOTBUILT NC_ENOTBUILT #define PIO_EDISKLESS NC_EDISKLESS - #define PIO_FILL_DOUBLE NC_FILL_DOUBLE #define PIO_FILL_FLOAT NC_FILL_FLOAT #define PIO_FILL_INT NC_FILL_INT #define PIO_FILL_CHAR NC_FILL_CHAR +#endif /* defined( _PNETCDF) || defined(_NETCDF) */ -#endif +/** Define the extra error codes for the parallel-netcdf library. */ #ifdef _PNETCDF #define PIO_EINDEP NC_EINDEP -#else +#else /* _PNETCDF */ #define PIO_EINDEP (-203) -#endif +#endif /* _PNETCDF */ +/** Define error codes for PIO. */ #define PIO_EBADIOTYPE -255 + +/** ??? */ #define PIO_REQ_NULL (NC_REQ_NULL-1) #if defined(__cplusplus) extern "C" { #endif + const char *PIOc_strerror(int pioerr); int PIOc_freedecomp(int iosysid, int ioid); int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp); int PIOc_inq_format (int ncid, int *formatp); diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index d4d7a292068..5c842d93984 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -17,6 +17,49 @@ int pio_log_level = 0; int my_rank; #endif /* PIO_ENABLE_LOGGING */ +/** Return a string description of an error code. If zero is passed, a + * null is returned. + * + * @param pioerr the error code returned by a PIO function call. + * + * @return Pointer to a constant string with the error message. + */ +const char *PIOc_strerror(int pioerr) +{ + + /* System error? */ + if(pioerr > 0) + { + const char *cp = (const char *)strerror(pioerr); + if(!cp) + return "Unknown Error"; + return cp; + } + + /* Not an error? */ + if (pioerr == PIO_NOERR) + return "No error"; + + /* NetCDF error? */ +#if defined( _PNETCDF) || defined(_NETCDF) + /* The if condition is somewhat confusing becuase netCDF uses + * negative error codes.*/ + if (pioerr <= NC2_ERR && pioerr >= NC4_LAST_ERROR) + return nc_strerror(pioerr); +#else /* defined( _PNETCDF) || defined(_NETCDF) */ + if (pioerr <= NC2_ERR && pioerr >= NC4_LAST_ERROR) + return "NetCDF error code, PIO not built with netCDF."; +#endif /* defined( _PNETCDF) || defined(_NETCDF) */ + + /* Handle PIO errors. */ + switch(pioerr) { + case PIO_EBADIOTYPE: + return "Bad IO type"; + default: + return "unknown PIO error"; + } +} + /** Set the logging level. Set to -1 for nothing, 0 for errors only, 1 * for important logging, and so on. Log levels below 1 are only * printed on the io/component root. If the library is not built with diff --git a/tests/unit/test_names.c b/tests/unit/test_names.c index ea493ddc0f5..5a191dcc06d 100644 --- a/tests/unit/test_names.c +++ b/tests/unit/test_names.c @@ -1,6 +1,6 @@ /** - * @file - * Tests for names of vars, atts, and dims. + * @file Tests for names of vars, atts, and dims. Also test the + * PIOc_strerror() function. * */ #include @@ -148,6 +148,50 @@ check_att_name(int my_rank, int ncid, int verbose) return 0; } +/** Check the PIOc_strerror() function. + * + * @param my_rank the rank of this process. + * @param verbose true to get printfs on stdout. + * + * @return 0 for success, error code otherwise. + */ +int +check_strerror(int my_rank, int verbose) { + +#define NUM_TRIES 7 + char errstr[PIO_MAX_NAME + 1]; + int errcode[NUM_TRIES] = {NC2_ERR, PIO_EBADID, + NC_ENOTNC3, NC4_LAST_ERROR - 1, 0, 1, + PIO_EBADIOTYPE}; + const char *expected[NUM_TRIES] = {"Unknown Error", "NetCDF: Not a valid ID", + "NetCDF: Attempting netcdf-3 operation on netcdf-4 file", + "unknown PIO error", "No error", + nc_strerror(1), "Bad IO type"}; + int ret = PIO_NOERR; + + for (int try = 0; try < NUM_TRIES; try++) + { + /* Get the error string for this errcode. */ + strcpy(errstr, PIOc_strerror(errcode[try])); + + /* Check that it was as expected. */ + if (strcmp(errstr, expected[try])) + ret = ERR_AWFUL; + + /* Print some output to stdout if required. */ + if (verbose) + { + printf("%d: PIO strerror(%d) = %s\n", my_rank, errcode[try], + errstr); + strcpy(errstr, nc_strerror(errcode[try])); + printf("%d: netCDF strerror(%d) = %s\n", my_rank, errcode[try], + errstr); + } + } + + return ret; +} + /** Run Tests for NetCDF-4 Functions. * * @param argc argument count @@ -289,6 +333,10 @@ main(int argc, char **argv) printf("%d: ParallelIO Library example1 running on %d processors.\n", my_rank, ntasks); + /* Check the error string function. */ + if ((ret = check_strerror(my_rank, verbose))) + ERR(ret); + /* keep things simple - 1 iotask per MPI process */ niotasks = ntasks; From c0dd8ddabe77b435d27f07bda113905751619334 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 3 Aug 2016 10:43:44 -0600 Subject: [PATCH 179/184] changed signature of PIOc_strerror() --- src/clib/pio.h | 2 +- src/clib/pioc_support.c | 54 +++++++++++++++++++++++------------------ src/flib/pio.F90 | 1 + src/flib/pio_nf.F90 | 30 ++++++++++++++++++++++- tests/unit/driver.F90 | 2 ++ tests/unit/test_names.c | 4 ++- 6 files changed, 66 insertions(+), 27 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index 2dd570831c5..07f4b8b20c5 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -440,7 +440,7 @@ enum PIO_ERROR_HANDLERS #if defined(__cplusplus) extern "C" { #endif - const char *PIOc_strerror(int pioerr); + int PIOc_strerror(int pioerr, char *errstr); int PIOc_freedecomp(int iosysid, int ioid); int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp); int PIOc_inq_format (int ncid, int *formatp); diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 5c842d93984..c60f4047a49 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -21,43 +21,49 @@ int my_rank; * null is returned. * * @param pioerr the error code returned by a PIO function call. + * @param errmsg Pointer that will get the error message. It will be + * PIO_MAX_NAME chars or less. * - * @return Pointer to a constant string with the error message. + * @return 0 on success */ -const char *PIOc_strerror(int pioerr) +int +PIOc_strerror(int pioerr, char *errmsg) { /* System error? */ if(pioerr > 0) { const char *cp = (const char *)strerror(pioerr); - if(!cp) - return "Unknown Error"; - return cp; + if (cp) + strncpy(errmsg, cp, PIO_MAX_NAME); + else + strcpy(errmsg, "Unknown Error"); } - - /* Not an error? */ - if (pioerr == PIO_NOERR) - return "No error"; - - /* NetCDF error? */ + else if (pioerr == PIO_NOERR) + { + strcpy(errmsg, "No error"); + } + else if (pioerr <= NC2_ERR && pioerr >= NC4_LAST_ERROR) /* NetCDF error? */ + { #if defined( _PNETCDF) || defined(_NETCDF) - /* The if condition is somewhat confusing becuase netCDF uses - * negative error codes.*/ - if (pioerr <= NC2_ERR && pioerr >= NC4_LAST_ERROR) - return nc_strerror(pioerr); + strncpy(errmsg, nc_strerror(pioerr), NC_MAX_NAME); #else /* defined( _PNETCDF) || defined(_NETCDF) */ - if (pioerr <= NC2_ERR && pioerr >= NC4_LAST_ERROR) - return "NetCDF error code, PIO not built with netCDF."; + strcpy(errmsg, "NetCDF error code, PIO not built with netCDF."); #endif /* defined( _PNETCDF) || defined(_NETCDF) */ - - /* Handle PIO errors. */ - switch(pioerr) { - case PIO_EBADIOTYPE: - return "Bad IO type"; - default: - return "unknown PIO error"; } + else + { + /* Handle PIO errors. */ + switch(pioerr) { + case PIO_EBADIOTYPE: + strcpy(errmsg, "Bad IO type"); + break; + default: + strcpy(errmsg, "unknown PIO error"); + } + } + + return PIO_NOERR; } /** Set the logging level. Set to -1 for nothing, 0 for errors only, 1 diff --git a/src/flib/pio.F90 b/src/flib/pio.F90 index c415f7a6fb3..f5115cc5722 100644 --- a/src/flib/pio.F90 +++ b/src/flib/pio.F90 @@ -60,6 +60,7 @@ module pio PIO_get_chunk_cache, & PIO_set_var_chunk_cache, & PIO_get_var_chunk_cache +! PIO_strerror, & use pionfatt_mod, only : PIO_put_att => put_att, & PIO_get_att => get_att diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90 index 8aa74d06f9f..a3cd06d92b9 100644 --- a/src/flib/pio_nf.F90 +++ b/src/flib/pio_nf.F90 @@ -37,7 +37,8 @@ module pio_nf pio_set_var_chunk_cache , & pio_get_var_chunk_cache , & pio_redef , & - pio_set_log_level + pio_set_log_level +! pio_strerror ! pio_copy_att to be done interface pio_def_var @@ -194,6 +195,11 @@ module pio_nf set_log_level end interface pio_set_log_level + ! interface pio_strerror + ! module procedure & + ! strerror + ! end interface pio_strerror + interface pio_inquire module procedure & inquire_desc , & @@ -678,6 +684,28 @@ end function PIOc_set_log_level end interface ierr = PIOc_set_log_level(log_level) end function set_log_level +!> +!! @defgroup PIO_strerror +!< +!> +!! @ingroup PIO_strerror +!! Returns a descriptive string for an error code. +!! +!! @param errcode the error code +!! @retval a description of the error + !< + ! function strerror(errcode) result(errmsg) + ! integer, intent(in) :: errcode + ! Character(LEN=80) :: errmsg + ! interface + ! Function PIOc_strerror(errcode) BIND(C) + ! USE ISO_C_BINDING, ONLY: C_INT, C_PTR + ! Integer(C_INT), VALUE :: errcode + ! Type(C_PTR) :: PIOc_strerror + ! End Function PIOc_strerror + ! end interface + ! strerror = PIOc_strerror(errcode) + ! end function strerror !> !! @public !! @ingroup PIO_redef diff --git a/tests/unit/driver.F90 b/tests/unit/driver.F90 index d12d1d29cc2..ced2c77af59 100644 --- a/tests/unit/driver.F90 +++ b/tests/unit/driver.F90 @@ -55,6 +55,8 @@ Program pio_unit_test_driver write(*,"(A,1x,I0,1x,A,1x,I0)") "Running unit tests with", ntasks, & "MPI tasks and stride of", stride +! print *, 'errcode =', -33, ' strerror = ', PIO_strerror(-33) + if (stride.gt.ntasks) then stride = ntasks write(*,"(A,1x,A,I0)") "WARNING: stride value in namelist is larger than", & diff --git a/tests/unit/test_names.c b/tests/unit/test_names.c index 5a191dcc06d..48884eedd82 100644 --- a/tests/unit/test_names.c +++ b/tests/unit/test_names.c @@ -171,8 +171,10 @@ check_strerror(int my_rank, int verbose) { for (int try = 0; try < NUM_TRIES; try++) { + char result[PIO_MAX_NAME]; + /* Get the error string for this errcode. */ - strcpy(errstr, PIOc_strerror(errcode[try])); + PIOc_strerror(errcode[try], errstr); /* Check that it was as expected. */ if (strcmp(errstr, expected[try])) From 7f877f842b1a19b8e909750c0f944db3b3b2a806 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 3 Aug 2016 11:03:23 -0600 Subject: [PATCH 180/184] adding fortran interface to PIOc_strerror() --- src/flib/pio_nf.F90 | 60 +++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90 index a3cd06d92b9..4db50a2a51d 100644 --- a/src/flib/pio_nf.F90 +++ b/src/flib/pio_nf.F90 @@ -37,8 +37,8 @@ module pio_nf pio_set_var_chunk_cache , & pio_get_var_chunk_cache , & pio_redef , & - pio_set_log_level -! pio_strerror + pio_set_log_level , & + pio_strerror ! pio_copy_att to be done interface pio_def_var @@ -195,10 +195,10 @@ module pio_nf set_log_level end interface pio_set_log_level - ! interface pio_strerror - ! module procedure & - ! strerror - ! end interface pio_strerror + interface pio_strerror + module procedure & + strerror + end interface pio_strerror interface pio_inquire module procedure & @@ -684,28 +684,34 @@ end function PIOc_set_log_level end interface ierr = PIOc_set_log_level(log_level) end function set_log_level -!> -!! @defgroup PIO_strerror -!< -!> -!! @ingroup PIO_strerror -!! Returns a descriptive string for an error code. -!! -!! @param errcode the error code -!! @retval a description of the error + + !> + !! @defgroup PIO_strerror !< - ! function strerror(errcode) result(errmsg) - ! integer, intent(in) :: errcode - ! Character(LEN=80) :: errmsg - ! interface - ! Function PIOc_strerror(errcode) BIND(C) - ! USE ISO_C_BINDING, ONLY: C_INT, C_PTR - ! Integer(C_INT), VALUE :: errcode - ! Type(C_PTR) :: PIOc_strerror - ! End Function PIOc_strerror - ! end interface - ! strerror = PIOc_strerror(errcode) - ! end function strerror + !> + !! @ingroup PIO_strerror + !! Returns a descriptive string for an error code. + !! + !! @param errcode the error code + !! @retval a description of the error + !< + integer function strerror(errcode, errmsg) result(ierr) + integer, intent(in) :: errcode + character(len=*), intent(out) :: errmsg + interface + integer(C_INT) function PIOc_strerror(errcode, errmsg) & + bind(C, name="PIOc_strerror") + use iso_c_binding + integer(C_INT), value :: errcode + character(C_CHAR) :: errmsg(*) + end function PIOc_strerror + end interface + errmsg = C_NULL_CHAR + ierr = PIOc_strerror(errcode, errmsg) + call replace_c_null(errmsg) + + end function strerror + !> !! @public !! @ingroup PIO_redef From 86d23b9bb42359c200daa5c9c5827eefbf1f90ff Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 3 Aug 2016 14:35:13 -0600 Subject: [PATCH 181/184] added test for fortran pio_strerror --- src/flib/pio.F90 | 4 ++-- tests/unit/driver.F90 | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/flib/pio.F90 b/src/flib/pio.F90 index f5115cc5722..739c0f8441e 100644 --- a/src/flib/pio.F90 +++ b/src/flib/pio.F90 @@ -59,8 +59,8 @@ module pio PIO_set_chunk_cache, & PIO_get_chunk_cache, & PIO_set_var_chunk_cache, & - PIO_get_var_chunk_cache -! PIO_strerror, & + PIO_get_var_chunk_cache, & + PIO_strerror use pionfatt_mod, only : PIO_put_att => put_att, & PIO_get_att => get_att diff --git a/tests/unit/driver.F90 b/tests/unit/driver.F90 index ced2c77af59..92af6cec3eb 100644 --- a/tests/unit/driver.F90 +++ b/tests/unit/driver.F90 @@ -27,6 +27,9 @@ Program pio_unit_test_driver integer, external :: nc_set_log_level2 #endif integer ret_val + character(len=80) :: errmsg + character(len=80) :: expected + ! Set up MPI call MPI_Init(ierr) call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) @@ -55,8 +58,6 @@ Program pio_unit_test_driver write(*,"(A,1x,I0,1x,A,1x,I0)") "Running unit tests with", ntasks, & "MPI tasks and stride of", stride -! print *, 'errcode =', -33, ' strerror = ', PIO_strerror(-33) - if (stride.gt.ntasks) then stride = ntasks write(*,"(A,1x,A,I0)") "WARNING: stride value in namelist is larger than", & @@ -124,6 +125,16 @@ Program pio_unit_test_driver fail_cnt = 0 test_cnt = 0 + ! Test pio_strerror. + ret_val = PIO_strerror(-33, errmsg); + print *, 'errcode =', -33, ' strerror = ', errmsg + expected = 'NetCDF: Not a valid ID' + if (trim(errmsg) .ne. expected) then + err_msg = 'expected ' // trim(expected) // ' and got ' // trim(errmsg) + print *, err_msg + call parse(err_msg, fail_cnt) + end if + do test_id=1,ntest if (ltest(test_id)) then ! Make sure i is a valid test number @@ -148,6 +159,7 @@ Program pio_unit_test_driver #if defined( _NETCDF4) && defined(LOGGING) if(master_task) ierr = nc_set_log_level2(3) #endif + ! test_create() if (master_task) write(*,"(3x,A,1x)") "testing PIO_createfile..." call test_create(test_id, err_msg) From 65e731cb507549bf555156382f3860bb7e24f8c1 Mon Sep 17 00:00:00 2001 From: ed Date: Wed, 10 Aug 2016 10:52:11 -0600 Subject: [PATCH 182/184] removed test that depended on changing netCDF error string --- tests/unit/test_names.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_names.c b/tests/unit/test_names.c index 48884eedd82..b9684d03628 100644 --- a/tests/unit/test_names.c +++ b/tests/unit/test_names.c @@ -158,12 +158,12 @@ check_att_name(int my_rank, int ncid, int verbose) int check_strerror(int my_rank, int verbose) { -#define NUM_TRIES 7 +#define NUM_TRIES 6 char errstr[PIO_MAX_NAME + 1]; - int errcode[NUM_TRIES] = {NC2_ERR, PIO_EBADID, + int errcode[NUM_TRIES] = {PIO_EBADID, NC_ENOTNC3, NC4_LAST_ERROR - 1, 0, 1, PIO_EBADIOTYPE}; - const char *expected[NUM_TRIES] = {"Unknown Error", "NetCDF: Not a valid ID", + const char *expected[NUM_TRIES] = {"NetCDF: Not a valid ID", "NetCDF: Attempting netcdf-3 operation on netcdf-4 file", "unknown PIO error", "No error", nc_strerror(1), "Bad IO type"}; From 941a4b8c8a46ad8dfc053ace29bfffc4c99b20a8 Mon Sep 17 00:00:00 2001 From: katetc Date: Fri, 12 Aug 2016 13:09:24 -0600 Subject: [PATCH 183/184] Changes required for the new Hobart nag 6.1 --- examples/f03/examplePio.f90 | 2 +- src/gptl/perf_mod.F90 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/f03/examplePio.f90 b/examples/f03/examplePio.f90 index f5eede9e26a..d2baddf2096 100644 --- a/examples/f03/examplePio.f90 +++ b/examples/f03/examplePio.f90 @@ -10,7 +10,7 @@ module pioExample use pio, only : PIO_nowrite, PIO_openfile implicit none - save + private include 'mpif.h' diff --git a/src/gptl/perf_mod.F90 b/src/gptl/perf_mod.F90 index 2e38c491c67..e62059de98e 100644 --- a/src/gptl/perf_mod.F90 +++ b/src/gptl/perf_mod.F90 @@ -30,7 +30,7 @@ module perf_mod !----------------------------------------------------------------------- implicit none private ! Make the default access private - save + !----------------------------------------------------------------------- ! Public interfaces ---------------------------------------------------- From 4d798adac1dbe3ecc7b70d3183ba6fc99ecd5fa6 Mon Sep 17 00:00:00 2001 From: katetc Date: Fri, 12 Aug 2016 13:25:50 -0600 Subject: [PATCH 184/184] Changes required for nightly cdash to work with new Hobart and Nag 6.1 --- ctest/runcdash-cgd-nag.sh | 6 +++--- ctest/runctest-cgd.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ctest/runcdash-cgd-nag.sh b/ctest/runcdash-cgd-nag.sh index 44ced786200..f81102d8ff1 100755 --- a/ctest/runcdash-cgd-nag.sh +++ b/ctest/runcdash-cgd-nag.sh @@ -8,8 +8,8 @@ else fi module purge -module load compiler/nag/6.0 -module load tool/parallel-netcdf/1.6.1/nag/openmpi +module load compiler/nag/6.1 +module load tool/parallel-netcdf/1.7.0/nag/mvapich2 export CC=mpicc export FC=mpif90 @@ -17,7 +17,7 @@ export PIO_DASHBOARD_SITE="cgd" export PIO_DASHBOARD_ROOT=/scratch/cluster/katec/dashboard export CTEST_SCRIPT_DIRECTORY=${PIO_DASHBOARD_ROOT}/src export PIO_DASHBOARD_SOURCE_DIR=${CTEST_SCRIPT_DIRECTORY} -export PIO_COMPILER_ID=Nag-6.0-gcc-`gcc --version | head -n 1 | cut -d' ' -f3` +export PIO_COMPILER_ID=Nag-6.1-gcc-`gcc --version | head -n 1 | cut -d' ' -f3` if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then mkdir "$PIO_DASHBOARD_ROOT" diff --git a/ctest/runctest-cgd.sh b/ctest/runctest-cgd.sh index 02c99d10b4a..34233e20cf0 100755 --- a/ctest/runctest-cgd.sh +++ b/ctest/runctest-cgd.sh @@ -29,7 +29,7 @@ echo "\$CTESTCMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.sh chmod +x runctest.sh # Submit the job to the queue -jobid=`/usr/local/bin/qsub -l nodes=1:ppn=4 runctest.sh` +jobid=`/usr/local/bin/qsub -l nodes=1:ppn=4 runctest.sh -q short` # Wait for the job to complete before exiting while true; do