From 8061b90a2039fdac2f00334be7f092db9713dc8e Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 14 Dec 2018 13:44:39 -0700 Subject: [PATCH 01/13] try to sort arrays - not working yet --- src/clib/pio.h | 9 ++ src/clib/pio_darray.c | 21 ++++- src/clib/pio_darray_int.c | 123 +++++++++++++++++++++++++++ src/clib/pio_internal.h | 5 +- src/clib/pioc.c | 45 +++++++++- tests/performance/pioperformance.F90 | 62 +++++++------- 6 files changed, 226 insertions(+), 39 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index 8973a9f9c01..8c2a8f96cb3 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -267,6 +267,12 @@ typedef struct io_desc_t * 1-based mappings to the global array for that task. */ PIO_Offset *map; + /** If the map passed in is not monotonically increasing + * then map is sorted and remap is an array of original + * indices of map. */ + + int *remap; + /** Number of tasks involved in the communication between comp and * io tasks. */ int nrecvs; @@ -294,6 +300,9 @@ typedef struct io_desc_t * everywhere (false) */ bool needsfill; + /** If the map is not monotonically increasing we will need to sort it. */ + bool needssort; + /** The maximum number of bytes of this iodesc before flushing. */ int maxbytes; diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index 116f0103770..014770b2f3f 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -113,6 +113,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, int fndims; /* Number of dims in the var in the file. */ int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function calls. */ int ierr; /* Return code. */ + void *tmparray; /* Get the file info. */ if ((ierr = pio_get_file(ncid, &file))) @@ -264,9 +265,18 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); LOG((3, "allocated token for variable buffer")); } + if (iodesc->needssort) + { + tmparray = pio_sorted_copy(iodesc, array, arraylen, nvars); + } + else + { + tmparray = array; + } + /* Move data from compute to IO tasks. */ - if ((ierr = rearrange_comp2io(ios, iodesc, array, file->iobuf, nvars))) + if ((ierr = rearrange_comp2io(ios, iodesc, tmparray, file->iobuf, nvars))) return pio_err(ios, file, ierr, __FILE__, __LINE__); /* Write the darray based on the iotype. */ @@ -364,6 +374,9 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, } } + if(iodesc->needssort && tmparray != NULL) + free(tmparray); + /* Flush data to disk for pnetcdf. */ if (ios->ioproc && file->iotype == PIO_IOTYPE_PNETCDF) if ((ierr = flush_output_buffer(file, flushtodisk, 0))) @@ -403,7 +416,7 @@ pio_inq_var_fill_expected(int ncid, int varid, int pio_type, PIO_Offset type_siz unsigned long long uint64_fill_value = NC_FILL_UINT64; char *string_fill_value = ""; int ret; - + /* Check inputs. */ assert(fillvalue); @@ -412,7 +425,7 @@ pio_inq_var_fill_expected(int ncid, int varid, int pio_type, PIO_Offset type_siz /* Is there a _FillValue attribute? */ ret = PIOc_inq_att_eh(ncid, varid, "_FillValue", 0, NULL, NULL); - + LOG((3, "pio_inq_var_fill_expected ret %d", ret)); /* If there is a fill value, get it. */ @@ -472,7 +485,7 @@ pio_inq_var_fill_expected(int ncid, int varid, int pio_type, PIO_Offset type_siz return PIO_EBADTYPE; } } - + return PIO_NOERR; } diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c index 37c9f85576f..337dba265e6 100644 --- a/src/clib/pio_darray_int.c +++ b/src/clib/pio_darray_int.c @@ -1905,3 +1905,126 @@ int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) return PIO_NOERR; } + +void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int nvars) +{ + void *tmparray; + if (!(tmparray = malloc(iodesc->piotype_size * arraylen))) + return NULL; + + int maplen = arraylen / nvars; + switch (iodesc->piotype) + { + case PIO_BYTE: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((signed char *)tmparray)[iodesc->remap[m]] = ((signed char *)array)[m+maplen*v]; + } + } + break; + case PIO_CHAR: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((char *)tmparray)[iodesc->remap[m]] = ((char *)array)[m+maplen*v]; + } + } + break; + case PIO_SHORT: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((short *)tmparray)[iodesc->remap[m]] = ((short *)array)[m+maplen*v]; + } + } + + break; + case PIO_INT: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((int *)tmparray)[iodesc->remap[m]] = ((int *)array)[m+maplen*v]; + } + } + break; + case PIO_FLOAT: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((float *)tmparray)[iodesc->remap[m]] = ((float *)array)[m+maplen*v]; + } + } + break; + case PIO_DOUBLE: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((double *)tmparray)[iodesc->remap[m]] = ((double *)array)[m+maplen*v]; + } + } + break; + case PIO_UBYTE: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((unsigned char *)tmparray)[iodesc->remap[m]] = ((unsigned char *)array)[m+maplen*v]; + } + } + break; + case PIO_USHORT: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((unsigned short *)tmparray)[iodesc->remap[m]] = ((unsigned short *)array)[m+maplen*v]; + } + } + break; + case PIO_UINT: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((unsigned int *)tmparray)[iodesc->remap[m]] = ((unsigned int *)array)[m+maplen*v]; + } + } + break; + case PIO_INT64: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((long long *)tmparray)[iodesc->remap[m]] = ((long long *)array)[m+maplen*v]; + } + } + break; + case PIO_UINT64: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((unsigned long long *)tmparray)[iodesc->remap[m]] = ((unsigned long long *)array)[m+maplen*v]; + } + } + break; + case PIO_STRING: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((char **)tmparray)[iodesc->remap[m]] = ((char **)array)[m+maplen*v]; + } + } + break; + default: + return NULL; + } +} diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index a8947135a24..11840018d17 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -145,7 +145,7 @@ extern "C" { int pio_get_file(int ncid, file_desc_t **filep); int pio_delete_file_from_list(int ncid); void pio_add_to_file_list(file_desc_t *file); - + /* List operations for var_desc_t list. */ int add_to_varlist(int varid, int rec_var, int pio_type, int pio_type_size, MPI_Datatype mpi_type, int mpi_type_size, var_desc_t **varlist); @@ -312,7 +312,7 @@ extern "C" { int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf); int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf); - int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc); + int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc); /* Read atts with type conversion. */ int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void *ip); @@ -361,6 +361,7 @@ extern "C" { int determine_procs(int num_io_procs, int component_count, int *num_procs_per_comp, int **proc_list, int **my_proc_list); + void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int nvars); #if defined(__cplusplus) } #endif diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 037b64720ed..39bdbb0212a 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -362,6 +362,27 @@ int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method) return PIO_NOERR; } +void pio_map_sort(const PIO_Offset *map, int *remap, int maplen) +{ + bool switched=false; + do + { + switched = false; + for(int i=1; imap = malloc(sizeof(PIO_Offset) * maplen))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + iodesc->needssort = false; + iodesc->remap = NULL; for (int m = 0; m < maplen; m++) { - iodesc->map[m] = compmap[m]; + if(m > 0 && compmap[m] > 0 && compmap[m] < compmap[m-1]) + iodesc->needssort = true; LOG((4, "compmap[%d] = %d", m, compmap[m])); } - + if (iodesc->needssort){ + if (!(iodesc->remap = malloc(sizeof(int) * maplen))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for (int m=0; m < maplen; m++) + iodesc->remap[m] = m; + pio_map_sort(compmap, iodesc->remap, maplen); + for (int m=0; m < maplen; m++) + { + iodesc->map[iodesc->remap[m]] = compmap[m]; + } + } + else + { + for (int m=0; m < maplen; m++) + { + iodesc->map[m] = compmap[m]; + } + } /* Remember the dim sizes. */ if (!(iodesc->dimlen = malloc(sizeof(int) * ndims))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); diff --git a/tests/performance/pioperformance.F90 b/tests/performance/pioperformance.F90 index f814a306973..828dd9ec959 100644 --- a/tests/performance/pioperformance.F90 +++ b/tests/performance/pioperformance.F90 @@ -1,4 +1,4 @@ -#define VARINT 1 +#define VARINT 1 !#define VARREAL 1 !#define VARDOUBLE 1 @@ -8,11 +8,11 @@ program pioperformance #endif use perf_mod, only : t_initf, t_finalizef use pio, only : pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4p, & - pio_iotype_netcdf4c, pio_rearr_subset, pio_rearr_box + pio_iotype_netcdf4c, pio_rearr_subset, pio_rearr_box, pio_set_log_level implicit none #ifdef NO_MPIMOD #include -#endif +#endif integer, parameter :: max_io_task_array_size=64, max_decomp_files=64 @@ -102,8 +102,8 @@ program pioperformance if(rearrangers(1)==0) then rearrangers(1)=1 rearrangers(2)=2 - endif - + endif +! i = pio_set_log_level(2) do i=1,max_decomp_files if(len_trim(decompfile(i))==0) exit if(mype == 0) print *, ' Testing decomp: ',trim(decompfile(i)) @@ -112,7 +112,7 @@ program pioperformance do nv=1,max_nvars if(nvars(nv)>0) then call pioperformancetest(decompfile(i), piotypes(1:niotypes), mype, npe, & - rearrangers, niotasks, nframes, nvars(nv), varsize(vs),unlimdimindof) + rearrangers, niotasks, nframes, nvars(nv), varsize(vs),unlimdimindof) endif enddo endif @@ -133,7 +133,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & integer, intent(in) :: piotypes(:) integer, intent(in) :: rearrangers(:) integer, intent(inout) :: niotasks(:) - integer, intent(in) :: nframes + integer, intent(in) :: nframes integer, intent(in) :: nvars integer, intent(in) :: varsize logical, intent(in) :: unlimdimindof @@ -206,7 +206,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & ! if(gmaplen /= product(gdims)) then ! print *,__FILE__,__LINE__,gmaplen,gdims ! endif - + allocate(ifld(maplen,nvars)) allocate(ifld_in(maplen,nvars,nframes)) @@ -255,9 +255,9 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & stride = max(1,npe/ntasks) call pio_init(mype, comm, ntasks, 0, stride, PIO_REARR_SUBSET, iosystem) - + write(fname, '(a,i1,a,i4.4,a,i1,a)') 'pioperf.',rearr,'-',ntasks,'-',iotype,'.nc' - + ierr = PIO_CreateFile(iosystem, File, iotype, trim(fname), mode) call WriteMetadata(File, gdims, vari, varr, vard, unlimdimindof) @@ -297,7 +297,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & endif if(mype==0) print *,__FILE__,__LINE__,'Frame: ',recnum - do nv=1,nvars + do nv=1,nvars if(mype==0) print *,__FILE__,__LINE__,'var: ',nv #ifdef VARINT call PIO_setframe(File, vari(nv), recnum) @@ -313,7 +313,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & #endif enddo if(unlimdimindof) then -#ifdef VARREAL +#ifdef VARREAL call PIO_freedecomp(File, iodesc_r4) #endif #ifdef VARDOUBLE @@ -321,7 +321,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & #endif #ifdef VARINT call PIO_freedecomp(File, iodesc_i4) -#endif +#endif endif enddo call pio_closefile(File) @@ -344,7 +344,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & #ifdef VARDOUBLE nvarmult = nvarmult+2 #endif - write(*,'(a15,a9,i10,i10,i10,f20.10)') & + write(*,'(a15,a9,i10,i10,i10,f20.10)') & 'RESULT: write ',rearr_name(rearr), piotypes(k), ntasks, nvars, & nvarmult*nvars*nframes*gmaplen*4.0/(1048576.0*wall(2)) #ifdef BGQTRY @@ -383,8 +383,8 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & call MPI_Barrier(comm,ierr) call t_stampf(wall(1), usr(1), sys(1)) - - do frame=1,nframes + + do frame=1,nframes do nv=1,nvars #ifdef VARINT call PIO_setframe(File, vari(nv), frame) @@ -400,7 +400,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & #endif enddo enddo - + call pio_closefile(File) call MPI_Barrier(comm,ierr) call t_stampf(wall(2), usr(2), sys(2)) @@ -413,7 +413,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & if(compmap(j)>0) then #ifdef VARINT #ifdef DEBUG - write(*,'(a11,i2,a9,i11,a9,i11,a9,i2)') & + write(*,'(a11,i2,a9,i11,a9,i11,a9,i2)') & ' Int PE=',mype,'ifld=',ifld(j,nv),' ifld_in=',ifld_in(j,nv,frame),' compmap=',compmap(j) #endif if(ifld(j,nv) /= ifld_in(j,nv,frame)) then @@ -421,7 +421,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & ! print *,__LINE__,'Int: ',mype,j,nv,ifld(j,nv),ifld_in(j,nv,frame),compmap(j) !endif write(*,*) '***ERROR:Mismatch!***' - write(*,'(a11,i2,a9,i11,a9,i11,a9,i2)') & + write(*,'(a11,i2,a9,i11,a9,i11,a9,i2)') & ' Int PE=',mype,'ifld=',ifld(j,nv),' ifld_in=',ifld_in(j,nv,frame),' compmap=',compmap(j) errorcnt = errorcnt+1 @@ -432,7 +432,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & write(*,'(a11,i2,a9,f11.2,a9,f11.2,a9,i2)') & ' Real PE=',mype,'rfld=',rfld(j,nv),' rfld_in=',rfld_in(j,nv,frame),' compmap=',compmap(j) #endif - + if(rfld(j,nv) /= rfld_in(j,nv,frame) ) then !if(errorcnt < 10) then ! print *,__LINE__,'Real:', mype,j,nv,rfld(j,nv),rfld_in(j,nv,frame),compmap(j) @@ -441,7 +441,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & write(*,'(a11,i2,a9,f11.2,a9,f11.2,a9,i2)') & ' Real PE=',mype,'rfld=',rfld(j,nv),' rfld_in=',rfld_in(j,nv,frame),' compmap=',compmap(j) - errorcnt = errorcnt+1 + errorcnt = errorcnt+1 endif #endif #ifdef VARDOUBLE @@ -466,7 +466,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & enddo j = errorcnt call MPI_Reduce(j, errorcnt, 1, MPI_INTEGER, MPI_SUM, 0, comm, ierr) - + if(mype==0) then if(errorcnt > 0) then print *,'ERROR: INPUT/OUTPUT data mismatch ',errorcnt @@ -484,11 +484,11 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & write(*,'(a15,a9,i10,i10,i10,f20.10)') & 'RESULT: read ',rearr_name(rearr), piotypes(k), ntasks, nvars, & nvarmult*nvars*nframes*gmaplen*4.0/(1048576.0*wall(2)) -#ifdef BGQTRY +#ifdef BGQTRY call print_memusage() #endif end if -#ifdef VARREAL +#ifdef VARREAL call PIO_freedecomp(iosystem, iodesc_r4) #endif #ifdef VARDOUBLE @@ -496,7 +496,7 @@ subroutine pioperformancetest(filename, piotypes, mype, npe_base, & #endif #ifdef VARINT call PIO_freedecomp(iosystem, iodesc_i4) -#endif +#endif call pio_finalize(iosystem, ierr) enddo enddo @@ -533,7 +533,7 @@ subroutine init_ideal_dof(doftype, mype, npe, ndims, gdims, compmap, varsize) allocate(compmap(varsize)) if(doftype .eq. 'ROUNDROBIN') then do i=1,varsize - compmap(i) = (i-1)*npe+mype+1 + compmap(i) = (i-1)*npe+mype+1 enddo else if(doftype .eq. 'BLOCK') then do i=1,varsize @@ -570,7 +570,7 @@ subroutine WriteMetadata(File, gdims, vari, varr, vard,unlimdimindof) do i=1,ndims - write(dimname,'(a,i6.6)') 'dim',i + write(dimname,'(a,i6.6)') 'dim',i iostat = PIO_def_dim(File, trim(dimname), int(gdims(i),pio_offset_kind), dimid(i)) enddo iostat = PIO_def_dim(File, 'time', PIO_UNLIMITED, dimid(ndims+1)) @@ -611,15 +611,15 @@ subroutine CheckMPIreturn(line,errcode) implicit none #ifdef NO_MPIMOD #include -#endif +#endif integer, intent(in) :: errcode integer, intent(in) :: line character(len=MPI_MAX_ERROR_STRING) :: errorstring - + integer :: errorlen - + integer :: ierr - + if (errcode .ne. MPI_SUCCESS) then call MPI_Error_String(errcode,errorstring,errorlen,ierr) write(*,*) errorstring(1:errorlen) From 87e57b5b89469327bd54a1fac88d58a4921d22c3 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 14 Dec 2018 18:43:11 -0700 Subject: [PATCH 02/13] better but still has issues --- src/clib/pio_darray.c | 25 ++++++++++++++++++++----- src/clib/pio_darray_int.c | 35 ++++++++++++++++------------------- src/clib/pio_internal.h | 2 +- src/clib/pio_rearrange.c | 1 + src/clib/pio_spmd.c | 2 +- src/clib/pioc.c | 4 +--- 6 files changed, 40 insertions(+), 29 deletions(-) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index 014770b2f3f..747bb27c94a 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -267,14 +267,15 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, } if (iodesc->needssort) { - tmparray = pio_sorted_copy(iodesc, array, arraylen, nvars); + if (!(tmparray = malloc(arraylen*nvars*iodesc->piotype_size))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + pio_sorted_copy(array, tmparray, iodesc, nvars); } else { tmparray = array; } - /* Move data from compute to IO tasks. */ if ((ierr = rearrange_comp2io(ios, iodesc, tmparray, file->iobuf, nvars))) return pio_err(ios, file, ierr, __FILE__, __LINE__); @@ -844,8 +845,8 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, io_desc_t *iodesc; /* Pointer to IO description information. */ void *iobuf = NULL; /* holds the data as read on the io node. */ size_t rlen = 0; /* the length of data in iobuf. */ - int ierr; /* Return code. */ - + int ierr; /* Return code. */ + void *tmparray; /* unsorted copy of array buf if required */ /* Get the file info. */ if ((ierr = pio_get_file(ncid, &file))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); @@ -885,10 +886,24 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, return pio_err(NULL, NULL, PIO_EBADIOTYPE, __FILE__, __LINE__); } + if (iodesc->needssort) + { + if (!(tmparray = malloc(iodesc->piotype_size*iodesc->maplen))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + } + else + tmparray = array; + /* Rearrange the data. */ - if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, array))) + if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, tmparray))) return pio_err(ios, file, ierr, __FILE__, __LINE__); + if (iodesc->needssort) + { + pio_sorted_copy(tmparray, array, iodesc, 1); + free(tmparray); + } + /* Free the buffer. */ if (rlen > 0) brel(iobuf); diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c index 337dba265e6..eed9d975231 100644 --- a/src/clib/pio_darray_int.c +++ b/src/clib/pio_darray_int.c @@ -1906,13 +1906,9 @@ int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) return PIO_NOERR; } -void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int nvars) +int pio_sorted_copy(const void *array, void *sortedarray, io_desc_t *iodesc, int nvars) { - void *tmparray; - if (!(tmparray = malloc(iodesc->piotype_size * arraylen))) - return NULL; - - int maplen = arraylen / nvars; + int maplen = iodesc->maplen; switch (iodesc->piotype) { case PIO_BYTE: @@ -1920,7 +1916,7 @@ void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int n { for (int m=0; m < maplen; m++) { - ((signed char *)tmparray)[iodesc->remap[m]] = ((signed char *)array)[m+maplen*v]; + ((signed char *)sortedarray)[iodesc->remap[m]] = ((signed char *)array)[m+maplen*v]; } } break; @@ -1929,7 +1925,7 @@ void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int n { for (int m=0; m < maplen; m++) { - ((char *)tmparray)[iodesc->remap[m]] = ((char *)array)[m+maplen*v]; + ((char *)sortedarray)[iodesc->remap[m]] = ((char *)array)[m+maplen*v]; } } break; @@ -1938,7 +1934,7 @@ void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int n { for (int m=0; m < maplen; m++) { - ((short *)tmparray)[iodesc->remap[m]] = ((short *)array)[m+maplen*v]; + ((short *)sortedarray)[iodesc->remap[m]] = ((short *)array)[m+maplen*v]; } } @@ -1948,7 +1944,7 @@ void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int n { for (int m=0; m < maplen; m++) { - ((int *)tmparray)[iodesc->remap[m]] = ((int *)array)[m+maplen*v]; + ((int *)sortedarray)[iodesc->remap[m]] = ((int *)array)[m+maplen*v]; } } break; @@ -1957,7 +1953,7 @@ void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int n { for (int m=0; m < maplen; m++) { - ((float *)tmparray)[iodesc->remap[m]] = ((float *)array)[m+maplen*v]; + ((float *)sortedarray)[iodesc->remap[m]] = ((float *)array)[m+maplen*v]; } } break; @@ -1966,7 +1962,7 @@ void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int n { for (int m=0; m < maplen; m++) { - ((double *)tmparray)[iodesc->remap[m]] = ((double *)array)[m+maplen*v]; + ((double *)sortedarray)[iodesc->remap[m]] = ((double *)array)[m+maplen*v]; } } break; @@ -1975,7 +1971,7 @@ void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int n { for (int m=0; m < maplen; m++) { - ((unsigned char *)tmparray)[iodesc->remap[m]] = ((unsigned char *)array)[m+maplen*v]; + ((unsigned char *)sortedarray)[iodesc->remap[m]] = ((unsigned char *)array)[m+maplen*v]; } } break; @@ -1984,7 +1980,7 @@ void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int n { for (int m=0; m < maplen; m++) { - ((unsigned short *)tmparray)[iodesc->remap[m]] = ((unsigned short *)array)[m+maplen*v]; + ((unsigned short *)sortedarray)[iodesc->remap[m]] = ((unsigned short *)array)[m+maplen*v]; } } break; @@ -1993,7 +1989,7 @@ void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int n { for (int m=0; m < maplen; m++) { - ((unsigned int *)tmparray)[iodesc->remap[m]] = ((unsigned int *)array)[m+maplen*v]; + ((unsigned int *)sortedarray)[iodesc->remap[m]] = ((unsigned int *)array)[m+maplen*v]; } } break; @@ -2002,7 +1998,7 @@ void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int n { for (int m=0; m < maplen; m++) { - ((long long *)tmparray)[iodesc->remap[m]] = ((long long *)array)[m+maplen*v]; + ((long long *)sortedarray)[iodesc->remap[m]] = ((long long *)array)[m+maplen*v]; } } break; @@ -2011,7 +2007,7 @@ void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int n { for (int m=0; m < maplen; m++) { - ((unsigned long long *)tmparray)[iodesc->remap[m]] = ((unsigned long long *)array)[m+maplen*v]; + ((unsigned long long *)sortedarray)[iodesc->remap[m]] = ((unsigned long long *)array)[m+maplen*v]; } } break; @@ -2020,11 +2016,12 @@ void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int n { for (int m=0; m < maplen; m++) { - ((char **)tmparray)[iodesc->remap[m]] = ((char **)array)[m+maplen*v]; + ((char **)sortedarray)[iodesc->remap[m]] = ((char **)array)[m+maplen*v]; } } break; default: - return NULL; + return pio_err(NULL, NULL, PIO_EBADTYPE, __FILE__, __LINE__); } + return PIO_NOERR; } diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 11840018d17..d2a85be6e2e 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -361,7 +361,7 @@ extern "C" { int determine_procs(int num_io_procs, int component_count, int *num_procs_per_comp, int **proc_list, int **my_proc_list); - void *pio_sorted_copy(io_desc_t *iodesc, void *array, PIO_Offset arraylen, int nvars); + int pio_sorted_copy(const void *array, void *tmparray, io_desc_t *iodesc, int nvars); #if defined(__cplusplus) } #endif diff --git a/src/clib/pio_rearrange.c b/src/clib/pio_rearrange.c index dbda0744bfe..1ad4c1817fc 100644 --- a/src/clib/pio_rearrange.c +++ b/src/clib/pio_rearrange.c @@ -986,6 +986,7 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, int niotasks; int mpierr; /* Return code from MPI calls. */ int ret; + void *tmparray; /* Check inputs. */ pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__); diff --git a/src/clib/pio_spmd.c b/src/clib/pio_spmd.c index da2eef333b0..515cdd7da8e 100644 --- a/src/clib/pio_spmd.c +++ b/src/clib/pio_spmd.c @@ -113,7 +113,7 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty #if PIO_ENABLE_LOGGING { for (int p = 0; p < ntasks; p++) - LOG((3, "sendcounts[%d] = %d sdispls[%d] = %d sendtypes[%d] = %d recvcounts[%d] = %d " + LOG((4, "sendcounts[%d] = %d sdispls[%d] = %d sendtypes[%d] = %d recvcounts[%d] = %d " "rdispls[%d] = %d recvtypes[%d] = %d", p, sendcounts[p], p, sdispls[p], p, sendtypes[p], p, recvcounts[p], p, rdispls[p], p, recvtypes[p])); } diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 39bdbb0212a..87cda84f37f 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -370,7 +370,7 @@ void pio_map_sort(const PIO_Offset *map, int *remap, int maplen) switched = false; for(int i=1; i map[remap[i]]) { int remaptemp = remap[i]; remap[i] = remap[i-1]; @@ -531,9 +531,7 @@ int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, in iodesc->remap[m] = m; pio_map_sort(compmap, iodesc->remap, maplen); for (int m=0; m < maplen; m++) - { iodesc->map[iodesc->remap[m]] = compmap[m]; - } } else { From 6bbc245e8173871244b1266f16badd7e63f984c1 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Sat, 15 Dec 2018 15:23:12 -0700 Subject: [PATCH 03/13] working now I think --- src/clib/pio_darray.c | 6 +- src/clib/pio_darray_int.c | 334 ++++++++++++++++++++++++++------------ src/clib/pio_internal.h | 2 +- src/clib/pioc.c | 23 ++- src/flib/piodarray.F90.in | 2 - 5 files changed, 257 insertions(+), 110 deletions(-) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index 747bb27c94a..7f7612034ff 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -269,7 +269,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, { if (!(tmparray = malloc(arraylen*nvars*iodesc->piotype_size))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - pio_sorted_copy(array, tmparray, iodesc, nvars); + pio_sorted_copy(array, tmparray, iodesc, nvars, 0); } else { @@ -890,6 +890,8 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, { if (!(tmparray = malloc(iodesc->piotype_size*iodesc->maplen))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + for(int m=0; mmaplen;m++) + ((int *) array)[m] = -1; } else tmparray = array; @@ -900,7 +902,7 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, if (iodesc->needssort) { - pio_sorted_copy(tmparray, array, iodesc, 1); + pio_sorted_copy(tmparray, array, iodesc, 1, 1); free(tmparray); } diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c index eed9d975231..3c723e94a61 100644 --- a/src/clib/pio_darray_int.c +++ b/src/clib/pio_darray_int.c @@ -397,8 +397,8 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * #if USE_VARD_WRITE if (!ios->async || !ios->ioproc) { - if ((ierr = get_gdim0(file, iodesc, varids[0], fndims, &gdim0))) - return pio_err(NULL, file, ierr, __FILE__, __LINE__); + if ((ierr = get_gdim0(file, iodesc, varids[0], fndims, &gdim0))) + return pio_err(NULL, file, ierr, __FILE__, __LINE__); } #endif @@ -618,7 +618,7 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * if(filetype != MPI_DATATYPE_NULL) { if((mpierr = MPI_Type_free(&filetype))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); + return check_mpi(NULL, mpierr, __FILE__, __LINE__); } vard_llen = 0; /* reset request size to 0 */ numReqs = 0; @@ -1291,23 +1291,23 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu if (regioncnt == iodesc->maxregions - 1) { #if USE_VARD_READ - MPI_Datatype filetype; - PIO_Offset unlimdimoffset; - int mpierr; - if (gdim0 == 0) /* if there is an unlimited dimension get the offset between records of a variable */ + MPI_Datatype filetype; + PIO_Offset unlimdimoffset; + int mpierr; + if (gdim0 == 0) /* if there is an unlimited dimension get the offset between records of a variable */ { - if((ierr = ncmpi_inq_recsize(file->fh, &unlimdimoffset))) - return pio_err(NULL, file, ierr, __FILE__, __LINE__); + if((ierr = ncmpi_inq_recsize(file->fh, &unlimdimoffset))) + return pio_err(NULL, file, ierr, __FILE__, __LINE__); } - else - unlimdimoffset = gdim0; + else + unlimdimoffset = gdim0; - ierr = get_vard_mpidatatype(iodesc, gdim0, unlimdimoffset, - rrlen, ndims, fndims, - vdesc->record, startlist, countlist, &filetype); - ierr = ncmpi_get_vard_all(file->fh, vid, filetype, iobuf, iodesc->llen, iodesc->mpitype); - if(filetype != MPI_DATATYPE_NULL && (mpierr = MPI_Type_free(&filetype))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); + ierr = get_vard_mpidatatype(iodesc, gdim0, unlimdimoffset, + rrlen, ndims, fndims, + vdesc->record, startlist, countlist, &filetype); + ierr = ncmpi_get_vard_all(file->fh, vid, filetype, iobuf, iodesc->llen, iodesc->mpitype); + if(filetype != MPI_DATATYPE_NULL && (mpierr = MPI_Type_free(&filetype))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); #else /* Read a list of subarrays. */ @@ -1906,122 +1906,252 @@ int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) return PIO_NOERR; } -int pio_sorted_copy(const void *array, void *sortedarray, io_desc_t *iodesc, int nvars) +int pio_sorted_copy(const void *array, void *sortedarray, io_desc_t *iodesc, int nvars,int direction) { int maplen = iodesc->maplen; - switch (iodesc->piotype) - { - case PIO_BYTE: - for (int v=0; v < nvars; v++) + + if (direction == 0){ + switch (iodesc->piotype) { - for (int m=0; m < maplen; m++) + case PIO_BYTE: + for (int v=0; v < nvars; v++) { - ((signed char *)sortedarray)[iodesc->remap[m]] = ((signed char *)array)[m+maplen*v]; + for (int m=0; m < maplen; m++) + { + ((signed char *)sortedarray)[m] = ((signed char *)array)[iodesc->remap[m]+maplen*v]; + } } - } - break; - case PIO_CHAR: - for (int v=0; v < nvars; v++) - { - for (int m=0; m < maplen; m++) + break; + case PIO_CHAR: + for (int v=0; v < nvars; v++) { - ((char *)sortedarray)[iodesc->remap[m]] = ((char *)array)[m+maplen*v]; + for (int m=0; m < maplen; m++) + { + ((char *)sortedarray)[m] = ((char *)array)[iodesc->remap[m]+maplen*v]; + } } - } - break; - case PIO_SHORT: - for (int v=0; v < nvars; v++) - { - for (int m=0; m < maplen; m++) + break; + case PIO_SHORT: + for (int v=0; v < nvars; v++) { - ((short *)sortedarray)[iodesc->remap[m]] = ((short *)array)[m+maplen*v]; + for (int m=0; m < maplen; m++) + { + ((short *)sortedarray)[m] = ((short *)array)[iodesc->remap[m]+maplen*v]; + } } - } - break; - case PIO_INT: - for (int v=0; v < nvars; v++) - { - for (int m=0; m < maplen; m++) + break; + case PIO_INT: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((int *)sortedarray)[m] = ((int *)array)[iodesc->remap[m]+maplen*v]; + } +/* for (int m=0; m < maplen; m++) + printf("0:%d: array=%d sorted=%d\n",m,((int *)array)[m], ((int *)sortedarray)[m]); +*/ } + break; + case PIO_FLOAT: + for (int v=0; v < nvars; v++) { - ((int *)sortedarray)[iodesc->remap[m]] = ((int *)array)[m+maplen*v]; + for (int m=0; m < maplen; m++) + { + ((float *)sortedarray)[m] = ((float *)array)[iodesc->remap[m]+maplen*v]; + } } - } - break; - case PIO_FLOAT: - for (int v=0; v < nvars; v++) - { - for (int m=0; m < maplen; m++) + break; + case PIO_DOUBLE: + for (int v=0; v < nvars; v++) { - ((float *)sortedarray)[iodesc->remap[m]] = ((float *)array)[m+maplen*v]; + for (int m=0; m < maplen; m++) + { + ((double *)sortedarray)[m] = ((double *)array)[iodesc->remap[m]+maplen*v]; + } } - } - break; - case PIO_DOUBLE: - for (int v=0; v < nvars; v++) - { - for (int m=0; m < maplen; m++) + break; + case PIO_UBYTE: + for (int v=0; v < nvars; v++) { - ((double *)sortedarray)[iodesc->remap[m]] = ((double *)array)[m+maplen*v]; + for (int m=0; m < maplen; m++) + { + ((unsigned char *)sortedarray)[m] = ((unsigned char *)array)[iodesc->remap[m]+maplen*v]; + } } - } - break; - case PIO_UBYTE: - for (int v=0; v < nvars; v++) - { - for (int m=0; m < maplen; m++) + break; + case PIO_USHORT: + for (int v=0; v < nvars; v++) { - ((unsigned char *)sortedarray)[iodesc->remap[m]] = ((unsigned char *)array)[m+maplen*v]; + for (int m=0; m < maplen; m++) + { + ((unsigned short *)sortedarray)[m] = ((unsigned short *)array)[iodesc->remap[m]+maplen*v]; + } } - } - break; - case PIO_USHORT: - for (int v=0; v < nvars; v++) - { - for (int m=0; m < maplen; m++) + break; + case PIO_UINT: + for (int v=0; v < nvars; v++) { - ((unsigned short *)sortedarray)[iodesc->remap[m]] = ((unsigned short *)array)[m+maplen*v]; + for (int m=0; m < maplen; m++) + { + ((unsigned int *)sortedarray)[m] = ((unsigned int *)array)[iodesc->remap[m]+maplen*v]; + } } - } - break; - case PIO_UINT: - for (int v=0; v < nvars; v++) - { - for (int m=0; m < maplen; m++) + break; + case PIO_INT64: + for (int v=0; v < nvars; v++) { - ((unsigned int *)sortedarray)[iodesc->remap[m]] = ((unsigned int *)array)[m+maplen*v]; + for (int m=0; m < maplen; m++) + { + ((long long *)sortedarray)[m] = ((long long *)array)[iodesc->remap[m]+maplen*v]; + } } - } - break; - case PIO_INT64: - for (int v=0; v < nvars; v++) - { - for (int m=0; m < maplen; m++) + break; + case PIO_UINT64: + for (int v=0; v < nvars; v++) { - ((long long *)sortedarray)[iodesc->remap[m]] = ((long long *)array)[m+maplen*v]; + for (int m=0; m < maplen; m++) + { + ((unsigned long long *)sortedarray)[m] = ((unsigned long long *)array)[iodesc->remap[m]+maplen*v]; + } } - } - break; - case PIO_UINT64: - for (int v=0; v < nvars; v++) - { - for (int m=0; m < maplen; m++) + break; + case PIO_STRING: + for (int v=0; v < nvars; v++) { - ((unsigned long long *)sortedarray)[iodesc->remap[m]] = ((unsigned long long *)array)[m+maplen*v]; + for (int m=0; m < maplen; m++) + { + ((char **)sortedarray)[m] = ((char **)array)[iodesc->remap[m]+maplen*v]; + } } + break; + default: + return pio_err(NULL, NULL, PIO_EBADTYPE, __FILE__, __LINE__); } - break; - case PIO_STRING: - for (int v=0; v < nvars; v++) + } + else + { + switch (iodesc->piotype) { - for (int m=0; m < maplen; m++) + case PIO_BYTE: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((signed char *)sortedarray)[iodesc->remap[m]] = ((signed char *)array)[m+maplen*v]; + } + } + break; + case PIO_CHAR: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((char *)sortedarray)[iodesc->remap[m]] = ((char *)array)[m+maplen*v]; + } + } + break; + case PIO_SHORT: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((short *)sortedarray)[iodesc->remap[m]] = ((short *)array)[m+maplen*v]; + } + } + + break; + case PIO_INT: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((int *)sortedarray)[iodesc->remap[m]] = ((int *)array)[m+maplen*v]; + } + int flag=0; + for (int m=0; m < maplen; m++) + if(((int *)sortedarray)[m] < 0) + flag =1; + if(flag==1) + for (int m=0; m < maplen; m++) + printf("%d %d %d %d\n",m, iodesc->remap[m], ((int *)sortedarray)[iodesc->remap[m]], ((int *)array)[m]); + + } + break; + case PIO_FLOAT: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((float *)sortedarray)[iodesc->remap[m]] = ((float *)array)[m+maplen*v]; + } + } + break; + case PIO_DOUBLE: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((double *)sortedarray)[iodesc->remap[m]] = ((double *)array)[m+maplen*v]; + } + } + break; + case PIO_UBYTE: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((unsigned char *)sortedarray)[iodesc->remap[m]] = ((unsigned char *)array)[m+maplen*v]; + } + } + break; + case PIO_USHORT: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((unsigned short *)sortedarray)[iodesc->remap[m]] = ((unsigned short *)array)[m+maplen*v]; + } + } + break; + case PIO_UINT: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((unsigned int *)sortedarray)[iodesc->remap[m]] = ((unsigned int *)array)[m+maplen*v]; + } + } + break; + case PIO_INT64: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((long long *)sortedarray)[iodesc->remap[m]] = ((long long *)array)[m+maplen*v]; + } + } + break; + case PIO_UINT64: + for (int v=0; v < nvars; v++) + { + for (int m=0; m < maplen; m++) + { + ((unsigned long long *)sortedarray)[iodesc->remap[m]] = ((unsigned long long *)array)[m+maplen*v]; + } + } + break; + case PIO_STRING: + for (int v=0; v < nvars; v++) { - ((char **)sortedarray)[iodesc->remap[m]] = ((char **)array)[m+maplen*v]; + for (int m=0; m < maplen; m++) + { + ((char **)sortedarray)[iodesc->remap[m]] = ((char **)array)[m+maplen*v]; + } } + break; + default: + return pio_err(NULL, NULL, PIO_EBADTYPE, __FILE__, __LINE__); } - break; - default: - return pio_err(NULL, NULL, PIO_EBADTYPE, __FILE__, __LINE__); } return PIO_NOERR; } diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index d2a85be6e2e..d18cc3b212a 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -361,7 +361,7 @@ extern "C" { int determine_procs(int num_io_procs, int component_count, int *num_procs_per_comp, int **proc_list, int **my_proc_list); - int pio_sorted_copy(const void *array, void *tmparray, io_desc_t *iodesc, int nvars); + int pio_sorted_copy(const void *array, void *tmparray, io_desc_t *iodesc, int nvars, int direction); #if defined(__cplusplus) } #endif diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 87cda84f37f..77b5b23006b 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -380,6 +380,20 @@ void pio_map_sort(const PIO_Offset *map, int *remap, int maplen) } } while(switched); +/* + for(int i=maplen-1; i>=0; i--) + { + for(int j = 1; j<=i; j++) + { + if (map[remap[j-1]] > map[remap[j]]) + { + int tmp = remap[j-1]; + remap[j-1] = remap[j]; + remap[j] = tmp; + } + } + } +*/ } @@ -531,7 +545,10 @@ int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, in iodesc->remap[m] = m; pio_map_sort(compmap, iodesc->remap, maplen); for (int m=0; m < maplen; m++) - iodesc->map[iodesc->remap[m]] = compmap[m]; + iodesc->map[m] = compmap[iodesc->remap[m]]; + for (int m=1; m < maplen; m++) + if (iodesc->map[m] < iodesc->map[m-1]) + printf("%d: compmap[%d] %ld map[%d] %ld remap[%d] %d\n",ios->comp_rank, m, compmap[m], m, iodesc->map[m], m, iodesc->remap[m]); } else { @@ -559,7 +576,7 @@ int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, in iodesc->num_aiotasks = ios->num_iotasks; LOG((2, "creating subset rearranger iodesc->num_aiotasks = %d", iodesc->num_aiotasks)); - if ((ierr = subset_rearrange_create(ios, maplen, (PIO_Offset *)compmap, gdimlen, + if ((ierr = subset_rearrange_create(ios, maplen, (PIO_Offset *)iodesc->map, gdimlen, ndims, iodesc))) return pio_err(ios, NULL, ierr, __FILE__, __LINE__); } @@ -605,7 +622,7 @@ int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, in /* Compute the communications pattern for this decomposition. */ if (iodesc->rearranger == PIO_REARR_BOX) - if ((ierr = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc))) + if ((ierr = box_rearrange_create(ios, maplen, iodesc->map, gdimlen, ndims, iodesc))) return pio_err(ios, NULL, ierr, __FILE__, __LINE__); } diff --git a/src/flib/piodarray.F90.in b/src/flib/piodarray.F90.in index 7c80e89df8a..ef49be8be57 100644 --- a/src/flib/piodarray.F90.in +++ b/src/flib/piodarray.F90.in @@ -368,7 +368,6 @@ contains integer(C_SIZE_T) :: tlen tlen = size(array) - call read_darray_internal_{TYPE} (File%fh, vardesc%varid, iodesc%ioid, tlen, array, iostat) end subroutine read_darray_{DIMS}d_{TYPE} @@ -399,4 +398,3 @@ contains end subroutine read_darray_internal_{TYPE} end module piodarray - From e77b2cf2c08ba408361dc918b829fc672f006a2b Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Sat, 15 Dec 2018 15:50:27 -0700 Subject: [PATCH 04/13] remove debug print statements --- src/clib/pio_darray_int.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c index 3c723e94a61..c48241927f5 100644 --- a/src/clib/pio_darray_int.c +++ b/src/clib/pio_darray_int.c @@ -1948,9 +1948,7 @@ int pio_sorted_copy(const void *array, void *sortedarray, io_desc_t *iodesc, int { ((int *)sortedarray)[m] = ((int *)array)[iodesc->remap[m]+maplen*v]; } -/* for (int m=0; m < maplen; m++) - printf("0:%d: array=%d sorted=%d\n",m,((int *)array)[m], ((int *)sortedarray)[m]); -*/ } + } break; case PIO_FLOAT: for (int v=0; v < nvars; v++) @@ -2067,14 +2065,6 @@ int pio_sorted_copy(const void *array, void *sortedarray, io_desc_t *iodesc, int { ((int *)sortedarray)[iodesc->remap[m]] = ((int *)array)[m+maplen*v]; } - int flag=0; - for (int m=0; m < maplen; m++) - if(((int *)sortedarray)[m] < 0) - flag =1; - if(flag==1) - for (int m=0; m < maplen; m++) - printf("%d %d %d %d\n",m, iodesc->remap[m], ((int *)sortedarray)[iodesc->remap[m]], ((int *)array)[m]); - } break; case PIO_FLOAT: From 3878280bca7d4d6f9d8de9a4498d75a7b865dd6b Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 17 Dec 2018 13:02:50 -0700 Subject: [PATCH 05/13] update cheyenne gnu build --- ctest/runcdash-nwsc-gnu.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ctest/runcdash-nwsc-gnu.sh b/ctest/runcdash-nwsc-gnu.sh index c79077b60dc..020fe4a557e 100755 --- a/ctest/runcdash-nwsc-gnu.sh +++ b/ctest/runcdash-nwsc-gnu.sh @@ -9,17 +9,17 @@ fi module reset module unload netcdf -module swap intel gnu/7.1.0 -module swap mpt openmpi/3.0.0 +module swap intel gnu/8.1.0 +module swap mpt openmpi/3.1.0 module load git/2.10.2 -module load cmake/3.9.1 -module load netcdf/4.4.1.1 -export PNETCDF=/glade/u/home/jedwards/pnetcdf/svn3652/openmpi300/gnu710 +module load cmake/3.12.1 +module load netcdf/4.6.1 +export PNETCDF=/glade/u/home/jedwards/pnetcdf/df0b42f19/gnu/8.1.0/openmpi/3.1.0 export CC=mpicc export FC=mpif90 -export PIO_DASHBOARD_ROOT=`pwd`/dashboard +export PIO_DASHBOARD_ROOT=/glade/u/home/jedwards/sandboxes/dashboard export PIO_COMPILER_ID=GNU-`$CC --version | head -n 1 | tail -n 1 | cut -d' ' -f3` if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then From fbd27d84bb6e16277a7dead306fa69fc63d1850d Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 17 Dec 2018 13:47:20 -0700 Subject: [PATCH 06/13] update sitename --- cmake/LibMPI.cmake | 46 ++++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/cmake/LibMPI.cmake b/cmake/LibMPI.cmake index 2dd9a7d27a8..9a1b73886e2 100644 --- a/cmake/LibMPI.cmake +++ b/cmake/LibMPI.cmake @@ -16,7 +16,7 @@ endif () # #============================================================================== -# - Get the machine platform-specific +# - Get the machine platform-specific # # Syntax: platform_name (RETURN_VARIABLE) # @@ -25,30 +25,24 @@ function (platform_name RETURN_VARIABLE) # Determine platform name from site name... site_name (SITENAME) - # UCAR/NWSC Machines - if (SITENAME MATCHES "^yslogin" OR - SITENAME MATCHES "^geyser" OR - SITENAME MATCHES "^caldera") - - set (${RETURN_VARIABLE} "nwsc" PARENT_SCOPE) - # New NWSC SGI machine - elseif (SITENAME MATCHES "^laramie") - + if (SITENAME MATCHES "^laramie" OR + SITENAME MATCHES "^cheyenne") + set (${RETURN_VARIABLE} "nwscla" PARENT_SCOPE) - + # ALCF/Argonne Machines elseif (SITENAME MATCHES "^mira" OR SITENAME MATCHES "^cetus" OR SITENAME MATCHES "^vesta" OR SITENAME MATCHES "^cooley") - + set (${RETURN_VARIABLE} "alcf" PARENT_SCOPE) - + # NERSC Machines elseif (SITENAME MATCHES "^edison" OR SITENAME MATCHES "^cori") - + set (${RETURN_VARIABLE} "nersc" PARENT_SCOPE) # NCSA Machine (Blue Waters) @@ -61,11 +55,11 @@ function (platform_name RETURN_VARIABLE) SITENAME MATCHES "^titan") set (${RETURN_VARIABLE} "olcf" PARENT_SCOPE) - + else () set (${RETURN_VARIABLE} "unknown" PARENT_SCOPE) - + endif () endfunction () @@ -85,35 +79,35 @@ function (add_mpi_test TESTNAME) set (oneValueArgs NUMPROCS TIMEOUT EXECUTABLE) set (multiValueArgs ARGUMENTS) cmake_parse_arguments (${TESTNAME} "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - + # Store parsed arguments for convenience set (exec_file ${${TESTNAME}_EXECUTABLE}) set (exec_args ${${TESTNAME}_ARGUMENTS}) set (num_procs ${${TESTNAME}_NUMPROCS}) set (timeout ${${TESTNAME}_TIMEOUT}) - + # Get the platform name platform_name (PLATFORM) - + # Default ("unknown" platform) execution if (PLATFORM STREQUAL "unknown") # Run tests directly from the command line - set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs} - ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} + set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs} + ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${MPIEXEC_POSTFLAGS} ${exec_args}) else () - + # Run tests from the platform-specific executable - set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM} + set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM} ${num_procs} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${exec_args}) - + endif () - + # Add the test to CTest add_test(NAME ${TESTNAME} COMMAND ${EXE_CMD}) - + # Adjust the test timeout set_tests_properties(${TESTNAME} PROPERTIES TIMEOUT ${timeout}) From 4bc9b011991ced34f649e2c450365e280f6294f3 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 17 Dec 2018 14:08:51 -0700 Subject: [PATCH 07/13] clean up ctest scripts --- cmake/mpiexec.nwsc | 8 ++++++-- cmake/mpiexec.nwscla | 8 ++++++-- ctest/CTestEnvironment-nwsc.cmake | 18 ------------------ ...ash-nwsc-gnu.sh => runcdash-nwscla-gnu.sh} | 0 ctest/runctest-nwsc.sh | 19 ------------------- 5 files changed, 12 insertions(+), 41 deletions(-) delete mode 100644 ctest/CTestEnvironment-nwsc.cmake rename ctest/{runcdash-nwsc-gnu.sh => runcdash-nwscla-gnu.sh} (100%) delete mode 100755 ctest/runctest-nwsc.sh diff --git a/cmake/mpiexec.nwsc b/cmake/mpiexec.nwsc index a6242b66712..1004e573333 100755 --- a/cmake/mpiexec.nwsc +++ b/cmake/mpiexec.nwsc @@ -8,5 +8,9 @@ NP=$1 shift - -mpirun.lsf $@ -n$NP +if [ $LMOD_FAMILY_MPI -eq "openmpi" ] +then + mpirun -np $NP $@ +else + mpiexec_mpt -n $NP $@ +fi diff --git a/cmake/mpiexec.nwscla b/cmake/mpiexec.nwscla index bb7018bf92c..1004e573333 100755 --- a/cmake/mpiexec.nwscla +++ b/cmake/mpiexec.nwscla @@ -8,5 +8,9 @@ NP=$1 shift - -mpiexec_mpt -n $NP $@ +if [ $LMOD_FAMILY_MPI -eq "openmpi" ] +then + mpirun -np $NP $@ +else + mpiexec_mpt -n $NP $@ +fi diff --git a/ctest/CTestEnvironment-nwsc.cmake b/ctest/CTestEnvironment-nwsc.cmake deleted file mode 100644 index 4a0d6fd3acd..00000000000 --- a/ctest/CTestEnvironment-nwsc.cmake +++ /dev/null @@ -1,18 +0,0 @@ -#============================================================================== -# -# This file sets the environment variables needed to configure and build -# on the NCAR Wyoming Supercomputing Center systems -# (yellowstone/caldera/geyser). -# -#============================================================================== - -# Assume all package locations (NetCDF, PnetCDF, HDF5, etc) are already -# set with existing environment variables: NETCDF, PNETCDF, HDF5, etc. - -# Define the extra CMake configure options -set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_DOC=OFF") - -# If MPISERIAL environment variable is set, then enable MPISERIAL -if (DEFINED ENV{MPISERIAL}) - set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MPISERIAL=ON -DPIO_ENABLE_EXAMPLES=OFF ") -endif () diff --git a/ctest/runcdash-nwsc-gnu.sh b/ctest/runcdash-nwscla-gnu.sh similarity index 100% rename from ctest/runcdash-nwsc-gnu.sh rename to ctest/runcdash-nwscla-gnu.sh diff --git a/ctest/runctest-nwsc.sh b/ctest/runctest-nwsc.sh deleted file mode 100755 index 64b8e9a8181..00000000000 --- a/ctest/runctest-nwsc.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -#============================================================================== -# -# This script defines how to run CTest on the NCAR Wyoming Supercomputing -# Center systems (yellowstone/caldera/geyser). -# -# This assumes the CTest model name (e.g., "Nightly") is passed to it when -# run. -# -#============================================================================== - -# Get the CTest script directory -scrdir=$1 - -# Get the CTest model name -model=$2 - -# Run the "ctest" command through an interactive parallel session -DAV_CORES=4 execca ctest -S ${scrdir}/CTestScript-Test.cmake,${model} -V From 0ba25df8c2f698d82deb4740cac8fae08d811a87 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 17 Dec 2018 15:14:04 -0700 Subject: [PATCH 08/13] fix syntax --- cmake/mpiexec.nwsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/mpiexec.nwsc b/cmake/mpiexec.nwsc index 1004e573333..15bd07a8937 100755 --- a/cmake/mpiexec.nwsc +++ b/cmake/mpiexec.nwsc @@ -8,7 +8,7 @@ NP=$1 shift -if [ $LMOD_FAMILY_MPI -eq "openmpi" ] +if [ "$LMOD_FAMILY_MPI" == "openmpi" ] then mpirun -np $NP $@ else From 9ee227c7f7aa3550dd998d189757c3b20c162788 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 17 Dec 2018 15:27:55 -0700 Subject: [PATCH 09/13] fix syntax --- cmake/mpiexec.nwsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/mpiexec.nwsc b/cmake/mpiexec.nwsc index 15bd07a8937..bf735175ed9 100755 --- a/cmake/mpiexec.nwsc +++ b/cmake/mpiexec.nwsc @@ -8,7 +8,7 @@ NP=$1 shift -if [ "$LMOD_FAMILY_MPI" == "openmpi" ] +if [[ "$LMOD_FAMILY_MPI" == "openmpi" ]] then mpirun -np $NP $@ else From 415fcc150c56dbb4e12e4cd379513ce1f44271e3 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 17 Dec 2018 15:34:51 -0700 Subject: [PATCH 10/13] try fixing the right file --- cmake/mpiexec.nwsc | 16 ---------------- cmake/mpiexec.nwscla | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) delete mode 100755 cmake/mpiexec.nwsc diff --git a/cmake/mpiexec.nwsc b/cmake/mpiexec.nwsc deleted file mode 100755 index bf735175ed9..00000000000 --- a/cmake/mpiexec.nwsc +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# -# Arguments: -# -# $1 - Number of MPI Tasks -# $2+ - Executable and its arguments -# - -NP=$1 -shift -if [[ "$LMOD_FAMILY_MPI" == "openmpi" ]] -then - mpirun -np $NP $@ -else - mpiexec_mpt -n $NP $@ -fi diff --git a/cmake/mpiexec.nwscla b/cmake/mpiexec.nwscla index 1004e573333..bf735175ed9 100755 --- a/cmake/mpiexec.nwscla +++ b/cmake/mpiexec.nwscla @@ -8,7 +8,7 @@ NP=$1 shift -if [ $LMOD_FAMILY_MPI -eq "openmpi" ] +if [[ "$LMOD_FAMILY_MPI" == "openmpi" ]] then mpirun -np $NP $@ else From b4be30dc0c05980e134b2082e845a5120236122c Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 17 Dec 2018 16:01:06 -0700 Subject: [PATCH 11/13] update pgi test --- ctest/{runcdash-nwsc-pgi.sh => runcdash-nwscla-pgi.sh} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename ctest/{runcdash-nwsc-pgi.sh => runcdash-nwscla-pgi.sh} (82%) diff --git a/ctest/runcdash-nwsc-pgi.sh b/ctest/runcdash-nwscla-pgi.sh similarity index 82% rename from ctest/runcdash-nwsc-pgi.sh rename to ctest/runcdash-nwscla-pgi.sh index 20c09d619e3..31fcae7fad6 100755 --- a/ctest/runcdash-nwsc-pgi.sh +++ b/ctest/runcdash-nwscla-pgi.sh @@ -9,11 +9,11 @@ fi module reset module unload netcdf -module swap intel pgi/16.5 -module load git/2.3.0 -module load cmake/3.0.2 -module load netcdf-mpi/4.4.1 -module load pnetcdf/1.7.0 +module swap intel pgi/17.9 +module load git/2.10.2 +module load cmake/3.12.1 +module load netcdf-mpi/4.5.0 +module load pnetcdf/1.9.0 export CC=mpicc export FC=mpif90 From 205567f5e5dfb6529902090bad1d6dd234fe07d4 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 18 Dec 2018 07:01:11 -0700 Subject: [PATCH 12/13] still working on it --- cmake/LibMPI.cmake | 3 ++- ctest/runcdash-nwscla-pgi.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmake/LibMPI.cmake b/cmake/LibMPI.cmake index 9a1b73886e2..f1116724831 100644 --- a/cmake/LibMPI.cmake +++ b/cmake/LibMPI.cmake @@ -27,7 +27,8 @@ function (platform_name RETURN_VARIABLE) if (SITENAME MATCHES "^laramie" OR - SITENAME MATCHES "^cheyenne") + SITENAME MATCHES "^cheyenne" OR + SITENAME MATCHES "^chadmin") set (${RETURN_VARIABLE} "nwscla" PARENT_SCOPE) diff --git a/ctest/runcdash-nwscla-pgi.sh b/ctest/runcdash-nwscla-pgi.sh index 31fcae7fad6..40514c1d9e1 100755 --- a/ctest/runcdash-nwscla-pgi.sh +++ b/ctest/runcdash-nwscla-pgi.sh @@ -18,7 +18,7 @@ module load pnetcdf/1.9.0 export CC=mpicc export FC=mpif90 -export PIO_DASHBOARD_ROOT=`pwd`/dashboard +export PIO_DASHBOARD_ROOT=/glade/u/home/jedwards/sandboxes/dashboard export PIO_COMPILER_ID=PGI-`$CC --version | head -n 2 | tail -n 1 | cut -d' ' -f2` if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then From 219699459c71af50276aa3a34400addfed2a338e Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 18 Dec 2018 11:06:06 -0700 Subject: [PATCH 13/13] add a test --- ctest/CTestEnvironment-nwscla.cmake | 2 +- tests/general/pio_decomp_tests_1d.F90.in | 103 +++++++++++++++++++++-- 2 files changed, 99 insertions(+), 6 deletions(-) diff --git a/ctest/CTestEnvironment-nwscla.cmake b/ctest/CTestEnvironment-nwscla.cmake index b7f1d1c9aef..efee6bf659d 100644 --- a/ctest/CTestEnvironment-nwscla.cmake +++ b/ctest/CTestEnvironment-nwscla.cmake @@ -10,7 +10,7 @@ # set with existing environment variables: NETCDF, PNETCDF, HDF5, etc. # Define the extra CMake configure options -set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_ASYNC=TRUE") +set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE ") # If MPISERIAL environment variable is set, then enable MPISERIAL if (DEFINED ENV{MPISERIAL}) diff --git a/tests/general/pio_decomp_tests_1d.F90.in b/tests/general/pio_decomp_tests_1d.F90.in index 48d4f95a101..fc903aab93b 100644 --- a/tests/general/pio_decomp_tests_1d.F90.in +++ b/tests/general/pio_decomp_tests_1d.F90.in @@ -122,7 +122,7 @@ END SUBROUTINE ! Test block cyclic interface ! Write with one decomp and read with another -! Test all combs +! Test all combs ! - no rearrage read + no rearrange write ! - rearrage read + no rearrange write ! - no rearrage read + rearrange write @@ -185,7 +185,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_rd_1d_bc filename = "test_pio_decomp_simple_tests.testfile" do i=1,num_iotypes PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) - ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim) @@ -210,7 +210,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_rd_1d_bc PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val") call PIO_closefile(pio_file) - + call PIO_deletefile(pio_tf_iosystem_, filename); end do @@ -266,7 +266,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_bc_with_holes filename = "test_pio_decomp_simple_tests.testfile" do i=1,num_iotypes PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) - ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim) @@ -290,7 +290,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_bc_with_holes PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val") call PIO_closefile(pio_file) - + call PIO_deletefile(pio_tf_iosystem_, filename); end do @@ -304,3 +304,96 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_bc_with_holes deallocate(rbuf) deallocate(wbuf) PIO_TF_AUTO_TEST_SUB_END nc_wr_1d_bc_with_holes + +PIO_TF_TEMPLATE +PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_bc_random + use mpi, only : MPI_INT, MPI_SCATTER + implicit none + type(var_desc_t) :: pio_var + type(file_desc_t) :: pio_file + character(len=PIO_TF_MAX_STR_LEN) :: filename + type(io_desc_t) :: wr_iodesc + integer, dimension(:), allocatable :: compdof, gcompdof + integer, dimension(1) :: count + PIO_TF_FC_DATA_TYPE, dimension(:), allocatable :: rbuf, wbuf + integer, dimension(1) :: dims + integer :: pio_dim + integer :: i, j, ierr, lsz + integer :: tmp + real :: u + ! iotypes = valid io types + integer, dimension(:), allocatable :: iotypes + character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs + integer :: num_iotypes + + ! Set the decomposition for writing data - random order same local size + count(1) = 4 + dims(1) = count(1)*pio_tf_world_sz_ + if(pio_tf_world_rank_ == 0) then + allocate(gcompdof(dims(1))) + gcompdof = 0 + do i=1,dims(1) + gcompdof(i) = i + enddo + do i=dims(1),1,-1 + call random_number(u) + j = CEILING(real(i)*u) + tmp = gcompdof(j) + gcompdof(j) = gcompdof(i) + gcompdof(i) = tmp + enddo + endif + allocate(compdof(count(1))) + call mpi_scatter(gcompdof, count(1), MPI_INT, compdof, 4, MPI_INT, 0, pio_tf_comm_, ierr) + if(allocated(gcompdof)) deallocate(gcompdof) + allocate(rbuf(count(1))) + allocate(wbuf(count(1))) + do i=1,count(1) + wbuf(i) = compdof(i) + end do + + call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc) + deallocate(compdof) + + num_iotypes = 0 + call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) + filename = "test_pio_decomp_simple_tests.testfile" + do i=1,num_iotypes + PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i) + ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) + PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename)) + + ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim) + PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename)) + + ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var) + PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename)) + + ierr = PIO_enddef(pio_file) + PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename)) + + ! Write the variable out + call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename)) + + call PIO_syncfile(pio_file) + + call PIO_read_darray(pio_file, pio_var, wr_iodesc, rbuf, ierr) + PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename)) + + PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val") + + call PIO_closefile(pio_file) + + call PIO_deletefile(pio_tf_iosystem_, filename); + end do + + if(allocated(iotypes)) then + deallocate(iotypes) + deallocate(iotype_descs) + end if + + call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc) + deallocate(rbuf) + deallocate(wbuf) +PIO_TF_AUTO_TEST_SUB_END nc_wr_1d_bc_random