From ee5c7ed7747c7969717bd0ded2a2df645c11d294 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 05:59:02 -0600 Subject: [PATCH 01/26] starting to add vard functions --- configure.ac | 2 +- src/clib/pio.h | 54 +++++++ src/clib/pio_get_vard.c | 290 ++++++++++++++++++++++++++++++++++++ src/clib/pio_getput_int.c | 49 +++++++ src/clib/pio_put_vard.c | 299 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 693 insertions(+), 1 deletion(-) create mode 100644 src/clib/pio_get_vard.c create mode 100644 src/clib/pio_put_vard.c diff --git a/configure.ac b/configure.ac index a61c1a0a98d..567c5e40f6e 100644 --- a/configure.ac +++ b/configure.ac @@ -136,7 +136,7 @@ if test "x$enable_docs" = xyes; then if test "x$enable_developer_docs" = xyes; then AC_SUBST([C_SRC_FILES], ["../src/clib"]) else - AC_SUBST([C_SRC_FILES], ["../src/clib/pio_nc.c ../src/clib/pio_nc4.c ../src/clib/pio_darray.c ../src/clib/pio_get_nc.c ../src/clib/pio_put_nc.c ../src/clib/pioc_support.c ../src/clib/pioc.c ../src/clib/pio_file.c ../src/clib/pio.h"]) + AC_SUBST([C_SRC_FILES], ["../src/clib/pio_nc.c ../src/clib/pio_nc4.c ../src/clib/pio_darray.c ../src/clib/pio_get_nc.c ../src/clib/pio_put_nc.c ../src/clib/pioc_support.c ../src/clib/pioc.c ../src/clib/pio_file.c ../src/clib/pio.h ../src/clib/pio_get_vard.c ../src/clib/pio_put_vard.c"]) fi AC_CONFIG_FILES([doc/Doxyfile]) fi diff --git a/src/clib/pio.h b/src/clib/pio.h index 980dc60070d..12dba4fba8e 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -1195,6 +1195,60 @@ extern "C" { const PIO_Offset *count, const PIO_Offset *stride, const unsigned long long *op); + /* Data reads - vard. */ + int PIOc_get_vard(int ncid, int varid, int decompid, const PIO_Offset recnum, void *buf); + int PIOc_get_vard_text(int ncid, int varid, int decompid, const PIO_Offset recnum, + char *buf); + int PIOc_get_vard_schar(int ncid, int varid, int decompid, const PIO_Offset recnum, + signed char *buf); + int PIOc_get_vard_short(int ncid, int varid, int decompid, const PIO_Offset recnum, + short *buf); + int PIOc_get_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum, + int *buf); + int PIOc_get_vard_float(int ncid, int varid, int decompid, const PIO_Offset recnum, + float *buf); + int PIOc_get_vard_long(int ncid, int varid, int decompid, const PIO_Offset recnum, + long *buf); + int PIOc_get_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum, + double *buf); + int PIOc_get_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum, + unsigned char *buf); + int PIOc_get_vard_ushort(int ncid, int varid, int decompid, const PIO_Offset recnum, + unsigned short *buf); + int PIOc_get_vard_uint(int ncid, int varid, int decompid, const PIO_Offset recnum, + unsigned int *buf); + int PIOc_get_vard_longlong(int ncid, int varid, int decompid, const PIO_Offset recnum, + long long *buf); + int PIOc_get_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum, + unsigned long long *buf); + + /* Data writes - vara. */ + int PIOc_put_vard(int ncid, int varid, int decompid, const PIO_Offset recnum, + const void *buf); + int PIOc_put_vard_text(int ncid, int varid, int decompid, const PIO_Offset recnum, + const char *op); + int PIOc_put_vard_schar(int ncid, int varid, int decompid, const PIO_Offset recnum, + const signed char *op); + int PIOc_put_vard_short(int ncid, int varid, int decompid, const PIO_Offset recnum, + const short *op); + int PIOc_put_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum, + const int *op); + int PIOc_put_vard_long(int ncid, int varid, int decompid, const PIO_Offset recnum, + const long *op); + int PIOc_put_vard_float(int ncid, int varid, int decompid, const PIO_Offset recnum, + const float *op); + int PIOc_put_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum, + const double *op); + int PIOc_put_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum, + const unsigned char *op); + int PIOc_put_vard_ushort(int ncid, int varid, int decompid, const PIO_Offset recnum, + const unsigned short *op); + int PIOc_put_vard_uint(int ncid, int varid, int decompid, const PIO_Offset recnum, + const unsigned int *op); + int PIOc_put_vard_longlong(int ncid, int varid, int decompid, const PIO_Offset recnum, + const long long *op); + int PIOc_put_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum, + const unsigned long long *op); #if defined(__cplusplus) } #endif diff --git a/src/clib/pio_get_vard.c b/src/clib/pio_get_vard.c new file mode 100644 index 00000000000..953a4df01ce --- /dev/null +++ b/src/clib/pio_get_vard.c @@ -0,0 +1,290 @@ +/** + * @file + * PIO functions to get data with distributed arrays. + * + * @author Ed Hartnett + * @date 2019 + * + * @see https://github.com/NCAR/ParallelIO + */ +#include +#include +#include + +/** + * @addtogroup PIO_get_var_c Read Data + * Read data from a variable in C. + * @{ + */ + +/** + * Get a muti-dimensional subset of a text variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard_text(int ncid, int varid, int decompid, + const PIO_Offset recnum, char *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_CHAR, buf); +} + +/** + * Get a muti-dimensional subset of an unsigned char variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard_uchar(int ncid, int varid, int decompid, + const PIO_Offset recnum, unsigned char *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_UBYTE, buf); +} + +/** + * Get a muti-dimensional subset of a signed char variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard_schar(int ncid, int varid, int decompid, + const PIO_Offset recnum, signed char *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_BYTE, buf); +} + +/** + * Get a muti-dimensional subset of an unsigned 16-bit integer + * variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard_ushort(int ncid, int varid, int decompid, + const PIO_Offset recnum, unsigned short *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_USHORT, + buf); +} + +/** + * Get a muti-dimensional subset of a 16-bit integer variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard_short(int ncid, int varid, int decompid, + const PIO_Offset recnum, short *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_SHORT, buf); +} + +/** + * Get a muti-dimensional subset of a 64-bit integer variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard_long(int ncid, int varid, int decompid, + const PIO_Offset recnum, long *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, + PIO_LONG_INTERNAL, buf); +} + +/** + * Get a muti-dimensional subset of an unsigned integer variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard_uint(int ncid, int varid, int decompid, + const PIO_Offset recnum, unsigned int *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_UINT, buf); +} + +/** + * Get a muti-dimensional subset of an integer variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard_int(int ncid, int varid, int decompid, + const PIO_Offset recnum, int *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_INT, buf); +} + +/** + * Get a muti-dimensional subset of a floating point variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard_float(int ncid, int varid, int decompid, + const PIO_Offset recnum, float *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_FLOAT, buf); +} + +/** + * Get a muti-dimensional subset of a 64-bit floating point variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard_double(int ncid, int varid, int decompid, + const PIO_Offset recnum, double *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_DOUBLE, + buf); +} + +/** + * Get a muti-dimensional subset of an unsigned 64-bit integer + * variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard_ulonglong(int ncid, int varid, int decompid, + const PIO_Offset recnum, unsigned long long *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_UINT64, + buf); +} + +/** + * Get a muti-dimensional subset of a 64-bit integer variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard_longlong(int ncid, int varid, int decompid, + const PIO_Offset recnum, long long *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_INT64, buf); +} + +/** + * Get a muti-dimensional subset of a variable the same type + * as the variable in the file. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param buf pointer that will get the data. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int PIOc_get_vard(int ncid, int varid, int decompid, + const PIO_Offset recnum, void *buf) +{ + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_NAT, buf); +} + + +/** + * @} + */ diff --git a/src/clib/pio_getput_int.c b/src/clib/pio_getput_int.c index b5aafae53af..eafde624226 100644 --- a/src/clib/pio_getput_int.c +++ b/src/clib/pio_getput_int.c @@ -1424,3 +1424,52 @@ PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op) return ierr; } +/** + * Internal PIO function which provides a type-neutral interface to + * nc_get_vard. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param xtype the netCDF type of the data being passed in buf. Data + * will be automatically covnerted from the type of the variable being + * read from to this type. If NC_NAT then the variable's file type + * will be used. Use special PIO_LONG_INTERNAL for _long() functions. + * @param buf pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_get_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum, + nc_type xtype, void *buf) +{ + return PIO_NOERR; +} + +/** + * Internal PIO function which provides a type-neutral interface to + * nc_put_vard. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param xtype the netCDF type of the data being passed in buf. Data + * will be automatically covnerted from this type to the type of the + * variable being written to. If NC_NAT then the variable's file type + * will be used. Use special PIO_LONG_INTERNAL for _long() functions. + * @param buf pointer to the data to be written. + * + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum, + nc_type xtype, const void *buf) +{ + return PIO_NOERR; +} diff --git a/src/clib/pio_put_vard.c b/src/clib/pio_put_vard.c new file mode 100644 index 00000000000..f68371020cf --- /dev/null +++ b/src/clib/pio_put_vard.c @@ -0,0 +1,299 @@ +/** + * @file + * PIO functions to write data with distributed arrays. + * + * @author Ed Hartnett + * @date 2019 + * @see https://github.com/NCAR/ParallelIO + */ +#include +#include +#include + +/** + * @addtogroup PIO_put_var_c Write Data + * Write data to a Variable in C. + * @{ + */ + +/** + * Get strided, muti-dimensional subset of a text variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_text(int ncid, int varid, int decompid, const PIO_Offset recnum, + const char *op) +{ + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_CHAR, op); +} + +/** + * Get strided, muti-dimensional subset of an unsigned char variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum, + const unsigned char *op) +{ + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_UBYTE, op); +} + +/** + * Get strided, muti-dimensional subset of a signed char variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_schar(int ncid, int varid, int decompid, const PIO_Offset recnum, + const signed char *op) +{ + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_BYTE, op); +} + +/** + * Get strided, muti-dimensional subset of an unsigned 16-bit integer + * variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_ushort(int ncid, int varid, int decompid, const PIO_Offset recnum, + const unsigned short *op) +{ + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_USHORT, op); +} + +/** + * Get strided, muti-dimensional subset of a 16-bit integer variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_short(int ncid, int varid, int decompid, const PIO_Offset recnum, + const short *op) +{ + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_SHORT, op); +} + +/** + * Get strided, muti-dimensional subset of an unsigned integer + * variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_uint(int ncid, int varid, int decompid, const PIO_Offset recnum, + const unsigned int *op) +{ + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_UINT, op); +} + +/** + * Get strided, muti-dimensional subset of an integer variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum, + const int *op) +{ + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_INT, op); +} + +/** + * Get strided, muti-dimensional subset of a 64-bit integer variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_long(int ncid, int varid, int decompid, const PIO_Offset recnum, + const long *op) +{ + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, PIO_LONG_INTERNAL, op); +} + +/** + * Get strided, muti-dimensional subset of a floating point variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_float(int ncid, int varid, int decompid, const PIO_Offset recnum, + const float *op) +{ + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_FLOAT, op); +} + +/** + * Get strided, muti-dimensional subset of a 64-bit unsigned integer + * variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_longlong(int ncid, int varid, int decompid, const PIO_Offset recnum, + const long long *op) +{ + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_INT64, op); +} + +/** + * Get strided, muti-dimensional subset of a 64-bit floating point + * variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum, + const double *op) +{ + return PIOc_put_vars_tc(ncid, varid, decompid, recnum, NC_DOUBLE, op); +} + +/** + * Get strided, muti-dimensional subset of an unsigned 64-bit integer + * variable. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum, + const unsigned long long *op) +{ + return PIOc_put_vars_tc(ncid, varid, decompid, recnum, NC_UINT64, op); +} + +/** + * Write strided, muti-dimensional subset of a variable of any type. + * + * This routine is called collectively by all tasks in the + * communicator ios.union_comm. + * + * @param ncid identifies the netCDF file + * @param varid the variable ID number + * @param decompid the decomposition ID. + * @param recnum the record number. + * @param op pointer to the data to be written. + * @return PIO_NOERR on success, error code otherwise. + * @author Ed Hartnett + */ +int +PIOc_put_vard(int ncid, int varid, int decompid, const PIO_Offset recnum, + const void *op) +{ + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_NAT, op); +} + +/** + * @} + */ From c2a24fb5abc829d7742c2a6c71eab7e87b5c136e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 06:24:50 -0600 Subject: [PATCH 02/26] more work on vard functions --- src/clib/pio_darray.c | 13 ++++++++----- src/clib/pio_getput_int.c | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index 0689634bd1f..c8e0f91ec64 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -853,12 +853,14 @@ PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *arra * * @param ncid identifies the netCDF file * @param varid the variable ID to be read - * @param ioid: the I/O description ID as passed back by + * @param ioid the I/O description ID as passed back by * PIOc_InitDecomp(). - * @param arraylen: the length of the array to be read. This - * is the length of the distrubited array. That is, the length of - * the portion of the data that is on the processor. - * @param array: pointer to the data to be read. This is a + * @param arraylen this parameter is ignored. Nominally it is the + * length of the array to be read. This is the length of the + * distrubited array. That is, the length of the portion of the data + * that is on the processor. This is already known because it is in + * the decomposition. + * @param array pointer to the data to be read. This is a * pointer to the distributed portion of the array that is on this * processor. * @return 0 for success, error code otherwise. @@ -876,6 +878,7 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, size_t rlen = 0; /* the length of data in iobuf. */ int ierr; /* Return code. */ void *tmparray; /* unsorted copy of array buf if required */ + /* Get the file info. */ if ((ierr = pio_get_file(ncid, &file))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); diff --git a/src/clib/pio_getput_int.c b/src/clib/pio_getput_int.c index eafde624226..5159790ad00 100644 --- a/src/clib/pio_getput_int.c +++ b/src/clib/pio_getput_int.c @@ -1426,7 +1426,8 @@ PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op) } /** * Internal PIO function which provides a type-neutral interface to - * nc_get_vard. + * PIOc_get_vard() and related functions. This function gets + * distributed arrays of any type, converting them to any type. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -1447,12 +1448,23 @@ int PIOc_get_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum, nc_type xtype, void *buf) { + int ret; + + /* Set the value of the record dimension. */ + if ((ret = PIOc_setframe(ncid, varid, recnum))) + return ret; + + /* Read the distributed array. */ + if ((ret = PIOc_read_darray(ncid, varid, decompid, 0, buf))) + return ret; + return PIO_NOERR; } /** * Internal PIO function which provides a type-neutral interface to - * nc_put_vard. + * PIOc_get_vard() and related functions. This function puts + * distributed arrays of any type, converting them to any type. * * @param ncid identifies the netCDF file * @param varid the variable ID number @@ -1471,5 +1483,21 @@ int PIOc_put_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum, nc_type xtype, const void *buf) { + io_desc_t *iodesc; /* The IO description. */ + int ret; + + /* Set the value of the record dimension. */ + if ((ret = PIOc_setframe(ncid, varid, recnum))) + return ret; + + /* Get decomposition information. */ + if (!(iodesc = pio_get_iodesc_from_id(decompid))) + return PIO_EBADID; + + /* Write the distributed array. */ + if ((ret = PIOc_write_darray(ncid, varid, decompid, iodesc->ndof, + (void *)buf, NULL))) + return ret; + return PIO_NOERR; } From f33f309376bcebbdc548017518f5f6036ad0bfd5 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 06:39:29 -0600 Subject: [PATCH 03/26] further work on vard functions --- src/clib/Makefile.am | 3 ++- src/clib/pio.h | 8 +++---- src/clib/pio_get_vard.c | 49 +++++++++++---------------------------- src/clib/pio_internal.h | 4 ++++ src/clib/pio_put_vard.c | 44 +++++++++++++++++------------------ tests/cunit/test_darray.c | 12 ++++++---- 6 files changed, 54 insertions(+), 66 deletions(-) diff --git a/src/clib/Makefile.am b/src/clib/Makefile.am index bb9edc51703..a2abefce34f 100644 --- a/src/clib/Makefile.am +++ b/src/clib/Makefile.am @@ -16,6 +16,7 @@ include_HEADERS = pio.h libpio_la_SOURCES = bget.c pioc_sc.c pio_darray.c pio_file.c \ pio_getput_int.c pio_msg.c pio_nc.c pio_rearrange.c pioc.c \ pioc_support.c pio_darray_int.c pio_get_nc.c pio_lists.c pio_nc4.c \ -pio_put_nc.c pio_spmd.c pio_internal.h bget.h uthash.h +pio_put_nc.c pio_spmd.c pio_get_vard.c pio_put_vard.c pio_internal.h \ +bget.h uthash.h EXTRA_DIST = CMakeLists.txt diff --git a/src/clib/pio.h b/src/clib/pio.h index 12dba4fba8e..bfe4cfb33a0 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -1207,8 +1207,8 @@ extern "C" { int *buf); int PIOc_get_vard_float(int ncid, int varid, int decompid, const PIO_Offset recnum, float *buf); - int PIOc_get_vard_long(int ncid, int varid, int decompid, const PIO_Offset recnum, - long *buf); + /* int PIOc_get_vard_long(int ncid, int varid, int decompid, const PIO_Offset recnum, */ + /* long *buf); */ int PIOc_get_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum, double *buf); int PIOc_get_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum, @@ -1233,8 +1233,8 @@ extern "C" { const short *op); int PIOc_put_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum, const int *op); - int PIOc_put_vard_long(int ncid, int varid, int decompid, const PIO_Offset recnum, - const long *op); + /* int PIOc_put_vard_long(int ncid, int varid, int decompid, const PIO_Offset recnum, */ + /* const long *op); */ int PIOc_put_vard_float(int ncid, int varid, int decompid, const PIO_Offset recnum, const float *op); int PIOc_put_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum, diff --git a/src/clib/pio_get_vard.c b/src/clib/pio_get_vard.c index 953a4df01ce..cccead9bc62 100644 --- a/src/clib/pio_get_vard.c +++ b/src/clib/pio_get_vard.c @@ -34,7 +34,7 @@ int PIOc_get_vard_text(int ncid, int varid, int decompid, const PIO_Offset recnum, char *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_CHAR, buf); + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_CHAR, buf); } /** @@ -54,7 +54,7 @@ int PIOc_get_vard_text(int ncid, int varid, int decompid, int PIOc_get_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum, unsigned char *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_UBYTE, buf); + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_UBYTE, buf); } /** @@ -74,7 +74,7 @@ int PIOc_get_vard_uchar(int ncid, int varid, int decompid, int PIOc_get_vard_schar(int ncid, int varid, int decompid, const PIO_Offset recnum, signed char *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_BYTE, buf); + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_BYTE, buf); } /** @@ -95,7 +95,7 @@ int PIOc_get_vard_schar(int ncid, int varid, int decompid, int PIOc_get_vard_ushort(int ncid, int varid, int decompid, const PIO_Offset recnum, unsigned short *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_USHORT, + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_USHORT, buf); } @@ -116,28 +116,7 @@ int PIOc_get_vard_ushort(int ncid, int varid, int decompid, int PIOc_get_vard_short(int ncid, int varid, int decompid, const PIO_Offset recnum, short *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_SHORT, buf); -} - -/** - * Get a muti-dimensional subset of a 64-bit integer variable. - * - * This routine is called collectively by all tasks in the - * communicator ios.union_comm. - * - * @param ncid identifies the netCDF file - * @param varid the variable ID number - * @param decompid the decomposition ID. - * @param recnum the record number. - * @param buf pointer that will get the data. - * @return PIO_NOERR on success, error code otherwise. - * @author Ed Hartnett - */ -int PIOc_get_vard_long(int ncid, int varid, int decompid, - const PIO_Offset recnum, long *buf) -{ - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, - PIO_LONG_INTERNAL, buf); + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_SHORT, buf); } /** @@ -157,7 +136,7 @@ int PIOc_get_vard_long(int ncid, int varid, int decompid, int PIOc_get_vard_uint(int ncid, int varid, int decompid, const PIO_Offset recnum, unsigned int *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_UINT, buf); + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_UINT, buf); } /** @@ -177,7 +156,7 @@ int PIOc_get_vard_uint(int ncid, int varid, int decompid, int PIOc_get_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum, int *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_INT, buf); + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_INT, buf); } /** @@ -197,7 +176,7 @@ int PIOc_get_vard_int(int ncid, int varid, int decompid, int PIOc_get_vard_float(int ncid, int varid, int decompid, const PIO_Offset recnum, float *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_FLOAT, buf); + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_FLOAT, buf); } /** @@ -217,7 +196,7 @@ int PIOc_get_vard_float(int ncid, int varid, int decompid, int PIOc_get_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum, double *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_DOUBLE, + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_DOUBLE, buf); } @@ -239,7 +218,7 @@ int PIOc_get_vard_double(int ncid, int varid, int decompid, int PIOc_get_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum, unsigned long long *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_UINT64, + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_UINT64, buf); } @@ -260,7 +239,7 @@ int PIOc_get_vard_ulonglong(int ncid, int varid, int decompid, int PIOc_get_vard_longlong(int ncid, int varid, int decompid, const PIO_Offset recnum, long long *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_INT64, buf); + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_INT64, buf); } /** @@ -278,10 +257,10 @@ int PIOc_get_vard_longlong(int ncid, int varid, int decompid, * @return PIO_NOERR on success, error code otherwise. * @author Ed Hartnett */ -int PIOc_get_vard(int ncid, int varid, int decompid, - const PIO_Offset recnum, void *buf) +int PIOc_get_vard(int ncid, int varid, int decompid, const PIO_Offset recnum, + void *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NULL, NC_NAT, buf); + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_NAT, buf); } diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index dd635b72d14..09ee497ab5b 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -324,6 +324,8 @@ extern "C" { int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype, void *buf); int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf); + int PIOc_get_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum, + nc_type xtype, void *buf); /* Generalized put functions. */ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, @@ -331,6 +333,8 @@ extern "C" { int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype, const void *op); int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op); + int PIOc_put_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum, + nc_type xtype, const void *buf); /* An internal replacement for a function pnetcdf does not * have. */ diff --git a/src/clib/pio_put_vard.c b/src/clib/pio_put_vard.c index f68371020cf..60161125d16 100644 --- a/src/clib/pio_put_vard.c +++ b/src/clib/pio_put_vard.c @@ -165,26 +165,26 @@ PIOc_put_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum, return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_INT, op); } -/** - * Get strided, muti-dimensional subset of a 64-bit integer variable. - * - * This routine is called collectively by all tasks in the - * communicator ios.union_comm. - * - * @param ncid identifies the netCDF file - * @param varid the variable ID number - * @param decompid the decomposition ID. - * @param recnum the record number. - * @param op pointer to the data to be written. - * @return PIO_NOERR on success, error code otherwise. - * @author Ed Hartnett - */ -int -PIOc_put_vard_long(int ncid, int varid, int decompid, const PIO_Offset recnum, - const long *op) -{ - return PIOc_put_vard_tc(ncid, varid, decompid, recnum, PIO_LONG_INTERNAL, op); -} +/* /\** */ +/* * Get strided, muti-dimensional subset of a 64-bit integer variable. */ +/* * */ +/* * This routine is called collectively by all tasks in the */ +/* * communicator ios.union_comm. */ +/* * */ +/* * @param ncid identifies the netCDF file */ +/* * @param varid the variable ID number */ +/* * @param decompid the decomposition ID. */ +/* * @param recnum the record number. */ +/* * @param op pointer to the data to be written. */ +/* * @return PIO_NOERR on success, error code otherwise. */ +/* * @author Ed Hartnett */ +/* *\/ */ +/* int */ +/* PIOc_put_vard_long(int ncid, int varid, int decompid, const PIO_Offset recnum, */ +/* const long *op) */ +/* { */ +/* return PIOc_put_vard_tc(ncid, varid, decompid, recnum, PIO_LONG_INTERNAL, op); */ +/* } */ /** * Get strided, muti-dimensional subset of a floating point variable. @@ -248,7 +248,7 @@ int PIOc_put_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum, const double *op) { - return PIOc_put_vars_tc(ncid, varid, decompid, recnum, NC_DOUBLE, op); + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_DOUBLE, op); } /** @@ -270,7 +270,7 @@ int PIOc_put_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum, const unsigned long long *op) { - return PIOc_put_vars_tc(ncid, varid, decompid, recnum, NC_UINT64, op); + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_UINT64, op); } /** diff --git a/tests/cunit/test_darray.c b/tests/cunit/test_darray.c index 70d7b9d2b82..784de2c2899 100644 --- a/tests/cunit/test_darray.c +++ b/tests/cunit/test_darray.c @@ -271,12 +271,16 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank test_data_in) != PIO_EBADID) ERR(ERR_WRONG); - /* Set the record number. */ - if ((ret = PIOc_setframe(ncid2, varid, 0))) - ERR(ret); + /* /\* Set the record number. *\/ */ + /* if ((ret = PIOc_setframe(ncid2, varid, 0))) */ + /* ERR(ret); */ + + /* /\* Read the data. *\/ */ + /* if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in))) */ + /* ERR(ret); */ /* Read the data. */ - if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in))) + if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, (void *)test_data_in))) ERR(ret); /* Check the results. */ From 9f842477dfc1f0a3f977f3580ebbeb5a5afb6fad Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 07:17:12 -0600 Subject: [PATCH 04/26] trying to fix cmake build --- src/clib/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt index baf4dcd3c75..83635d9f685 100644 --- a/src/clib/CMakeLists.txt +++ b/src/clib/CMakeLists.txt @@ -9,7 +9,7 @@ project (PIOC C) add_library (pioc topology.c pio_file.c pioc_support.c pio_lists.c pioc.c pioc_sc.c pio_spmd.c pio_rearrange.c pio_nc4.c bget.c pio_nc.c pio_put_nc.c pio_get_nc.c pio_getput_int.c pio_msg.c - pio_darray.c pio_darray_int.c) + pio_darray.c pio_darray_int.c pio_get_vard.c pio_put_vard.c) # set up include-directories include_directories( From b280e4a39431126cc3f703861944667aee133863 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 07:46:05 -0600 Subject: [PATCH 05/26] disallow type conversion in vard writes (for now) --- src/clib/pio_getput_int.c | 20 ++++++++++++++-- src/clib/pio_put_nc.c | 50 +++++++++++++++++++-------------------- src/clib/pio_put_vard.c | 26 ++++++++++---------- 3 files changed, 56 insertions(+), 40 deletions(-) diff --git a/src/clib/pio_getput_int.c b/src/clib/pio_getput_int.c index 5159790ad00..8fa281bfd59 100644 --- a/src/clib/pio_getput_int.c +++ b/src/clib/pio_getput_int.c @@ -1483,16 +1483,32 @@ int PIOc_put_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum, nc_type xtype, const void *buf) { + iosystem_desc_t *ios; /* Pointer to io system information. */ io_desc_t *iodesc; /* The IO description. */ + file_desc_t *file; /* Pointer to file information. */ + var_desc_t *vdesc; /* Pointer to var information. */ int ret; + /* Get file info. */ + if ((ret = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ret, __FILE__, __LINE__); + ios = file->iosystem; + /* Set the value of the record dimension. */ if ((ret = PIOc_setframe(ncid, varid, recnum))) - return ret; + return pio_err(ios, file, ret, __FILE__, __LINE__); /* Get decomposition information. */ if (!(iodesc = pio_get_iodesc_from_id(decompid))) - return PIO_EBADID; + return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__); + + /* Get var info. */ + if ((ret = get_var_desc(varid, &file->varlist, &vdesc))) + return pio_err(ios, file, ret, __FILE__, __LINE__); + + /* Disallow type conversion for now. */ + if (xtype != vdesc->pio_type) + return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__); /* Write the distributed array. */ if ((ret = PIOc_write_darray(ncid, varid, decompid, iodesc->ndof, diff --git a/src/clib/pio_put_nc.c b/src/clib/pio_put_nc.c index 9835f067f84..76aea931731 100644 --- a/src/clib/pio_put_nc.c +++ b/src/clib/pio_put_nc.c @@ -17,7 +17,7 @@ */ /** - * Get strided, muti-dimensional subset of a text variable. + * Put strided, muti-dimensional subset of a text variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -45,7 +45,7 @@ PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offse } /** - * Get strided, muti-dimensional subset of an unsigned char variable. + * Put strided, muti-dimensional subset of an unsigned char variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -74,7 +74,7 @@ PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset *start, } /** - * Get strided, muti-dimensional subset of a signed char variable. + * Put strided, muti-dimensional subset of a signed char variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -102,7 +102,7 @@ PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offs } /** - * Get strided, muti-dimensional subset of an unsigned 16-bit integer + * Put strided, muti-dimensional subset of an unsigned 16-bit integer * variable. * * This routine is called collectively by all tasks in the @@ -131,7 +131,7 @@ PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Off } /** - * Get strided, muti-dimensional subset of a 16-bit integer variable. + * Put strided, muti-dimensional subset of a 16-bit integer variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -159,7 +159,7 @@ PIOc_put_vars_short(int ncid, int varid, const PIO_Offset *start, } /** - * Get strided, muti-dimensional subset of an unsigned integer + * Put strided, muti-dimensional subset of an unsigned integer * variable. * * This routine is called collectively by all tasks in the @@ -188,7 +188,7 @@ PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offse } /** - * Get strided, muti-dimensional subset of an integer variable. + * Put strided, muti-dimensional subset of an integer variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -216,7 +216,7 @@ PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset } /** - * Get strided, muti-dimensional subset of a 64-bit integer variable. + * Put strided, muti-dimensional subset of a 64-bit integer variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -244,7 +244,7 @@ PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offse } /** - * Get strided, muti-dimensional subset of a floating point variable. + * Put strided, muti-dimensional subset of a floating point variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -272,7 +272,7 @@ PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offs } /** - * Get strided, muti-dimensional subset of a 64-bit unsigned integer + * Put strided, muti-dimensional subset of a 64-bit unsigned integer * variable. * * This routine is called collectively by all tasks in the @@ -301,7 +301,7 @@ PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_O } /** - * Get strided, muti-dimensional subset of a 64-bit floating point + * Put strided, muti-dimensional subset of a 64-bit floating point * variable. * * This routine is called collectively by all tasks in the @@ -330,7 +330,7 @@ PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Off } /** - * Get strided, muti-dimensional subset of an unsigned 64-bit integer + * Put strided, muti-dimensional subset of an unsigned 64-bit integer * variable. * * This routine is called collectively by all tasks in the @@ -359,7 +359,7 @@ PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_ } /** - * Get one value from an text variable. + * Put one value from an text variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -380,7 +380,7 @@ PIOc_put_var1_text(int ncid, int varid, const PIO_Offset *index, const char *op) } /** - * Get one value from an text variable. + * Put one value from an text variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -402,7 +402,7 @@ PIOc_put_var1_uchar(int ncid, int varid, const PIO_Offset *index, } /** - * Get one value from an signed char variable. + * Put one value from an signed char variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -424,7 +424,7 @@ PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset *index, } /** - * Get one value from an unsigned 16-bit integer variable. + * Put one value from an unsigned 16-bit integer variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -446,7 +446,7 @@ PIOc_put_var1_ushort(int ncid, int varid, const PIO_Offset *index, } /** - * Get one value from a 16-bit integer variable. + * Put one value from a 16-bit integer variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -468,7 +468,7 @@ PIOc_put_var1_short(int ncid, int varid, const PIO_Offset *index, } /** - * Get one value from an unsigned integer variable. + * Put one value from an unsigned integer variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -490,7 +490,7 @@ PIOc_put_var1_uint(int ncid, int varid, const PIO_Offset *index, } /** - * Get one value from an integer variable. + * Put one value from an integer variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -511,7 +511,7 @@ PIOc_put_var1_int(int ncid, int varid, const PIO_Offset *index, const int *op) } /** - * Get one value from an floating point variable. + * Put one value from an floating point variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -532,7 +532,7 @@ PIOc_put_var1_float(int ncid, int varid, const PIO_Offset *index, const float *o } /** - * Get one value from an integer variable. + * Put one value from an integer variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -553,7 +553,7 @@ PIOc_put_var1_long(int ncid, int varid, const PIO_Offset *index, const long *op) } /** - * Get one value from an 64-bit floating point variable. + * Put one value from an 64-bit floating point variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -575,7 +575,7 @@ PIOc_put_var1_double(int ncid, int varid, const PIO_Offset *index, } /** - * Get one value from an unsigned 64-bit integer variable. + * Put one value from an unsigned 64-bit integer variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -597,7 +597,7 @@ PIOc_put_var1_ulonglong(int ncid, int varid, const PIO_Offset *index, } /** - * Get one value from a 64-bit integer variable. + * Put one value from a 64-bit integer variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -1153,7 +1153,7 @@ PIOc_put_var(int ncid, int varid, const void *op) } /** - * Get one value from a variable of any type. + * Put one value from a variable of any type. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. diff --git a/src/clib/pio_put_vard.c b/src/clib/pio_put_vard.c index 60161125d16..ac30207727f 100644 --- a/src/clib/pio_put_vard.c +++ b/src/clib/pio_put_vard.c @@ -17,7 +17,7 @@ */ /** - * Get strided, muti-dimensional subset of a text variable. + * Put distributed array subset of a text variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -38,7 +38,7 @@ PIOc_put_vard_text(int ncid, int varid, int decompid, const PIO_Offset recnum, } /** - * Get strided, muti-dimensional subset of an unsigned char variable. + * Put distributed array subset of an unsigned char variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -59,7 +59,7 @@ PIOc_put_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum, } /** - * Get strided, muti-dimensional subset of a signed char variable. + * Put distributed array subset of a signed char variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -80,7 +80,7 @@ PIOc_put_vard_schar(int ncid, int varid, int decompid, const PIO_Offset recnum, } /** - * Get strided, muti-dimensional subset of an unsigned 16-bit integer + * Put distributed array subset of an unsigned 16-bit integer * variable. * * This routine is called collectively by all tasks in the @@ -102,7 +102,7 @@ PIOc_put_vard_ushort(int ncid, int varid, int decompid, const PIO_Offset recnum, } /** - * Get strided, muti-dimensional subset of a 16-bit integer variable. + * Put distributed array subset of a 16-bit integer variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -123,7 +123,7 @@ PIOc_put_vard_short(int ncid, int varid, int decompid, const PIO_Offset recnum, } /** - * Get strided, muti-dimensional subset of an unsigned integer + * Put distributed array subset of an unsigned integer * variable. * * This routine is called collectively by all tasks in the @@ -145,7 +145,7 @@ PIOc_put_vard_uint(int ncid, int varid, int decompid, const PIO_Offset recnum, } /** - * Get strided, muti-dimensional subset of an integer variable. + * Put distributed array subset of an integer variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -166,7 +166,7 @@ PIOc_put_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum, } /* /\** */ -/* * Get strided, muti-dimensional subset of a 64-bit integer variable. */ +/* * Put distributed array subset of a 64-bit integer variable. */ /* * */ /* * This routine is called collectively by all tasks in the */ /* * communicator ios.union_comm. */ @@ -187,7 +187,7 @@ PIOc_put_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum, /* } */ /** - * Get strided, muti-dimensional subset of a floating point variable. + * Put distributed array subset of a floating point variable. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. @@ -208,7 +208,7 @@ PIOc_put_vard_float(int ncid, int varid, int decompid, const PIO_Offset recnum, } /** - * Get strided, muti-dimensional subset of a 64-bit unsigned integer + * Put distributed array subset of a 64-bit unsigned integer * variable. * * This routine is called collectively by all tasks in the @@ -230,7 +230,7 @@ PIOc_put_vard_longlong(int ncid, int varid, int decompid, const PIO_Offset recnu } /** - * Get strided, muti-dimensional subset of a 64-bit floating point + * Put distributed array subset of a 64-bit floating point * variable. * * This routine is called collectively by all tasks in the @@ -252,7 +252,7 @@ PIOc_put_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum, } /** - * Get strided, muti-dimensional subset of an unsigned 64-bit integer + * Put distributed array subset of an unsigned 64-bit integer * variable. * * This routine is called collectively by all tasks in the @@ -274,7 +274,7 @@ PIOc_put_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recn } /** - * Write strided, muti-dimensional subset of a variable of any type. + * Write distributed array subset of a variable of any type. * * This routine is called collectively by all tasks in the * communicator ios.union_comm. From aa26496b9f47d4c49e0f9a6afa2368280319bfb2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 07:50:09 -0600 Subject: [PATCH 06/26] disallow type conversion in vard reads (for now) --- src/clib/pio_getput_int.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/clib/pio_getput_int.c b/src/clib/pio_getput_int.c index 8fa281bfd59..04b5277df30 100644 --- a/src/clib/pio_getput_int.c +++ b/src/clib/pio_getput_int.c @@ -1448,12 +1448,28 @@ int PIOc_get_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum, nc_type xtype, void *buf) { + iosystem_desc_t *ios; /* Pointer to io system information. */ + file_desc_t *file; /* Pointer to file information. */ + var_desc_t *vdesc; /* Pointer to var information. */ int ret; + /* Get file info. */ + if ((ret = pio_get_file(ncid, &file))) + return pio_err(NULL, NULL, ret, __FILE__, __LINE__); + ios = file->iosystem; + /* Set the value of the record dimension. */ if ((ret = PIOc_setframe(ncid, varid, recnum))) return ret; + /* Get var info. */ + if ((ret = get_var_desc(varid, &file->varlist, &vdesc))) + return pio_err(ios, file, ret, __FILE__, __LINE__); + + /* Disallow type conversion for now. */ + if (xtype != NC_NAT && xtype != vdesc->pio_type) + return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__); + /* Read the distributed array. */ if ((ret = PIOc_read_darray(ncid, varid, decompid, 0, buf))) return ret; @@ -1507,7 +1523,7 @@ PIOc_put_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum, return pio_err(ios, file, ret, __FILE__, __LINE__); /* Disallow type conversion for now. */ - if (xtype != vdesc->pio_type) + if (xtype != NC_NAT && xtype != vdesc->pio_type) return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__); /* Write the distributed array. */ From 2ec36c71640ebaa04b5a88f6a09d19f6008561db Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 08:04:35 -0600 Subject: [PATCH 07/26] adding vard testing --- tests/cunit/Makefile.am | 4 +- tests/cunit/run_tests.sh | 2 +- tests/cunit/test_darray.c | 16 +- tests/cunit/test_darray_vard.c | 362 +++++++++++++++++++++++++++++++++ 4 files changed, 374 insertions(+), 10 deletions(-) create mode 100644 tests/cunit/test_darray_vard.c diff --git a/tests/cunit/Makefile.am b/tests/cunit/Makefile.am index d79c9f7d163..0b7042883b4 100644 --- a/tests/cunit/Makefile.am +++ b/tests/cunit/Makefile.am @@ -19,7 +19,8 @@ test_darray_multivar3 test_darray_1d test_darray_3d \ test_decomp_uneven test_decomps test_rearr test_darray_async_simple \ test_darray_async test_darray_async_many test_darray_2sync \ test_async_multicomp test_async_multi2 test_async_manyproc \ -test_darray_fill test_decomp_frame test_perf2 test_async_perf +test_darray_fill test_decomp_frame test_perf2 test_async_perf \ +test_darray_vard if RUN_TESTS # Tests will run from a bash script. @@ -63,6 +64,7 @@ test_darray_fill_SOURCES = test_darray_fill.c test_common.c pio_tests.h test_decomp_frame_SOURCES = test_decomp_frame.c test_common.c pio_tests.h test_perf2_SOURCES = test_perf2.c test_common.c pio_tests.h test_async_perf_SOURCES = test_async_perf.c test_common.c pio_tests.h +test_darray_vard_SOURCES = test_darray_vard.c test_common.c pio_tests.h # Distribute the test script. EXTRA_DIST = run_tests.sh diff --git a/tests/cunit/run_tests.sh b/tests/cunit/run_tests.sh index 356eee29d3a..c9106617ae7 100755 --- a/tests/cunit/run_tests.sh +++ b/tests/cunit/run_tests.sh @@ -17,7 +17,7 @@ PIO_TESTS='test_intercomm2 test_async_mpi test_spmd test_rearr test_async_simple 'test_darray_multivar test_darray_multivar2 test_darray_multivar3 test_darray_1d '\ 'test_darray_3d test_decomp_uneven test_decomps test_darray_async_simple '\ 'test_darray_async test_darray_async_many test_darray_2sync test_async_multicomp '\ -'test_darray_fill' +'test_darray_fill test_darray_vard' success1=true success2=true diff --git a/tests/cunit/test_darray.c b/tests/cunit/test_darray.c index 784de2c2899..40c4f7e289f 100644 --- a/tests/cunit/test_darray.c +++ b/tests/cunit/test_darray.c @@ -271,18 +271,18 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank test_data_in) != PIO_EBADID) ERR(ERR_WRONG); - /* /\* Set the record number. *\/ */ - /* if ((ret = PIOc_setframe(ncid2, varid, 0))) */ - /* ERR(ret); */ - - /* /\* Read the data. *\/ */ - /* if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in))) */ - /* ERR(ret); */ + /* Set the record number. */ + if ((ret = PIOc_setframe(ncid2, varid, 0))) + ERR(ret); /* Read the data. */ - if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, (void *)test_data_in))) + if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in))) ERR(ret); + /* /\* Read the data. *\/ */ + /* if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, (void *)test_data_in))) */ + /* ERR(ret); */ + /* Check the results. */ for (int f = 0; f < arraylen; f++) { diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c new file mode 100644 index 00000000000..1051fc716f7 --- /dev/null +++ b/tests/cunit/test_darray_vard.c @@ -0,0 +1,362 @@ +/* + * Tests for PIO distributed arrays. + * + * @author Ed Hartnett + * @date 6/4/19 + */ +#include +#include +#include +#include + +/* The number of tasks this test should run on. */ +#define TARGET_NTASKS 4 + +/* The minimum number of tasks this test should run on. */ +#define MIN_NTASKS 4 + +/* The name of this test. */ +#define TEST_NAME "test_darray_vard" + +/* Number of processors that will do IO. */ +#define NUM_IO_PROCS 1 + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +/* The number of dimensions in the example data. In this test, we + * are using three-dimensional data. */ +#define NDIM 3 + +/* But sometimes we need arrays of the non-record dimensions. */ +#define NDIM2 2 + +/* The length of our sample data along each dimension. */ +#define X_DIM_LEN 4 +#define Y_DIM_LEN 4 + +/* The number of timesteps of data to write. */ +#define NUM_TIMESTEPS 2 + +/* The names of variables in the netCDF output files. */ +#define VAR_NAME "Billy-Bob" +#define VAR_NAME2 "Sally-Sue" + +/* Test with and without specifying a fill value to + * PIOc_write_darray(). */ +#define NUM_TEST_CASES_FILLVALUE 2 + +/* The dimension names. */ +char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"}; + +/* Length of the dimensions in the sample data. */ +int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; + +/** + * Test the darray functionality. Create a netCDF file with 3 + * dimensions and 1 PIO_INT variable, and use darray to write some + * data. + * + * @param iosysid the IO system ID. + * @param ioid the ID of the decomposition. + * @param num_flavors the number of IOTYPES available in this build. + * @param flavor array of available iotypes. + * @param my_rank rank of this task. + * @param pio_type the type of the data. + * @returns 0 for success, error code otherwise. + */ +int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, + int pio_type) +{ + char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ + int dimids[NDIM]; /* The dimension IDs. */ + int ncid; /* The ncid of the netCDF file. */ + int ncid2; /* The ncid of the re-opened netCDF file. */ + int varid; /* The ID of the netCDF varable. */ + int varid2; /* The ID of a netCDF varable of different type. */ + int ret; /* Return code. */ + MPI_Datatype mpi_type; + int type_size; /* size of a variable of type pio_type */ + int other_type; /* another variable of the same size but different type */ + PIO_Offset arraylen = 4; + void *fillvalue, *ofillvalue; + void *test_data; + void *test_data_in; + int fillvalue_int = NC_FILL_INT; + int test_data_int[arraylen]; + int test_data_int_in[arraylen]; + float fillvalue_float = NC_FILL_FLOAT; + float test_data_float[arraylen]; + float test_data_float_in[arraylen]; + double fillvalue_double = NC_FILL_DOUBLE; + double test_data_double[arraylen]; + double test_data_double_in[arraylen]; + + /* Initialize some data. */ + for (int f = 0; f < arraylen; f++) + { + test_data_int[f] = my_rank * 10 + f; + test_data_float[f] = my_rank * 10 + f + 0.5; + test_data_double[f] = my_rank * 100000 + f + 0.5; + } + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + { + /* Test with/without providing a fill value to PIOc_write_darray(). */ + for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) + { + /* Create the filename. */ + sprintf(filename, "data_%s_iotype_%d_pio_type_%d_provide_fill_%d.nc", TEST_NAME, + flavor[fmt], pio_type, provide_fill); + /* Select the fill value and data. */ + switch (pio_type) + { + case PIO_INT: + fillvalue = provide_fill ? &fillvalue_int : NULL; + test_data = test_data_int; + test_data_in = test_data_int_in; + break; + case PIO_FLOAT: + fillvalue = provide_fill ? &fillvalue_float : NULL; + test_data = test_data_float; + test_data_in = test_data_float_in; + break; + case PIO_DOUBLE: + fillvalue = provide_fill ? &fillvalue_double : NULL; + test_data = test_data_double; + test_data_in = test_data_double_in; + break; + default: + ERR(ERR_WRONG); + } + + /* Create the netCDF output file. */ + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + for (int d = 0; d < NDIM; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + + /* Define a variable. */ + if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, &varid))) + ERR(ret); + + /* Define a variable with a different type but same size. */ + if ((ret = find_mpi_type(pio_type, &mpi_type, &type_size))) + ERR(ret); + if (type_size == NETCDF_INT_FLOAT_SIZE) + other_type = pio_type == PIO_INT ? PIO_FLOAT : PIO_INT; +// else if(type_size == NETCDF_DOUBLE_INT64_SIZE) +// other_type = pio_type == PIO_INT64 ? PIO_DOUBLE : PIO_INT64; + else + other_type = 0; /* skip the test */ + switch (other_type) + { + case PIO_INT: + ofillvalue = provide_fill ? &fillvalue_int : NULL; + break; + case PIO_FLOAT: + ofillvalue = provide_fill ? &fillvalue_float : NULL; + break; + default: + break; + } + if (other_type && (ret = PIOc_def_var(ncid, VAR_NAME2, other_type, NDIM, dimids, &varid2))) + ERR(ret); + + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Set the value of the record dimension. */ + if ((ret = PIOc_setframe(ncid, varid, 0))) + ERR(ret); + if (other_type && (ret = PIOc_setframe(ncid, varid2, 0))) + ERR(ret); + + /* These should not work. */ + if (PIOc_write_darray(ncid + TEST_VAL_42, varid, ioid, arraylen, test_data, fillvalue) != PIO_EBADID) + ERR(ERR_WRONG); + if (PIOc_write_darray(ncid, varid, ioid + TEST_VAL_42, arraylen, test_data, fillvalue) != PIO_EBADID) + ERR(ERR_WRONG); + if (PIOc_write_darray(ncid, varid, ioid, arraylen - 1, test_data, fillvalue) != PIO_EINVAL) + ERR(ERR_WRONG); + if (PIOc_write_darray(ncid, TEST_VAL_42, ioid, arraylen, test_data, fillvalue) != PIO_ENOTVAR) + ERR(ERR_WRONG); + + /* This should work - library type conversion */ + if (other_type && (ret = PIOc_write_darray(ncid, varid2, ioid, arraylen, test_data, ofillvalue))) + ERR(ret); + + /* Write the data. */ + if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue))) + ERR(ret); + + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) + ERR(ret); + + PIO_Offset dimlen; + /* check the unlimited dim size - it should be 1 */ + if ((ret = PIOc_inq_dimlen(ncid2, dimids[0], &dimlen))) + ERR(ret); + if (dimlen != 1) + ERR(ERR_WRONG); + + /* These should not work. */ + if ((ret = PIOc_get_vard(ncid2 + TEST_VAL_42, varid, ioid, 0, test_data_in)) != PIO_EBADID) + ERR(ret); + if ((ret = PIOc_get_vard(ncid2, varid, ioid + TEST_VAL_42, 0, test_data_in)) != PIO_EBADID) + ERR(ret); + + /* Read the data. */ + if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, (void *)test_data_in))) + ERR(ret); + + /* Check the results. */ + for (int f = 0; f < arraylen; f++) + { + switch (pio_type) + { + case PIO_INT: + if (test_data_int_in[f] != test_data_int[f]) + return ERR_WRONG; + break; + case PIO_FLOAT: + if (test_data_float_in[f] != test_data_float[f]) + return ERR_WRONG; + break; + case PIO_DOUBLE: + if (test_data_double_in[f] != test_data_double[f]) + return ERR_WRONG; + break; + default: + ERR(ERR_WRONG); + } + } + + /* Try to write, but it won't work, because we opened file read-only. */ + { + if (PIOc_write_darray(ncid2, varid, ioid, arraylen, test_data, fillvalue) != PIO_EPERM) + ERR(ERR_WRONG); + } + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid2))) + ERR(ret); + } /* next fillvalue test case */ + } /* next test multi */ + } /* next iotype */ + + return PIO_NOERR; +} + +/** + * Run all the tests. + * + * @param iosysid the IO system ID. + * @param num_flavors number of available iotypes in the build. + * @param flavor pointer to array of the available iotypes. + * @param my_rank rank of this task. + * @param test_comm the communicator the test is running on. + * @returns 0 for success, error code otherwise. + */ +int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, + MPI_Comm test_comm) +{ +#define NUM_TYPES_TO_TEST 3 + int ioid; + char filename[PIO_MAX_NAME + 1]; + int pio_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE}; + int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; + int ret; /* Return code. */ + + for (int t = 0; t < NUM_TYPES_TO_TEST; t++) + { + /* This will be our file name for writing out decompositions. */ + sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", TEST_NAME, my_rank, + *flavor, pio_type[t]); + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, + &ioid, pio_type[t]))) + return ret; + + /* Run a simple darray test. */ + if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, pio_type[t]))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + } + + return PIO_NOERR; +} + +/* Run tests for darray functions. */ +int main(int argc, char **argv) +{ +#define NUM_REARRANGERS_TO_TEST 2 + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; + int my_rank; + int ntasks; + int num_flavors; /* Number of PIO netCDF flavors in this build. */ + int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ + MPI_Comm test_comm; /* A communicator for this test. */ + int ret; /* Return code. */ + + /* Initialize test. */ + if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, + MIN_NTASKS, -1, &test_comm))) + ERR(ERR_INIT); + + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Only do something on max_ntasks tasks. */ + if (my_rank < TARGET_NTASKS) + { + int iosysid; /* The ID for the parallel I/O system. */ + int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ + int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ + int ret; /* Return code. */ + + /* Figure out iotypes. */ + if ((ret = get_iotypes(&num_flavors, flavor))) + ERR(ret); + + for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++) + { + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, + ioproc_start, rearranger[r], &iosysid))) + return ret; + + /* Run tests. */ + if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm))) + return ret; + + /* Finalize PIO system. */ + if ((ret = PIOc_free_iosystem(iosysid))) + return ret; + } /* next rearranger */ + } /* endif my_rank < TARGET_NTASKS */ + + /* Finalize the MPI library. */ + if ((ret = pio_test_finalize(&test_comm))) + return ret; + + printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); + return 0; +} From 328afad24761def0f8c96430a99bb7a828484e3b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 08:13:05 -0600 Subject: [PATCH 08/26] working on vard test --- tests/cunit/test_darray_vard.c | 36 +--------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index 1051fc716f7..b4c96689ecd 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -73,13 +73,9 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank int ncid; /* The ncid of the netCDF file. */ int ncid2; /* The ncid of the re-opened netCDF file. */ int varid; /* The ID of the netCDF varable. */ - int varid2; /* The ID of a netCDF varable of different type. */ int ret; /* Return code. */ - MPI_Datatype mpi_type; - int type_size; /* size of a variable of type pio_type */ - int other_type; /* another variable of the same size but different type */ PIO_Offset arraylen = 4; - void *fillvalue, *ofillvalue; + void *fillvalue; void *test_data; void *test_data_in; int fillvalue_int = NC_FILL_INT; @@ -146,29 +142,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, &varid))) ERR(ret); - /* Define a variable with a different type but same size. */ - if ((ret = find_mpi_type(pio_type, &mpi_type, &type_size))) - ERR(ret); - if (type_size == NETCDF_INT_FLOAT_SIZE) - other_type = pio_type == PIO_INT ? PIO_FLOAT : PIO_INT; -// else if(type_size == NETCDF_DOUBLE_INT64_SIZE) -// other_type = pio_type == PIO_INT64 ? PIO_DOUBLE : PIO_INT64; - else - other_type = 0; /* skip the test */ - switch (other_type) - { - case PIO_INT: - ofillvalue = provide_fill ? &fillvalue_int : NULL; - break; - case PIO_FLOAT: - ofillvalue = provide_fill ? &fillvalue_float : NULL; - break; - default: - break; - } - if (other_type && (ret = PIOc_def_var(ncid, VAR_NAME2, other_type, NDIM, dimids, &varid2))) - ERR(ret); - /* End define mode. */ if ((ret = PIOc_enddef(ncid))) ERR(ret); @@ -176,8 +149,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank /* Set the value of the record dimension. */ if ((ret = PIOc_setframe(ncid, varid, 0))) ERR(ret); - if (other_type && (ret = PIOc_setframe(ncid, varid2, 0))) - ERR(ret); /* These should not work. */ if (PIOc_write_darray(ncid + TEST_VAL_42, varid, ioid, arraylen, test_data, fillvalue) != PIO_EBADID) @@ -189,10 +160,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank if (PIOc_write_darray(ncid, TEST_VAL_42, ioid, arraylen, test_data, fillvalue) != PIO_ENOTVAR) ERR(ERR_WRONG); - /* This should work - library type conversion */ - if (other_type && (ret = PIOc_write_darray(ncid, varid2, ioid, arraylen, test_data, ofillvalue))) - ERR(ret); - /* Write the data. */ if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue))) ERR(ret); @@ -201,7 +168,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank if ((ret = PIOc_closefile(ncid))) ERR(ret); - /* Reopen the file. */ if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) ERR(ret); From b6ea1d4c8a116702644cf740e7548ea85ceead0c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 08:15:50 -0600 Subject: [PATCH 09/26] working on vard test --- tests/cunit/test_darray_vard.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index b4c96689ecd..ed5f2d6bd20 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -146,22 +146,16 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank if ((ret = PIOc_enddef(ncid))) ERR(ret); - /* Set the value of the record dimension. */ - if ((ret = PIOc_setframe(ncid, varid, 0))) - ERR(ret); - /* These should not work. */ - if (PIOc_write_darray(ncid + TEST_VAL_42, varid, ioid, arraylen, test_data, fillvalue) != PIO_EBADID) - ERR(ERR_WRONG); - if (PIOc_write_darray(ncid, varid, ioid + TEST_VAL_42, arraylen, test_data, fillvalue) != PIO_EBADID) + if (PIOc_put_vard(ncid + TEST_VAL_42, varid, ioid, 0, test_data) != PIO_EBADID) ERR(ERR_WRONG); - if (PIOc_write_darray(ncid, varid, ioid, arraylen - 1, test_data, fillvalue) != PIO_EINVAL) + if (PIOc_put_vard(ncid + TEST_VAL_42, varid, ioid + TEST_VAL_42, 0, test_data) != PIO_EBADID) ERR(ERR_WRONG); - if (PIOc_write_darray(ncid, TEST_VAL_42, ioid, arraylen, test_data, fillvalue) != PIO_ENOTVAR) + if (PIOc_put_vard(ncid, TEST_VAL_42, ioid + TEST_VAL_42, 0, test_data) != PIO_ENOTVAR) ERR(ERR_WRONG); /* Write the data. */ - if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue))) + if ((ret = PIOc_put_vard(ncid, varid, ioid, 0, test_data))) ERR(ret); /* Close the netCDF file. */ From 8114b34425fcff9d4f65b99800001174fd96a793 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 11:53:41 -0600 Subject: [PATCH 10/26] cleanup --- tests/cunit/test_darray_vard.c | 209 +++++++++++++++++---------------- 1 file changed, 105 insertions(+), 104 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index ed5f2d6bd20..1ba39e780e5 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -87,9 +87,10 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank double fillvalue_double = NC_FILL_DOUBLE; double test_data_double[arraylen]; double test_data_double_in[arraylen]; + int f, fmt, provide_fill, d; /* Initialize some data. */ - for (int f = 0; f < arraylen; f++) + for (f = 0; f < arraylen; f++) { test_data_int[f] = my_rank * 10 + f; test_data_float[f] = my_rank * 10 + f + 0.5; @@ -98,123 +99,121 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank /* Use PIO to create the example file in each of the four * available ways. */ - for (int fmt = 0; fmt < num_flavors; fmt++) + for (fmt = 0; fmt < num_flavors; fmt++) { + /* Test with/without providing a fill value to PIOc_write_darray(). */ + for (provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) { - /* Test with/without providing a fill value to PIOc_write_darray(). */ - for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) + /* Create the filename. */ + sprintf(filename, "data_%s_iotype_%d_pio_type_%d_provide_fill_%d.nc", TEST_NAME, + flavor[fmt], pio_type, provide_fill); + /* Select the fill value and data. */ + switch (pio_type) + { + case PIO_INT: + fillvalue = provide_fill ? &fillvalue_int : NULL; + test_data = test_data_int; + test_data_in = test_data_int_in; + break; + case PIO_FLOAT: + fillvalue = provide_fill ? &fillvalue_float : NULL; + test_data = test_data_float; + test_data_in = test_data_float_in; + break; + case PIO_DOUBLE: + fillvalue = provide_fill ? &fillvalue_double : NULL; + test_data = test_data_double; + test_data_in = test_data_double_in; + break; + default: + ERR(ERR_WRONG); + } + + /* Create the netCDF output file. */ + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + for (d = 0; d < NDIM; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + + /* Define a variable. */ + if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, &varid))) + ERR(ret); + + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* These should not work. */ + if (PIOc_put_vard(ncid + TEST_VAL_42, varid, ioid, 0, test_data) != PIO_EBADID) + ERR(ERR_WRONG); + if (PIOc_put_vard(ncid + TEST_VAL_42, varid, ioid + TEST_VAL_42, 0, test_data) != PIO_EBADID) + ERR(ERR_WRONG); + if (PIOc_put_vard(ncid, TEST_VAL_42, ioid + TEST_VAL_42, 0, test_data) != PIO_ENOTVAR) + ERR(ERR_WRONG); + + /* Write the data. */ + if ((ret = PIOc_put_vard(ncid, varid, ioid, 0, test_data))) + ERR(ret); + + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) + ERR(ret); + + PIO_Offset dimlen; + /* check the unlimited dim size - it should be 1 */ + if ((ret = PIOc_inq_dimlen(ncid2, dimids[0], &dimlen))) + ERR(ret); + if (dimlen != 1) + ERR(ERR_WRONG); + + /* These should not work. */ + if ((ret = PIOc_get_vard(ncid2 + TEST_VAL_42, varid, ioid, 0, test_data_in)) != PIO_EBADID) + ERR(ret); + if ((ret = PIOc_get_vard(ncid2, varid, ioid + TEST_VAL_42, 0, test_data_in)) != PIO_EBADID) + ERR(ret); + + /* Read the data. */ + if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, (void *)test_data_in))) + ERR(ret); + + /* Check the results. */ + for (f = 0; f < arraylen; f++) { - /* Create the filename. */ - sprintf(filename, "data_%s_iotype_%d_pio_type_%d_provide_fill_%d.nc", TEST_NAME, - flavor[fmt], pio_type, provide_fill); - /* Select the fill value and data. */ switch (pio_type) { case PIO_INT: - fillvalue = provide_fill ? &fillvalue_int : NULL; - test_data = test_data_int; - test_data_in = test_data_int_in; + if (test_data_int_in[f] != test_data_int[f]) + return ERR_WRONG; break; case PIO_FLOAT: - fillvalue = provide_fill ? &fillvalue_float : NULL; - test_data = test_data_float; - test_data_in = test_data_float_in; + if (test_data_float_in[f] != test_data_float[f]) + return ERR_WRONG; break; case PIO_DOUBLE: - fillvalue = provide_fill ? &fillvalue_double : NULL; - test_data = test_data_double; - test_data_in = test_data_double_in; + if (test_data_double_in[f] != test_data_double[f]) + return ERR_WRONG; break; default: ERR(ERR_WRONG); } + } - /* Create the netCDF output file. */ - if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) - ERR(ret); - - /* Define netCDF dimensions and variable. */ - for (int d = 0; d < NDIM; d++) - if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) - ERR(ret); - - /* Define a variable. */ - if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, &varid))) - ERR(ret); - - /* End define mode. */ - if ((ret = PIOc_enddef(ncid))) - ERR(ret); - - /* These should not work. */ - if (PIOc_put_vard(ncid + TEST_VAL_42, varid, ioid, 0, test_data) != PIO_EBADID) - ERR(ERR_WRONG); - if (PIOc_put_vard(ncid + TEST_VAL_42, varid, ioid + TEST_VAL_42, 0, test_data) != PIO_EBADID) - ERR(ERR_WRONG); - if (PIOc_put_vard(ncid, TEST_VAL_42, ioid + TEST_VAL_42, 0, test_data) != PIO_ENOTVAR) + /* Try to write, but it won't work, because we opened file read-only. */ + { + if (PIOc_write_darray(ncid2, varid, ioid, arraylen, test_data, fillvalue) != PIO_EPERM) ERR(ERR_WRONG); - - /* Write the data. */ - if ((ret = PIOc_put_vard(ncid, varid, ioid, 0, test_data))) - ERR(ret); - - /* Close the netCDF file. */ - if ((ret = PIOc_closefile(ncid))) - ERR(ret); - - /* Reopen the file. */ - if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) - ERR(ret); - - PIO_Offset dimlen; - /* check the unlimited dim size - it should be 1 */ - if ((ret = PIOc_inq_dimlen(ncid2, dimids[0], &dimlen))) - ERR(ret); - if (dimlen != 1) - ERR(ERR_WRONG); - - /* These should not work. */ - if ((ret = PIOc_get_vard(ncid2 + TEST_VAL_42, varid, ioid, 0, test_data_in)) != PIO_EBADID) - ERR(ret); - if ((ret = PIOc_get_vard(ncid2, varid, ioid + TEST_VAL_42, 0, test_data_in)) != PIO_EBADID) - ERR(ret); - - /* Read the data. */ - if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, (void *)test_data_in))) - ERR(ret); - - /* Check the results. */ - for (int f = 0; f < arraylen; f++) - { - switch (pio_type) - { - case PIO_INT: - if (test_data_int_in[f] != test_data_int[f]) - return ERR_WRONG; - break; - case PIO_FLOAT: - if (test_data_float_in[f] != test_data_float[f]) - return ERR_WRONG; - break; - case PIO_DOUBLE: - if (test_data_double_in[f] != test_data_double[f]) - return ERR_WRONG; - break; - default: - ERR(ERR_WRONG); - } - } - - /* Try to write, but it won't work, because we opened file read-only. */ - { - if (PIOc_write_darray(ncid2, varid, ioid, arraylen, test_data, fillvalue) != PIO_EPERM) - ERR(ERR_WRONG); - } - /* Close the netCDF file. */ - if ((ret = PIOc_closefile(ncid2))) - ERR(ret); - } /* next fillvalue test case */ - } /* next test multi */ + } + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid2))) + ERR(ret); + } /* next fillvalue test case */ } /* next iotype */ return PIO_NOERR; @@ -238,9 +237,10 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, char filename[PIO_MAX_NAME + 1]; int pio_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE}; int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; + int t; int ret; /* Return code. */ - for (int t = 0; t < NUM_TYPES_TO_TEST; t++) + for (t = 0; t < NUM_TYPES_TO_TEST; t++) { /* This will be our file name for writing out decompositions. */ sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", TEST_NAME, my_rank, @@ -289,13 +289,14 @@ int main(int argc, char **argv) int iosysid; /* The ID for the parallel I/O system. */ int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ + int r; int ret; /* Return code. */ /* Figure out iotypes. */ if ((ret = get_iotypes(&num_flavors, flavor))) ERR(ret); - for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++) + for (r = 0; r < NUM_REARRANGERS_TO_TEST; r++) { /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ From 586b09cd188fa0a314085df82044045ff3e79e15 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 11:59:47 -0600 Subject: [PATCH 11/26] cleanup --- tests/cunit/test_darray_vard.c | 90 ++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 30 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index 1ba39e780e5..d231291c54e 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -65,8 +65,8 @@ int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; * @param pio_type the type of the data. * @returns 0 for success, error code otherwise. */ -int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, - int pio_type) +int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, + int my_rank, int pio_type) { char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ int dimids[NDIM]; /* The dimension IDs. */ @@ -78,6 +78,9 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank void *fillvalue; void *test_data; void *test_data_in; + short fillvalue_short = NC_FILL_SHORT; + short test_data_short[arraylen]; + short test_data_short_in[arraylen]; int fillvalue_int = NC_FILL_INT; int test_data_int[arraylen]; int test_data_int_in[arraylen]; @@ -92,6 +95,7 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank /* Initialize some data. */ for (f = 0; f < arraylen; f++) { + test_data_short[f] = my_rank + f; test_data_int[f] = my_rank * 10 + f; test_data_float[f] = my_rank * 10 + f + 0.5; test_data_double[f] = my_rank * 100000 + f + 0.5; @@ -102,14 +106,20 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank for (fmt = 0; fmt < num_flavors; fmt++) { /* Test with/without providing a fill value to PIOc_write_darray(). */ - for (provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) + for (provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; + provide_fill++) { /* Create the filename. */ - sprintf(filename, "data_%s_iotype_%d_pio_type_%d_provide_fill_%d.nc", TEST_NAME, - flavor[fmt], pio_type, provide_fill); + sprintf(filename, "data_%s_iotype_%d_pio_type_%d_provide_fill_%d.nc", + TEST_NAME, flavor[fmt], pio_type, provide_fill); /* Select the fill value and data. */ switch (pio_type) { + case PIO_SHORT: + fillvalue = provide_fill ? &fillvalue_short : NULL; + test_data = test_data_short; + test_data_in = test_data_short_in; + break; case PIO_INT: fillvalue = provide_fill ? &fillvalue_int : NULL; test_data = test_data_int; @@ -130,16 +140,19 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank } /* Create the netCDF output file. */ - if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, + PIO_CLOBBER))) ERR(ret); /* Define netCDF dimensions and variable. */ for (d = 0; d < NDIM; d++) - if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + if ((ret = PIOc_def_dim(ncid, dim_name[d], + (PIO_Offset)dim_len[d], &dimids[d]))) ERR(ret); /* Define a variable. */ - if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, &varid))) + if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, + &varid))) ERR(ret); /* End define mode. */ @@ -147,11 +160,14 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank ERR(ret); /* These should not work. */ - if (PIOc_put_vard(ncid + TEST_VAL_42, varid, ioid, 0, test_data) != PIO_EBADID) + if (PIOc_put_vard(ncid + TEST_VAL_42, varid, ioid, 0, + test_data) != PIO_EBADID) ERR(ERR_WRONG); - if (PIOc_put_vard(ncid + TEST_VAL_42, varid, ioid + TEST_VAL_42, 0, test_data) != PIO_EBADID) + if (PIOc_put_vard(ncid + TEST_VAL_42, varid, ioid + TEST_VAL_42, 0, + test_data) != PIO_EBADID) ERR(ERR_WRONG); - if (PIOc_put_vard(ncid, TEST_VAL_42, ioid + TEST_VAL_42, 0, test_data) != PIO_ENOTVAR) + if (PIOc_put_vard(ncid, TEST_VAL_42, ioid + TEST_VAL_42, 0, + test_data) != PIO_ENOTVAR) ERR(ERR_WRONG); /* Write the data. */ @@ -163,7 +179,8 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank ERR(ret); /* Reopen the file. */ - if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) + if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, + PIO_NOWRITE))) ERR(ret); PIO_Offset dimlen; @@ -174,13 +191,16 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank ERR(ERR_WRONG); /* These should not work. */ - if ((ret = PIOc_get_vard(ncid2 + TEST_VAL_42, varid, ioid, 0, test_data_in)) != PIO_EBADID) + if ((ret = PIOc_get_vard(ncid2 + TEST_VAL_42, varid, ioid, 0, + test_data_in)) != PIO_EBADID) ERR(ret); - if ((ret = PIOc_get_vard(ncid2, varid, ioid + TEST_VAL_42, 0, test_data_in)) != PIO_EBADID) + if ((ret = PIOc_get_vard(ncid2, varid, ioid + TEST_VAL_42, 0, + test_data_in)) != PIO_EBADID) ERR(ret); /* Read the data. */ - if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, (void *)test_data_in))) + if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, + (void *)test_data_in))) ERR(ret); /* Check the results. */ @@ -188,6 +208,10 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank { switch (pio_type) { + case PIO_SHORT: + if (test_data_short_in[f] != test_data_short[f]) + return ERR_WRONG; + break; case PIO_INT: if (test_data_int_in[f] != test_data_int[f]) return ERR_WRONG; @@ -205,9 +229,11 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank } } - /* Try to write, but it won't work, because we opened file read-only. */ + /* Try to write, but it won't work, because we opened file + * read-only. */ { - if (PIOc_write_darray(ncid2, varid, ioid, arraylen, test_data, fillvalue) != PIO_EPERM) + if (PIOc_write_darray(ncid2, varid, ioid, arraylen, test_data, + fillvalue) != PIO_EPERM) ERR(ERR_WRONG); } /* Close the netCDF file. */ @@ -232,10 +258,10 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm) { -#define NUM_TYPES_TO_TEST 3 +#define NUM_TYPES_TO_TEST 4 int ioid; char filename[PIO_MAX_NAME + 1]; - int pio_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE}; + int pio_type[NUM_TYPES_TO_TEST] = {PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE}; int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int t; int ret; /* Return code. */ @@ -243,16 +269,17 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, for (t = 0; t < NUM_TYPES_TO_TEST; t++) { /* This will be our file name for writing out decompositions. */ - sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", TEST_NAME, my_rank, - *flavor, pio_type[t]); + sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", + TEST_NAME, my_rank, *flavor, pio_type[t]); /* Decompose the data over the tasks. */ - if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, - &ioid, pio_type[t]))) + if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, + dim_len_2d, &ioid, pio_type[t]))) return ret; /* Run a simple darray test. */ - if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, pio_type[t]))) + if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, + pio_type[t]))) return ret; /* Free the PIO decomposition. */ @@ -280,15 +307,16 @@ int main(int argc, char **argv) MIN_NTASKS, -1, &test_comm))) ERR(ERR_INIT); - if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, + PIO_RETURN_ERROR, NULL))) return ret; /* Only do something on max_ntasks tasks. */ if (my_rank < TARGET_NTASKS) { int iosysid; /* The ID for the parallel I/O system. */ - int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ - int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ + int ioproc_stride = 1; /* Stride in rank between io tasks. */ + int ioproc_start = 0; /* Zero based rank of first I/O task. */ int r; int ret; /* Return code. */ @@ -300,12 +328,14 @@ int main(int argc, char **argv) { /* Initialize the PIO IO system. This specifies how * many and which processors are involved in I/O. */ - if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, - ioproc_start, rearranger[r], &iosysid))) + if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, + ioproc_stride, ioproc_start, + rearranger[r], &iosysid))) return ret; /* Run tests. */ - if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm))) + if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, + test_comm))) return ret; /* Finalize PIO system. */ From 9f2a33a5a354d4a1501a8f2145c375c1284727ea Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 12:13:35 -0600 Subject: [PATCH 12/26] adding types --- tests/cunit/test_darray_vard.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index d231291c54e..fdd0c9a0c57 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -78,6 +78,9 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, void *fillvalue; void *test_data; void *test_data_in; + signed char fillvalue_byte = NC_FILL_BYTE; + signed char test_data_byte[arraylen]; + signed char test_data_byte_in[arraylen]; short fillvalue_short = NC_FILL_SHORT; short test_data_short[arraylen]; short test_data_short_in[arraylen]; @@ -95,6 +98,7 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, /* Initialize some data. */ for (f = 0; f < arraylen; f++) { + test_data_byte[f] = my_rank - f; test_data_short[f] = my_rank + f; test_data_int[f] = my_rank * 10 + f; test_data_float[f] = my_rank * 10 + f + 0.5; @@ -105,16 +109,24 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, * available ways. */ for (fmt = 0; fmt < num_flavors; fmt++) { + /* Pnetcdf cannot handle 1-byte types. */ + if (fmt == 0 && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) continue; + /* Test with/without providing a fill value to PIOc_write_darray(). */ for (provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) { /* Create the filename. */ - sprintf(filename, "data_%s_iotype_%d_pio_type_%d_provide_fill_%d.nc", + sprintf(filename, "%s_iotype_%d_pio_type_%d_provide_fill_%d.nc", TEST_NAME, flavor[fmt], pio_type, provide_fill); /* Select the fill value and data. */ switch (pio_type) { + case PIO_BYTE: + fillvalue = provide_fill ? &fillvalue_byte : NULL; + test_data = test_data_byte; + test_data_in = test_data_byte_in; + break; case PIO_SHORT: fillvalue = provide_fill ? &fillvalue_short : NULL; test_data = test_data_short; @@ -184,6 +196,7 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, ERR(ret); PIO_Offset dimlen; + /* check the unlimited dim size - it should be 1 */ if ((ret = PIOc_inq_dimlen(ncid2, dimids[0], &dimlen))) ERR(ret); @@ -208,6 +221,10 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, { switch (pio_type) { + case PIO_BYTE: + if (test_data_byte_in[f] != test_data_byte[f]) + return ERR_WRONG; + break; case PIO_SHORT: if (test_data_short_in[f] != test_data_short[f]) return ERR_WRONG; @@ -258,10 +275,11 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm) { -#define NUM_TYPES_TO_TEST 4 +#define NUM_TYPES_TO_TEST 5 int ioid; char filename[PIO_MAX_NAME + 1]; - int pio_type[NUM_TYPES_TO_TEST] = {PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE}; + int pio_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_SHORT, PIO_INT, + PIO_FLOAT, PIO_DOUBLE}; int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int t; int ret; /* Return code. */ From 86f8a17262b61441f28b3dca0b8d71e4abc215ec Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 12:15:43 -0600 Subject: [PATCH 13/26] adding types --- tests/cunit/test_darray_vard.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index fdd0c9a0c57..d0abecd72fe 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -78,6 +78,9 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, void *fillvalue; void *test_data; void *test_data_in; + unsigned char fillvalue_char = NC_FILL_BYTE; + unsigned char test_data_char[arraylen]; + unsigned char test_data_char_in[arraylen]; signed char fillvalue_byte = NC_FILL_BYTE; signed char test_data_byte[arraylen]; signed char test_data_byte_in[arraylen]; @@ -98,6 +101,7 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, /* Initialize some data. */ for (f = 0; f < arraylen; f++) { + test_data_char[f] = my_rank; test_data_byte[f] = my_rank - f; test_data_short[f] = my_rank + f; test_data_int[f] = my_rank * 10 + f; @@ -122,6 +126,11 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, /* Select the fill value and data. */ switch (pio_type) { + case PIO_CHAR: + fillvalue = provide_fill ? &fillvalue_char : NULL; + test_data = test_data_char; + test_data_in = test_data_char_in; + break; case PIO_BYTE: fillvalue = provide_fill ? &fillvalue_byte : NULL; test_data = test_data_byte; @@ -221,6 +230,10 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, { switch (pio_type) { + case PIO_CHAR: + if (test_data_char_in[f] != test_data_char[f]) + return ERR_WRONG; + break; case PIO_BYTE: if (test_data_byte_in[f] != test_data_byte[f]) return ERR_WRONG; @@ -275,11 +288,11 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm) { -#define NUM_TYPES_TO_TEST 5 +#define NUM_TYPES_TO_TEST 6 int ioid; char filename[PIO_MAX_NAME + 1]; - int pio_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_SHORT, PIO_INT, - PIO_FLOAT, PIO_DOUBLE}; + int pio_type[NUM_TYPES_TO_TEST] = {PIO_CHAR, PIO_BYTE, PIO_SHORT, + PIO_INT, PIO_FLOAT, PIO_DOUBLE}; int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int t; int ret; /* Return code. */ From 44030a29c5596f59fff5958533763d5825f4af81 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 12:24:46 -0600 Subject: [PATCH 14/26] starting to add netcdf4 types --- tests/cunit/test_darray_vard.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index d0abecd72fe..df1aee792d8 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -52,6 +52,12 @@ char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"}; /* Length of the dimensions in the sample data. */ int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; +#ifdef _NETCDF4 +#define NUM_TYPES_TO_TEST 6 +#else +#define NUM_TYPES_TO_TEST 6 +#endif /* _NETCDF4 */ + /** * Test the darray functionality. Create a netCDF file with 3 * dimensions and 1 PIO_INT variable, and use darray to write some @@ -288,11 +294,16 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm) { -#define NUM_TYPES_TO_TEST 6 int ioid; char filename[PIO_MAX_NAME + 1]; - int pio_type[NUM_TYPES_TO_TEST] = {PIO_CHAR, PIO_BYTE, PIO_SHORT, +#ifdef _NETCDF4 + int pio_type[NUM_TYPES_TO_TEST] = {PIO_CHAR, PIO_BYTE, PIO_SHORT, + PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#else + int pio_type[NUM_TYPES_TO_TEST] = {PIO_CHAR, PIO_BYTE, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#endif /* _NETCDF4 */ + int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int t; int ret; /* Return code. */ From bc03efaf6a5deeb6d863a9978565392ad61cc792 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 16:24:33 -0600 Subject: [PATCH 15/26] adding more testing, fixing some type stuff --- src/clib/pio_get_vard.c | 2 +- src/clib/pio_getput_int.c | 4 ++++ tests/cunit/test_darray_vard.c | 38 +++++++++++++++++++++++++++++++--- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/clib/pio_get_vard.c b/src/clib/pio_get_vard.c index cccead9bc62..87b4e09b254 100644 --- a/src/clib/pio_get_vard.c +++ b/src/clib/pio_get_vard.c @@ -54,7 +54,7 @@ int PIOc_get_vard_text(int ncid, int varid, int decompid, int PIOc_get_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum, unsigned char *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_UBYTE, buf); + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_CHAR, buf); } /** diff --git a/src/clib/pio_getput_int.c b/src/clib/pio_getput_int.c index 04b5277df30..6b71c2eaed1 100644 --- a/src/clib/pio_getput_int.c +++ b/src/clib/pio_getput_int.c @@ -1453,6 +1453,9 @@ PIOc_get_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum, var_desc_t *vdesc; /* Pointer to var information. */ int ret; + LOG((1, "PIOc_get_vard_tc ncid %d varid %d decompid %d recnum %d " + "xtype %d", ncid, varid, decompid, recnum, xtype)); + /* Get file info. */ if ((ret = pio_get_file(ncid, &file))) return pio_err(NULL, NULL, ret, __FILE__, __LINE__); @@ -1465,6 +1468,7 @@ PIOc_get_vard_tc(int ncid, int varid, int decompid, const PIO_Offset recnum, /* Get var info. */ if ((ret = get_var_desc(varid, &file->varlist, &vdesc))) return pio_err(ios, file, ret, __FILE__, __LINE__); + LOG((2, "vdesc->pio_type %d", vdesc->pio_type)); /* Disallow type conversion for now. */ if (xtype != NC_NAT && xtype != vdesc->pio_type) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index df1aee792d8..6a2a447ad8c 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -227,9 +227,41 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, ERR(ret); /* Read the data. */ - if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, - (void *)test_data_in))) - ERR(ret); + switch (pio_type) + { + case PIO_CHAR: + if ((ret = PIOc_get_vard_uchar(ncid2, varid, ioid, 0, + test_data_char_in))) + ERR(ret); + break; + case PIO_BYTE: + if ((ret = PIOc_get_vard_schar(ncid2, varid, ioid, 0, + test_data_byte_in))) + ERR(ret); + break; + case PIO_SHORT: + if ((ret = PIOc_get_vard_short(ncid2, varid, ioid, 0, + test_data_short_in))) + ERR(ret); + break; + case PIO_INT: + if ((ret = PIOc_get_vard_int(ncid2, varid, ioid, 0, + test_data_int_in))) + ERR(ret); + break; + case PIO_FLOAT: + if ((ret = PIOc_get_vard_float(ncid2, varid, ioid, 0, + test_data_float_in))) + ERR(ret); + break; + case PIO_DOUBLE: + if ((ret = PIOc_get_vard_double(ncid2, varid, ioid, 0, + test_data_double_in))) + ERR(ret); + break; + default: + ERR(ERR_WRONG); + } /* Check the results. */ for (f = 0; f < arraylen; f++) From e98dff3b7122ffc855ce7d440b0220d02e80ce9d Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 5 Jun 2019 16:40:55 -0600 Subject: [PATCH 16/26] more tests, fixing char type --- src/clib/pio_put_vard.c | 2 +- tests/cunit/test_darray_vard.c | 38 +++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/clib/pio_put_vard.c b/src/clib/pio_put_vard.c index ac30207727f..09044e7f0c0 100644 --- a/src/clib/pio_put_vard.c +++ b/src/clib/pio_put_vard.c @@ -55,7 +55,7 @@ int PIOc_put_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum, const unsigned char *op) { - return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_UBYTE, op); + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_CHAR, op); } /** diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index 6a2a447ad8c..bbdb0f6572b 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -197,9 +197,41 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, test_data) != PIO_ENOTVAR) ERR(ERR_WRONG); - /* Write the data. */ - if ((ret = PIOc_put_vard(ncid, varid, ioid, 0, test_data))) - ERR(ret); + switch (pio_type) + { + case PIO_CHAR: + if ((ret = PIOc_put_vard_uchar(ncid, varid, ioid, 0, + test_data_char))) + ERR(ret); + break; + case PIO_BYTE: + if ((ret = PIOc_put_vard_schar(ncid, varid, ioid, 0, + test_data_byte))) + ERR(ret); + break; + case PIO_SHORT: + if ((ret = PIOc_put_vard_short(ncid, varid, ioid, 0, + test_data_short))) + ERR(ret); + break; + case PIO_INT: + if ((ret = PIOc_put_vard_int(ncid, varid, ioid, 0, + test_data_int))) + ERR(ret); + break; + case PIO_FLOAT: + if ((ret = PIOc_put_vard_float(ncid, varid, ioid, 0, + test_data_float))) + ERR(ret); + break; + case PIO_DOUBLE: + if ((ret = PIOc_put_vard_double(ncid, varid, ioid, 0, + test_data_double))) + ERR(ret); + break; + default: + ERR(ERR_WRONG); + } /* Close the netCDF file. */ if ((ret = PIOc_closefile(ncid))) From f5b5d567e85b9639ed9bb9e8b457bb6c522bfae5 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 6 Jun 2019 06:50:16 -0600 Subject: [PATCH 17/26] moving iotype for loop --- tests/cunit/test_darray_vard.c | 51 +++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index bbdb0f6572b..0b9eecaacb8 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -65,14 +65,15 @@ int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; * * @param iosysid the IO system ID. * @param ioid the ID of the decomposition. + * @param fmt the index of the IOTYPE to test. * @param num_flavors the number of IOTYPES available in this build. * @param flavor array of available iotypes. * @param my_rank rank of this task. * @param pio_type the type of the data. * @returns 0 for success, error code otherwise. */ -int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, - int my_rank, int pio_type) +int test_darray(int iosysid, int ioid, int fmt, int num_flavors, + int *flavor, int my_rank, int pio_type) { char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ int dimids[NDIM]; /* The dimension IDs. */ @@ -102,7 +103,7 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, double fillvalue_double = NC_FILL_DOUBLE; double test_data_double[arraylen]; double test_data_double_in[arraylen]; - int f, fmt, provide_fill, d; + int f, provide_fill, d; /* Initialize some data. */ for (f = 0; f < arraylen; f++) @@ -117,10 +118,10 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, /* Use PIO to create the example file in each of the four * available ways. */ - for (fmt = 0; fmt < num_flavors; fmt++) { /* Pnetcdf cannot handle 1-byte types. */ - if (fmt == 0 && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) continue; + if (fmt == 0 && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) + return PIO_NOERR; /* Test with/without providing a fill value to PIOc_write_darray(). */ for (provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; @@ -370,27 +371,31 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int t; + int fmt; int ret; /* Return code. */ - for (t = 0; t < NUM_TYPES_TO_TEST; t++) + for (fmt = 0; fmt < num_flavors; fmt++) { - /* This will be our file name for writing out decompositions. */ - sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", - TEST_NAME, my_rank, *flavor, pio_type[t]); - - /* Decompose the data over the tasks. */ - if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, - dim_len_2d, &ioid, pio_type[t]))) - return ret; - - /* Run a simple darray test. */ - if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, - pio_type[t]))) - return ret; - - /* Free the PIO decomposition. */ - if ((ret = PIOc_freedecomp(iosysid, ioid))) - ERR(ret); + for (t = 0; t < NUM_TYPES_TO_TEST; t++) + { + /* This will be our file name for writing out decompositions. */ + sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", + TEST_NAME, my_rank, *flavor, pio_type[t]); + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, + dim_len_2d, &ioid, pio_type[t]))) + return ret; + + /* Run a simple darray test. */ + if ((ret = test_darray(iosysid, ioid, fmt, num_flavors, flavor, + my_rank, pio_type[t]))) + return ret; + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + } } return PIO_NOERR; From 341d753662108af70bb2aab8c2d87d7ff611192e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 6 Jun 2019 06:52:54 -0600 Subject: [PATCH 18/26] moving iotype for loop --- tests/cunit/test_darray_vard.c | 45 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index 0b9eecaacb8..91d6612ae33 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -350,14 +350,15 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, * Run all the tests. * * @param iosysid the IO system ID. + * @param fmt index into array of IOTYPEs. * @param num_flavors number of available iotypes in the build. * @param flavor pointer to array of the available iotypes. * @param my_rank rank of this task. * @param test_comm the communicator the test is running on. * @returns 0 for success, error code otherwise. */ -int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, - MPI_Comm test_comm) +int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, + int my_rank, MPI_Comm test_comm) { int ioid; char filename[PIO_MAX_NAME + 1]; @@ -371,10 +372,8 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int t; - int fmt; int ret; /* Return code. */ - for (fmt = 0; fmt < num_flavors; fmt++) { for (t = 0; t < NUM_TYPES_TO_TEST; t++) { @@ -429,30 +428,34 @@ int main(int argc, char **argv) int ioproc_stride = 1; /* Stride in rank between io tasks. */ int ioproc_start = 0; /* Zero based rank of first I/O task. */ int r; + int fmt; int ret; /* Return code. */ /* Figure out iotypes. */ if ((ret = get_iotypes(&num_flavors, flavor))) ERR(ret); - for (r = 0; r < NUM_REARRANGERS_TO_TEST; r++) + for (fmt = 0; fmt < num_flavors; fmt++) { - /* Initialize the PIO IO system. This specifies how - * many and which processors are involved in I/O. */ - if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, - ioproc_stride, ioproc_start, - rearranger[r], &iosysid))) - return ret; - - /* Run tests. */ - if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, - test_comm))) - return ret; - - /* Finalize PIO system. */ - if ((ret = PIOc_free_iosystem(iosysid))) - return ret; - } /* next rearranger */ + for (r = 0; r < NUM_REARRANGERS_TO_TEST; r++) + { + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, + ioproc_stride, ioproc_start, + rearranger[r], &iosysid))) + return ret; + + /* Run tests. */ + if ((ret = test_all_darray(iosysid, fmt, num_flavors, flavor, + my_rank, test_comm))) + return ret; + + /* Finalize PIO system. */ + if ((ret = PIOc_free_iosystem(iosysid))) + return ret; + } /* next rearranger */ + } } /* endif my_rank < TARGET_NTASKS */ /* Finalize the MPI library. */ From 6618962e950e05b585a6f394e92ce5a4e328603c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 6 Jun 2019 06:58:49 -0600 Subject: [PATCH 19/26] test development --- tests/cunit/test_darray_vard.c | 51 ++++++++++++++-------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index 91d6612ae33..2728af5ff34 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -52,11 +52,7 @@ char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"}; /* Length of the dimensions in the sample data. */ int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; -#ifdef _NETCDF4 #define NUM_TYPES_TO_TEST 6 -#else -#define NUM_TYPES_TO_TEST 6 -#endif /* _NETCDF4 */ /** * Test the darray functionality. Create a netCDF file with 3 @@ -362,39 +358,34 @@ int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, { int ioid; char filename[PIO_MAX_NAME + 1]; -#ifdef _NETCDF4 - int pio_type[NUM_TYPES_TO_TEST] = {PIO_CHAR, PIO_BYTE, PIO_SHORT, - PIO_INT, PIO_FLOAT, PIO_DOUBLE}; -#else - int pio_type[NUM_TYPES_TO_TEST] = {PIO_CHAR, PIO_BYTE, PIO_SHORT, - PIO_INT, PIO_FLOAT, PIO_DOUBLE}; -#endif /* _NETCDF4 */ - + /* int pio_type[NUM_NETCDF4_TYPES - 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, */ + /* PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, */ + /* PIO_UINT, PIO_INT64, PIO_UINT64}; */ + int pio_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, + PIO_FLOAT, PIO_DOUBLE}; int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int t; int ret; /* Return code. */ + for (t = 0; t < NUM_TYPES_TO_TEST; t++) { - for (t = 0; t < NUM_TYPES_TO_TEST; t++) - { - /* This will be our file name for writing out decompositions. */ - sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", - TEST_NAME, my_rank, *flavor, pio_type[t]); - - /* Decompose the data over the tasks. */ - if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, - dim_len_2d, &ioid, pio_type[t]))) + /* This will be our file name for writing out decompositions. */ + sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", + TEST_NAME, my_rank, *flavor, pio_type[t]); + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, + dim_len_2d, &ioid, pio_type[t]))) + return ret; + + /* Run a simple darray test. */ + if ((ret = test_darray(iosysid, ioid, fmt, num_flavors, flavor, + my_rank, pio_type[t]))) return ret; - /* Run a simple darray test. */ - if ((ret = test_darray(iosysid, ioid, fmt, num_flavors, flavor, - my_rank, pio_type[t]))) - return ret; - - /* Free the PIO decomposition. */ - if ((ret = PIOc_freedecomp(iosysid, ioid))) - ERR(ret); - } + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); } return PIO_NOERR; From f85ff96105fe0aa00d3274e1a240e792c52955f0 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 6 Jun 2019 07:09:13 -0600 Subject: [PATCH 20/26] test development --- tests/cunit/test_darray_vard.c | 110 ++++++++++++--------------------- 1 file changed, 39 insertions(+), 71 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index 2728af5ff34..7e9744a9ef7 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -78,27 +78,20 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, int varid; /* The ID of the netCDF varable. */ int ret; /* Return code. */ PIO_Offset arraylen = 4; - void *fillvalue; - void *test_data; - void *test_data_in; - unsigned char fillvalue_char = NC_FILL_BYTE; unsigned char test_data_char[arraylen]; unsigned char test_data_char_in[arraylen]; - signed char fillvalue_byte = NC_FILL_BYTE; signed char test_data_byte[arraylen]; signed char test_data_byte_in[arraylen]; - short fillvalue_short = NC_FILL_SHORT; short test_data_short[arraylen]; short test_data_short_in[arraylen]; - int fillvalue_int = NC_FILL_INT; int test_data_int[arraylen]; int test_data_int_in[arraylen]; - float fillvalue_float = NC_FILL_FLOAT; float test_data_float[arraylen]; float test_data_float_in[arraylen]; - double fillvalue_double = NC_FILL_DOUBLE; double test_data_double[arraylen]; double test_data_double_in[arraylen]; + unsigned char test_data_ubyte[arraylen]; + unsigned char test_data_ubyte_in[arraylen]; int f, provide_fill, d; /* Initialize some data. */ @@ -126,42 +119,6 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, /* Create the filename. */ sprintf(filename, "%s_iotype_%d_pio_type_%d_provide_fill_%d.nc", TEST_NAME, flavor[fmt], pio_type, provide_fill); - /* Select the fill value and data. */ - switch (pio_type) - { - case PIO_CHAR: - fillvalue = provide_fill ? &fillvalue_char : NULL; - test_data = test_data_char; - test_data_in = test_data_char_in; - break; - case PIO_BYTE: - fillvalue = provide_fill ? &fillvalue_byte : NULL; - test_data = test_data_byte; - test_data_in = test_data_byte_in; - break; - case PIO_SHORT: - fillvalue = provide_fill ? &fillvalue_short : NULL; - test_data = test_data_short; - test_data_in = test_data_short_in; - break; - case PIO_INT: - fillvalue = provide_fill ? &fillvalue_int : NULL; - test_data = test_data_int; - test_data_in = test_data_int_in; - break; - case PIO_FLOAT: - fillvalue = provide_fill ? &fillvalue_float : NULL; - test_data = test_data_float; - test_data_in = test_data_float_in; - break; - case PIO_DOUBLE: - fillvalue = provide_fill ? &fillvalue_double : NULL; - test_data = test_data_double; - test_data_in = test_data_double_in; - break; - default: - ERR(ERR_WRONG); - } /* Create the netCDF output file. */ if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, @@ -183,20 +140,22 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, if ((ret = PIOc_enddef(ncid))) ERR(ret); - /* These should not work. */ - if (PIOc_put_vard(ncid + TEST_VAL_42, varid, ioid, 0, - test_data) != PIO_EBADID) - ERR(ERR_WRONG); - if (PIOc_put_vard(ncid + TEST_VAL_42, varid, ioid + TEST_VAL_42, 0, - test_data) != PIO_EBADID) - ERR(ERR_WRONG); - if (PIOc_put_vard(ncid, TEST_VAL_42, ioid + TEST_VAL_42, 0, - test_data) != PIO_ENOTVAR) - ERR(ERR_WRONG); switch (pio_type) { case PIO_CHAR: + /* These should not work. */ + if (PIOc_put_vard_uchar(ncid + TEST_VAL_42, varid, ioid, 0, + test_data_char) != PIO_EBADID) + ERR(ERR_WRONG); + if (PIOc_put_vard_uchar(ncid + TEST_VAL_42, varid, ioid + TEST_VAL_42, 0, + test_data_char) != PIO_EBADID) + ERR(ERR_WRONG); + if (PIOc_put_vard_uchar(ncid, TEST_VAL_42, ioid + TEST_VAL_42, 0, + test_data_char) != PIO_ENOTVAR) + ERR(ERR_WRONG); + + /* This will work. */ if ((ret = PIOc_put_vard_uchar(ncid, varid, ioid, 0, test_data_char))) ERR(ret); @@ -247,18 +206,19 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, if (dimlen != 1) ERR(ERR_WRONG); - /* These should not work. */ - if ((ret = PIOc_get_vard(ncid2 + TEST_VAL_42, varid, ioid, 0, - test_data_in)) != PIO_EBADID) - ERR(ret); - if ((ret = PIOc_get_vard(ncid2, varid, ioid + TEST_VAL_42, 0, - test_data_in)) != PIO_EBADID) - ERR(ret); - /* Read the data. */ switch (pio_type) { case PIO_CHAR: + /* These should not work. */ + if ((ret = PIOc_get_vard_uchar(ncid2 + TEST_VAL_42, varid, ioid, 0, + test_data_char_in)) != PIO_EBADID) + ERR(ret); + if ((ret = PIOc_get_vard_uchar(ncid2, varid, ioid + TEST_VAL_42, 0, + test_data_char_in)) != PIO_EBADID) + ERR(ret); + + /* This will work. */ if ((ret = PIOc_get_vard_uchar(ncid2, varid, ioid, 0, test_data_char_in))) ERR(ret); @@ -328,11 +288,10 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, /* Try to write, but it won't work, because we opened file * read-only. */ - { - if (PIOc_write_darray(ncid2, varid, ioid, arraylen, test_data, - fillvalue) != PIO_EPERM) - ERR(ERR_WRONG); - } + if (PIOc_write_darray(ncid2, varid, ioid, arraylen, test_data_char, + NULL) != PIO_EPERM) + ERR(ERR_WRONG); + /* Close the netCDF file. */ if ((ret = PIOc_closefile(ncid2))) ERR(ret); @@ -361,13 +320,22 @@ int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, /* int pio_type[NUM_NETCDF4_TYPES - 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, */ /* PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, */ /* PIO_UINT, PIO_INT64, PIO_UINT64}; */ - int pio_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, - PIO_FLOAT, PIO_DOUBLE}; + int pio_type[NUM_TYPES_TO_TEST + 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, + PIO_FLOAT, PIO_DOUBLE, NC_UBYTE}; int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; + int num_types; int t; int ret; /* Return code. */ - for (t = 0; t < NUM_TYPES_TO_TEST; t++) + /* Based on the IOTYPE, decide how many types to check. */ + if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P) + /* num_types = NUM_NETCDF4_TYPES - 1; */ + num_types = NUM_TYPES_TO_TEST; + else + num_types = NUM_CLASSIC_TYPES; + + /* Check each type. */ + for (t = 0; t < num_types; t++) { /* This will be our file name for writing out decompositions. */ sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", From d83ca72726ea1b88221216b965824398e843bebb Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 6 Jun 2019 07:29:46 -0600 Subject: [PATCH 21/26] starting to add netcdf-4 atomic type tests --- tests/cunit/test_darray_vard.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index 7e9744a9ef7..84a2dc2e890 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -185,6 +185,11 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, test_data_double))) ERR(ret); break; + case PIO_UBYTE: + if ((ret = PIOc_put_vard_uchar(ncid, varid, ioid, 0, + test_data_ubyte))) + ERR(ret); + break; default: ERR(ERR_WRONG); } @@ -248,6 +253,11 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, test_data_double_in))) ERR(ret); break; + case PIO_UBYTE: + if ((ret = PIOc_get_vard_uchar(ncid2, varid, ioid, 0, + test_data_ubyte_in))) + ERR(ret); + break; default: ERR(ERR_WRONG); } @@ -281,6 +291,10 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, if (test_data_double_in[f] != test_data_double[f]) return ERR_WRONG; break; + case PIO_UBYTE: + if (test_data_ubyte_in[f] != test_data_ubyte[f]) + return ERR_WRONG; + break; default: ERR(ERR_WRONG); } From f2133553f69e7c6682f1fc7a8a15725717959921 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 6 Jun 2019 07:52:10 -0600 Subject: [PATCH 22/26] adding netcdf-4 UBYTE type to test --- src/clib/pio_get_vard.c | 2 +- src/clib/pio_nc.c | 2 ++ src/clib/pio_put_vard.c | 2 +- tests/cunit/test_darray_vard.c | 22 +++++++++++----------- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/clib/pio_get_vard.c b/src/clib/pio_get_vard.c index 87b4e09b254..cccead9bc62 100644 --- a/src/clib/pio_get_vard.c +++ b/src/clib/pio_get_vard.c @@ -54,7 +54,7 @@ int PIOc_get_vard_text(int ncid, int varid, int decompid, int PIOc_get_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum, unsigned char *buf) { - return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_CHAR, buf); + return PIOc_get_vard_tc(ncid, varid, decompid, recnum, NC_UBYTE, buf); } /** diff --git a/src/clib/pio_nc.c b/src/clib/pio_nc.c index 854e943391b..7223025e0fc 100644 --- a/src/clib/pio_nc.c +++ b/src/clib/pio_nc.c @@ -2273,6 +2273,8 @@ PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims, if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io) ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, &varid); + LOG((3, "defined var ierr %d file->iotype %d", ierr, file->iotype)); + #ifdef _NETCDF4 /* For netCDF-4 serial files, turn on compression for this variable. */ if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4C) diff --git a/src/clib/pio_put_vard.c b/src/clib/pio_put_vard.c index 09044e7f0c0..ac30207727f 100644 --- a/src/clib/pio_put_vard.c +++ b/src/clib/pio_put_vard.c @@ -55,7 +55,7 @@ int PIOc_put_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum, const unsigned char *op) { - return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_CHAR, op); + return PIOc_put_vard_tc(ncid, varid, decompid, recnum, NC_UBYTE, op); } /** diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index 84a2dc2e890..42ceb83ac1c 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -78,8 +78,8 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, int varid; /* The ID of the netCDF varable. */ int ret; /* Return code. */ PIO_Offset arraylen = 4; - unsigned char test_data_char[arraylen]; - unsigned char test_data_char_in[arraylen]; + char test_data_char[arraylen]; + char test_data_char_in[arraylen]; signed char test_data_byte[arraylen]; signed char test_data_byte_in[arraylen]; short test_data_short[arraylen]; @@ -145,18 +145,18 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, { case PIO_CHAR: /* These should not work. */ - if (PIOc_put_vard_uchar(ncid + TEST_VAL_42, varid, ioid, 0, + if (PIOc_put_vard_text(ncid + TEST_VAL_42, varid, ioid, 0, test_data_char) != PIO_EBADID) ERR(ERR_WRONG); - if (PIOc_put_vard_uchar(ncid + TEST_VAL_42, varid, ioid + TEST_VAL_42, 0, + if (PIOc_put_vard_text(ncid + TEST_VAL_42, varid, ioid + TEST_VAL_42, 0, test_data_char) != PIO_EBADID) ERR(ERR_WRONG); - if (PIOc_put_vard_uchar(ncid, TEST_VAL_42, ioid + TEST_VAL_42, 0, + if (PIOc_put_vard_text(ncid, TEST_VAL_42, ioid + TEST_VAL_42, 0, test_data_char) != PIO_ENOTVAR) ERR(ERR_WRONG); /* This will work. */ - if ((ret = PIOc_put_vard_uchar(ncid, varid, ioid, 0, + if ((ret = PIOc_put_vard_text(ncid, varid, ioid, 0, test_data_char))) ERR(ret); break; @@ -216,15 +216,15 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, { case PIO_CHAR: /* These should not work. */ - if ((ret = PIOc_get_vard_uchar(ncid2 + TEST_VAL_42, varid, ioid, 0, + if ((ret = PIOc_get_vard_text(ncid2 + TEST_VAL_42, varid, ioid, 0, test_data_char_in)) != PIO_EBADID) ERR(ret); - if ((ret = PIOc_get_vard_uchar(ncid2, varid, ioid + TEST_VAL_42, 0, + if ((ret = PIOc_get_vard_text(ncid2, varid, ioid + TEST_VAL_42, 0, test_data_char_in)) != PIO_EBADID) ERR(ret); /* This will work. */ - if ((ret = PIOc_get_vard_uchar(ncid2, varid, ioid, 0, + if ((ret = PIOc_get_vard_text(ncid2, varid, ioid, 0, test_data_char_in))) ERR(ret); break; @@ -335,7 +335,7 @@ int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, /* PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, */ /* PIO_UINT, PIO_INT64, PIO_UINT64}; */ int pio_type[NUM_TYPES_TO_TEST + 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, - PIO_FLOAT, PIO_DOUBLE, NC_UBYTE}; + PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE}; int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int num_types; int t; @@ -344,7 +344,7 @@ int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, /* Based on the IOTYPE, decide how many types to check. */ if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P) /* num_types = NUM_NETCDF4_TYPES - 1; */ - num_types = NUM_TYPES_TO_TEST; + num_types = NUM_TYPES_TO_TEST + 1; else num_types = NUM_CLASSIC_TYPES; From b7aad139a84552816b96845e13a89a5434316aca Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 6 Jun 2019 08:01:24 -0600 Subject: [PATCH 23/26] adding more netcdf-4 atomic types to test --- tests/cunit/test_darray_vard.c | 41 +++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index 42ceb83ac1c..ecb61307c08 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -92,6 +92,14 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, double test_data_double_in[arraylen]; unsigned char test_data_ubyte[arraylen]; unsigned char test_data_ubyte_in[arraylen]; + unsigned short test_data_ushort[arraylen]; + unsigned short test_data_ushort_in[arraylen]; + unsigned int test_data_uint[arraylen]; + unsigned int test_data_uint_in[arraylen]; + long long int test_data_int64[arraylen]; + long long int test_data_int64_in[arraylen]; + unsigned long long int test_data_uint64[arraylen]; + unsigned long long int test_data_uint64_in[arraylen]; int f, provide_fill, d; /* Initialize some data. */ @@ -103,6 +111,11 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, test_data_int[f] = my_rank * 10 + f; test_data_float[f] = my_rank * 10 + f + 0.5; test_data_double[f] = my_rank * 100000 + f + 0.5; + test_data_ubyte[f] = my_rank + f + 2; + test_data_ushort[f] = my_rank + f + 20; + test_data_uint[f] = my_rank + f + 200; + test_data_int64[f] = my_rank - f - 20000; + test_data_uint64[f] = my_rank + f + 20000; } /* Use PIO to create the example file in each of the four @@ -190,6 +203,26 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, test_data_ubyte))) ERR(ret); break; + case PIO_USHORT: + if ((ret = PIOc_put_vard_ushort(ncid, varid, ioid, 0, + test_data_ushort))) + ERR(ret); + break; + case PIO_UINT: + if ((ret = PIOc_put_vard_uint(ncid, varid, ioid, 0, + test_data_uint))) + ERR(ret); + break; + case PIO_INT64: + if ((ret = PIOc_put_vard_longlong(ncid, varid, ioid, 0, + test_data_int64))) + ERR(ret); + break; + case PIO_UINT64: + if ((ret = PIOc_put_vard_ulonglong(ncid, varid, ioid, 0, + test_data_uint64))) + ERR(ret); + break; default: ERR(ERR_WRONG); } @@ -331,11 +364,9 @@ int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, { int ioid; char filename[PIO_MAX_NAME + 1]; - /* int pio_type[NUM_NETCDF4_TYPES - 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, */ - /* PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, */ - /* PIO_UINT, PIO_INT64, PIO_UINT64}; */ - int pio_type[NUM_TYPES_TO_TEST + 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, - PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE}; + int pio_type[NUM_NETCDF4_TYPES - 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, + PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, + PIO_UINT, PIO_INT64, PIO_UINT64}; int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int num_types; int t; From e49c2b98551e4d835d687c5b0d4aadaeac047d30 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 6 Jun 2019 08:08:19 -0600 Subject: [PATCH 24/26] added rest of netcdf-4 atomic types --- tests/cunit/test_darray_vard.c | 52 ++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index ecb61307c08..8dca446a7d9 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -35,12 +35,8 @@ #define X_DIM_LEN 4 #define Y_DIM_LEN 4 -/* The number of timesteps of data to write. */ -#define NUM_TIMESTEPS 2 - /* The names of variables in the netCDF output files. */ #define VAR_NAME "Billy-Bob" -#define VAR_NAME2 "Sally-Sue" /* Test with and without specifying a fill value to * PIOc_write_darray(). */ @@ -100,7 +96,7 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, long long int test_data_int64_in[arraylen]; unsigned long long int test_data_uint64[arraylen]; unsigned long long int test_data_uint64_in[arraylen]; - int f, provide_fill, d; + int f, d; /* Initialize some data. */ for (f = 0; f < arraylen; f++) @@ -125,13 +121,10 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, if (fmt == 0 && (pio_type == PIO_BYTE || pio_type == PIO_CHAR)) return PIO_NOERR; - /* Test with/without providing a fill value to PIOc_write_darray(). */ - for (provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; - provide_fill++) { /* Create the filename. */ - sprintf(filename, "%s_iotype_%d_pio_type_%d_provide_fill_%d.nc", - TEST_NAME, flavor[fmt], pio_type, provide_fill); + sprintf(filename, "%s_iotype_%d_pio_type_%d.nc", + TEST_NAME, flavor[fmt], pio_type); /* Create the netCDF output file. */ if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, @@ -291,6 +284,26 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, test_data_ubyte_in))) ERR(ret); break; + case PIO_USHORT: + if ((ret = PIOc_get_vard_ushort(ncid2, varid, ioid, 0, + test_data_ushort_in))) + ERR(ret); + break; + case PIO_UINT: + if ((ret = PIOc_get_vard_uint(ncid2, varid, ioid, 0, + test_data_uint_in))) + ERR(ret); + break; + case PIO_INT64: + if ((ret = PIOc_get_vard_longlong(ncid2, varid, ioid, 0, + test_data_int64_in))) + ERR(ret); + break; + case PIO_UINT64: + if ((ret = PIOc_get_vard_ulonglong(ncid2, varid, ioid, 0, + test_data_uint64_in))) + ERR(ret); + break; default: ERR(ERR_WRONG); } @@ -328,6 +341,22 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, if (test_data_ubyte_in[f] != test_data_ubyte[f]) return ERR_WRONG; break; + case PIO_USHORT: + if (test_data_ushort_in[f] != test_data_ushort[f]) + return ERR_WRONG; + break; + case PIO_UINT: + if (test_data_uint_in[f] != test_data_uint[f]) + return ERR_WRONG; + break; + case PIO_INT64: + if (test_data_int64_in[f] != test_data_int64[f]) + return ERR_WRONG; + break; + case PIO_UINT64: + if (test_data_uint64_in[f] != test_data_uint64[f]) + return ERR_WRONG; + break; default: ERR(ERR_WRONG); } @@ -374,8 +403,7 @@ int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, /* Based on the IOTYPE, decide how many types to check. */ if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P) - /* num_types = NUM_NETCDF4_TYPES - 1; */ - num_types = NUM_TYPES_TO_TEST + 1; + num_types = NUM_NETCDF4_TYPES - 1; else num_types = NUM_CLASSIC_TYPES; From 4819d30e9bbcbcb40b81aecb2cc6f12495b2d14b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 6 Jun 2019 08:29:20 -0600 Subject: [PATCH 25/26] adding tests for void * versions of vard functions --- tests/cunit/test_darray_vard.c | 36 ++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index 8dca446a7d9..5eaf2743f4d 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -73,6 +73,7 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, int ncid2; /* The ncid of the re-opened netCDF file. */ int varid; /* The ID of the netCDF varable. */ int ret; /* Return code. */ + int type_to_use; PIO_Offset arraylen = 4; char test_data_char[arraylen]; char test_data_char_in[arraylen]; @@ -138,7 +139,8 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, ERR(ret); /* Define a variable. */ - if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, + type_to_use = (pio_type == NC_NAT) ? PIO_INT : pio_type; + if ((ret = PIOc_def_var(ncid, VAR_NAME, type_to_use, NDIM, dimids, &varid))) ERR(ret); @@ -216,6 +218,11 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, test_data_uint64))) ERR(ret); break; + case NC_NAT: + /* Using NAT to test void * version, using int data. */ + if ((ret = PIOc_put_vard(ncid, varid, ioid, 0, test_data_int))) + ERR(ret); + break; default: ERR(ERR_WRONG); } @@ -304,6 +311,12 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, test_data_uint64_in))) ERR(ret); break; + case NC_NAT: + /* Using NAT to test void * version, using int data. */ + if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, + test_data_int_in))) + ERR(ret); + break; default: ERR(ERR_WRONG); } @@ -357,6 +370,11 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, if (test_data_uint64_in[f] != test_data_uint64[f]) return ERR_WRONG; break; + case NC_NAT: + /* Using NAT to test void * version, using int data. */ + if (test_data_int_in[f] != test_data_int[f]) + return ERR_WRONG; + break; default: ERR(ERR_WRONG); } @@ -393,9 +411,9 @@ int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, { int ioid; char filename[PIO_MAX_NAME + 1]; - int pio_type[NUM_NETCDF4_TYPES - 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, - PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, - PIO_UINT, PIO_INT64, PIO_UINT64}; + int pio_type[NUM_NETCDF4_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, + PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, + PIO_UINT, PIO_INT64, PIO_UINT64, NC_NAT}; int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN}; int num_types; int t; @@ -403,20 +421,26 @@ int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, /* Based on the IOTYPE, decide how many types to check. */ if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P) - num_types = NUM_NETCDF4_TYPES - 1; + num_types = NUM_NETCDF4_TYPES; else num_types = NUM_CLASSIC_TYPES; /* Check each type. */ for (t = 0; t < num_types; t++) { + int type_to_use; + + /* Using NAT to test generic versions of vard functions, so + * substiture PIO_INT. */ + type_to_use = (pio_type[t] == NC_NAT) ? PIO_INT : pio_type[t]; + /* This will be our file name for writing out decompositions. */ sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", TEST_NAME, my_rank, *flavor, pio_type[t]); /* Decompose the data over the tasks. */ if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, - dim_len_2d, &ioid, pio_type[t]))) + dim_len_2d, &ioid, type_to_use))) return ret; /* Run a simple darray test. */ From 5aeae4ef1c508c12a23d68ba7cb998d56e3fc3f9 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 6 Jun 2019 08:48:58 -0600 Subject: [PATCH 26/26] clean up --- src/clib/pio_getput_int.c | 1 + tests/cunit/test_darray_vard.c | 36 +++++++++++++++++----------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/clib/pio_getput_int.c b/src/clib/pio_getput_int.c index 6b71c2eaed1..2eb37064bb8 100644 --- a/src/clib/pio_getput_int.c +++ b/src/clib/pio_getput_int.c @@ -1424,6 +1424,7 @@ PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op) return ierr; } + /** * Internal PIO function which provides a type-neutral interface to * PIOc_get_vard() and related functions. This function gets diff --git a/tests/cunit/test_darray_vard.c b/tests/cunit/test_darray_vard.c index 5eaf2743f4d..1a94ecdf301 100644 --- a/tests/cunit/test_darray_vard.c +++ b/tests/cunit/test_darray_vard.c @@ -154,18 +154,18 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, case PIO_CHAR: /* These should not work. */ if (PIOc_put_vard_text(ncid + TEST_VAL_42, varid, ioid, 0, - test_data_char) != PIO_EBADID) + test_data_char) != PIO_EBADID) ERR(ERR_WRONG); - if (PIOc_put_vard_text(ncid + TEST_VAL_42, varid, ioid + TEST_VAL_42, 0, - test_data_char) != PIO_EBADID) + if (PIOc_put_vard_text(ncid, varid + TEST_VAL_42, ioid, 0, + test_data_char) != PIO_ENOTVAR) ERR(ERR_WRONG); - if (PIOc_put_vard_text(ncid, TEST_VAL_42, ioid + TEST_VAL_42, 0, - test_data_char) != PIO_ENOTVAR) + if (PIOc_put_vard_text(ncid, varid, ioid + TEST_VAL_42, 0, + test_data_char) != PIO_EBADID) ERR(ERR_WRONG); /* This will work. */ if ((ret = PIOc_put_vard_text(ncid, varid, ioid, 0, - test_data_char))) + test_data_char))) ERR(ret); break; case PIO_BYTE: @@ -180,7 +180,7 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, break; case PIO_INT: if ((ret = PIOc_put_vard_int(ncid, varid, ioid, 0, - test_data_int))) + test_data_int))) ERR(ret); break; case PIO_FLOAT: @@ -190,7 +190,7 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, break; case PIO_DOUBLE: if ((ret = PIOc_put_vard_double(ncid, varid, ioid, 0, - test_data_double))) + test_data_double))) ERR(ret); break; case PIO_UBYTE: @@ -200,12 +200,12 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, break; case PIO_USHORT: if ((ret = PIOc_put_vard_ushort(ncid, varid, ioid, 0, - test_data_ushort))) + test_data_ushort))) ERR(ret); break; case PIO_UINT: if ((ret = PIOc_put_vard_uint(ncid, varid, ioid, 0, - test_data_uint))) + test_data_uint))) ERR(ret); break; case PIO_INT64: @@ -250,15 +250,15 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, case PIO_CHAR: /* These should not work. */ if ((ret = PIOc_get_vard_text(ncid2 + TEST_VAL_42, varid, ioid, 0, - test_data_char_in)) != PIO_EBADID) + test_data_char_in)) != PIO_EBADID) ERR(ret); if ((ret = PIOc_get_vard_text(ncid2, varid, ioid + TEST_VAL_42, 0, - test_data_char_in)) != PIO_EBADID) + test_data_char_in)) != PIO_EBADID) ERR(ret); /* This will work. */ if ((ret = PIOc_get_vard_text(ncid2, varid, ioid, 0, - test_data_char_in))) + test_data_char_in))) ERR(ret); break; case PIO_BYTE: @@ -273,7 +273,7 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, break; case PIO_INT: if ((ret = PIOc_get_vard_int(ncid2, varid, ioid, 0, - test_data_int_in))) + test_data_int_in))) ERR(ret); break; case PIO_FLOAT: @@ -283,7 +283,7 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, break; case PIO_DOUBLE: if ((ret = PIOc_get_vard_double(ncid2, varid, ioid, 0, - test_data_double_in))) + test_data_double_in))) ERR(ret); break; case PIO_UBYTE: @@ -293,12 +293,12 @@ int test_darray(int iosysid, int ioid, int fmt, int num_flavors, break; case PIO_USHORT: if ((ret = PIOc_get_vard_ushort(ncid2, varid, ioid, 0, - test_data_ushort_in))) + test_data_ushort_in))) ERR(ret); break; case PIO_UINT: if ((ret = PIOc_get_vard_uint(ncid2, varid, ioid, 0, - test_data_uint_in))) + test_data_uint_in))) ERR(ret); break; case PIO_INT64: @@ -446,7 +446,7 @@ int test_all_darray(int iosysid, int fmt, int num_flavors, int *flavor, /* Run a simple darray test. */ if ((ret = test_darray(iosysid, ioid, fmt, num_flavors, flavor, my_rank, pio_type[t]))) - return ret; + return ret; /* Free the PIO decomposition. */ if ((ret = PIOc_freedecomp(iosysid, ioid)))