Merge pull request ESMCI#1294 from NCAR/ejh_back

Fix darray fill value problem
rfiorella · May 6, 2018 · efd9b10 · efd9b10
2 parents 8405ee0 + 1ec8fea
commit efd9b10
Show file tree

Hide file tree

Showing 6 changed files with 507 additions and 13 deletions.
diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c
@@ -372,6 +372,104 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
     return PIO_NOERR;
 }
 
+/**
+ * Find the fill value that would be used for a variable, if fill mode
+ * was turned on.
+ *
+ * @param ncid File ID.
+ * @param varid Variable ID.
+ * @param pio_type Type of the variable.
+ * @param type_size Size of one element of this type.
+ * @param fillvalue Pointer that will get the fill value.
+ *
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ * @author Ed Hartnett
+ */
+static int
+pio_inq_var_fill_expected(int ncid, int varid, int pio_type, PIO_Offset type_size,
+                          void *fillvalue)
+{
+    signed char byte_fill_value = NC_FILL_BYTE;
+    char char_fill_value = NC_FILL_CHAR;
+    short short_fill_value = NC_FILL_SHORT;
+    int int_fill_value = NC_FILL_INT;
+    float float_fill_value = NC_FILL_FLOAT;
+    double double_fill_value = NC_FILL_DOUBLE;
+    unsigned char ubyte_fill_value = NC_FILL_UBYTE;
+    unsigned short ushort_fill_value = NC_FILL_USHORT;
+    unsigned int uint_fill_value = NC_FILL_UINT;
+    long long int64_fill_value = NC_FILL_INT64;
+    unsigned long long uint64_fill_value = NC_FILL_UINT64;
+    char *string_fill_value = "";
+    int ret;
+
+    /* Check inputs. */
+    assert(fillvalue);
+
+    LOG((2, "pio_inq_var_fill_expected ncid %d varid %d pio_type %d type_size %d",
+         ncid, varid, pio_type, type_size));
+
+    /* Is there a _FillValue attribute? */
+    ret = PIOc_get_att(ncid, varid, "_FillValue", fillvalue);
+    LOG((3, "pio_inq_var_fill_expected ret %d", ret));
+
+    /* If no _FillValue at was found we still have work to do. */
+    if (ret)
+    {
+        /* Did we get some other error? */
+        if (ret != PIO_ENOTATT)
+            return ret;
+
+        /* What is the default fill value for this type? */
+        switch (pio_type)
+        {
+        case PIO_BYTE:
+            memcpy(fillvalue, &byte_fill_value, type_size);
+            break;
+        case PIO_CHAR:
+            memcpy(fillvalue, &char_fill_value, type_size);
+            break;
+        case PIO_SHORT:
+            memcpy(fillvalue, &short_fill_value, type_size);
+            break;
+        case PIO_INT:
+            memcpy(fillvalue, &int_fill_value, type_size);
+            break;
+        case PIO_FLOAT:
+            memcpy(fillvalue, &float_fill_value, type_size);
+            break;
+        case PIO_DOUBLE:
+            memcpy(fillvalue, &double_fill_value, type_size);
+            break;
+#ifdef _NETCDF4
+        case PIO_UBYTE:
+            memcpy(fillvalue, &ubyte_fill_value, type_size);
+            break;
+        case PIO_USHORT:
+            memcpy(fillvalue, &ushort_fill_value, type_size);
+            break;
+        case PIO_UINT:
+            memcpy(fillvalue, &uint_fill_value, type_size);
+            break;
+        case PIO_INT64:
+            memcpy(fillvalue, &int64_fill_value, type_size);
+            break;
+        case PIO_UINT64:
+            memcpy(fillvalue, &uint64_fill_value, type_size);
+            break;
+        case PIO_STRING:
+            memcpy(fillvalue, string_fill_value, type_size);
+            break;
+#endif /* _NETCDF4 */
+        default:
+            return PIO_EBADTYPE;
+        }
+    }
+
+    return PIO_NOERR;
+}
+
 /**
  * Find the fillvalue that should be used for a variable.
  *
@@ -410,12 +508,20 @@ int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
     if (!(vdesc->fillvalue = malloc(type_size)))
         return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
 
-    /* Get the fill value. */
+    /* Get the fill mode and value, if fill mode is on (which is will
+     * not be, because it is turned off at open/create). */
     if ((ierr = PIOc_inq_var_fill(file->pio_ncid, varid, &no_fill, vdesc->fillvalue)))
         return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
     vdesc->use_fill = no_fill ? 0 : 1;
     LOG((3, "vdesc->use_fill = %d", vdesc->use_fill));
 
+    /* Get the fill value one would expect, if NOFILL were not turned
+     * on. */
+    if (!vdesc->use_fill)
+        if ((ierr = pio_inq_var_fill_expected(file->pio_ncid, varid, pio_type, type_size,
+                                              vdesc->fillvalue)))
+            return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+
     return PIO_NOERR;
 }
 

diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c
@@ -226,18 +226,17 @@ void pio_log(int severity, const char *fmt, ...)
     ptr += strlen(rank_str);
     rem_len -= strlen(rank_str);
 
-    /* Show the severity. */
-    snprintf(rank_str, MAX_RANK_STR, ":%d ", severity);
-    strncpy(ptr, rank_str, (rem_len > 0) ? rem_len : 0);
-    ptr += strlen(rank_str);
-    rem_len -= strlen(rank_str);
+    /* /\* Show the severity. *\/ */
+    /* snprintf(rank_str, MAX_RANK_STR, ":%d ", severity); */
+    /* strncpy(ptr, rank_str, (rem_len > 0) ? rem_len : 0); */
+    /* ptr += strlen(rank_str); */
+    /* rem_len -= strlen(rank_str); */
 
     /* Print out the variable list of args with vprintf. */
     va_start(argp, fmt);
     vsnprintf(ptr, ((rem_len > 0) ? rem_len : 0), fmt, argp);
     va_end(argp);
 
-
     /* Put on a final linefeed. */
     ptr = msg + strlen(msg);
     rem_len = MAX_LOG_MSG - strlen(msg);
@@ -420,10 +419,12 @@ int check_netcdf2(iosystem_desc_t *ios, file_desc_t *file, int status,
 
     if (file && file->iosystem->ioproc &&
 	(file->iotype == PIO_IOTYPE_PNETCDF || file->iotype == PIO_IOTYPE_NETCDF4P))
-      if (file->iosystem->io_rank == 0)
-	MPI_Reduce(MPI_IN_PLACE, &status, 1, MPI_INT, MPI_MIN, 0, file->iosystem->io_comm);
-      else
-	MPI_Reduce(&status, &rbuf, 1, MPI_INT, MPI_MIN, 0, file->iosystem->io_comm);
+    {
+        if (file->iosystem->io_rank == 0)
+            MPI_Reduce(MPI_IN_PLACE, &status, 1, MPI_INT, MPI_MIN, 0, file->iosystem->io_comm);
+        else
+            MPI_Reduce(&status, &rbuf, 1, MPI_INT, MPI_MIN, 0, file->iosystem->io_comm);
+    }
 
     LOG((1, "check_netcdf2 status = %d fname = %s line = %d", status, fname, line));
 

diff --git a/tests/cunit/CMakeLists.txt b/tests/cunit/CMakeLists.txt
@@ -88,6 +88,8 @@ if (NOT PIO_USE_MPISERIAL)
   target_link_libraries (test_decomps pioc)
   add_executable (test_rearr EXCLUDE_FROM_ALL test_rearr.c test_common.c)
   target_link_libraries (test_rearr pioc)
+  add_executable (test_darray_fill EXCLUDE_FROM_ALL test_darray_fill.c test_common.c)
+  target_link_libraries (test_darray_fill pioc)
   if (PIO_USE_MALLOC)
     add_executable (test_darray_async_simple EXCLUDE_FROM_ALL test_darray_async_simple.c test_common.c)
     target_link_libraries (test_darray_async_simple pioc)
@@ -123,6 +125,7 @@ add_dependencies (tests test_darray_1d)
 add_dependencies (tests test_darray_3d)
 add_dependencies (tests test_decomp_uneven)
 add_dependencies (tests test_decomps)
+add_dependencies (tests test_darray_fill)
 if(PIO_USE_MALLOC)
   add_dependencies (tests test_darray_async_simple)
   add_dependencies (tests test_darray_async)
@@ -252,6 +255,10 @@ else ()
     EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_3d
     NUMPROCS ${AT_LEAST_FOUR_TASKS}
     TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray_fill
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_fill
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
   if(PIO_USE_MALLOC)
     add_mpi_test(test_darray_2sync
       EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_2sync

diff --git a/tests/cunit/Makefile.am b/tests/cunit/Makefile.am
@@ -18,7 +18,8 @@ test_darray_multi test_darray_multivar test_darray_multivar2		\
 test_darray_multivar3 test_darray_1d test_darray_3d			\
 test_decomp_uneven test_decomps test_rearr test_darray_async_simple	\
 test_darray_async test_darray_async_many test_darray_2sync		\
-test_async_multicomp test_async_multi2 test_async_manyproc
+test_async_multicomp test_async_multi2 test_async_manyproc		\
+test_darray_fill
 
 # Tests will run from a bash script.
 TESTS = run_tests.sh
@@ -56,6 +57,7 @@ test_async_3proc_SOURCES = test_async_3proc.c test_common.c pio_tests.h
 test_async_multicomp_SOURCES = test_async_multicomp.c test_common.c pio_tests.h
 test_async_multi2_SOURCES = test_async_multi2.c test_common.c pio_tests.h
 test_async_manyproc_SOURCES = test_async_manyproc.c test_common.c pio_tests.h
+test_darray_fill_SOURCES = test_darray_fill.c test_common.c pio_tests.h
 
 # Distribute the test script.
 EXTRA_DIST = run_tests.sh

diff --git a/tests/cunit/run_tests.sh b/tests/cunit/run_tests.sh
@@ -15,7 +15,8 @@ PIO_TESTS='test_intercomm2 test_async_mpi test_spmd test_rearr test_async_simple
 'test_pioc_unlim test_pioc_putget test_pioc_fill test_darray test_darray_multi '\
 'test_darray_multivar test_darray_multivar2 test_darray_multivar3 test_darray_1d '\
 'test_darray_3d test_decomp_uneven test_decomps test_darray_async_simple '\
-'test_darray_async test_darray_async_many test_darray_2sync test_async_multicomp '
+'test_darray_async test_darray_async_many test_darray_2sync test_async_multicomp '\
+'test_darray_fill'
 
 success1=true
 success2=true