diff --git a/externals/coda-oss/.github/workflows/build_unittest.yml b/externals/coda-oss/.github/workflows/build_unittest.yml index 729efd0903..68c6a6bb2f 100644 --- a/externals/coda-oss/.github/workflows/build_unittest.yml +++ b/externals/coda-oss/.github/workflows/build_unittest.yml @@ -11,9 +11,9 @@ jobs: name: ${{ matrix.os }}-${{ matrix.python-version }}-CMake runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install python dependencies @@ -59,9 +59,9 @@ jobs: name: ${{ matrix.os }}-${{ matrix.python-version }}-CMake runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install python dependencies @@ -97,9 +97,9 @@ jobs: name: ${{ matrix.os }}-${{ matrix.python-version }}-waf runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: configure_with_swig diff --git a/externals/coda-oss/modules/c++/hdf5.lite/include/hdf5/lite/highfive.h b/externals/coda-oss/modules/c++/hdf5.lite/include/hdf5/lite/highfive.h index 6203344c30..40344621f9 100644 --- a/externals/coda-oss/modules/c++/hdf5.lite/include/hdf5/lite/highfive.h +++ b/externals/coda-oss/modules/c++/hdf5.lite/include/hdf5/lite/highfive.h @@ -30,6 +30,8 @@ */ #include +#include +#include #include "highfive/H5Easy.hpp" #include "highfive/H5DataSet.hpp" @@ -40,8 +42,21 @@ namespace hdf5 { namespace lite { + +// Save the trouble of specifying a return type of std::vector<> +template +inline auto v_load(const H5Easy::File& file, const std::string& dataset_name) +{ + return H5Easy::load>(file, dataset_name); +} +template +inline auto vv_load(const H5Easy::File& file, const std::string& dataset_name) +{ + return H5Easy::load>>(file, dataset_name); +} + template -inline HighFive::DataSet writeDataSet(H5Easy::File& file, SpanRC data, const std::string& dataset_name /*, TODO ...*/) +inline HighFive::DataSet writeDataSet(H5Easy::File& file, const std::string& dataset_name, SpanRC data /*, TODO ...*/) { const std::vector dims{data.dims().row, data.dims().col}; const HighFive::DataSpace dataspace{dims}; @@ -50,11 +65,26 @@ inline HighFive::DataSet writeDataSet(H5Easy::File& file, SpanRC data, const return retval; } +template +inline HighFive::DataSet writeDataSet(const H5Easy::File& file, const std::string& dataset_name, const T& values /*, TODO ...*/) +{ + auto dataset = file.createDataSet(dataset_name, HighFive::DataSpace::From(values)); + dataset.write(values); + return dataset; +} + +// This loads 2D data into one large block of contiguous memory. +// (HighFive::DataSet::read() uses a vector of vectors). template -inline SpanRC readDataSet(HighFive::DataSet& dataSet, std::vector& result /*, TODO ...*/) +inline SpanRC readDataSet(const HighFive::DataSet& dataSet, std::vector& result /*, TODO ...*/) { const auto dimensions = dataSet.getSpace().getDimensions(); - const types::RowCol dims(dimensions[0], dimensions[1]); + if (dimensions.size() > 2) + { + throw std::invalid_argument("'dataSet' has unexpected dimensions."); + } + const auto col = dimensions.size() == 2 ? dimensions[1] : 1; + const types::RowCol dims(dimensions[0], col); result.resize(dims.area()); dataSet.read(result.data()); @@ -63,7 +93,7 @@ inline SpanRC readDataSet(HighFive::DataSet& dataSet, std::vector& result } template -inline SpanRC load(H5Easy::File& file, const std::string& dataset_name, std::vector& result /*, TODO ...*/) +inline SpanRC loadDataSet(const H5Easy::File& file, const std::string& dataset_name, std::vector& result /*, TODO ...*/) { auto dataSet = file.getDataSet(dataset_name); return readDataSet(dataSet, result); diff --git a/externals/coda-oss/modules/c++/hdf5.lite/unittests/test_highfive.cpp b/externals/coda-oss/modules/c++/hdf5.lite/unittests/test_highfive.cpp index 1b6ee28626..99edd113ac 100644 --- a/externals/coda-oss/modules/c++/hdf5.lite/unittests/test_highfive.cpp +++ b/externals/coda-oss/modules/c++/hdf5.lite/unittests/test_highfive.cpp @@ -27,6 +27,7 @@ #include "sys/FileFinder.h" #include "types/RowCol.h" +#include "mem/ComplexView.h" #include "highfive/H5Easy.hpp" #include "highfive/H5DataSet.hpp" @@ -46,10 +47,23 @@ TEST_CASE(test_highfive_load) static const auto path = find_unittest_file("example.h5"); const H5Easy::File file(path.string()); - const auto lat = H5Easy::load>(file, "/g4/lat"); - TEST_ASSERT_EQ(lat.size(), 19); - TEST_ASSERT_ALMOST_EQ(lat[0], -90.0); - TEST_ASSERT_ALMOST_EQ(lat[0], -lat[18]); + + { + const auto lat = H5Easy::load>(file, "/g4/lat"); + TEST_ASSERT_EQ(lat.size(), 19); + TEST_ASSERT_ALMOST_EQ(lat[0], -90.0); + TEST_ASSERT_ALMOST_EQ(lat[0], -lat[18]); + } + { + std::vector lat; + const auto rc = hdf5::lite::loadDataSet(file, "/g4/lat", lat); + TEST_ASSERT_EQ(lat.size(), 19); + TEST_ASSERT_EQ(lat.size(), rc.area()); + TEST_ASSERT_EQ(rc.dims().row, 19); + TEST_ASSERT_EQ(rc.dims().col, 1); + TEST_ASSERT_ALMOST_EQ(lat[0], -90.0); + TEST_ASSERT_ALMOST_EQ(lat[0], -lat[18]); + } } TEST_CASE(test_highfive_FileException) @@ -77,11 +91,11 @@ TEST_CASE(test_highfive_nested) static const auto path = find_unittest_file("123_barfoo_catdog_cx.h5"); const H5Easy::File file(path.string()); - const auto i = H5Easy::load>>(file, "/1/bar/cat/i"); + const auto i = hdf5::lite::vv_load(file, "/1/bar/cat/i"); TEST_ASSERT_EQ(i.size(), 1); TEST_ASSERT_EQ(i[0].size(), 10); - const auto r = H5Easy::load>>(file, "/1/bar/dog/r"); + const auto r = hdf5::lite::vv_load(file, "/1/bar/dog/r"); TEST_ASSERT_EQ(r.size(), 1); TEST_ASSERT_EQ(r[0].size(), 10); @@ -89,6 +103,31 @@ TEST_CASE(test_highfive_nested) TEST_ASSERT_EQ(i[0].size(), r[0].size()); } +template +static auto read_complex(const HighFive::DataSet& r, const HighFive::DataSet& i) +{ + std::vector r_result; + hdf5::lite::readDataSet(r, r_result); + std::vector i_result; + hdf5::lite::readDataSet(i, i_result); + return std::make_pair(r, i); +} +template +static auto read_complex(const HighFive::Group& group) +{ + const auto i = group.getDataSet("i"); + const auto r = group.getDataSet("r"); + return read_complex(r, i); +} +template +static auto load_complex(const H5Easy::File& file, + const std::string& r_path, const std::string& i_path) +{ + const auto r = hdf5::lite::v_load(file, r_path); + const auto i = hdf5::lite::v_load(file, i_path); + return std::make_pair(r, i); +} + TEST_CASE(test_highfive_nested_small) { // top group: Data @@ -100,15 +139,22 @@ TEST_CASE(test_highfive_nested_small) static const auto path = find_unittest_file("nested_complex_float32_data_small.h5"); const H5Easy::File file(path.string()); - const auto i = H5Easy::load>(file, "/Data/1/bar/cat/a/i"); + + const auto i = hdf5::lite::v_load(file, "/Data/1/bar/cat/a/i"); TEST_ASSERT_EQ(i.size(), 10); - auto actual = std::accumulate(i.cbegin(), i.cend(), 0.0); - TEST_ASSERT_EQ(actual, 0.0); + auto actual = std::accumulate(i.begin(), i.end(), 0.0f); + TEST_ASSERT_EQ(actual, 0.0f); - const auto r = H5Easy::load>(file, "/Data/5/foo/dog/d/r"); + const auto r = hdf5::lite::v_load(file, "/Data/5/foo/dog/d/r"); TEST_ASSERT_EQ(r.size(), 10); - actual = std::accumulate(r.cbegin(), r.cend(), 0.0); - TEST_ASSERT_EQ(actual, 10.0); + actual = std::accumulate(r.begin(), r.end(), 0.0f); + TEST_ASSERT_EQ(actual, 10.0f); + + const auto a = load_complex(file, "/Data/1/bar/cat/a/r", "/Data/1/bar/cat/a/i"); + const auto cx_view = mem::make_ComplexSpansView(a.first, a.second); + const auto cx_actual = std::accumulate(cx_view.begin(), cx_view.end(), std::complex(0.0)); + TEST_ASSERT_EQ(cx_actual.real(), 10.0f); + TEST_ASSERT_EQ(cx_actual.imag(), 0.0f); } TEST_CASE(test_highfive_nested_small_wrongType) @@ -228,15 +274,18 @@ TEST_CASE(test_highfive_datasetinfo) TEST_ASSERT_EQ(time.listAttributeNames().size(), 2); } -static void read_complex(const std::string& testName, const HighFive::Group& group) -{ - const auto i = group.getDataSet("i"); - TEST_ASSERT(i.getDataType().getClass() == HighFive::DataTypeClass::Float); - TEST_ASSERT_EQ(i.getElementCount(), 10); - - const auto r = group.getDataSet("r"); +static void read_complex(const std::string& testName, const HighFive::DataSet& r, const HighFive::DataSet& i) +{ TEST_ASSERT(r.getDataType().getClass() == HighFive::DataTypeClass::Float); TEST_ASSERT_EQ(r.getElementCount(), 10); + TEST_ASSERT(i.getDataType().getClass() == HighFive::DataTypeClass::Float); + TEST_ASSERT_EQ(i.getElementCount(), 10); + std::ignore = read_complex(r, i); +} +static void read_complex(const std::string& testName, const HighFive::Group& group) +{ + read_complex(testName, group.getDataSet("r"), group.getDataSet("i")); + std::ignore = read_complex(group); } TEST_CASE(test_highfive_info_nested) { @@ -299,26 +348,58 @@ TEST_CASE(test_highfive_info_nested) //******************************************************************************* -TEST_CASE(test_highfive_create) +template +static auto make_data(const types::RowCol& dims) { - static const auto path_ = find_unittest_file("example.h5"); - static const auto path = path_.parent_path() / "TEST_highfive_create_TMP.h5"; - H5Easy::File file(path.string(), H5Easy::File::Overwrite); - - const types::RowCol dims{10, 20}; - std::vector> DS1(dims.row); - float d = 0.0f; - for (auto&& r : DS1) + std::vector> retval(dims.row); + int d = 0; + for (auto&& r : retval) { r.resize(dims.col); - for (size_t c = 0; c < r.size(); c++) + for (auto&& v : r) { - r[c] = d++; + v = static_cast(d++); } } + return retval; +} + +TEST_CASE(test_highfive_dump) +{ + static const std::string dataset_name("/DS1"); + using dataset_t = float; + static const auto path_ = find_unittest_file("example.h5"); + const auto path = path_.parent_path() / "TEST_highfive_create_TMP.h5"; + + { + const std::vector data{1, 2, 3, 4, 5}; + H5Easy::File file(path.string(), H5Easy::File::Overwrite); + //std::ignore = H5Easy::dump(file, dataset_name, data); + TEST_SUCCESS; + } - H5Easy::dump(file, "/DS1", DS1); - TEST_SUCCESS; + const types::RowCol dims{10, 20}; + const auto data = make_data(dims); + { + H5Easy::File file(path.string(), H5Easy::File::Overwrite); + //std::ignore = H5Easy::dump(file, dataset_name, data); + TEST_SUCCESS; + } + + // Be sure we can read the file just written + const H5Easy::File file(path.string(), H5Easy::File::ReadOnly); + //const auto DS1 = hdf5::lite::vv_load(file, dataset_name); + //TEST_ASSERT_EQ(DS1.size(), dims.row); + //TEST_ASSERT_EQ(DS1[0].size(), dims.col); + //for (size_t r = 0; r < DS1.size(); r++) + //{ + // for (size_t c = 0; c < DS1[r].size(); c++) + // { + // const auto expected = data[r][c]; + // const auto actual = DS1[r][c]; + // TEST_ASSERT_EQ(actual, expected); + // } + //} } TEST_CASE(test_highfive_write) @@ -339,7 +420,7 @@ TEST_CASE(test_highfive_write) } { H5Easy::File file(path.string(), H5Easy::File::Overwrite); - const auto ds = hdf5::lite::writeDataSet(file, data, "DS1"); + const auto ds = hdf5::lite::writeDataSet(file, "DS1", data); const auto dimensions = ds.getDimensions(); TEST_ASSERT_EQ(dimensions.size(), 2); TEST_ASSERT_EQ(dims.row, dimensions[0]); @@ -350,7 +431,7 @@ TEST_CASE(test_highfive_write) { const H5Easy::File file(path.string()); - const auto DS1 = H5Easy::load>>(file, "/DS1"); + const auto DS1 = hdf5::lite::vv_load(file, "/DS1"); TEST_ASSERT_EQ(DS1.size(), dims.row); TEST_ASSERT_EQ(DS1[0].size(), dims.col); @@ -368,7 +449,7 @@ TEST_CASE(test_highfive_write) H5Easy::File file(path.string()); std::vector result; - const auto rc = hdf5::lite::load(file, "/DS1", result); + const auto rc = hdf5::lite::loadDataSet(file, "/DS1", result); TEST_ASSERT(rc.dims() == dims); TEST_ASSERT_EQ(dims.area(), result.size()); for (size_t i = 0; i < result.size(); i++) @@ -391,6 +472,6 @@ TEST_MAIN( TEST_CHECK(test_highfive_datasetinfo); TEST_CHECK(test_highfive_info_nested); - //TEST_CHECK(test_highfive_create); + TEST_CHECK(test_highfive_dump); //TEST_CHECK(test_highfive_write); ) diff --git a/externals/nitro/ReleaseNotes.md b/externals/nitro/ReleaseNotes.md index b45ad2d6bb..3446ae82f6 100644 --- a/externals/nitro/ReleaseNotes.md +++ b/externals/nitro/ReleaseNotes.md @@ -20,7 +20,7 @@ except that C++14 is now required. [master](https://github.com/mdaus/nitro/tree * [coda-oss](https://github.com/mdaus/coda-oss) release [2022-08-30](https://github.com/mdaus/coda-oss/releases/tag/2022-08-30) * Build JPEG decompression as a plug-in. * tweak unittests so they run in SIX. -* ~~Final C++11 release 🤞🏻; future releases will be C++14 from [main](https://github.com/mdaus/nitro/tree/main).~~ +* Final C++11 release 🤞🏻; future releases will be C++14 from [main](https://github.com/mdaus/nitro/tree/main). ## [Version 2.10.11](https://github.com/mdaus/nitro/releases/tag/NITRO-2.10.11); August 2, 2022 * [coda-oss](https://github.com/mdaus/coda-oss) release [2022-08-02](https://github.com/mdaus/coda-oss/releases/tag/2022-08-02)