Pair developing HDF5 #181

.github/workflows/ci.yml

@@ -56,6 +56,11 @@ jobs:
       - name: Set up MATLAB
         uses: matlab-actions/[email protected]
 
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
       # -------------------------------------------------------------------------------
       # Ubuntu
       - name: Install dependencies for Ubuntu
@@ -123,6 +128,20 @@ jobs:
         run: |
           cmake --build . --config ${{ matrix.build_type }}
 
+      # -------------------------------------------------------------------------------
+      # Unit tests
+      - name: Produce MATLAB unit test data
+        if: matrix.build_testing == 'ON'
+        uses: matlab-actions/run-command@v1
+        with:
+          command: cd('tdms/tests/unit/benchmark_scripts/'), setup_unit_tests
+
+      - name: Produce hdf5 unit test data
+        if: matrix.build_testing == 'ON'
+        run: |
+          pip install -r ${{ github.workspace }}/tdms/tests/requirements.txt
+          python ${{ github.workspace }}/tdms/tests/unit/benchmark_scripts/create_hdf5_test_file.py
+
       - name: Run TDMS unit tests
         if: matrix.build_testing == 'ON'
         working-directory: ${{ runner.workspace }}/build
@@ -148,6 +167,8 @@ jobs:
         if: matrix.build_testing == 'ON' && contains(matrix.os, 'ubuntu')
         uses: codecov/codecov-action@v3
 
+      # -------------------------------------------------------------------------------
+      # Upload build artefact for system tests
       - name: Tar the build result to maintain permissions
         # https://github.com/actions/upload-artifact#maintaining-file-permissions-and-case-sensitive-files
         # https://github.com/actions/upload-artifact/issues/38

.gitignore

@@ -15,6 +15,7 @@ html/
 **/.pytest_cache/
 tdms/tests/system/data/*.zip
 **.mat
+tdms/tests/unit/benchmark_scripts/unit_test_data/**
 
 # text editor files
 .idea/

doc/developers.md

@@ -254,7 +254,22 @@ The doxygen-style comments will be included in this developer documentation.
 
 To run the unit tests, [compile](#compiling) with `-DBUILD_TESTING=ON`. Then run `ctest` from the build directory or execute the test executable `./tdms_tests`.
 
-It's good practice, and reassuring for your pull-request reviewers, if new C++ functionality is at covered by unit tests.
+It's good practice, and reassuring for your pull-request reviewers, if new C++ functionality is covered by unit tests.
+
+#### Benchmark Scripts and Data Generation
+
+The `tdms/tests/unit/benchmarking` directory contains scripts that produce input data for the unit tests, or that provide benchmarks for the units that are tested.
+
+The `C++` unit tests require the presence of a `.mat` or `.hdf5` file to read/write data from/to during testing.
+The locations of these files are coded into `tests/include/unit_test_utils.h`, but the files themselves are not committed to the repository - they can be created by running the `setup_unit_tests.m` and `create_hdf5_data.py` scripts.
+These scripts can then be updated to add/remove/edit the data available to the unit tests:
+- `create_hdf5_test_file.py` : Produces `hdf5_test_file.hdf5`; used when testing read/writing from `.hdf5` files.
+- `create_structure_array.m` : Produces `structure_array.mat`; used when testing reading/writing MATLAB `struct` arrays.
+- `create_class_data.m` : Produces `class_data.mat`; used when testing reading/writing `tdms` objects to/from `.mat` files.
+- `create_bad_class_data.m` : Produces `bad_class_data.mat`; used for testing failure cases when reading/writing `tdms` objects to/from `.mat` files.
+
+The `benchmark_` scripts perform band-limited interpolation (BLi) using `MATLAB`'s `interp` function.
+`TDMS`'s interpolation schemes are based off this `MATLAB` function (specficially, in the coefficients the scheme uses to interpolate), and are thus used to benchmark the accuracy of the scheme.
 
 ### Test coverage {#coverage}
 

tdms/CMakeLists.txt

@@ -1,7 +1,7 @@
 # General setup ---------------------------------------------------------------
 cmake_minimum_required(VERSION 3.21)
 
-project(tdms LANGUAGES CXX)
+project(tdms LANGUAGES CXX C)
 set(CMAKE_CXX_STANDARD 17)
 
 # Allow building with testing, and default to off
@@ -84,6 +84,12 @@ else()
     add_definitions(-DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_INFO)
 endif()
 
+# Compile definintions
+if (CMAKE_SOURCE_DIR)
+    add_compile_definitions(CMAKE_SOURCE_DIR="${CMAKE_SOURCE_DIR}")
+    message(STATUS "Unit tests will use ${CMAKE_SOURCE_DIR} as a reference for finding data files.")
+endif()
+
 
 # TDMS version ----------------------------------------------------------------
 # if supplied via the cmake configuration (-DTDMS_VERSION=whatever_I_want) then

tdms/include/arrays.h

@@ -7,13 +7,21 @@
 #include <complex>
 #include <stdexcept>
 #include <string>
+#include <vector>
 
 #include <fftw3.h>
 
 #include "globals.h"
 #include "matlabio.h"
 #include "utils.h"
 
+template<typename T>
+struct xyz_vector {
+  std::vector<T> x = {};
+  std::vector<T> y = {};
+  std::vector<T> z = {};
+};
+
 template<typename T>
 class XYZTensor3D {
 public:
@@ -201,28 +209,23 @@ class DMaterial : public DCollectionBase, MaterialCollection {
   explicit DMaterial(const mxArray *ptr);
 };
 
-class DispersiveMultiLayer {
-public:
-  double *alpha = nullptr;
-  double *beta = nullptr;
-  double *gamma = nullptr;
-  XYZVectors kappa;
-  XYZVectors sigma;
-
+struct DispersiveMultiLayer {
 public:
-  explicit DispersiveMultiLayer(const mxArray *ptr);
+  std::vector<double> alpha;
+  std::vector<double> beta;
+  std::vector<double> gamma;
+  xyz_vector<double> kappa;
+  xyz_vector<double> sigma;
 
   /**
    * @brief Determines whether the (background) medium is dispersive
    *
-   * @param K_tot Number of Yee cells in the z-direction (number of entries in
-   * this->gamma)
    * @param near_zero_tolerance Tolerance for non-zero gamma (attenuation)
    * values
    * @return true Background is dispersive
    * @return false Background is not dispersive
    */
-  bool is_dispersive(int K_tot, double near_zero_tolerance = 1e-15);
+  bool is_dispersive(double near_zero_tolerance = 1e-15) const;
 };
 
 template<typename T>
@@ -272,6 +275,9 @@ class Matrix {
     }
   };
 
+  int get_n_cols() const { return n_cols; }
+  int get_n_rows() const { return n_rows; }
+
   /**
    * Destructor. Must be defined in the header
    */
@@ -322,12 +328,9 @@ class FrequencyExtractVector : public Vector<double> {
   double max();
 };
 
-class FrequencyVectors {
-public:
-  Vector<double> x;
-  Vector<double> y;
-
-  void initialise(const mxArray *ptr);
+struct FrequencyVectors {
+  std::vector<double> x;
+  std::vector<double> y;
 };
 
 /**

tdms/include/cuboid.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "mat_io.h"
+
+/**
+ * @brief Defines a cuboid by specifying the first and last Yee cells in each
+axial direction that form part of the cube.
+ *
+ * @details For example, { 0, 5, 2, 7, 1, 10 } corresponds to the cuboid that
+contains all Yee cells indexed (i,j,k) where;
+ * 0 <= i <= 5,
+ * 2 <= j <= 7,
+ * 1 <= k <= 10.
+ *
+ * TODO: Check inclusivity of the inequalities above.
+ */
+struct Cuboid {
+  int array[6] = {0, 0, 0, 0, 0, 0};
+
+  int operator[](int index) const { return array[index]; }
+};

tdms/include/fdtd_grid_initialiser.h

@@ -15,9 +15,9 @@
 class fdtdGridInitialiser {
 
 private:
-  const mxArray *pointer;        //< Pointer to the array
-  const char *mat_filename;      //< Filename of the MATLAB file
-  std::vector<mwSize> dimensions;//< The dimensions of the array
+  const mxArray *pointer = nullptr;          //< Pointer to the array
+  const char *mat_filename = nullptr;        //< Filename of the MATLAB file
+  std::vector<mwSize> dimensions = {0, 0, 0};//< The dimensions of the array
 
   /**
    * @brief Get a value from a integer attribute of the FDTD grid defined in a
@@ -29,6 +29,8 @@ class fdtdGridInitialiser {
   mwSize value_of_attribute(const std::string &key);
 
 public:
+  /** @brief Construct a new fdtd Grid Initialiser object */
+  fdtdGridInitialiser() {}
   /**
    * @brief Construct a new fdtd Grid Initialiser object
    *

tdms/include/hdf5_io/hdf5_base.h

@@ -0,0 +1,93 @@
+/**
+ * @file hdf5_io.h
+ * @brief Helper classes for HDF5 file I/O.
+ * @details The main classes are `HDF5Reader` and `HDF5Writer` with the methods
+ * `HDF5Reader::read` and `HDF5Writer::write` respectively.
+ */
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <H5Cpp.h>
+
+#include "cell_coordinate.h"
+#include "hdf5_io/hdf5_dimension.h"
+
+/**
+ * @brief The base class for HDF5 I/O.
+ * @details Common functionality and wraps handling the std::unique_ptr to hold
+ * the H5::File object.
+ */
+class HDF5Base {
+
+protected:
+  std::string filename_;             /**< The name of the file. */
+  std::shared_ptr<H5::H5File> file_; /**< Pointer to the underlying H5::File. */
+
+  /**
+   * @brief Construct a new HDF5{Reader/Writer} for a named file.
+   * @param filename The name of the file.
+   * @param mode The H5 file access mode (RDONLY for a HDF5Reader, TRUNC for a
+   * HDF5Writer.)
+   * @throws H5::FileIException if the file doesn't exist or can't be created.
+   */
+  HDF5Base(const std::string &filename, int mode = H5F_ACC_RDONLY)
+      : filename_(filename) {
+    file_ = std::make_unique<H5::H5File>(filename, mode);
+  }
+
+  /**
+   * @brief Destructor closes the file.
+   * @details Closes file when HDF5Reader(or HDF5Writer) goes out of scope.
+   * Since the file pointer is a smart pointer it is deallocated automatically.
+   */
+  ~HDF5Base() { file_->close(); }
+
+public:
+  /**
+   * @brief Get the name of the file.
+   * @return std::string the filename.
+   */
+  std::string get_filename() const { return filename_; }
+
+  /**
+   * @brief Get the names of all datasets (data tables) currently in the file.
+   * @return std::vector<std::string> A vector of their names.
+   */
+  std::vector<std::string> get_datanames() const;
+
+  /**
+   * @brief Print the names of all datasets to std::out.
+   */
+  void ls() const;
+
+  /**
+   * @brief Return shape/dimensionality information about the array data stored
+   * with `dataname`.
+   * @param dataname The name of the data table.
+   * @return The dimensions of the data.
+   */
+  H5Dimension shape_of(const std::string &dataname) const;
+  /**
+   * @brief Return shape/dimensionality information about array data stored
+   * within a group.
+   *
+   * @param group_name The name of the HDF5 Group in which the data array is
+   * stored.
+   * @param dataname The name of the data array to check dimensions of.
+   * @return The dimensions of the data.
+   */
+  H5Dimension shape_of(const std::string &group_name,
+                       const std::string &dataname) const;
+
+  /**
+   * @brief Checks the file is a valid HDF5 file, and everything is OK.
+   * TODO: Can perhaps remove.
+   *
+   * @return true If all is well.
+   * @return false Otherwise.
+   */
+  bool is_ok() const;
+};

tdms/include/hdf5_io/hdf5_dimension.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <algorithm>
+#include <vector>
+
+#include <H5Cpp.h>
+
+class H5Dimension : public std::vector<hsize_t> {
+public:
+  H5Dimension() = default;
+  H5Dimension(const H5::DataSpace &data_space);
+  H5Dimension(const H5::DataSet &data_set) : H5Dimension(data_set.getSpace()){};
+
+  /**
+   * @brief Whether these dimensions describe an array that is castable to a 1D
+   * array.
+   * @details In the event that these dimensions only have one entry, or at
+   * most one of the entries is greater than 1, the shape described can be cast
+   * to a 1D-array of length max_dim().
+   *
+   * @return true These dimensions describe (an object castable to) a 1D-array
+   * @return false Otherwise
+   */
+  bool is_1D() const;
+
+  /**
+   * @brief Returns the dimension of the greatest extent.
+   * @details For instances where is_1D() returns true, this conincides with the
+   * number of elements in the array, and the length of a 1D array necessary to
+   * hold all the elements.
+   *
+   * @return hsize_t
+   */
+  hsize_t max_dim() const { return *std::max_element(begin(), end()); };
+
+  /** @brief The total number of elements (product of the dimensions) */
+  int number_of_elements() const {
+    int product = 1;
+    for (hsize_t axis_size : *this) { product *= axis_size; }
+    return product;
+  }
+};