wip: cleanup

docs/source/backends/json.rst

@@ -54,9 +54,10 @@ Stored as an actual dataset, an **openPMD dataset** is a JSON object with three
 
 Stored as a **dataset template**, an openPMD dataset is represented by three JSON keys:
 
-* ``datatype`` (required): As above.
-* ``extent`` (required): A list of integers, describing the extent of the dataset.
-* ``attributes``: As above.
+ * ``datatype`` (required): As above.
+ * ``extent`` (required): A list of integers, describing the extent of the dataset.
+   This replaces the ``data`` key from the non-template representation.
+ * ``attributes``: As above.
 
 This mode stores only the dataset metadata.
 Chunk load/store operations are ignored.

docs/source/details/backendconfig.rst

@@ -243,11 +243,11 @@ A full configuration of the JSON backend:
 
 The TOML backend is configured analogously, replacing the ``"json"`` key with ``"toml"``.
 
-All keys found under ``hdf5.dataset`` are applicable globally as well as per dataset.
+All keys found under ``json.dataset`` are applicable globally as well as per dataset.
 Explanation of the single keys:
 
 * ``json.dataset.mode`` / ``toml.dataset.mode``: One of ``"dataset"`` (default) or ``"template"``.
   In "dataset" mode, the dataset will be written as an n-dimensional (recursive) array, padded with nulls (JSON) or zeroes (TOML) for missing values.
-  In "template" mode, only the dataset metadata (type, extent and attributes) are stored and no chunks can be written or read.
-* ``json.attribute.mode`` / ``toml.attribute.mode``: One of ``"long"`` (default in openPMD 1.*) or ``"short"`` (default in openPMD 2.*).
+  In "template" mode, only the dataset metadata (type, extent and attributes) are stored and no chunks can be written or read (i.e. write/read operations will be skipped).
+* ``json.attribute.mode`` / ``toml.attribute.mode``: One of ``"long"`` (default in openPMD 1.*) or ``"short"`` (default in openPMD 2.* and generally in TOML).
   The long format explicitly encodes the attribute type in the dataset on disk, the short format only writes the actual attribute as a JSON/TOML value, requiring readers to recover the type.

include/openPMD/Dataset.hpp

@@ -41,7 +41,23 @@ class Dataset
 public:
     enum : std::uint64_t
     {
+        /**
+         * Setting one dimension of the extent as JOINED_DIMENSION means that
+         * the extent along that dimension will be defined by the sum of all
+         * parallel processes' contributions.
+         * Only one dimension can be joined. For store operations, the offset
+         * should be an empty array and the extent should give the actual
+         * extent of the chunk (i.e. the number of joined elements along the
+         * joined dimension, equal to the global extent in all other
+         * dimensions). For more details, refer to
+         * docs/source/usage/workflow.rst.
+         */
         JOINED_DIMENSION = std::numeric_limits<std::uint64_t>::max(),
+        /**
+         * Some backends (i.e. JSON and TOML in template mode) support the
+         * creation of dataset with undefined datatype and extent.
+         * The extent should be given as {UNDEFINED_EXTENT} for that.
+         */
         UNDEFINED_EXTENT = std::numeric_limits<std::uint64_t>::max() - 1
     };
 

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -267,6 +267,10 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
      */
     FileFormat m_fileFormat{};
 
+    /*
+     * Under which key do we find the backend configuration?
+     * -> "json" for the JSON backend, "toml" for the TOML backend.
+     */
     std::string backendConfigKey() const;
 
     /*
@@ -278,6 +282,10 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
 
     std::string m_originalExtension;
 
+    /*
+     * Was the config value explicitly user-chosen, or are we still working with
+     * defaults?
+     */
     enum class SpecificationVia
     {
         DefaultValue,
@@ -288,30 +296,33 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     // Dataset IO mode //
     /////////////////////
 
-    enum class IOMode
+    enum class DatasetMode
     {
         Dataset,
         Template
     };
 
-    IOMode m_mode = IOMode::Dataset;
-    SpecificationVia m_IOModeSpecificationVia = SpecificationVia::DefaultValue;
-    bool m_printedSkippedWriteWarningAlready = false;
+    // IOMode m_mode{};
+    // SpecificationVia m_IOModeSpecificationVia =
+    // SpecificationVia::DefaultValue; bool m_printedSkippedWriteWarningAlready
+    // = false;
 
-    struct DatasetMode
+    struct DatasetMode_s
     {
-        IOMode m_IOMode;
+        // Initialized in init()
+        DatasetMode m_mode{};
         SpecificationVia m_specificationVia;
         bool m_skipWarnings;
 
         template <typename A, typename B, typename C>
         operator std::tuple<A, B, C>()
         {
             return std::tuple<A, B, C>{
-                m_IOMode, m_specificationVia, m_skipWarnings};
+                m_mode, m_specificationVia, m_skipWarnings};
         }
     };
-    DatasetMode retrieveDatasetMode(openPMD::json::TracingJSON &config) const;
+    DatasetMode_s m_datasetMode;
+    DatasetMode_s retrieveDatasetMode(openPMD::json::TracingJSON &config) const;
 
     ///////////////////////
     // Attribute IO mode //
@@ -323,11 +334,16 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
         Long
     };
 
-    AttributeMode m_attributeMode = AttributeMode::Long;
-    SpecificationVia m_attributeModeSpecificationVia =
-        SpecificationVia::DefaultValue;
+    struct AttributeMode_s
+    {
+        // Will be modified in init() based on the openPMD version and the
+        // active file format (JSON/TOML)
+        AttributeMode m_mode{};
+        SpecificationVia m_specificationVia = SpecificationVia::DefaultValue;
+    };
+    AttributeMode_s m_attributeMode;
 
-    std::pair<AttributeMode, SpecificationVia>
+    AttributeMode_s
     retrieveAttributeMode(openPMD::json::TracingJSON &config) const;
 
     // HELPER FUNCTIONS
@@ -376,7 +392,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     // essentially: m_i = \prod_{j=0}^{i-1} extent_j
     static Extent getMultiplicators(Extent const &extent);
 
-    static std::pair<Extent, IOMode> getExtent(nlohmann::json &j);
+    static std::pair<Extent, DatasetMode> getExtent(nlohmann::json &j);
 
     // remove single '/' in the beginning and end of a string
     static std::string removeSlashes(std::string);
@@ -434,7 +450,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
 
     // check whether the json reference contains a valid dataset
     template <typename Param>
-    IOMode verifyDataset(Param const &parameters, nlohmann::json &);
+    DatasetMode verifyDataset(Param const &parameters, nlohmann::json &);
 
     static nlohmann::json platformSpecifics();