Feature/sina/hdf5 output support

src/axom/sina/core/Document.cpp

@@ -15,13 +15,24 @@
 
 #include "axom/sina/core/Document.hpp"
 
+#include "axom/config.hpp"
+#include "axom/core/Path.hpp"
+#include "axom/core/utilities/StringUtilities.hpp"
+
+#include "conduit.hpp"
+#ifdef AXOM_USE_HDF5
+  #include "conduit_relay.hpp"
+  #include "conduit_relay_io.hpp"
+#endif
+
 #include <cstdio>
 #include <fstream>
 #include <ios>
 #include <iostream>
 #include <utility>
 #include <sstream>
 #include <stdexcept>
+#include <algorithm>
 
 namespace axom
 {
@@ -35,6 +46,26 @@ char const RELATIONSHIPS_KEY[] = "relationships";
 char const SAVE_TMP_FILE_EXTENSION[] = ".sina.tmp";
 }  // namespace
 
+void protocolWarn(std::string const protocol, std::string const &name)
+{
+  std::unordered_map<std::string, std::string> protocolMessages = {
+    {".json", ".json extension not found, did you mean to save to this format?"},
+    {".hdf5",
+     ".hdf5 extension not found, did you use one of its other supported types? "
+     "(h5, hdf, ...)"}};
+
+  Path path(name, '.');
+
+  if(protocol != '.' + path.baseName())
+  {
+    auto messageIt = protocolMessages.find(protocol);
+    if(messageIt != protocolMessages.end())
+    {
+      std::cerr << messageIt->second;
+    }
+  }
+}
+
 void Document::add(std::unique_ptr<Record> record)
 {
   records.emplace_back(std::move(record));
@@ -63,63 +94,184 @@ conduit::Node Document::toNode() const
   return document;
 }
 
-void Document::createFromNode(conduit::Node const &asNode,
-                              RecordLoader const &recordLoader)
+void Document::createFromNode(const conduit::Node &asNode,
+                              const RecordLoader &recordLoader)
 {
-  if(asNode.has_child(RECORDS_KEY))
-  {
-    conduit::Node record_nodes = asNode[RECORDS_KEY];
-    if(record_nodes.dtype().is_list())
+  conduit::Node nodeCopy = asNode;
+
+  auto processChildNodes = [&](const char *key,
+                               std::function<void(conduit::Node &)> addFunc) {
+    if(nodeCopy.has_child(key))
     {
-      auto recordIter = record_nodes.children();
-      while(recordIter.has_next())
+      conduit::Node &childNodes = nodeCopy[key];
+
+      // -- 1. Check if this node is a primitive leaf (throw immediately if so)
+      // Customize these checks to match exactly what you consider "primitive."
+      if(childNodes.dtype().is_number() || childNodes.dtype().is_char8_str() ||
+         childNodes.dtype().is_string())
+      {
+        std::ostringstream message;
+        message << "The '" << key
+                << "' element of a document cannot be a primitive value.";
+        throw std::invalid_argument(message.str());
+      }
+
+      // -- 2. Not a primitive. Check if it has no children.
+      if(childNodes.number_of_children() == 0)
+      {
+        // Turn it into an empty list
+        childNodes.set(conduit::DataType::list());
+      }
+
+      // -- 3. If it's still not a list, throw
+      if(!childNodes.dtype().is_list())
+      {
+        std::ostringstream message;
+        message << "The '" << key
+                << "' element of a document must be an array/list.";
+        throw std::invalid_argument(message.str());
+      }
+
+      // -- 4. Now it's guaranteed to be a list, so iterate
+      auto childIter = childNodes.children();
+      while(childIter.has_next())
       {
-        auto record = recordIter.next();
-        add(recordLoader.load(record));
+        conduit::Node child = childIter.next();
+        addFunc(child);
       }
     }
-    else
+  };
+  processChildNodes(RECORDS_KEY, [&](conduit::Node &record) {
+    add(recordLoader.load(record));
+  });
+
+  processChildNodes(RELATIONSHIPS_KEY, [&](conduit::Node &relationship) {
+    add(Relationship {relationship});
+  });
+}
+
+Document::Document(conduit::Node const &asNode, RecordLoader const &recordLoader)
+{
+  this->createFromNode(asNode, recordLoader);
+}
+
+Document::Document(std::string const &asJson, RecordLoader const &recordLoader)
+{
+  conduit::Node asNode;
+  asNode.parse(asJson, "json");
+  this->createFromNode(asNode, recordLoader);
+}
+
+#ifdef AXOM_USE_HDF5
+void removeSlashes(const conduit::Node &originalNode, conduit::Node &modifiedNode)
+{
+  for(auto it = originalNode.children(); it.has_next();)
+  {
+    it.next();
+    std::string key = it.name();
+    std::string modifiedKey =
+      axom::utilities::string::replaceAllInstances(key, "/", slashSubstitute);
+
+    modifiedNode[modifiedKey] = it.node();
+
+    if(it.node().dtype().is_object())
     {
-      std::ostringstream message;
-      message << "The '" << RECORDS_KEY
-              << "' element of a document must be an array";
-      throw std::invalid_argument(message.str());
+      conduit::Node nestedNode;
+      removeSlashes(it.node(), nestedNode);
+      modifiedNode[modifiedKey].set(nestedNode);
     }
   }
+}
+
+void restoreSlashes(const conduit::Node &modifiedNode, conduit::Node &restoredNode)
+{
+  // Check if List or Object, if its a list the else statement would turn it into an object
+  // which breaks the Document
 
-  if(asNode.has_child(RELATIONSHIPS_KEY))
+  if(modifiedNode.dtype().is_list())
   {
-    conduit::Node relationship_nodes = asNode[RELATIONSHIPS_KEY];
-    if(relationship_nodes.dtype().is_list())
+    // If its empty with no children it's the end of a tree
+
+    for(auto it = modifiedNode.children(); it.has_next();)
     {
-      auto relationshipsIter = relationship_nodes.children();
-      while(relationshipsIter.has_next())
+      it.next();
+      conduit::Node &newChild = restoredNode.append();
+
+      // Leaves empty nodes empty, if null data is set the
+      // Document breaks
+
+      if(it.node().dtype().is_string() || it.node().dtype().is_number())
       {
-        auto &relationship = relationshipsIter.next();
-        add(Relationship {relationship});
+        newChild.set(it.node());  // Lists need .set
+      }
+
+      // Recursive Call
+      if(it.node().number_of_children() > 0)
+      {
+        restoreSlashes(it.node(), newChild);
       }
     }
-    else
+  }
+  else
+  {
+    for(auto it = modifiedNode.children(); it.has_next();)
     {
-      std::ostringstream message;
-      message << "The '" << RELATIONSHIPS_KEY
-              << "' element of a document must be an array";
-      throw std::invalid_argument(message.str());
+      it.next();
+      std::string key = it.name();
+      std::string restoredKey =
+        axom::utilities::string::replaceAllInstances(key, slashSubstitute, "/");
+
+      // Initialize a new node for the restored key
+      conduit::Node &newChild = restoredNode.add_child(restoredKey);
+
+      // Leaves empty keys empty but continues recursive call if its a list
+      if(it.node().dtype().is_string() || it.node().dtype().is_number() ||
+         it.node().dtype().is_object())
+      {
+        newChild.set(it.node());
+      }
+      else if(it.node().dtype().is_list())
+      {
+        restoreSlashes(it.node(), newChild);  // Handle nested lists
+      }
+
+      // If the node has children, recursively restore them
+      if(it.node().number_of_children() > 0)
+      {
+        conduit::Node nestedNode;
+        restoreSlashes(it.node(), nestedNode);
+        newChild.set(nestedNode);
+      }
     }
   }
 }
 
-Document::Document(conduit::Node const &asNode, RecordLoader const &recordLoader)
+void Document::toHDF5(const std::string &filename) const
 {
-  this->createFromNode(asNode, recordLoader);
-}
+  conduit::Node node;
+  conduit::Node &recordsNode = node["records"];
+  conduit::Node &relationshipsNode = node["relationships"];
 
-Document::Document(std::string const &asJson, RecordLoader const &recordLoader)
-{
-  conduit::Node asNode;
-  asNode.parse(asJson, "json");
-  this->createFromNode(asNode, recordLoader);
+  for(const auto &record : getRecords())
+  {
+    conduit::Node recordNode = record->toNode();
+
+    removeSlashes(recordNode, recordsNode.append());
+  }
+
+  // Process relationships
+  for(const auto &relationship : getRelationships())
+  {
+    conduit::Node relationshipNode = relationship.toNode();
+
+    removeSlashes(relationshipNode, relationshipsNode.append());
+  }
+
+  conduit::relay::io::save(node, filename, "hdf5");
 }
+#endif
+
+//
 
 std::string Document::toJson(conduit::index_t indent,
                              conduit::index_t depth,
@@ -129,7 +281,9 @@ std::string Document::toJson(conduit::index_t indent,
   return this->toNode().to_json("json", indent, depth, pad, eoe);
 }
 
-void saveDocument(Document const &document, std::string const &fileName)
+void saveDocument(Document const &document,
+                  std::string const &fileName,
+                  Protocol protocol)
 {
   // It is a common use case for users to want to overwrite their files as
   // the simulation progresses. However, this operation should be atomic so
@@ -138,12 +292,30 @@ void saveDocument(Document const &document, std::string const &fileName)
   // file is in the same directory to ensure that it is part of the same
   // file system as the destination file so that the move operation is
   // atomic.
+
   std::string tmpFileName = fileName + SAVE_TMP_FILE_EXTENSION;
-  auto asJson = document.toJson();
-  std::ofstream fout {tmpFileName};
-  fout.exceptions(std::ostream::failbit | std::ostream::badbit);
-  fout << asJson;
-  fout.close();
+
+  if(protocol == Protocol::JSON)
+  {
+    protocolWarn(".json", fileName);
+    auto asJson = document.toJson();
+    std::ofstream fout {tmpFileName};
+    fout.exceptions(std::ostream::failbit | std::ostream::badbit);
+    fout << asJson;
+    fout.close();
+  }
+#ifdef AXOM_USE_HDF5
+  else if(protocol == Protocol::HDF5)
+  {
+    protocolWarn(".hdf5", fileName);
+    document.toHDF5(tmpFileName);
+  }
+#endif
+  else
+  {
+    throw std::invalid_argument(
+      "Invalid format choice. Please enter 'json' or 'hdf5'.");
+  }
 
   if(rename(tmpFileName.c_str(), fileName.c_str()) != 0)
   {
@@ -154,20 +326,39 @@ void saveDocument(Document const &document, std::string const &fileName)
   }
 }
 
-Document loadDocument(std::string const &path)
+Document loadDocument(std::string const &path, Protocol protocol)
 {
-  return loadDocument(path, createRecordLoaderWithAllKnownTypes());
+  return loadDocument(path, createRecordLoaderWithAllKnownTypes(), protocol);
 }
 
-Document loadDocument(std::string const &path, RecordLoader const &recordLoader)
+Document loadDocument(std::string const &path,
+                      RecordLoader const &recordLoader,
+                      Protocol protocol)
 {
-  conduit::Node nodeFromJson;
-  std::ifstream file_in {path};
+  conduit::Node node, modifiedNode;
   std::ostringstream file_contents;
-  file_contents << file_in.rdbuf();
-  file_in.close();
-  nodeFromJson.parse(file_contents.str(), "json");
-  return Document {nodeFromJson, recordLoader};
+  std::ifstream file_in {path};
+
+  // Load the file depending on the protocol
+  switch(protocol)
+  {
+  case Protocol::JSON:
+    file_contents << file_in.rdbuf();
+    file_in.close();
+    node.parse(file_contents.str(), "json");
+    return Document {node, recordLoader};
+
+#ifdef AXOM_USE_HDF5
+  case Protocol::HDF5:
+    file_in.close();
+    conduit::relay::io::load(path, "hdf5", node);
+    restoreSlashes(node, modifiedNode);
+    return Document {modifiedNode, recordLoader};
+#endif
+
+  default:
+    break;
+  }
 }
 
 }  // namespace sina