repo manifest upload (reworked)

src/libaktualizr/utilities/CMakeLists.txt

@@ -17,7 +17,8 @@ set(HEADERS apiqueue.h
             sockaddr_io.h
             timer.h
             types.h
-            utils.h)
+            utils.h
+            xml2json.h)
 
 set_property(SOURCE aktualizr_version.cc PROPERTY COMPILE_DEFINITIONS AKTUALIZR_VERSION="${AKTUALIZR_VERSION}")
 
@@ -29,5 +30,6 @@ add_aktualizr_test(NAME timer SOURCES timer_test.cc)
 add_aktualizr_test(NAME types SOURCES types_test.cc)
 add_aktualizr_test(NAME utils SOURCES utils_test.cc PROJECT_WORKING_DIRECTORY)
 add_aktualizr_test(NAME sighandler SOURCES sighandler_test.cc)
+add_aktualizr_test(NAME xml2json SOURCES xml2json_test.cc)
 
 aktualizr_source_file_checks(${SOURCES} ${HEADERS} ${TEST_SOURCES})

src/libaktualizr/utilities/xml2json.h

@@ -0,0 +1,99 @@
+#include <boost/optional.hpp>
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/xml_parser.hpp>
+#include <istream>
+
+#include "json/json.h"
+
+namespace xml2json {
+
+static inline void addSubArray(Json::Value &d, const std::string &key, const Json::Value &arr) {
+  if (arr.size() == 0) {
+    return;
+  } else if (arr.size() == 1) {
+    d[key] = arr[0];
+  } else {
+    d[key] = arr;
+  }
+}
+
+static const int MAX_DEPTH = 10;
+
+static inline Json::Value treeJson(const boost::property_tree::ptree &tree, int depth = 0) {
+  namespace bpt = boost::property_tree;
+
+  if (depth > MAX_DEPTH) {
+    throw std::runtime_error("parse error");
+  }
+
+  bool leaf = true;
+  Json::Value output;
+
+  struct {
+    // used to collasce same-key children into lists
+    std::string key;
+    Json::Value list = Json::Value(Json::arrayValue);
+  } cur;
+
+  for (auto it = tree.ordered_begin(); it != tree.not_found(); it++) {
+    const std::string &val = it->first;
+    const bpt::ptree &subtree = it->second;
+    leaf = false;
+
+    // xml attributes
+    if (val == "<xmlattr>") {
+      for (const bpt::ptree::value_type &attr : subtree) {
+        output[std::string("@") + attr.first] = attr.second.data();
+      }
+      continue;
+    }
+
+    if (cur.key == "") {
+      cur.key = val;
+    } else if (cur.key != val) {
+      addSubArray(output, cur.key, cur.list);
+
+      cur.key = val;
+      cur.list = Json::Value(Json::arrayValue);
+    }
+    cur.list.append(treeJson(subtree, depth + 1));
+  }
+
+  if (cur.key != "") {
+    addSubArray(output, cur.key, cur.list);
+  }
+
+  {
+    auto val = tree.get_value_optional<std::string>();
+    if (!!val && val.get() != "") {
+      if (leaf) {
+        // <e>c</e> -> { "e": "c" }
+        return val.get();
+      } else {
+        // <e a=b>c</e> -> { "e": { "@a": "b", "#text": "c" } }
+        output["#text"] = val.get();
+      }
+    }
+  }
+
+  return output;
+}
+
+static inline Json::Value xml2json(std::istream &is) {
+  namespace bpt = boost::property_tree;
+
+  try {
+    bpt::ptree pt;
+    bpt::read_xml(is, pt, bpt::xml_parser::trim_whitespace);
+
+    if (pt.size() != 1) {
+      throw std::runtime_error("parse error");
+    }
+
+    return treeJson(pt);
+  } catch (std::exception &e) {
+    throw std::runtime_error("parse error");
+  }
+}
+
+}  // namespace xml2json

src/libaktualizr/utilities/xml2json_test.cc

@@ -0,0 +1,99 @@
+#include <gtest/gtest.h>
+#include <sstream>
+
+#include "utilities/utils.h"
+#include "utilities/xml2json.h"
+
+TEST(xml2json, simple) {
+  {
+    std::stringstream inxml("<a/>");
+    auto j = xml2json::xml2json(inxml);
+    EXPECT_EQ(Utils::jsonToCanonicalStr(j), R"({"a":null})");
+  }
+  {
+    std::stringstream inxml(R"(<a b="xxx"/>)");
+    auto j = xml2json::xml2json(inxml);
+    EXPECT_EQ(Utils::jsonToCanonicalStr(j), R"({"a":{"@b":"xxx"}})");
+  }
+  {
+    std::stringstream inxml(R"(<a b="xxx" c="rrr"></a>)");
+    auto j = xml2json::xml2json(inxml);
+    EXPECT_EQ(Utils::jsonToCanonicalStr(j), R"({"a":{"@b":"xxx","@c":"rrr"}})");
+  }
+  {
+    std::stringstream inxml("<a>xxx</a>");
+    auto j = xml2json::xml2json(inxml);
+    EXPECT_EQ(Utils::jsonToCanonicalStr(j), R"({"a":"xxx"})");
+  }
+  {
+    std::stringstream inxml("<a><b>xxx</b></a>");
+    auto j = xml2json::xml2json(inxml);
+    EXPECT_EQ(Utils::jsonToCanonicalStr(j), R"({"a":{"b":"xxx"}})");
+  }
+  {
+    std::stringstream inxml("<a><b>xxx</b><c>yyy</c></a>");
+    auto j = xml2json::xml2json(inxml);
+    EXPECT_EQ(Utils::jsonToCanonicalStr(j), R"({"a":{"b":"xxx","c":"yyy"}})");
+  }
+  {
+    std::stringstream inxml(R"(<a xxx="1">yy</a>)");
+    auto j = xml2json::xml2json(inxml);
+    EXPECT_EQ(Utils::jsonToCanonicalStr(j), R"({"a":{"#text":"yy","@xxx":"1"}})");
+  }
+  {
+    std::stringstream inxml(R"(<a><b>1</b><c>xx</c><b>2</b></a>)");
+    auto j = xml2json::xml2json(inxml);
+    EXPECT_EQ(Utils::jsonToCanonicalStr(j), R"({"a":{"b":["1","2"],"c":"xx"}})");
+  }
+}
+
+static const std::string example_manifest = R"(
+<?xml version="1.0" encoding="UTF-8"?>
+<manifest>
+  <remote fetch="https://github.com" name="github" pushurl="ssh://[email protected]"/>
+  <remote fetch="git://git.openembedded.org/" name="openembedded"/>
+  <remote fetch="https://git.yoctoproject.org/git/" name="yocto"/>
+
+  <default remote="github" revision="master"/>
+
+  <project name="advancedtelematic/meta-updater" path="meta-updater" revision="6e1c9cf5cc59437ce07f5aec2dc62d665d218bdb" upstream="master"/>
+  <project name="advancedtelematic/meta-updater-minnowboard" path="meta-updater-minnowboard" revision="c822d05f860c3a2437236696b22ef7536c0a1311" upstream="master"/>
+  <project name="advancedtelematic/meta-updater-qemux86-64" path="meta-updater-qemux86-64" revision="162d1378659343a3ad34569c1315babe7246ec86" upstream="master"/>
+  <project name="advancedtelematic/meta-updater-raspberrypi" path="meta-updater-raspberrypi" revision="501156e6d12e3207a5acb611984dce1856a7729c" upstream="master"/>
+  <project name="meta-intel" remote="yocto" revision="eacd8eb9f762c90cec2825736e8c4d483966c4d4" upstream="master"/>
+  <project name="meta-openembedded" remote="openembedded" revision="18506b797bcfe162999223b79919e7c730875bb4" upstream="master"/>
+  <project name="meta-raspberrypi" remote="yocto" revision="254c9366b9c3309db6dc07beb80aba55e0c87f94" upstream="master"/>
+  <project name="poky" remote="yocto" revision="3a751d5564fc6ee9aef225653cc7b8630fd25a35" upstream="master"/>
+  <project name="ricardosalveti/meta-updater-riscv" path="meta-updater-riscv" revision="8164a21c04a7de91f90ada763104063540a84961" upstream="master"/>
+  <project name="riscv/meta-riscv" path="meta-riscv" revision="0ba537b9270046b1c08d2b2f1cc9a9ca96ea0328" upstream="master"/>
+</manifest>
+)";
+
+static const std::string example_json =
+    R"({"manifest":{"default":{"@remote":"github","@revision":"master"},"project":[{"@name":"advancedtelematic/meta-updater","@path":"meta-updater","@revision":"6e1c9cf5cc59437ce07f5aec2dc62d665d218bdb","@upstream":"master"},{"@name":"advancedtelematic/meta-updater-minnowboard","@path":"meta-updater-minnowboard","@revision":"c822d05f860c3a2437236696b22ef7536c0a1311","@upstream":"master"},{"@name":"advancedtelematic/meta-updater-qemux86-64","@path":"meta-updater-qemux86-64","@revision":"162d1378659343a3ad34569c1315babe7246ec86","@upstream":"master"},{"@name":"advancedtelematic/meta-updater-raspberrypi","@path":"meta-updater-raspberrypi","@revision":"501156e6d12e3207a5acb611984dce1856a7729c","@upstream":"master"},{"@name":"meta-intel","@remote":"yocto","@revision":"eacd8eb9f762c90cec2825736e8c4d483966c4d4","@upstream":"master"},{"@name":"meta-openembedded","@remote":"openembedded","@revision":"18506b797bcfe162999223b79919e7c730875bb4","@upstream":"master"},{"@name":"meta-raspberrypi","@remote":"yocto","@revision":"254c9366b9c3309db6dc07beb80aba55e0c87f94","@upstream":"master"},{"@name":"poky","@remote":"yocto","@revision":"3a751d5564fc6ee9aef225653cc7b8630fd25a35","@upstream":"master"},{"@name":"ricardosalveti/meta-updater-riscv","@path":"meta-updater-riscv","@revision":"8164a21c04a7de91f90ada763104063540a84961","@upstream":"master"},{"@name":"riscv/meta-riscv","@path":"meta-riscv","@revision":"0ba537b9270046b1c08d2b2f1cc9a9ca96ea0328","@upstream":"master"}],"remote":[{"@fetch":"https://github.com","@name":"github","@pushurl":"ssh://[email protected]"},{"@fetch":"git://git.openembedded.org/","@name":"openembedded"},{"@fetch":"https://git.yoctoproject.org/git/","@name":"yocto"}]}})";
+
+TEST(xml2json, manifest) {
+  std::stringstream inxml(example_manifest);
+  auto j = xml2json::xml2json(inxml);
+  EXPECT_EQ(Utils::jsonToCanonicalStr(j), example_json);
+}
+
+TEST(xml2json, bad_input) {
+  {
+    // wrong xml
+    std::stringstream inxml("<a");
+    EXPECT_THROW(xml2json::xml2json(inxml), std::runtime_error);
+  }
+  {
+    // too deep
+    std::stringstream inxml("<a><a><a><a><a><a><a><a><a><a><a>xxx</a></a></a></a></a></a></a></a></a></a></a>");
+    EXPECT_THROW(xml2json::xml2json(inxml), std::runtime_error);
+  }
+}
+
+#ifndef __NO_MAIN__
+int main(int argc, char **argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+#endif

src/sota_tools/CMakeLists.txt

@@ -1,4 +1,3 @@
-include_directories("${PROJECT_SOURCE_DIR}/src/libaktualizr/third_party/jsoncpp")
 set(SOTA_TOOLS_LIB_SRC
     authenticate.cc
     check.cc

src/sota_tools/garage_push.cc

@@ -11,6 +11,7 @@
 #include "logging/logging.h"
 #include "ostree_dir_repo.h"
 #include "ostree_repo.h"
+#include "utilities/xml2json.h"
 
 namespace po = boost::program_options;
 
@@ -22,6 +23,7 @@ int main(int argc, char **argv) {
   std::string ref;
   boost::filesystem::path credentials_path;
   std::string cacerts;
+  boost::filesystem::path manifest_path;
   int max_curl_requests;
   RunMode mode = RunMode::kDefault;
   po::options_description desc("garage-push command line options");
@@ -35,6 +37,7 @@ int main(int argc, char **argv) {
     ("ref,r", po::value<std::string>(&ref)->required(), "OSTree ref to push (or commit refhash)")
     ("credentials,j", po::value<boost::filesystem::path>(&credentials_path)->required(), "credentials (json or zip containing json)")
     ("cacert", po::value<std::string>(&cacerts), "override path to CA root certificates, in the same format as curl --cacert")
+    ("repo-manifest", po::value<boost::filesystem::path>(&manifest_path), "manifest describing repository branches used in the image, to be sent as attached metadata")
     ("jobs", po::value<int>(&max_curl_requests)->default_value(30), "maximum number of parallel requests")
     ("dry-run,n", "check arguments and authenticate but don't upload")
     ("walk-tree,w", "walk entire tree and upload all missing objects");
@@ -147,6 +150,35 @@ int main(int argc, char **argv) {
         return EXIT_FAILURE;
       }
     }
+
+    if (manifest_path != "") {
+      try {
+        std::string manifest_json_str;
+        std::ifstream ifs(manifest_path.string());
+        std::stringstream ss;
+        auto manifest_json = xml2json::xml2json(ifs);
+        ss << manifest_json;
+        manifest_json_str = ss.str();
+
+        LOG_INFO << "Sending manifest:\n" << manifest_json_str;
+        if (mode != RunMode::kDryRun) {
+          CURL *curl = curl_easy_init();
+          curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
+          push_server.SetContentType("Content-Type: application/json");
+          push_server.InjectIntoCurl("manifests/" + commit->string(), curl);
+          curlEasySetoptWrapper(curl, CURLOPT_CUSTOMREQUEST, "PUT");
+          curlEasySetoptWrapper(curl, CURLOPT_POSTFIELDS, manifest_json_str.c_str());
+          CURLcode rc = curl_easy_perform(curl);
+
+          if (rc != CURLE_OK) {
+            LOG_ERROR << "Error pushing repo manifest to Treehub";
+          }
+          curl_easy_cleanup(curl);
+        }
+      } catch (std::exception &e) {
+        LOG_ERROR << "Could not send repo manifest to Treehub";
+      }
+    }
   } catch (const BadCredentialsArchive &e) {
     LOG_FATAL << e.what();
     return EXIT_FAILURE;