Add capability for downloading collections (#98)

Adds support for downloading collections via ign fuel download. By default it downloads all items in a collection, but a -t option can be given to only download models or worlds in the collection. A -j option can be used to specify the number of parallel downloads.

Signed-off-by: Addisu Z. Taddese <[email protected]>

Co-authored-by: Louise Poubel <[email protected]>

include/ignition/fuel_tools/CollectionIdentifier.hh

@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2020 Open Source Robotics Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef IGNITION_FUEL_TOOLS_COLLECTIONIDENTIFIER_HH_
+#define IGNITION_FUEL_TOOLS_COLLECTIONIDENTIFIER_HH_
+
+#include <memory>
+#include <string>
+
+#include "ignition/fuel_tools/Helpers.hh"
+
+namespace ignition
+{
+  namespace fuel_tools
+  {
+    /// \brief Forward Declaration
+    class CollectionIdentifierPrivate;
+    class ServerConfig;
+
+    /// \brief Defines how to identify a collection
+    class IGNITION_FUEL_TOOLS_VISIBLE CollectionIdentifier
+    {
+      /// \brief Constructor.
+      public: CollectionIdentifier();
+
+      /// \brief Copy Constructor.
+      /// \param[in] _orig CollectionIdentifier to copy.
+      public: CollectionIdentifier(const CollectionIdentifier &_orig);
+
+      /// \brief Destructor.
+      public: ~CollectionIdentifier();
+
+      /// \brief Assignment operator
+      /// \param[in] _orig CollectionIdentifier to copy.
+      /// \return Reference to this object.
+      public: CollectionIdentifier
+              &operator=(const CollectionIdentifier &_orig);
+
+        /// \brief Equality operator.
+        /// \param[in] _rhs CollectionIdentifier to compare.
+        /// \return True if the CollectionIdentifier names are equal.
+      public: bool operator==(const CollectionIdentifier &_rhs) const;
+
+      /// \brief Returns the collection name
+      /// \return Collection name.
+      public: std::string Name() const;
+
+      /// \brief Set the name of the collection.
+      /// \param[in] _name The name to set.
+      /// \return True if successful.
+      public: bool SetName(const std::string &_name);
+
+      /// \brief Returns owner to attribute collection to.
+      /// \return Owner name.
+      public: std::string Owner() const;
+
+      /// \brief Set the owner of the collection
+      /// \param[in] _name The name to set. Must be ascii and pass [-_a-z0-9]+
+      /// \return true if successful
+      public: bool SetOwner(const std::string &_name);
+
+      /// \brief Returns server information to retrieve collection from.
+      /// \return Server information.
+      public: ServerConfig &Server() const;
+
+      /// \brief Set the server from which this collection comes.
+      /// \param[in] _server The server to retrieve the collection from.
+      /// \return True if successful.
+      public: bool SetServer(const ServerConfig &_server);
+
+      /// \brief Returns a unique name for the collection.
+      /// \remarks this is Server/Owner/Name.
+      /// \return Unique collection name.
+      public: std::string UniqueName() const;
+
+      /// \brief Returns all the collection information as a string. Convenient
+      /// for debugging.
+      /// \param[in] _prefix Optional prefix for every line of the string.
+      /// \return Collection information string
+      public: std::string AsString(const std::string &_prefix = "") const;
+
+      /// \brief Returns all the available collection information as a string
+      /// using colors for better human parsing.
+      /// \param[in] _prefix Optional prefix for every line of the string.
+      /// \return Collection information string
+      public: std::string AsPrettyString(const std::string &_prefix = "") const;
+
+      /// \brief PIMPL
+      private: std::unique_ptr<CollectionIdentifierPrivate> dataPtr;
+    };
+  }
+}
+
+#endif

include/ignition/fuel_tools/FuelClient.hh

@@ -35,6 +35,7 @@ namespace ignition
   {
     /// \brief Forward Declaration
     class ClientConfig;
+    class CollectionIdentifier;
     class FuelClientPrivate;
     class LocalCache;
     class ModelIdentifier;
@@ -128,11 +129,23 @@ namespace ignition
       /// \return An iterator of models with names matching the criteria
       public: ModelIter Models(const ModelIdentifier &_id) const;
 
+      /// \brief Returns an iterator for the models found in a collection.
+      /// \param[in] _id a partially filled out identifier used to fetch a
+      /// collection.
+      /// \return An iterator of models in the collection.
+      public: ModelIter Models(const CollectionIdentifier &_id) const;
+
       /// \brief Returns worlds matching a given identifying criteria
       /// \param[in] _id A partially filled out identifier used to fetch worlds
       /// \return An iterator of worlds with names matching the criteria
       public: WorldIter Worlds(const WorldIdentifier &_id) const;
 
+      /// \brief Returns an iterator for the worlds found in a collection.
+      /// \param[in] _id a partially filled out identifier used to fetch a
+      /// collection.
+      /// \return An iterator of words in the collection.
+      public: WorldIter Worlds(const CollectionIdentifier &_id) const;
+
       /// \brief Upload a directory as a new model
       /// \param[in] _pathToModelDir a path to a directory containing a model
       /// \param[in] _id An identifier to assign to this new model
@@ -279,6 +292,17 @@ namespace ignition
                                      WorldIdentifier &_id,
                                      std::string &_filePath);
 
+      /// \brief Parse Collection identifer from URL.
+      /// \param[in] _url The unique URL of a collection. It may also be a
+      /// unique name, which is a URL without the server version.
+      /// \param[out] _id The collection identifier. It may contain incomplete
+      /// information based on the passed URL and the current client config.
+      /// The server version will be overridden if that server is in the config
+      /// file.
+      /// \return True if parsed successfully.
+      public: bool ParseCollectionUrl(const common::URI &_url,
+                                      CollectionIdentifier &_id);
+
       /// \brief PIMPL
       private: std::unique_ptr<FuelClientPrivate> dataPtr;
     };

src/CMakeLists.txt

@@ -1,5 +1,6 @@
 set (sources
   ClientConfig.cc
+  CollectionIdentifier.cc
   FuelClient.cc
   ign.cc
   Interface.cc
@@ -17,6 +18,7 @@ set (sources
 
 set (gtest_sources
   ClientConfig_TEST.cc
+  CollectionIdentifier_TEST.cc
   FuelClient_TEST.cc
   ign_src_TEST.cc
   Interface_TEST.cc

src/CollectionIdentifier.cc

@@ -0,0 +1,159 @@
+/*
+ * Copyright (C) 2020 Open Source Robotics Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include <ignition/common/Filesystem.hh>
+
+#include "ignition/fuel_tools/ClientConfig.hh"
+#include "ignition/fuel_tools/CollectionIdentifier.hh"
+
+using namespace ignition;
+using namespace fuel_tools;
+
+class ignition::fuel_tools::CollectionIdentifierPrivate
+{
+  /// \brief a name given to this collection by a user
+  public: std::string name;
+
+  /// \brief owner who this collection is attributed to
+  public: std::string owner;
+
+  /// \brief Server of this collection
+  public: ServerConfig server;
+};
+
+//////////////////////////////////////////////////
+CollectionIdentifier::CollectionIdentifier()
+  : dataPtr(std::make_unique<CollectionIdentifierPrivate>())
+{
+}
+
+//////////////////////////////////////////////////
+CollectionIdentifier::CollectionIdentifier(const CollectionIdentifier &_orig)
+    : dataPtr(std::make_unique<CollectionIdentifierPrivate>(*_orig.dataPtr))
+{
+}
+
+//////////////////////////////////////////////////
+CollectionIdentifier &CollectionIdentifier::operator=(
+    const CollectionIdentifier &_orig)
+{
+  this->dataPtr = std::make_unique<CollectionIdentifierPrivate>(*_orig.dataPtr);
+  return *this;
+}
+
+//////////////////////////////////////////////////
+bool CollectionIdentifier::operator==(const CollectionIdentifier &_rhs) const
+{
+  return this->UniqueName() == _rhs.UniqueName();
+}
+
+//////////////////////////////////////////////////
+CollectionIdentifier::~CollectionIdentifier() = default;
+
+//////////////////////////////////////////////////
+std::string CollectionIdentifier::UniqueName() const
+{
+  return common::joinPaths(this->dataPtr->server.Url().Str(),
+                           this->dataPtr->owner, "collections",
+                           this->dataPtr->name);
+}
+
+//////////////////////////////////////////////////
+std::string CollectionIdentifier::Name() const
+{
+  return this->dataPtr->name;
+}
+
+//////////////////////////////////////////////////
+bool CollectionIdentifier::SetName(const std::string &_name)
+{
+  this->dataPtr->name = _name;
+  return true;
+}
+
+//////////////////////////////////////////////////
+std::string CollectionIdentifier::Owner() const
+{
+  return this->dataPtr->owner;
+}
+
+//////////////////////////////////////////////////
+bool CollectionIdentifier::SetOwner(const std::string &_name)
+{
+  this->dataPtr->owner = _name;
+  return true;
+}
+
+//////////////////////////////////////////////////
+bool CollectionIdentifier::SetServer(const ServerConfig &_server)
+{
+  bool success = _server.Url().Valid();
+  if (success)
+    this->dataPtr->server = _server;
+
+  return success;
+}
+
+//////////////////////////////////////////////////
+ServerConfig &CollectionIdentifier::Server() const
+{
+  return this->dataPtr->server;
+}
+
+//////////////////////////////////////////////////
+std::string CollectionIdentifier::AsString(const std::string &_prefix) const
+{
+  std::stringstream out;
+  out << _prefix << "Name: " << this->Name() << std::endl
+      << _prefix << "Owner: " << this->Owner() << std::endl
+      << _prefix << "Unique name: " << this->UniqueName() << std::endl
+      << _prefix << "Server:" << std::endl
+      << this->Server().AsString(_prefix + "  ");
+  return out.str();
+}
+
+//////////////////////////////////////////////////
+std::string CollectionIdentifier::AsPrettyString(
+    const std::string &_prefix) const
+{
+  std::string prop = "\033[96m\033[1m";
+  std::string value = "\033[37m";
+  std::string reset = "\033[0m";
+
+  std::stringstream out;
+
+  if (!this->Name().empty())
+  {
+    out << _prefix << prop << "Name: " << reset
+        << value << this->Name() << reset << std::endl;
+  }
+
+  if (!this->Owner().empty())
+  {
+    out << _prefix << prop << "Owner: " << reset
+        << value << this->Owner() << reset << std::endl;
+  }
+
+  out << _prefix << prop << "Server:" << reset << std::endl
+      << this->Server().AsPrettyString(_prefix + "  ");
+  return out.str();
+}
+