3ds: initial unistore support, repo refactors

Makefile

@@ -1,6 +1,7 @@
 ifneq ($(SOURCES),)
 	# we're in a chesto-style lib
-	LIBGET  := $(CURDIR)/libs/get/src
+	LIBGET  		:= $(CURDIR)/libs/get/src
+	LIBGET_REPOS	:= $(LIBGET)/repos
 else
 	# we are in the libget folder already
 	LIBGET := ./src
@@ -11,10 +12,10 @@ RAPIDJSON   := $(LIBGET)/libs/rapidjson/include
 MINIZIP     := $(LIBGET)/libs/minizip
 TINYXML     := $(LIBGET)/libs/tinyxml
 
-SOURCES     += $(LIBGET) $(MINIZIP) $(TINYXML)
+SOURCES     += $(LIBGET) $(LIBGET_REPOS) $(MINIZIP) $(TINYXML)
 INCLUDES    += $(RAPIDJSON) $(MINIZIP) $(TINYXML)
 
-VPATH       += $(LIBGET) $(MINIZIP) $(TINYXML)
+VPATH       += $(LIBGET) $(LIBGET_REPOS) $(MINIZIP) $(TINYXML)
 
 CFLAGS      += -DNETWORK
 LDFLAGS     += -lcurl
@@ -24,7 +25,7 @@ MINIZIP_O   :=  zip.o ioapi.o unzip.o
 ifeq ($(LIBGET),./src)
 build:
 	gcc -c $(MINIZIP)/*.c
-	g++ -g cli/*.cpp src/*.cpp -std=gnu++20 -lm -I $(RAPIDJSON) $(MINIZIP_O) -I $(MINIZIP) -lz -lcurl -o get -DAPP_VERSION=\"$(APP_VERSION)\"
+	g++ -g cli/*.cpp src/*.cpp src/repos/*.cpp -std=gnu++20 -lm -I $(RAPIDJSON) $(MINIZIP_O) -I $(MINIZIP) -lz -lcurl -o get -DAPP_VERSION=\"$(APP_VERSION)\"
 
 run_tests:
 	rm -rf tests/.get/packages tests/.get/tmp

README.md

@@ -15,7 +15,7 @@ Versions of the get client 2.0.0 and after can support non-libget repo formats.
 A "repos.json" file should be present on the local machine, in the same directory as the `get` binary. See [repos.json](https://github.com/vgmoose/get/blob/master/.get/repos.json) for what this file should look like. If this file does not exist, depending on the use case (such as on the Wii U) a default repos.json should automatically be generated.
 
 #### Example Local Repo JSONs
-Below are three example repos. The first two are libget repos used by the Switch and WiiU homebrew appstores, and the third is a OSC repo used by the Wii.
+Below are three example repos. The first two are libget repos used by the Switch and WiiU homebrew appstores, and the third and fourth are OSC / Universal-DB repos used by the Wii / 3DS.
 
 <details>
   <summary>WiiU Homebrew CDN (libget Repo)</summary>
@@ -54,7 +54,7 @@ Below are three example repos. The first two are libget repos used by the Switch
   {
     "repos": [{
        "name":"Wii OSC Repo",
-       "url":"https://hbb1.oscwii.org",
+       "url":"https://hbb1.oscwii.org/api/v3/contents",
        "type":"osc",
        "enabled":true
     }]
@@ -63,6 +63,21 @@ Below are three example repos. The first two are libget repos used by the Switch
 </details>
 
 
+<details>
+  <summary>3DS Universal-DB (Unistore Repo) (WIP)</summary>
+
+  ```javascript
+  {
+    "repos": [{
+       "name":"3DS Universal-DB",
+       "url":"https://db.universal-team.net/unistore/universal-db.unistore",
+       "type":"unistore",
+       "enabled":true
+    }]
+  }
+  ```
+</details>
+
 ### Setting up remote repos (libget repos)
 When `get` runs, it will go through the enabled repos in that config file, and try to make a GET request to `/repo.json`, which should contain (on the remote server) a listing of all of the packages and descriptions. Here is an example of what the remote's repo.json with one package looks like:
 ```

src/Get.cpp

@@ -12,8 +12,8 @@
 #include "rapidjson/writer.h"
 
 #include "Get.hpp"
-#include "GetRepo.hpp"
-#include "LocalRepo.hpp"
+#include "./repos/GetRepo.hpp"
+#include "./repos/LocalRepo.hpp"
 #include "Utils.hpp"
 
 using namespace rapidjson;
@@ -199,7 +199,7 @@ void Get::loadRepos()
 	{
 		printf("--> Could not load repos from %s, generating default GET repos.json\n", config_path.c_str());
 
-#if defined(WII)
+#if defined(WII) || defined(_3DS)
 		auto defaultRepo = GetRepo::createRepo("Default Repo", this->mDefaultRepo, true, this->mDefaultRepoType, mPkg_path);
 #else
 		auto defaultRepo = std::make_unique<GetRepo>("Default Repo", this->mDefaultRepo, true);

src/Get.hpp

@@ -1,7 +1,7 @@
 #ifndef GET_H
 #define GET_H
-#include "Repo.hpp"
-#include "constants.h"
+#include "./repos/Repo.hpp"
+#include "./repos/constants.h"
 #include <optional>
 #include <vector>
 #include <unordered_set>

src/Package.cpp

@@ -1,7 +1,7 @@
-#include "Repo.hpp"
+#include "./repos/Repo.hpp"
 #include "Utils.hpp"
 #include "ZipUtil.hpp"
-#include "constants.h"
+#include "./repos/constants.h"
 #include "rapidjson/document.h"
 #include "rapidjson/istreamwrapper.h"
 #include <algorithm>

src/Package.hpp

@@ -26,6 +26,7 @@ class Repo;
 class GetRepo;
 class LocalRepo;
 class OSCRepo;
+class UniStoreRepo;
 
 /**
  * A package is a single application that can be installed. It contains the URL to the zip file and any instructions to install it (like a GET manifest).
@@ -175,6 +176,7 @@ class Package
 	friend GetRepo;
 	friend LocalRepo;
 	friend OSCRepo;
+	friend UniStoreRepo;
 };
 
 #endif

src/GetRepo.cpp → src/repos/GetRepo.cpp

@@ -1,5 +1,5 @@
 #include "GetRepo.hpp"
-#include "Utils.hpp"
+#include "../Utils.hpp"
 #include "constants.h"
 #include "rapidjson/document.h"
 #include <regex>

src/GetRepo.hpp → src/repos/GetRepo.hpp

@@ -1,6 +1,6 @@
 #ifndef GET_REPO_H
 #define GET_REPO_H
-#include "Package.hpp"
+#include "../Package.hpp"
 #include "Repo.hpp"
 #include <iostream>
 #include <vector>

src/LocalRepo.cpp → src/repos/LocalRepo.cpp

@@ -1,7 +1,7 @@
 #include "LocalRepo.hpp"
-#include "Get.hpp"
-#include "Package.hpp"
-#include "Utils.hpp"
+#include "../Get.hpp"
+#include "../Package.hpp"
+#include "../Utils.hpp"
 #include "rapidjson/document.h"
 #include "rapidjson/istreamwrapper.h"
 #include <cstring>

src/LocalRepo.hpp → src/repos/LocalRepo.hpp

@@ -1,6 +1,6 @@
 #ifndef LOCAL_REPO_H
 #define LOCAL_REPO_H
-#include "Package.hpp"
+#include "../Package.hpp"
 #include "Repo.hpp"
 #include <iostream>
 #include <vector>

src/OSCRepo.cpp → src/repos/OSCRepo.cpp

@@ -1,5 +1,5 @@
 #include "OSCRepo.hpp"
-#include "Utils.hpp"
+#include "../Utils.hpp"
 #include "constants.h"
 #include "rapidjson/document.h"
 #include "rapidjson/rapidjson.h"
@@ -18,7 +18,7 @@ using namespace rapidjson;
 std::vector<std::unique_ptr<Package>> OSCRepo::loadPackages()
 {
 	std::vector<std::unique_ptr<Package>> result;
-	std::string directoryUrl = this->url + "/api/v3/contents";
+	std::string directoryUrl = this->url;
 
 	std::string response;
 	bool success = downloadFileToMemory(directoryUrl, &response);
@@ -31,7 +31,7 @@ std::vector<std::unique_ptr<Package>> OSCRepo::loadPackages()
 
 		// update repo url
 		this->url.replace(0, 5, "http");
-		directoryUrl = this->url + "/api/v3/contents";
+		directoryUrl = this->url;
 
 		// retry fetch
 		success = downloadFileToMemory(directoryUrl, &response);
@@ -55,7 +55,7 @@ std::vector<std::unique_ptr<Package>> OSCRepo::loadPackages()
 
 	if (!ok || !doc.IsArray())
 	{
-		printf("--> Invalid format in downloaded repo.json for %s\n", this->url.c_str());
+		printf("--> Invalid format in downloaded data for %s\n", this->url.c_str());
 		this->loaded = false;
 		return {};
 	}

src/OSCRepo.hpp → src/repos/OSCRepo.hpp

@@ -1,6 +1,6 @@
 #ifndef OSC_REPO_H
 #define OSC_REPO_H
-#include "Package.hpp"
+#include "../Package.hpp"
 #include "Repo.hpp"
 #include <iostream>
 #include <vector>

src/Repo.cpp → src/repos/Repo.cpp

@@ -2,7 +2,8 @@
 #include "GetRepo.hpp"
 #include "LocalRepo.hpp"
 #include "OSCRepo.hpp"
-#include "Package.hpp"
+#include "UniStoreRepo.hpp"
+#include "../Package.hpp"
 #include <iostream>
 #include <sstream>
 
@@ -39,7 +40,24 @@ std::unique_ptr<Repo> Repo::createRepo(std::string_view name, std::string_view u
 		return std::make_unique<LocalRepo>(std::string(package_path));
 	else if (type == "osc")
 		return std::make_unique<OSCRepo>(name, url, enabled);
-	// TODO: add more supported repo formats here
+	else if (type == "unistore")
+		return std::make_unique<UniStoreRepo>(name, url, enabled);
+	// else if (type == "github")
+	// 	return std::make_unique<GitHubRepo>(name, url, enabled);
+	// else if (type == "aroma")
+	// 	return std::make_unique<AromaRepo>(name, url, enabled);
+
+	// TODO: add more repo types from down below
+	// else if (type == "gitlab")
+	// 	return std::make_unique<GitLabRepo>(name, url, enabled);
+	// else if (type == "apk")
+	// 	return std::make_unique<APKRepo>(name, url, enabled);
+	// else if (type == "apt")
+	// 	return std::make_unique<AptRepo>(name, url, enabled);
+	// else if (type == "pacman")
+	// 	return std::make_unique<PacmanRepo>(name, url, enabled);
+	// else if (type == "dnf")
+	// 	return std::make_unique<DnfRepo>(name, url, enabled);
 
 	return nullptr;
 }

src/Repo.hpp → src/repos/Repo.hpp

@@ -1,6 +1,6 @@
 #ifndef REPO_H
 #define REPO_H
-#include "Package.hpp"
+#include "../Package.hpp"
 #include <iostream>
 #include <vector>
 

src/repos/UniStoreRepo.cpp

@@ -0,0 +1,127 @@
+#include "UniStoreRepo.hpp"
+#include "../Utils.hpp"
+#include "constants.h"
+#include "rapidjson/document.h"
+#include "rapidjson/rapidjson.h"
+#include <iomanip>
+#include <regex>
+#include <sstream>
+#include <map>
+
+using namespace rapidjson;
+
+std::vector<std::unique_ptr<Package>> UniStoreRepo::loadPackages()
+{
+	std::vector<std::unique_ptr<Package>> result;
+	std::string directoryUrl = this->url;
+
+	std::string response;
+	bool success = downloadFileToMemory(directoryUrl, &response);
+
+	// attempt fallback to http in case of https repos failure
+	// TODO: Abstract this out, and allow to be optional
+	if (!success && (this->url.rfind("https", 0) == 0))
+	{
+		printf("--> Attempting http fallback for https repo \"%s\" after loading failure\n", this->name.c_str());
+
+		// update repo url
+		this->url.replace(0, 5, "http");
+		directoryUrl = this->url;
+
+		// retry fetch
+		success = downloadFileToMemory(directoryUrl, &response);
+	}
+
+	if (!success)
+	{
+		printf("--> Could not update repository metadata for \"%s\" repo!\n", this->name.c_str());
+		this->loaded = false;
+		return {};
+	}
+
+	if (libget_status_callback != nullptr)
+	{
+		libget_status_callback(STATUS_UPDATING_STATUS, 1, 1);
+	}
+
+	// extract out packages, append to package list
+	Document doc;
+	ParseResult ok = doc.Parse(response.c_str());
+
+	if (!ok || !doc.IsObject() || !doc.HasMember("storeContent"))
+	{
+		printf("--> Invalid format in downloaded data for %s\n", this->url.c_str());
+		this->loaded = false;
+		return {};
+	}
+
+	const Value& packages_doc = doc["storeContent"];
+
+	// for every package in the repo
+	auto total = (int32_t)packages_doc.Size();
+	for (int i = 0; i < total; i++)
+	{
+		if (networking_callback != nullptr)
+			networking_callback(nullptr, total, i + 1, 0, 0);
+
+		std::vector<std::string> keys;
+		auto start = packages_doc[i].GetObject().MemberBegin();
+		auto end = packages_doc[i].GetObject().MemberEnd();
+		for (auto it = start; it != end; ++it) {
+			keys.push_back(it->name.GetString());
+		}
+
+		if (packages_doc[i].HasMember("info"))
+		{
+			auto package = std::make_unique<Package>(GET);
+			auto& cur = packages_doc[i]["info"];
+
+			if (cur.HasMember("title")) {
+				package->pkg_name = cur["title"].GetString();
+				package->title = package->pkg_name;
+			}
+			if (cur.HasMember("author"))
+				package->author = cur["author"].GetString();
+			if (cur.HasMember("description")) {
+				package->short_desc = cur["description"].GetString();
+				package->long_desc = package->short_desc;
+			}
+			if (cur.HasMember("version"))
+				package->version = cur["version"].GetString();
+			if (cur.HasMember("license"))
+				package->license = cur["license"].GetString();
+			if (cur.HasMember("releasenotes"))
+				package->changelog = cur["releasenotes"].GetString();
+			if (cur.HasMember("category")) {
+				// category is an array of strings in the JSON, we'll take just the first one
+				const Value& category = cur["category"];
+				if (category.Size() > 0)
+					package->category = category[0].GetString();
+			}
+
+			// TODO: screen shots and icons
+			// TODO: address needing to follow unistore script instructions properly
+
+			result.push_back(std::move(package));
+		}
+
+	}
+	return result;
+}
+
+std::string UniStoreRepo::getType() const
+{
+	return "unistore";
+}
+
+std::string UniStoreRepo::getZipUrl(const Package& package) const
+{
+	// OSC packages just use the url field directly
+	return "";
+}
+
+std::string UniStoreRepo::getIconUrl(const Package& package) const
+{
+	// OSC packages just use the url field directly
+	return "";
+}

src/repos/UniStoreRepo.hpp

@@ -0,0 +1,19 @@
+#ifndef UNISTORE_REPO_H
+#define UNISTORE_REPO_H
+#include "../Package.hpp"
+#include "Repo.hpp"
+#include <iostream>
+#include <vector>
+
+class UniStoreRepo : public Repo
+{
+public:
+	using Repo::Repo;
+	[[nodiscard]] std::string getType() const override;
+	[[nodiscard]] std::string getZipUrl(const Package& package) const override;
+	[[nodiscard]] std::string getIconUrl(const Package& package) const override;
+
+private:
+	[[maybe_unused]] std::vector<std::unique_ptr<Package>> loadPackages() override;
+};
+#endif