repo: add/remove methods for meta-repo

src/Get.cpp

@@ -20,7 +20,11 @@ using namespace rapidjson;
 
 bool debug = false;
 
-Get::Get(std::string_view config_dir, std::string_view defaultRepo)
+Get::Get(
+	std::string_view config_dir,
+	std::string_view defaultRepo,
+	bool performInitialLoad
+)
 	: mDefaultRepo(defaultRepo)
 {
 
@@ -40,9 +44,12 @@ Get::Get(std::string_view config_dir, std::string_view defaultRepo)
 
 	printf("--> Using \"%s\" as repo list\n", mRepos_path.c_str());
 
-	// load repo info
 	this->loadRepos();
-	this->update();
+
+	if (performInitialLoad) {
+		// load repo info
+		this->update();
+	}
 }
 
 int Get::install(Package& package)
@@ -90,6 +97,93 @@ void Get::addLocalRepo()
 	update();
 }
 
+void Get::addAndRemoveReposByURL(
+	const std::unordered_set<std::string>& reposToAdd,
+	const std::unordered_set<std::string>& reposToRemove
+)
+{
+	int reposLen = repos.size();
+
+	bool madeChanges = false;
+
+	repos.erase(std::remove_if(repos.begin(), repos.end(), 
+		[reposToRemove](auto curRepo) {
+			std::string curUrl = curRepo->getUrl();
+			return reposToRemove.find(curUrl) != reposToRemove.end();
+		}), repos.end()
+	);
+
+	madeChanges = reposLen != repos.size();
+
+	std::unordered_set<std::string> currentUrls;
+	for (auto& curRepo : repos) {
+		currentUrls.insert(curRepo->getUrl());
+	}
+
+	for (auto& url : reposToAdd) {
+		if (currentUrls.find(url) == currentUrls.end()) {
+			// extract domain from url string
+			std::string nameSummary;
+			size_t start = url.find("//");
+
+			if (start != std::string::npos) {
+				nameSummary = url.substr(start + 2);
+				size_t end = nameSummary.find("/");
+				if (end != std::string::npos) {
+					nameSummary = nameSummary.substr(0, end);
+				}
+			}
+
+			// if nameSummary is still empty, provide a fallback
+			if (nameSummary.empty()) {
+				nameSummary = "Auto-added from Meta";
+			}
+
+			repos.push_back(std::make_unique<GetRepo>(nameSummary, url, true));
+		}
+	}
+
+	madeChanges = madeChanges || reposLen != repos.size();
+
+	// save the repos to disk, if we've made any changes
+	if (madeChanges) {
+		saveRepos();
+		loadRepos();
+	}
+}
+
+// Saves the repos from our current Get object to disk
+void Get::saveRepos() {
+	Document d;
+	d.SetObject();
+	Document::AllocatorType& allocator = d.GetAllocator();
+
+	Value reposOut(kArrayType);
+
+	for (auto& repo : repos) {
+		Value repoObj(kObjectType);
+		printf("--> Saving repo %s\n", repo->getName().c_str());
+		printf("--> Saving repo %s\n", repo->getUrl().c_str());
+		printf("--> Saving repo %s\n", repo->getType().c_str());
+
+		repoObj.AddMember("name", rapidjson::Value(repo->getName().c_str(), allocator), allocator);
+		repoObj.AddMember("url",  rapidjson::Value(repo->getUrl().c_str(), allocator), allocator);
+		repoObj.AddMember("type", rapidjson::Value(repo->getType().c_str(), allocator), allocator);
+		repoObj.AddMember("enabled", repo->isEnabled(), allocator);
+		reposOut.PushBack(repoObj, allocator);
+	}
+
+	d.AddMember("repos", reposOut, allocator);
+
+	std::ofstream file(mRepos_path);
+	StringBuffer buffer;
+	Writer<StringBuffer> writer(buffer);
+	d.Accept(writer);
+	std::cout << buffer.GetString() << std::endl;
+	file << buffer.GetString();
+	file.close();
+}
+
 /**
 Load any repos from a config file into the repos vector.
 **/

src/Get.hpp

@@ -4,14 +4,19 @@
 #include "constants.h"
 #include <optional>
 #include <vector>
+#include <unordered_set>
 
 void info(const char* format, ...);
 
 class Get
 {
 public:
 	// constructor takes path to the .get directory, and a fallback default repo url
-	Get(std::string_view config_dir, std::string_view defaultRepo);
+	Get(
+		std::string_view config_dir,
+		std::string_view defaultRepo,
+		bool performInitialLoad = true
+	);
 
 	int install(Package& pkg_name); // download the given package name and manifest data
 	int remove(Package& pkg_name);	// delete and remove all files for the given package name
@@ -24,6 +29,12 @@ class Get
 	void removeDuplicates();
 	void update();
 
+	void saveRepos();
+	void addAndRemoveReposByURL(
+		const std::unordered_set<std::string>& reposToAdd,
+		const std::unordered_set<std::string>& reposToRemove
+	);
+
 	// map of word -> list of packages whose info matches that word
 	// TODO: this
 	// std::map<std::string, std::vector<Package*>>;
@@ -46,10 +57,10 @@ class Get
 		return packages;
 	}
 
-private:
 	void loadRepos();
 	int validateRepos() const;
-
+
+private:
 	// the remote repos and packages
 	std::vector<std::shared_ptr<Repo>> repos;
 	std::vector<std::shared_ptr<Package>> packages;

tests/12_RepoAddAndRemove.hpp

@@ -0,0 +1,106 @@
+#include "tests.hpp"
+#include "../src/Utils.hpp"
+
+// this test removes all the test repos, and then adds the two production CDN ones
+// using the same method (addAndRemoveReposByURL) used by the hbas meta-repo
+class RepoAddAndRemoveTest : public Test {
+	public:
+	RepoAddAndRemoveTest() {
+		purpose = "Bulk add and remove repos (meta-repo test)";
+	}
+	bool execute()
+	{
+		// get all the repos
+		auto repos = get->getRepos();
+
+		// add their URLs to the remove list
+		std::unordered_set<std::string> removeSet;
+		for (auto& repo : repos)
+			removeSet.insert(repo->getUrl());
+
+		// create some test repos to add
+		std::unordered_set<std::string> addSet;
+		addSet.insert("https://wiiu.cdn.fortheusers.org");
+		addSet.insert("https://switch.cdn.fortheusers.org");
+
+		// call the add and remove functions
+		get->addAndRemoveReposByURL(addSet, removeSet);
+
+		// we should end up with only the repos we added
+		repos = get->getRepos();
+		if (repos.size() != addSet.size())
+		{
+			error << "Expected " << addSet.size() << " repos, got " << repos.size();
+			return false;
+		}
+
+		// check that there are like, a lot of packages (since these are the real repos)
+		int packageCount = get->getPackages().size();
+
+		if (packageCount < 200)
+		{
+			error << "Expected at least 200 packages, got " << packageCount;
+			return false;
+		}
+
+		// check that our repo json looks how we expect
+		auto path = get->mRepos_path;
+		std::string expectedFile = R""""({"repos":[{"name":"switch.cdn.fortheusers.org","url":"https://switch.cdn.fortheusers.org","type":"get","enabled":true},{"name":"wiiu.cdn.fortheusers.org","url":"https://wiiu.cdn.fortheusers.org","type":"get","enabled":true}]})"""";
+
+		std::ifstream t(path);
+		std::stringstream buffer;
+		buffer << t.rdbuf();
+		std::string fileContents = buffer.str();
+
+		// compare the file to what we expect
+		if (fileContents != expectedFile)
+		{
+			error << "Expected file to be:\n" << expectedFile << "\n\nGot:\n" << fileContents;
+			return false;
+		}
+
+		// remove one of the repos we added, again using a repo set
+		removeSet.clear();
+		removeSet.insert("https://switch.cdn.fortheusers.org");
+		get->addAndRemoveReposByURL({}, removeSet);
+
+		// make sure we only have one repo left
+		repos = get->getRepos();
+		if (repos.size() != 1)
+		{
+			error << "Expected 1 repo, got " << repos.size();
+			return false;
+		}
+
+		// make sure it's the right one
+		if (repos[0]->getUrl() != "https://wiiu.cdn.fortheusers.org")
+		{
+			error << "Expected repo to be https://wiiu.cdn.fortheusers.org, got " << repos[0]->getUrl();
+			return false;
+		}
+
+		// and just make sure we have less packages, as a sanity check
+		int wiiuPackagesCount = get->getPackages().size();
+
+		if (wiiuPackagesCount > packageCount)
+		{
+			error << "Expected less packages, got " << wiiuPackagesCount;
+			return false;
+		}
+
+		expectedFile = R""""({"repos":[{"name":"wiiu.cdn.fortheusers.org","url":"https://wiiu.cdn.fortheusers.org","type":"get","enabled":true}]})"""";
+		t = std::ifstream(path);
+		buffer = std::stringstream();
+		buffer << t.rdbuf();
+		fileContents = buffer.str();
+
+		if (fileContents != expectedFile)
+		{
+			error << "Expected file to be:\n" << expectedFile << "\n\nGot:\n" << fileContents;
+			return false;
+		}
+
+
+		return true;
+	}
+};

tests/main.cpp

@@ -13,6 +13,8 @@
 #include "9_FakeManifestTest.hpp"
 #include "10_FakeManifestUpgrade.hpp"
 #include "11_PackagesSorted.hpp"
+#include "12_RepoAddAndRemove.hpp"
+
 
 using namespace std;
 
@@ -49,17 +51,18 @@ int main()
 	// due to this dependency, if one fails, the rest don't execute
 	// they will always run in the order defined here, though
 	vector<std::unique_ptr<Test>> tests ;
-	tests.push_back(std::make_unique<InitCheck>());
-	tests.push_back(std::make_unique<CheckPackages>());
-	tests.push_back(std::make_unique<InstallPackages>());
-	tests.push_back(std::make_unique<InstallPackages2>());
-	tests.push_back(std::make_unique<Search>());
-	tests.push_back(std::make_unique<RemovePackages>());
-	tests.push_back(std::make_unique<UpgradePackages>());
-	tests.push_back(std::make_unique<ContentTest>());
-	tests.push_back(std::make_unique<FakeManifestTest>());
-	tests.push_back(std::make_unique<FakeManifestUpgradeTest>());
+	// tests.push_back(std::make_unique<InitCheck>());
+	// tests.push_back(std::make_unique<CheckPackages>());
+	// tests.push_back(std::make_unique<InstallPackages>());
+	// tests.push_back(std::make_unique<InstallPackages2>());
+	// tests.push_back(std::make_unique<Search>());
+	// tests.push_back(std::make_unique<RemovePackages>());
+	// tests.push_back(std::make_unique<UpgradePackages>());
+	// tests.push_back(std::make_unique<ContentTest>());
+	// tests.push_back(std::make_unique<FakeManifestTest>());
+	// tests.push_back(std::make_unique<FakeManifestUpgradeTest>());
 	// tests.push_back(std::make_unique<PackagesSortedTest>());
+	tests.push_back(std::make_unique<RepoAddAndRemoveTest>());
 
 	// main test loop that goes through all our tests, and prints out
 	// their status with a happy friendly emoji