From 09a5f8c6b5b63bd1f14d3e0567e2dd735a5fcf5c Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 24 Dec 2021 02:31:37 -0800 Subject: [PATCH 1/2] FEX: Move common config loading to a helper --- Source/Common/Config.cpp | 39 ++++++++++++++++++++++++++++++++++++++ Source/Common/Config.h | 8 ++++++++ Source/Tests/FEXLoader.cpp | 33 ++++++++++---------------------- 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/Source/Common/Config.cpp b/Source/Common/Config.cpp index da25dc6652..16f84adb14 100644 --- a/Source/Common/Config.cpp +++ b/Source/Common/Config.cpp @@ -1,8 +1,10 @@ +#include "Common/ArgumentLoader.h" #include "Common/Config.h" #include #include +#include #include #include #include @@ -38,4 +40,41 @@ namespace FEX::Config { } } + std::string LoadConfig( + bool NoFEXArguments, + bool LoadProgramConfig, + int argc, + char **argv, + char **const envp) { + FEXCore::Config::Initialize(); + FEXCore::Config::AddLayer(FEXCore::Config::CreateMainLayer()); + + if (NoFEXArguments) { + FEX::ArgLoader::LoadWithoutArguments(argc, argv); + } + else { + FEXCore::Config::AddLayer(std::make_unique(argc, argv)); + } + + FEXCore::Config::AddLayer(FEXCore::Config::CreateEnvironmentLayer(envp)); + FEXCore::Config::Load(); + + auto Args = FEX::ArgLoader::Get(); + + if (LoadProgramConfig) { + if (Args.empty()) { + // Early exit if we weren't passed an argument + return {}; + } + + std::string Program = Args[0]; + + // These layers load on initialization + auto ProgramName = std::filesystem::path(Program).filename(); + FEXCore::Config::AddLayer(FEXCore::Config::CreateAppLayer(ProgramName, true)); + FEXCore::Config::AddLayer(FEXCore::Config::CreateAppLayer(ProgramName, false)); + return Program; + } + return {}; + } } diff --git a/Source/Common/Config.h b/Source/Common/Config.h index 9654bae8c5..8ae617eec4 100644 --- a/Source/Common/Config.h +++ b/Source/Common/Config.h @@ -18,4 +18,12 @@ namespace FEX::Config { }; void SaveLayerToJSON(const std::string& Filename, FEXCore::Config::Layer *const Layer); + + std::string LoadConfig( + bool NoFEXArguments, + bool LoadProgramConfig, + int argc, + char **argv, + char **const envp + ); } diff --git a/Source/Tests/FEXLoader.cpp b/Source/Tests/FEXLoader.cpp index f67a8f1779..3946c2f453 100644 --- a/Source/Tests/FEXLoader.cpp +++ b/Source/Tests/FEXLoader.cpp @@ -234,33 +234,19 @@ int main(int argc, char **argv, char **const envp) { } #endif - FEXCore::Config::Initialize(); - FEXCore::Config::AddLayer(FEXCore::Config::CreateMainLayer()); + auto Program = FEX::Config::LoadConfig( + IsInterpreter, + true, + argc, argv, envp + ); - if (IsInterpreter) { - FEX::ArgLoader::LoadWithoutArguments(argc, argv); - } - else { - FEXCore::Config::AddLayer(std::make_unique(argc, argv)); - } - - FEXCore::Config::AddLayer(FEXCore::Config::CreateEnvironmentLayer(envp)); - FEXCore::Config::Load(); - - auto Args = FEX::ArgLoader::Get(); - auto ParsedArgs = FEX::ArgLoader::GetParsedArgs(); - - if (Args.empty()) { + if (Program.empty()) { // Early exit if we weren't passed an argument return 0; } - std::string Program = Args[0]; - - // These layers load on initialization - auto ProgramName = std::filesystem::path(Program).filename(); - FEXCore::Config::AddLayer(FEXCore::Config::CreateAppLayer(ProgramName, true)); - FEXCore::Config::AddLayer(FEXCore::Config::CreateAppLayer(ProgramName, false)); + auto Args = FEX::ArgLoader::Get(); + auto ParsedArgs = FEX::ArgLoader::GetParsedArgs(); // Reload the meta layer FEXCore::Config::ReloadMetaLayer(); @@ -367,6 +353,7 @@ int main(int argc, char **argv, char **const envp) { if (LDPath().empty() || std::filesystem::exists(LDPath(), ec) == false) { fmt::print(stderr, "RootFS path doesn't exist. This is required on AArch64 hosts\n"); + fmt::print(stderr, "Use FEXRootFSFetcher to download a RootFS\n"); } #endif return -ENOEXEC; @@ -552,7 +539,7 @@ int main(int argc, char **argv, char **const envp) { FEXCore::Allocator::ClearHooks(); FEXCore::Allocator::ReclaimMemoryRegion(Base48Bit); // Allocator is now original system allocator - + auto ProgramName = std::filesystem::path(Program).filename(); FEXCore::Telemetry::Shutdown(ProgramName); if (ShutdownReason == FEXCore::Context::ExitReason::EXIT_SHUTDOWN) { return ProgramStatus; From 2c31080fd4e77714b3bf184b9a55fc4b3135de99 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 24 Dec 2021 02:33:00 -0800 Subject: [PATCH 2/2] FEXRootFSFetcher: Adds a new tool to help set up a new RootFS This tool supports both a zenity and tty interface. TTY will be presented if available while Zenity will be used otherwise. This tool pulls a rootfs list from https://rootfs.fex-emu.org/ It then allows you to select a rootfs from the list, download it, place it in to the correct working folder, extract it if desired, and set it as the current default RootFS. This requires curl and potentially zenity installed to use. Maybe also unsquashfs if the user chooses to extract the image. This is an all in one tool to quickly get a new user up and running. Additionally if you pass in a file path in to the tool, it will generate an xxhash of the file and exit out. This is the hash used to ensure the files are valid. --- Source/Tools/CMakeLists.txt | 1 + Source/Tools/FEXRootFSFetcher/CMakeLists.txt | 24 + Source/Tools/FEXRootFSFetcher/Main.cpp | 851 +++++++++++++++++++ Source/Tools/FEXRootFSFetcher/XXFileHash.cpp | 84 ++ Source/Tools/FEXRootFSFetcher/XXFileHash.h | 7 + 5 files changed, 967 insertions(+) create mode 100644 Source/Tools/FEXRootFSFetcher/CMakeLists.txt create mode 100644 Source/Tools/FEXRootFSFetcher/Main.cpp create mode 100644 Source/Tools/FEXRootFSFetcher/XXFileHash.cpp create mode 100644 Source/Tools/FEXRootFSFetcher/XXFileHash.h diff --git a/Source/Tools/CMakeLists.txt b/Source/Tools/CMakeLists.txt index 406de7dc13..e872fe9809 100644 --- a/Source/Tools/CMakeLists.txt +++ b/Source/Tools/CMakeLists.txt @@ -6,6 +6,7 @@ add_subdirectory(FEXConfig/) add_subdirectory(FEXGetConfig/) add_subdirectory(FEXMountDaemon/) add_subdirectory(FEXLogServer/) +add_subdirectory(FEXRootFSFetcher/) set(NAME Opt) set(SRCS Opt.cpp) diff --git a/Source/Tools/FEXRootFSFetcher/CMakeLists.txt b/Source/Tools/FEXRootFSFetcher/CMakeLists.txt new file mode 100644 index 0000000000..05b71270d1 --- /dev/null +++ b/Source/Tools/FEXRootFSFetcher/CMakeLists.txt @@ -0,0 +1,24 @@ +set(NAME FEXRootFSFetcher) +set(SRCS Main.cpp + XXFileHash.cpp) + +add_executable(${NAME} ${SRCS}) +list(APPEND LIBS FEXCore Common CommonCore) + +target_include_directories(${NAME} PRIVATE ${CMAKE_SOURCE_DIR}/Source/) + +if (CMAKE_BUILD_TYPE MATCHES "RELEASE") + target_link_options(${NAME} + PRIVATE + "LINKER:--gc-sections" + "LINKER:--strip-all" + "LINKER:--as-needed" + ) +endif() + +install(TARGETS ${NAME} + RUNTIME + DESTINATION bin + COMPONENT runtime) + +target_link_libraries(${NAME} PRIVATE ${LIBS} ${STATIC_PIE_OPTIONS} ${PTHREAD_LIB}) diff --git a/Source/Tools/FEXRootFSFetcher/Main.cpp b/Source/Tools/FEXRootFSFetcher/Main.cpp new file mode 100644 index 0000000000..628d8c949f --- /dev/null +++ b/Source/Tools/FEXRootFSFetcher/Main.cpp @@ -0,0 +1,851 @@ +#include "XXFileHash.h" + +#include "Common/ArgumentLoader.h" +#include "Common/Config.h" + +#include +#include +#include +#include +#include + +namespace Exec { + int32_t ExecAndWaitForResponse(const char *path, char* const* args) { + pid_t pid = fork(); + if (pid == 0) { + execv(path, args); + _exit(-1); + } + else { + int32_t Status{}; + waitpid(pid, &Status, 0); + if (WIFEXITED(Status)) { + return WEXITSTATUS(Status); + } + } + + return -1; + } + + std::string ExecAndWaitForResponseText(const char *path, char* const* args) { + int fd[2]; + pipe(fd); + + pid_t pid = fork(); + + if (pid == 0) { + close(fd[0]); // Close read side + + // Redirect stdout to pipe + dup2(fd[1], STDOUT_FILENO); + + // Close stderr + close(STDERR_FILENO); + + // We can now close the pipe since the duplications take care of the rest + close(fd[1]); + + execv(path, args); + _exit(-1); + } + else { + close(fd[1]); // Close write side + + // Nothing larger than this + char Buffer[1024]{}; + + // Read the pipe until it closes + while (read(fd[0], Buffer, sizeof(Buffer))); + + int32_t Status{}; + waitpid(pid, &Status, 0); + if (WIFEXITED(Status)) { + // Return what we've read + close(fd[0]); + return Buffer; + } + } + + return {}; + } +} + +namespace DistroQuery { + struct DistroInfo { + std::string DistroName; + std::string DistroVersion; + bool Unknown; + }; + + DistroInfo GetDistroInfo() { + // Detect these files in order + // + // /etc/lsb-release + // eg: + // DISTRIB_ID=Ubuntu + // DISTRIB_RELEASE=21.10 + // DISTRIB_CODENAME=impish + // DISTRIB_DESCRIPTION="Ubuntu 21.10" + // + // /etc/os-release + // eg: + // PRETTY_NAME="Ubuntu 21.10" + // NAME="Ubuntu" + // VERSION_ID="21.10" + // VERSION="21.10 (Impish Indri)" + // VERSION_CODENAME=impish + // ID=ubuntu + // ID_LIKE=debian + // HOME_URL="https://www.ubuntu.com/" + // SUPPORT_URL="https://help.ubuntu.com/" + // BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" + // PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" + // UBUNTU_CODENAME=impish + // + // /etc/debian_version + // eg: + // 11.0 + // + // uname -r + // eg: + // 5.13.0-22-generic + DistroInfo Info{}; + uint32_t FoundCount{}; + + if (std::filesystem::exists("/etc/lsb-release")) { + std::fstream File ("/etc/lsb-release", std::fstream::in); + std::string Line; + while (std::getline(File, Line)) { + if (File.eof() || FoundCount == 2) { + break; + } + + std::stringstream ss(Line); + std::string Key, Value; + std::getline(ss, Key, '='); + std::getline(ss, Value, '='); + + if (Key == "DISTRIB_ID") { + auto ToLower = [](auto Str) { + std::transform(Str.begin(), Str.end(), Str.begin(), + [](unsigned char c){ return std::tolower(c); }); + return Str; + }; + Info.DistroName = ToLower(Value); + ++FoundCount; + } + else if (Key == "DISTRIB_RELEASE") { + Info.DistroVersion = Value; + ++FoundCount; + } + } + } + + if (FoundCount == 2) { + Info.Unknown = false; + return Info; + } + FoundCount = 0; + + if (std::filesystem::exists("/etc/os-release")) { + std::fstream File ("/etc/os-release", std::fstream::in); + std::string Line; + while (std::getline(File, Line)) { + if (File.eof() || FoundCount == 2) { + break; + } + + std::stringstream ss(Line); + std::string Key, Value; + std::getline(ss, Key, '='); + std::getline(ss, Value, '='); + + if (Key == "ID") { + Info.DistroName = Value; + ++FoundCount; + } + else if (Key == "VERSION_ID") { + // Strip the two quotes from the VERSION_ID + Value = Value.substr(1, Value.size() - 2); + Info.DistroVersion = Value; + ++FoundCount; + } + } + } + + if (FoundCount == 2) { + Info.Unknown = false; + return Info; + } + FoundCount = 0; + + if (std::filesystem::exists("/etc/debian_version")) { + std::fstream File ("/etc/debian_version", std::fstream::in); + std::string Line; + + Info.DistroName = "debian"; + ++FoundCount; + while (std::getline(File, Line)) { + Info.DistroVersion = Line; + ++FoundCount; + } + } + + if (FoundCount == 2) { + Info.Unknown = false; + return Info; + } + + Info.DistroName = "Unknown"; + Info.DistroVersion = {}; + Info.Unknown = true; + return Info; + } +} + +namespace WebFileFetcher { + struct FileTargets { + // These two are for matching version checks + std::string DistroMatch; + std::string VersionMatch; + + // This is a human readable name + std::string DistroName; + + // This is the URL + std::string URL; + + // This is the hash of the file + std::string Hash; + }; + + const static std::string DownloadURL = "https://rootfs.fex-emu.org/file/fex-rootfs/RootFS_links.txt"; + + std::string DownloadToString(const std::string &URL) { + std::string BigArgs = + fmt::format("curl {}", URL); + std::vector ExecveArgs = { + "/bin/sh", + "-c", + BigArgs.c_str(), + nullptr, + }; + + return Exec::ExecAndWaitForResponseText(ExecveArgs[0], const_cast(ExecveArgs.data())); + } + + bool DownloadToPath(const std::string &URL, const std::string &Path) { + auto filename = URL.substr(URL.find_last_of('/') + 1); + auto PathName = Path + filename; + + std::string BigArgs = + fmt::format("curl {} -o {}", URL, PathName); + std::vector ExecveArgs = { + "/bin/sh", + "-c", + BigArgs.c_str(), + nullptr, + }; + + return Exec::ExecAndWaitForResponse(ExecveArgs[0], const_cast(ExecveArgs.data())) == 0; + } + + bool DownloadToPathWithZenityProgress(const std::string &URL, const std::string &Path) { + auto filename = URL.substr(URL.find_last_of('/') + 1); + auto PathName = Path + filename; + + // -# for progress bar + // -o for output file + // -f for silent fail + std::string CurlPipe = fmt::format("curl -#f {} -o {} 2>&1", URL, PathName); + const std::string StdBuf = "stdbuf -oL tr '\\r' '\\n'"; + const std::string SedBuf = "sed -u 's/[^0-9]*\\([0-9]*\\).*/\\1/'"; + const std::string ZenityBuf = "zenity --time-remaining --progress --auto-close --no-cancel --title 'Downloading'"; + std::string BigArgs = + fmt::format("{} | {} | {} | {}", CurlPipe, StdBuf, SedBuf, ZenityBuf); + std::vector ExecveArgs = { + "/bin/sh", + "-c", + BigArgs.c_str(), + nullptr, + }; + + return Exec::ExecAndWaitForResponse(ExecveArgs[0], const_cast(ExecveArgs.data())) == 0; + } + + std::vector GetRootFSLinks() { + // Decode the filetargets + std::string Data = DownloadToString(DownloadURL); + std::stringstream ss(Data); + std::vector Targets; + + while (!ss.eof()) { + FileTargets Target; + if (!std::getline(ss, Target.DistroMatch)) + break; + if (!std::getline(ss, Target.VersionMatch)) + break; + if (!std::getline(ss, Target.DistroName)) + break; + if (!std::getline(ss, Target.URL)) + break; + if (!std::getline(ss, Target.Hash)) + break; + + Targets.emplace_back(Target); + } + return Targets; + } +} + +namespace Zenity { + bool ExecWithQuestion(const std::string &Question) { + std::string TextArg = "--text=" + Question; + const char *Args[] = { + "/usr/bin/zenity", + "--question", + TextArg.c_str(), + nullptr, + }; + + int32_t Result = Exec::ExecAndWaitForResponse(Args[0], const_cast(Args)); + // 0 on Yes, 1 on No + return Result == 0; + } + + void ExecWithInfo(const std::string &Text) { + std::string TextArg = "--text=" + Text; + const char *Args[] = { + "/usr/bin/zenity", + "--info", + TextArg.c_str(), + nullptr, + }; + + Exec::ExecAndWaitForResponse(Args[0], const_cast(Args)); + } + + bool AskForConfirmation(const std::string &Question) { + return ExecWithQuestion(Question); + } + + int32_t AskForConfirmationList(const std::string &Text, const std::vector &Arguments) { + std::string TextArg = "--text=" + Text; + + std::vector ExecveArgs = { + "/usr/bin/zenity", + "--list", + TextArg.c_str(), + "--hide-header", + "--column=Index", + "--column=Text", + "--hide-column=1", + }; + + std::vector NumberArgs; + for (size_t i = 0; i < Arguments.size(); ++i) { + NumberArgs.emplace_back(std::to_string(i)); + } + + for (size_t i = 0; i < Arguments.size(); ++i) { + const auto &Arg = Arguments[i]; + ExecveArgs.emplace_back(NumberArgs[i].c_str()); + ExecveArgs.emplace_back(Arg.c_str()); + } + ExecveArgs.emplace_back(nullptr); + + auto Result = Exec::ExecAndWaitForResponseText(ExecveArgs[0], const_cast(ExecveArgs.data())); + if (Result.empty()) { + return -1; + } + return std::stoi(Result); + } + + int32_t AskForComplexConfirmationList(const std::string &Text, const std::vector &Arguments) { + std::string TextArg = "--text=" + Text; + + std::vector ExecveArgs = { + "/usr/bin/zenity", + "--list", + TextArg.c_str(), + }; + + for (auto &Arg : Arguments) { + ExecveArgs.emplace_back(Arg.c_str()); + } + ExecveArgs.emplace_back(nullptr); + + auto Result = Exec::ExecAndWaitForResponseText(ExecveArgs[0], const_cast(ExecveArgs.data())); + if (Result.empty()) { + return -1; + } + return std::stoi(Result); + } + + int32_t AskForDistroSelection(DistroQuery::DistroInfo &Info, std::vector &Targets) { + // Search for an exact match + int32_t DistroIndex = -1; + if (!Info.Unknown) { + for (size_t i = 0; i < Targets.size(); ++i) { + const auto &Target = Targets[i]; + + bool ExactMatch = Target.DistroMatch == Info.DistroName && + Target.VersionMatch == Info.DistroVersion; + if (ExactMatch) { + std::string Question = fmt::format("Found exact match for distro '{}'. Do you want to select this image?", Target.DistroName); + if (ExecWithQuestion(Question)) { + DistroIndex = i; + break; + } + } + } + } + + if (DistroIndex != -1) { + return DistroIndex; + } + + std::vector Args; + + Args.emplace_back("--column=Index"); + Args.emplace_back("--column=Distro"); + Args.emplace_back("--hide-column=1"); + for (size_t i = 0; i < Targets.size(); ++i) { + const auto &Target = Targets[i]; + Args.emplace_back(std::to_string(i)); + Args.emplace_back(Target.DistroName); + } + + std::string Text = "RootFS list selection"; + return AskForComplexConfirmationList(Text, Args); + } + + bool ValidateCheckExists(const WebFileFetcher::FileTargets &Target) { + std::string RootFS = FEXCore::Config::GetDataDirectory() + "RootFS/"; + auto filename = Target.URL.substr(Target.URL.find_last_of('/') + 1); + auto PathName = RootFS + filename; + uint64_t ExpectedHash = std::stoul(Target.Hash, nullptr, 16); + + std::error_code ec; + if (std::filesystem::exists(PathName, ec)) { + const std::vector Args { + "Overwrite", + "Validate", + }; + std::string Text = filename + " already exists. What do you want to do?"; + int Result = AskForConfirmationList(Text, Args); + if (Result == -1) { + return false; + } + + auto Res = XXFileHash::HashFile(PathName); + if (Result == 0) { + if (Res.first == true && + Res.second == ExpectedHash) { + std::string Text = fmt::format("{} matches expected hash. Skipping download", filename); + ExecWithInfo(Text); + return false; + } + } + else if (Result == 1) { + if (Res.first == false || + Res.second != ExpectedHash) { + return AskForConfirmation("RootFS doesn't match hash!\nDo you want to redownload?"); + } + else { + std::string Text = fmt::format("{} matches expected hash", filename); + ExecWithInfo(Text); + return false; + } + } + } + + return true; + } + + bool ValidateDownloadSelection(const WebFileFetcher::FileTargets &Target) { + std::string Text = fmt::format("Selected Rootfs: {}\n", Target.DistroName); + Text += fmt::format("\tURL: {}\n", Target.URL); + Text += fmt::format("Are you sure that you want to download this image"); + + if (AskForConfirmation(Text)) { + std::string RootFS = FEXCore::Config::GetDataDirectory() + "RootFS/"; + std::error_code ec{}; + if (!std::filesystem::exists(RootFS, ec)) { + // Doesn't exist, create the the folder as a user convenience + if (!std::filesystem::create_directories(RootFS, ec)) { + // Well I guess we failed + Text = fmt::format("Couldn't create {} path for storing RootFS", RootFS); + ExecWithInfo(Text); + return false; + } + } + + if (!WebFileFetcher::DownloadToPathWithZenityProgress(Target.URL, RootFS)) { + ExecWithInfo("Couldn't download RootFS"); + return false; + } + + return true; + } + return false; + } +} + +namespace TTY { + bool AskForConfirmation(const std::string &Question) { + auto ToLowerInPlace = [](auto &Str) { + std::transform(Str.begin(), Str.end(), Str.begin(), + [](unsigned char c){ return std::tolower(c); }); + }; + + std::cout << Question << std::endl; + std::cout << "Response {y,yes,1} or {n,no,0}" << std::endl; + std::string Response; + std::cin >> Response; + + ToLowerInPlace(Response); + if (Response == "y" || + Response == "yes" || + Response == "1") { + return true; + } + else if (Response == "n" || + Response == "no" || + Response == "0") { + return false; + } + else { + std::cout << "Unknown response. Assuming no" << std::endl; + return false; + } + } + + void ExecWithInfo(const std::string &Text) { + std::cout << Text << std::endl; + } + + int32_t AskForConfirmationList(const std::string &Text, const std::vector &List) { + fmt::print("{}\n", Text); + fmt::print("Options:\n"); + fmt::print("\t0: Cancel\n"); + + for (size_t i = 0; i < List.size(); ++i) { + fmt::print("\t{}: {}\n", i+1, List[i]); + } + + fmt::print("\t\nResponse {{1-{}}} or 0 to cancel\n", List.size()); + std::string Response; + std::cin >> Response; + + int32_t ResponseInt = std::stoi(Response); + if (ResponseInt == 0) { + return -1; + } + else if (ResponseInt >= 1 && + (ResponseInt - 1) < List.size()) { + return ResponseInt - 1; + } + else { + std::cout << "Unknown response. Assuming cancel" << std::endl; + return -1; + } + } + + int32_t AskForDistroSelection(DistroQuery::DistroInfo &Info, std::vector &Targets) { + // Search for an exact match + int32_t DistroIndex = -1; + if (!Info.Unknown) { + for (size_t i = 0; i < Targets.size(); ++i) { + const auto &Target = Targets[i]; + + bool ExactMatch = Target.DistroMatch == Info.DistroName && + Target.VersionMatch == Info.DistroVersion; + if (ExactMatch) { + std::string Question = fmt::format("Found exact match for distro '{}'. Do you want to select this image?", Target.DistroName); + if (AskForConfirmation(Question)) { + DistroIndex = i; + break; + } + } + } + } + + if (DistroIndex != -1) { + return DistroIndex; + } + + std::vector Args; + for (size_t i = 0; i < Targets.size(); ++i) { + const auto &Target = Targets[i]; + Args.emplace_back(Target.DistroName); + } + + std::string Text = "RootFS list selection"; + return AskForConfirmationList(Text, Args); + } + + bool ValidateCheckExists(const WebFileFetcher::FileTargets &Target) { + std::string RootFS = FEXCore::Config::GetDataDirectory() + "RootFS/"; + auto filename = Target.URL.substr(Target.URL.find_last_of('/') + 1); + auto PathName = RootFS + filename; + uint64_t ExpectedHash = std::stoul(Target.Hash, nullptr, 16); + + std::error_code ec; + if (std::filesystem::exists(PathName, ec)) { + const std::vector Args { + "Overwrite", + "Validate", + }; + std::string Text = filename + " already exists. What do you want to do?"; + int Result = AskForConfirmationList(Text, Args); + if (Result == -1) { + return false; + } + fmt::print("Validating RootFS hash...\n"); + auto Res = XXFileHash::HashFile(PathName); + if (Result == 0) { + if (Res.first == true && + Res.second == ExpectedHash) { + fmt::print("{} matches expected hash. Skipping downloading\n", filename); + return false; + } + } + else if (Result == 1) { + if (Res.first == false || + Res.second != ExpectedHash) { + fmt::print("RootFS doesn't match hash!\n"); + return AskForConfirmation("Do you want to redownload?"); + } + else { + fmt::print("{} matches expected hash\n", filename); + return false; + } + } + } + + return true; + } + + bool ValidateDownloadSelection(const WebFileFetcher::FileTargets &Target) { + fmt::print("Selected Rootfs: {}\n", Target.DistroName); + fmt::print("\tURL: {}\n", Target.URL); + + if (AskForConfirmation("Are you sure that you want to download this image")) { + std::string RootFS = FEXCore::Config::GetDataDirectory() + "RootFS/"; + std::error_code ec{}; + if (!std::filesystem::exists(RootFS, ec)) { + // Doesn't exist, create the the folder as a user convenience + if (!std::filesystem::create_directories(RootFS, ec)) { + // Well I guess we failed + fmt::print("Couldn't create {} path for storing RootFS\n", RootFS); + return false; + } + } + + if (!WebFileFetcher::DownloadToPath(Target.URL, RootFS)) { + fmt::print("Couldn't download RootFS\n"); + return false; + } + + return true; + } + return false; + } +} + +namespace { + bool IsTTY{}; + + std::function _AskForConfirmation; + std::function _ExecWithInfo; + std::function &List)> _AskForConfirmationList; + std::function &Targets)> _AskForDistroSelection; + std::function _ValidateCheckExists; + std::function _ValidateDownloadSelection; + + void CheckTTY() { + IsTTY = isatty(STDOUT_FILENO); + + if (IsTTY) { + _AskForConfirmation = TTY::AskForConfirmation; + _ExecWithInfo = TTY::ExecWithInfo; + _AskForConfirmationList = TTY::AskForConfirmationList; + _AskForDistroSelection = TTY::AskForDistroSelection; + _ValidateCheckExists = TTY::ValidateCheckExists; + _ValidateDownloadSelection = TTY::ValidateDownloadSelection; + } + else { + _AskForConfirmation = Zenity::AskForConfirmation; + _ExecWithInfo = Zenity::ExecWithInfo; + _AskForConfirmationList = Zenity::AskForConfirmationList; + _AskForDistroSelection = Zenity::AskForDistroSelection; + _ValidateCheckExists = Zenity::ValidateCheckExists; + _ValidateDownloadSelection = Zenity::ValidateDownloadSelection; + } + } + + bool AskForConfirmation(const std::string &Question) { + return _AskForConfirmation(Question); + } + + void ExecWithInfo(const std::string &Text) { + _ExecWithInfo(Text); + } + + int32_t AskForConfirmationList(const std::string &Text, const std::vector &Arguments) { + return _AskForConfirmationList(Text, Arguments); + } + + int32_t AskForDistroSelection(std::vector &Targets) { + auto Info = DistroQuery::GetDistroInfo(); + return _AskForDistroSelection(Info, Targets); + } + + bool ValidateCheckExists(const WebFileFetcher::FileTargets &Target) { + return _ValidateCheckExists(Target); + } + + bool ValidateDownloadSelection(const WebFileFetcher::FileTargets &Target) { + return _ValidateDownloadSelection(Target); + } +} + +namespace ConfigSetter { + void SetRootFSAsDefault(const std::string &RootFS) { + std::string Filename = FEXCore::Config::GetConfigFileLocation(); + auto LoadedConfig = FEXCore::Config::CreateMainLayer(&Filename); + LoadedConfig->Load(); + LoadedConfig->EraseSet(FEXCore::Config::ConfigOption::CONFIG_ROOTFS, RootFS); + FEX::Config::SaveLayerToJSON(Filename, LoadedConfig.get()); + } +} + +namespace UnSquash { + bool UnsquashRootFS(const std::string &Path, const std::string &RootFS, const std::string &FolderName) { + auto TargetFolder = Path + FolderName; + + bool Extract = true; + std::error_code ec; + if (std::filesystem::exists(TargetFolder, ec)) { + std::string Question = FolderName + " Already exists. Overwrite?"; + if (AskForConfirmation(Question)) { + if (std::filesystem::remove_all(TargetFolder, ec) != ~0ULL) { + Extract = true; + } + } + } + + if (Extract) { + const std::vector ExecveArgs = { + "/usr/bin/unsquashfs", + "-f", + "-d", + TargetFolder.c_str(), + RootFS.c_str(), + nullptr, + }; + + return Exec::ExecAndWaitForResponse(ExecveArgs[0], const_cast(ExecveArgs.data())) == 0; + } + + return false; + } +} + +int main(int argc, char **argv, char **const envp) { + CheckTTY(); + FEX::Config::LoadConfig( + true, + false, + argc, argv, envp + ); + + // Reload the meta layer + FEXCore::Config::ReloadMetaLayer(); + + auto Args = FEX::ArgLoader::Get(); + auto ParsedArgs = FEX::ArgLoader::GetParsedArgs(); + + if (Args.size()) { + auto Res = XXFileHash::HashFile(Args[0]); + if (Res.first) { + fmt::print("{} has hash: 0x{:x}\n", Args[0], Res.second); + } + else { + fmt::print("Couldn't generate hash for {}\n", Args[0]); + } + return 0; + } + + FEX_CONFIG_OPT(LDPath, ROOTFS); + + std::error_code ec; + std::string Question{}; + if (LDPath().empty() || + std::filesystem::exists(LDPath(), ec) == false) { + Question = "RootFS not found. Do you want to try and download one?"; + } + else { + Question = "RootFS is already in use. Do you want to check the download list?"; + } + + if (AskForConfirmation(Question)) { + auto Targets = WebFileFetcher::GetRootFSLinks(); + int32_t DistroIndex = AskForDistroSelection(Targets); + if (DistroIndex != -1) { + const auto &Target = Targets[DistroIndex]; + std::string RootFS = FEXCore::Config::GetDataDirectory() + "RootFS/"; + auto filename = Target.URL.substr(Target.URL.find_last_of('/') + 1); + auto PathName = RootFS + filename; + + if (!ValidateCheckExists(Target)) { + // Keep going + } + else if (ValidateDownloadSelection(Target)) { + uint64_t ExpectedHash = std::stoul(Target.Hash, nullptr, 16); + + if (std::filesystem::exists(PathName, ec)) { + auto Res = XXFileHash::HashFile(PathName); + if (Res.first == false || + Res.second != ExpectedHash) { + std::string Text = fmt::format("Couldn't hash the rootfs or hash didn't match\n"); + Text += fmt::format("Hash 0x{:x} != Expected Hash 0x{:x}\n", Res.second, ExpectedHash); + ExecWithInfo(Text); + return -1; + } + } + else { + ExecWithInfo("Correctly downloaded RootFS but doesn't exist?"); + return -1; + } + } + + std::vector Args = { + "Extract", + "As-Is", + }; + auto Result = AskForConfirmationList("Do you wish to extract the squashfs file or use it as-is?", Args); + if (Result == -1 || + Result == 1) { + // Nothing + // As-Is + } + else if (Result == 0) { + auto FolderName = filename.substr(0, filename.find_last_of('.')); + if (UnSquash::UnsquashRootFS(RootFS, PathName, FolderName)) { + // Remove the .sqsh suffix since we extracted to that + filename = FolderName; + } + } + + if (AskForConfirmation("Do you wish to set this RootFS as default?")) { + ConfigSetter::SetRootFSAsDefault(filename); + auto Text = fmt::format("{} set as default RootFS\n", filename); + ExecWithInfo(Text); + } + } + } + + return 0; +} diff --git a/Source/Tools/FEXRootFSFetcher/XXFileHash.cpp b/Source/Tools/FEXRootFSFetcher/XXFileHash.cpp new file mode 100644 index 0000000000..87a37adecf --- /dev/null +++ b/Source/Tools/FEXRootFSFetcher/XXFileHash.cpp @@ -0,0 +1,84 @@ +#include "XXFileHash.h" + +#include +#include +#include +#include +#include +#include + +namespace XXFileHash { + // 32MB blocks + constexpr static size_t BLOCK_SIZE = 32 * 1024 * 1024; + std::pair HashFile(const std::string &Filepath) { + int fd = open(Filepath.c_str(), O_RDONLY); + if (fd == -1) { + return {false, 0}; + } + + auto HadError = [fd]() -> std::pair { + close(fd); + return {false, 0}; + }; + // Get file size + off_t Size = lseek(fd, 0, SEEK_END); + double SizeD = Size; + + // Reset to beginning + lseek(fd, 0, SEEK_SET); + + // Set up XXHash state + XXH64_state_t* const State = XXH64_createState(); + XXH64_hash_t const Seed = 0; + + if (!State) { + return HadError(); + } + + if (XXH64_reset(State, Seed) == XXH_ERROR) { + return HadError(); + } + + std::vector Data(BLOCK_SIZE); + off_t DataRemaining = Size - BLOCK_SIZE; + off_t DataTail = Size - DataRemaining; + off_t CurrentOffset = 0; + auto Now = std::chrono::high_resolution_clock::now(); + + // Let the kernel know that we will be reading linearly + posix_fadvise(fd, 0, Size, POSIX_FADV_SEQUENTIAL); + while (CurrentOffset < DataRemaining) { + ssize_t Result = pread(fd, Data.data(), BLOCK_SIZE, CurrentOffset); + if (Result == -1) { + return HadError(); + } + + if (XXH64_update(State, Data.data(), BLOCK_SIZE) == XXH_ERROR) { + return HadError(); + } + auto Cur = std::chrono::high_resolution_clock::now(); + auto Dur = Cur - Now; + if (Dur >= std::chrono::seconds(5)) { + fmt::print("{}% hashed\n", (double)CurrentOffset / SizeD); + Now = Cur; + } + CurrentOffset += BLOCK_SIZE; + } + + // Finish the tail + ssize_t Result = pread(fd, Data.data(), DataTail, CurrentOffset); + if (Result == -1) { + return HadError(); + } + + if (XXH64_update(State, Data.data(), DataTail) == XXH_ERROR) { + return HadError(); + } + + XXH64_hash_t const Hash = XXH64_digest(State); + XXH64_freeState(State); + + close(fd); + return {true, Hash}; + } +} diff --git a/Source/Tools/FEXRootFSFetcher/XXFileHash.h b/Source/Tools/FEXRootFSFetcher/XXFileHash.h new file mode 100644 index 0000000000..e1b8e5c983 --- /dev/null +++ b/Source/Tools/FEXRootFSFetcher/XXFileHash.h @@ -0,0 +1,7 @@ +#pragma once +#include + +namespace XXFileHash { + std::pair HashFile(const std::string &Filepath); +} +