Merge pull request #18742 from hrydgard/memstick-move-cleanup

Preparations for reworking the memstick folder move functionality

CMakeLists.txt

@@ -2276,6 +2276,8 @@ add_library(${CoreLibName} ${CoreLinkType}
 	Core/Util/AudioFormat.h
 	Core/Util/GameManager.cpp
 	Core/Util/GameManager.h
+	Core/Util/MemStick.cpp
+	Core/Util/MemStick.h
 	Core/Util/GameDB.cpp
 	Core/Util/GameDB.h
 	Core/Util/PortManager.cpp

Core/Core.vcxproj

@@ -1079,6 +1079,7 @@
     <ClCompile Include="Util\DisArm64.cpp" />
     <ClCompile Include="Util\GameDB.cpp" />
     <ClCompile Include="Util\GameManager.cpp" />
+    <ClCompile Include="Util\MemStick.cpp" />
     <ClCompile Include="Util\PortManager.cpp" />
     <ClCompile Include="Util\PPGeDraw.cpp" />
     <ClCompile Include="..\ext\xxhash.c">
@@ -1449,6 +1450,7 @@
     <ClInclude Include="Util\DisArm64.h" />
     <ClInclude Include="Util\GameDB.h" />
     <ClInclude Include="Util\GameManager.h" />
+    <ClInclude Include="Util\MemStick.h" />
     <ClInclude Include="Util\PortManager.h" />
     <ClInclude Include="Util\PPGeDraw.h" />
     <ClInclude Include="..\ext\xxhash.h" />

Core/Core.vcxproj.filters

@@ -1303,6 +1303,9 @@
     <ClCompile Include="Util\GameDB.cpp">
       <Filter>Util</Filter>
     </ClCompile>
+    <ClCompile Include="Util\MemStick.cpp">
+      <Filter>Util</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="ELF\ElfReader.h">
@@ -2082,6 +2085,9 @@
     <ClInclude Include="Util\GameDB.h">
       <Filter>Util</Filter>
     </ClInclude>
+    <ClInclude Include="Util\MemStick.h">
+      <Filter>Util</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <None Include="..\LICENSE.TXT" />

Core/Util/MemStick.cpp

@@ -0,0 +1,205 @@
+#include "Common/File/Path.h"
+#include "Common/File/FileUtil.h"
+#include "Common/File/DirListing.h"
+#include "Common/Log.h"
+#include "Common/StringUtils.h"
+#include "Common/Data/Text/I18n.h"
+#include "Common/Data/Text/Parsers.h"
+
+#include "Core/Util/MemStick.h"
+#include "Core/Config.h"
+#include "Core/System.h"
+#include "Core/Reporting.h"
+
+bool FolderSeemsToBeUsed(const Path &newMemstickFolder) {
+	// Inspect the potential new folder, quickly.
+	if (File::Exists(newMemstickFolder / "PSP/SAVEDATA") || File::Exists(newMemstickFolder / "SAVEDATA")) {
+		// Does seem likely. We could add more criteria like checking for actual savegames or something.
+		return true;
+	} else {
+		return false;
+	}
+}
+
+bool SwitchMemstickFolderTo(Path newMemstickFolder) {
+	// Doesn't already exist, create.
+	// Should this ever happen?
+	if (newMemstickFolder.Type() == PathType::NATIVE) {
+		if (!File::Exists(newMemstickFolder)) {
+			File::CreateFullPath(newMemstickFolder);
+		}
+		Path testWriteFile = newMemstickFolder / ".write_verify_file";
+		if (!File::WriteDataToFile(true, "1", 1, testWriteFile)) {
+			return false;
+		}
+		File::Delete(testWriteFile);
+	} else {
+		// TODO: Do the same but with scoped storage? Not really necessary, right? If it came from a browse
+		// for folder, we can assume it exists and is writable, barring wacky race conditions like the user
+		// being connected by USB and deleting it.
+	}
+
+	Path memStickDirFile = g_Config.internalDataDirectory / "memstick_dir.txt";
+#if PPSSPP_PLATFORM(UWP)
+	File::Delete(memStickDirFile);
+	if (newMemstickFolder != g_Config.internalDataDirectory) {
+#endif
+
+		std::string str = newMemstickFolder.ToString();
+		if (!File::WriteDataToFile(true, str.c_str(), (unsigned int)str.size(), memStickDirFile)) {
+			ERROR_LOG(SYSTEM, "Failed to write memstick path '%s' to '%s'", newMemstickFolder.c_str(), memStickDirFile.c_str());
+			// Not sure what to do if this file can't be written.  Disk full?
+		}
+
+#if PPSSPP_PLATFORM(UWP)
+	}
+#endif
+
+	// Save so the settings, at least, are transferred.
+	g_Config.memStickDirectory = newMemstickFolder;
+	g_Config.SetSearchPath(GetSysDirectory(DIRECTORY_SYSTEM));
+	g_Config.UpdateIniLocation();
+	return true;
+}
+
+
+// Keep the size with the file, so we can skip overly large ones in the move.
+// The user will have to take care of them afterwards, it'll just take too long probably.
+struct FileSuffix {
+	std::string suffix;
+	u64 fileSize;
+};
+
+static bool ListFileSuffixesRecursively(const Path &root, const Path &folder, std::vector<std::string> &dirSuffixes, std::vector<FileSuffix> &fileSuffixes) {
+	std::vector<File::FileInfo> files;
+	if (!File::GetFilesInDir(folder, &files)) {
+		return false;
+	}
+
+	for (auto &file : files) {
+		if (file.isDirectory) {
+			std::string dirSuffix;
+			if (root.ComputePathTo(file.fullName, dirSuffix)) {
+				if (!dirSuffix.empty()) {
+					dirSuffixes.push_back(dirSuffix);
+					ListFileSuffixesRecursively(root, folder / file.name, dirSuffixes, fileSuffixes);
+				}
+			} else {
+				ERROR_LOG_REPORT(SYSTEM, "Failed to compute PathTo from '%s' to '%s'", root.c_str(), folder.c_str());
+			}
+		} else {
+			std::string fileSuffix;
+			if (root.ComputePathTo(file.fullName, fileSuffix)) {
+				if (!fileSuffix.empty()) {
+					fileSuffixes.push_back(FileSuffix{ fileSuffix, file.size });
+				}
+			}
+		}
+	}
+
+	return true;
+}
+
+MoveResult *MoveDirectoryContentsSafe(Path moveSrc, Path moveDest, MoveProgressReporter &progressReporter) {
+	auto ms = GetI18NCategory(I18NCat::MEMSTICK);
+	if (moveSrc.GetFilename() != "PSP") {
+		moveSrc = moveSrc / "PSP";
+	}
+	if (moveDest.GetFilename() != "PSP") {
+		moveDest = moveDest / "PSP";
+		File::CreateDir(moveDest);
+	}
+
+	INFO_LOG(SYSTEM, "About to move PSP data from '%s' to '%s'", moveSrc.c_str(), moveDest.c_str());
+
+	// Search through recursively, listing the files to move and also summing their sizes.
+	std::vector<FileSuffix> fileSuffixesToMove;
+	std::vector<std::string> directorySuffixesToCreate;
+
+	// NOTE: It's correct to pass moveSrc twice here, it's to keep the root in the recursion.
+	if (!ListFileSuffixesRecursively(moveSrc, moveSrc, directorySuffixesToCreate, fileSuffixesToMove)) {
+		// TODO: Handle failure listing files.
+		std::string error = "Failed to read old directory";
+		INFO_LOG(SYSTEM, "%s", error.c_str());
+		progressReporter.Set(ms->T(error.c_str()));
+		return new MoveResult{ false, error };
+	}
+
+	bool dryRun = false;  // Useful for debugging.
+
+	size_t failedFiles = 0;
+	size_t skippedFiles = 0;
+
+	// We're not moving huge files like ISOs during this process, unless
+	// they can be directly moved, without rewriting the file.
+	const uint64_t BIG_FILE_THRESHOLD = 24 * 1024 * 1024;
+
+	if (!moveSrc.empty()) {
+		// Better not interrupt the app while this is happening!
+
+		// Create all the necessary directories.
+		for (auto &dirSuffix : directorySuffixesToCreate) {
+			Path dir = moveDest / dirSuffix;
+			if (dryRun) {
+				INFO_LOG(SYSTEM, "dry run: Would have created dir '%s'", dir.c_str());
+			} else {
+				INFO_LOG(SYSTEM, "Creating dir '%s'", dir.c_str());
+				progressReporter.Set(dirSuffix);
+				// Just ignore already-exists errors.
+				File::CreateDir(dir);
+			}
+		}
+
+		for (auto &fileSuffix : fileSuffixesToMove) {
+			progressReporter.Set(StringFromFormat("%s (%s)", fileSuffix.suffix.c_str(), NiceSizeFormat(fileSuffix.fileSize).c_str()));
+
+			Path from = moveSrc / fileSuffix.suffix;
+			Path to = moveDest / fileSuffix.suffix;
+
+			if (fileSuffix.fileSize > BIG_FILE_THRESHOLD) {
+				// We only move big files if it's fast to do so.
+				if (dryRun) {
+					INFO_LOG(SYSTEM, "dry run: Would have moved '%s' to '%s' (%d bytes) if fast", from.c_str(), to.c_str(), (int)fileSuffix.fileSize);
+				} else {
+					if (!File::MoveIfFast(from, to)) {
+						INFO_LOG(SYSTEM, "Skipped moving file '%s' to '%s' (%s)", from.c_str(), to.c_str(), NiceSizeFormat(fileSuffix.fileSize).c_str());
+						skippedFiles++;
+					} else {
+						INFO_LOG(SYSTEM, "Moved file '%s' to '%s'", from.c_str(), to.c_str());
+					}
+				}
+			} else {
+				if (dryRun) {
+					INFO_LOG(SYSTEM, "dry run: Would have moved '%s' to '%s' (%d bytes)", from.c_str(), to.c_str(), (int)fileSuffix.fileSize);
+				} else {
+					// Remove the "from" prefix from the path.
+					// We have to drop down to string operations for this.
+					if (!File::Move(from, to)) {
+						ERROR_LOG(SYSTEM, "Failed to move file '%s' to '%s'", from.c_str(), to.c_str());
+						failedFiles++;
+						// Should probably just bail?
+					} else {
+						INFO_LOG(SYSTEM, "Moved file '%s' to '%s'", from.c_str(), to.c_str());
+					}
+				}
+			}
+		}
+
+		// Delete all the old, now hopefully empty, directories.
+		// Hopefully DeleteDir actually fails if it contains a file...
+		for (auto &dirSuffix : directorySuffixesToCreate) {
+			Path dir = moveSrc / dirSuffix;
+			if (dryRun) {
+				INFO_LOG(SYSTEM, "dry run: Would have deleted dir '%s'", dir.c_str());
+			} else {
+				INFO_LOG(SYSTEM, "Deleting dir '%s'", dir.c_str());
+				progressReporter.Set(dirSuffix);
+				if (File::Exists(dir)) {
+					File::DeleteDir(dir);
+				}
+			}
+		}
+	}
+
+	return new MoveResult{ true, "", failedFiles };
+}

Core/Util/MemStick.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "Common/File/Path.h"
+
+#include <mutex>
+
+// Utility functions moved out from MemstickScreen.
+
+class MoveProgressReporter {
+public:
+	void Set(const std::string &value) {
+		std::lock_guard<std::mutex> guard(mutex_);
+		progress_ = value;
+	}
+
+	std::string Get() {
+		std::lock_guard<std::mutex> guard(mutex_);
+		return progress_;
+	}
+
+private:
+	std::string progress_;
+	std::mutex mutex_;
+};
+
+struct MoveResult {
+	bool success;  // Got through the whole move.
+	std::string errorMessage;
+	size_t failedFiles;
+	size_t skippedFiles;
+};
+
+bool FolderSeemsToBeUsed(const Path &newMemstickFolder);
+bool SwitchMemstickFolderTo(Path newMemstickFolder);
+MoveResult *MoveDirectoryContentsSafe(Path moveSrc, Path moveDest, MoveProgressReporter &progressReporter);

GPU/Common/TextureReplacer.cpp

@@ -735,6 +735,7 @@ bool TextureReplacer::WillSave(const ReplacedTextureDecodeInfo &replacedInfo) {
 
 void TextureReplacer::NotifyTextureDecoded(ReplacedTexture *texture, const ReplacedTextureDecodeInfo &replacedInfo, const void *data, int pitch, int level, int origW, int origH, int scaledW, int scaledH) {
 	_assert_msg_(enabled_, "Replacement not enabled");
+	_assert_(pitch >= 0);
 
 	if (!WillSave(replacedInfo)) {
 		// Ignore.
@@ -798,7 +799,7 @@ void TextureReplacer::NotifyTextureDecoded(ReplacedTexture *texture, const Repla
 	// while we're at it.
 	saveBuf.resize(w * h * 4);
 	for (int y = 0; y < h; y++) {
-		memcpy((u8 *)saveBuf.data() + y * w * 4, (const u8 *)data + y * pitch, w * sizeof(u32));
+		memcpy((u8 *)saveBuf.data() + y * w * 4, (const u8 *)data + y * pitch, w * 4);
 	}
 	pitch = w * 4;