From dbf43763508d212507b20107a4facf6bce115c41 Mon Sep 17 00:00:00 2001 From: Alan Tse Date: Mon, 18 Mar 2024 05:07:54 -0700 Subject: [PATCH] feat: disable fileWatcher by default (#225) * fix: fix detection of changed files * feat: queue filewatcher changes This helps when many files are changed and prevents constant recompiling as the cache is invalidated. * feat: disable fileWatcher by default FileWatcher is now under the Advanced Menu as a setting. This should disable the code that may be killing bad Linux installs by gating the use of `std::filesystem::last_write_time` and `esfw` behind a setting. --- src/Menu.cpp | 11 +++ src/ShaderCache.cpp | 202 +++++++++++++++++++++++++++++++------------- src/ShaderCache.h | 29 ++++++- src/State.cpp | 3 + src/XSEPlugin.cpp | 3 +- 5 files changed, 185 insertions(+), 63 deletions(-) diff --git a/src/Menu.cpp b/src/Menu.cpp index 2ed4aa669..019f1358a 100644 --- a/src/Menu.cpp +++ b/src/Menu.cpp @@ -361,6 +361,17 @@ void Menu::DrawSettings() "Enabling will save current settings as TEST config. " "This has no impact if no settings are changed. "); } + bool useFileWatcher = shaderCache.UseFileWatcher(); + ImGui::TableNextColumn(); + if (ImGui::Checkbox("Enable File Watcher", &useFileWatcher)) { + shaderCache.SetFileWatcher(useFileWatcher); + } + if (auto _tt = Util::HoverTooltipWrapper()) { + ImGui::Text( + "Automatically recompile shaders on file change. " + "Intended for developing."); + } + if (ImGui::Button("Dump Ini Settings", { -1, 0 })) { Util::DumpSettingsOptions(); } diff --git a/src/ShaderCache.cpp b/src/ShaderCache.cpp index 7edbfd4e8..14b5a2539 100644 --- a/src/ShaderCache.cpp +++ b/src/ShaderCache.cpp @@ -987,7 +987,7 @@ namespace SIE if (!shaderBlob && useDiskCache && std::filesystem::exists(diskPath)) { shaderBlob = nullptr; // check build time of cache - auto diskCacheTime = std::chrono::clock_cast(std::filesystem::last_write_time(diskPath)); + auto diskCacheTime = cache.UseFileWatcher() ? std::chrono::clock_cast(std::filesystem::last_write_time(diskPath)) : system_clock::now(); if (cache.ShaderModifiedSince(shader.fxpFilename, diskCacheTime)) { logger::debug("Diskcached shader {} older than {}", SIE::SShaderCache::GetShaderString(shaderClass, shader, descriptor, true), std::format("{:%Y%m%d%H%M}", diskCacheTime)); } else if (FAILED(D3DReadFileToBlob(diskPath.c_str(), &shaderBlob))) { @@ -1312,24 +1312,29 @@ namespace SIE ShaderCache::~ShaderCache() { Clear(); - fileWatcher->removeWatch(watchID); + StopFileWatcher(); } void ShaderCache::Clear() { - for (auto& shaders : vertexShaders) { - for (auto& [id, shader] : shaders) { - shader->shader->Release(); + std::lock_guard lockGuardV(vertexShadersMutex); + { + for (auto& shaders : vertexShaders) { + for (auto& [id, shader] : shaders) { + shader->shader->Release(); + } + shaders.clear(); } - shaders.clear(); } - for (auto& shaders : pixelShaders) { - for (auto& [id, shader] : shaders) { - shader->shader->Release(); + std::lock_guard lockGuardP(pixelShadersMutex); + { + for (auto& shaders : pixelShaders) { + for (auto& [id, shader] : shaders) { + shader->shader->Release(); + } + shaders.clear(); } - shaders.clear(); } - compilationSet.Clear(); std::unique_lock lock{ mapMutex }; shaderMap.clear(); @@ -1368,6 +1373,7 @@ namespace SIE ID3DBlob* ShaderCache::GetCompletedShader(const std::string a_key) { std::string type = SIE::SShaderCache::GetTypeFromShaderString(a_key); + UpdateShaderModifiedTime(a_key); std::scoped_lock lock{ mapMutex }; if (!shaderMap.empty() && shaderMap.contains(a_key)) { if (ShaderModifiedSince(type, shaderMap.at(a_key).compileTime)) { @@ -1504,33 +1510,82 @@ namespace SIE compilationPool.push_task(&ShaderCache::ManageCompilationSet, this, ssource.get_token()); } + bool ShaderCache::UseFileWatcher() const + { + return useFileWatcher; + } + + void ShaderCache::SetFileWatcher(bool value) + { + auto oldValue = useFileWatcher; + useFileWatcher = value; + if (useFileWatcher && !oldValue) + StartFileWatcher(); + else if (!useFileWatcher && oldValue) + StopFileWatcher(); + } + void ShaderCache::StartFileWatcher() { - fileWatcher = new efsw::FileWatcher(); - listener = new UpdateListener(); - // Add a folder to watch, and get the efsw::WatchID - // Reporting the files and directories changes to the instance of the listener - watchID = fileWatcher->addWatch("Data\\Shaders", listener, true); - // Start watching asynchronously the directories - fileWatcher->watch(); - std::string pathStr = ""; - for (auto path : fileWatcher->directories()) { - pathStr += std::format("{}; ", path); + logger::info("Starting FileWatcher"); + if (!fileWatcher) { + fileWatcher = new efsw::FileWatcher(); + listener = new UpdateListener(); + // Add a folder to watch, and get the efsw::WatchID + // Reporting the files and directories changes to the instance of the listener + watchID = fileWatcher->addWatch("Data\\Shaders", listener, true); + // Start watching asynchronously the directories + fileWatcher->watch(); + std::string pathStr = ""; + for (auto path : fileWatcher->directories()) { + pathStr += std::format("{}; ", path); + } + logger::debug("ShaderCache watching for changes in {}", pathStr); + compilationPool.push_task(&SIE::UpdateListener::processQueue, listener); + } else { + logger::debug("ShaderCache already enabled"); } - logger::debug("ShaderCache watching for changes in {}", pathStr); } - bool ShaderCache::ShaderModifiedSince(std::string a_type, system_clock::time_point a_current) + void ShaderCache::StopFileWatcher() + { + logger::info("Stopping FileWatcher"); + if (fileWatcher) { + fileWatcher->removeWatch(watchID); + fileWatcher = nullptr; + } + if (listener) { + listener = nullptr; + } + } + + bool ShaderCache::UpdateShaderModifiedTime(std::string a_type) { - if (a_type.empty() || magic_enum::enum_cast(a_type, magic_enum::case_insensitive).has_value()) // type is invalid + if (!UseFileWatcher()) + return false; + if (a_type.empty() || !magic_enum::enum_cast(a_type, magic_enum::case_insensitive).has_value()) // type is invalid return false; std::filesystem::path filePath{ SIE::SShaderCache::GetShaderPath(a_type) }; std::lock_guard lockGuard(modifiedMapMutex); - if (std::filesystem::exists(filePath) && - (modifiedShaderMap.empty() || !modifiedShaderMap.contains(a_type))) // insert timestamp when first seen; rely on filewatcher for subsequent changes - modifiedShaderMap.insert_or_assign(a_type, std::chrono::clock_cast(std::filesystem::last_write_time(filePath))); + if (std::filesystem::exists(filePath)) { + auto fileTime = std::chrono::clock_cast(std::filesystem::last_write_time(filePath)); + if (!modifiedShaderMap.contains(a_type) || modifiedShaderMap.at(a_type) != fileTime) { // insert if new or timestamp changed + modifiedShaderMap.insert_or_assign(a_type, fileTime); + return true; + } + } + return false; + } + + bool ShaderCache::ShaderModifiedSince(std::string a_type, system_clock::time_point a_current) + { + if (!UseFileWatcher()) + return false; + if (a_type.empty() || !magic_enum::enum_cast(a_type, magic_enum::case_insensitive).has_value()) // type is invalid + return false; + std::lock_guard lockGuard(modifiedMapMutex); return !modifiedShaderMap.empty() && modifiedShaderMap.contains(a_type) // map has Type - && modifiedShaderMap.at(a_type) > a_current; //modification time is older than a_current + && modifiedShaderMap.at(a_type) > a_current; //modification time is newer than a_current } RE::BSGraphics::VertexShader* ShaderCache::MakeAndAddVertexShader(const RE::BSShader& shader, @@ -1820,40 +1875,67 @@ namespace SIE GetHumanTime(GetEta() + totalMs)); } - void UpdateListener::handleFileAction(efsw::WatchID, const std::string& dir, const std::string& filename, efsw::Action action, std::string) + void UpdateListener::processQueue() { + SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL); + std::unique_lock lock(actionMutex, std::defer_lock); auto& cache = SIE::ShaderCache::Instance(); - const std::filesystem::path filePath = std::filesystem::path(std::format("{}\\{}", dir, filename)); - std::chrono::time_point modifiedTime{}; - std::string extension = filePath.extension().string(); - std::string parentDir = filePath.parent_path().string(); - std::string shaderTypeString = filePath.stem().string(); - auto shaderType = magic_enum::enum_cast(shaderTypeString, magic_enum::case_insensitive); - switch (action) { - case efsw::Actions::Add: - break; - case efsw::Actions::Delete: - break; - case efsw::Actions::Modified: - logger::debug("Detected changed path {}", filePath.string()); - if (std::filesystem::exists(filePath)) - modifiedTime = std::chrono::clock_cast(std::filesystem::last_write_time(filePath)); - else // if file doesn't exist, don't do anything - return; - if (!std::filesystem::is_directory(filePath) && extension.starts_with(".hlsl") && parentDir.ends_with("Shaders") && shaderType.has_value()) { // TODO: Case insensitive checks - // Shader types, so only invalidate specific shader type (e.g,. Lighting) - cache.InsertModifiedShaderMap(shaderTypeString, modifiedTime); - cache.Clear(shaderType.value()); - } else if (!std::filesystem::is_directory(filePath) && extension.starts_with(".hlsl")) { // TODO: Case insensitive checks - // all other shaders, since we don't know what is using it, clear everything - cache.DeleteDiskCache(); - cache.Clear(); - } - break; - case efsw::Actions::Moved: - break; - default: - logger::error("Filewatcher received invalid action {}", magic_enum::enum_name(action)); + while (cache.UseFileWatcher()) { + lock.lock(); + if (!queue.empty() && queue.size() == lastQueueSize) { + bool clearCache = false; + for (fileAction fAction : queue) { + const std::filesystem::path filePath = std::filesystem::path(std::format("{}\\{}", fAction.dir, fAction.filename)); + std::chrono::time_point modifiedTime{}; + std::string extension = filePath.extension().string(); + std::string parentDir = filePath.parent_path().string(); + std::string shaderTypeString = filePath.stem().string(); + auto shaderType = magic_enum::enum_cast(shaderTypeString, magic_enum::case_insensitive); + switch (fAction.action) { + case efsw::Actions::Add: + break; + case efsw::Actions::Delete: + break; + case efsw::Actions::Modified: + logger::debug("Detected changed path {}", filePath.string()); + if (std::filesystem::exists(filePath)) + modifiedTime = std::chrono::clock_cast(std::filesystem::last_write_time(filePath)); + else // if file doesn't exist, don't do anything + return; + if (!std::filesystem::is_directory(filePath) && extension.starts_with(".hlsl") && parentDir.ends_with("Shaders") && shaderType.has_value()) { // TODO: Case insensitive checks + // Shader types, so only invalidate specific shader type (e.g,. Lighting) + cache.InsertModifiedShaderMap(shaderTypeString, modifiedTime); + cache.Clear(shaderType.value()); + } else if (!std::filesystem::is_directory(filePath) && extension.starts_with(".hlsl")) { // TODO: Case insensitive checks + // all other shaders, since we don't know what is using it, clear everything + clearCache = true; + } + break; + case efsw::Actions::Moved: + break; + default: + logger::error("Filewatcher received invalid action {}", magic_enum::enum_name(fAction.action)); + } + } + if (clearCache) { + cache.DeleteDiskCache(); + cache.Clear(); + } + queue.clear(); + } + lastQueueSize = queue.size(); + lock.unlock(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + queue.clear(); + } + + void UpdateListener::handleFileAction(efsw::WatchID watchid, const std::string& dir, const std::string& filename, efsw::Action action, std::string oldFilename) + { + std::lock_guard lock(actionMutex); + if (queue.empty() || (queue.back().action != action && queue.back().filename != filename)) { + // only add if not a duplicate; esfw is very spammy + queue.push_back({ watchid, dir, filename, action, oldFilename }); } } } \ No newline at end of file diff --git a/src/ShaderCache.h b/src/ShaderCache.h index be394941d..5db50d0f7 100644 --- a/src/ShaderCache.h +++ b/src/ShaderCache.h @@ -136,7 +136,17 @@ namespace SIE void DeleteDiskCache(); void ValidateDiskCache(); void WriteDiskCacheInfo(); + bool UseFileWatcher() const; + void SetFileWatcher(bool value); + void StartFileWatcher(); + void StopFileWatcher(); + + /** @brief Update the RE::BSShader::Type timestamp based on timestamp. + @param a_type Case insensitive string for the type of shader. E.g., Lighting + @return True if the shader for the type (i.e., Lighting.hlsl) timestamp was updated + */ + bool UpdateShaderModifiedTime(std::string a_type); /** @brief Whether the ShaderFile for RE::BSShader::Type has been modified since the timestamp. @param a_type Case insensitive string for the type of shader. E.g., Lighting @param a_current The current time in system_clock::time_point. @@ -306,6 +316,7 @@ namespace SIE bool isAsync = true; bool isDump = false; bool hideError = false; + bool useFileWatcher = false; std::stop_source ssource; std::mutex vertexShadersMutex; @@ -317,15 +328,29 @@ namespace SIE std::mutex modifiedMapMutex; // efsw file watcher - efsw::FileWatcher* fileWatcher; + efsw::FileWatcher* fileWatcher = nullptr; efsw::WatchID watchID; - UpdateListener* listener; + UpdateListener* listener = nullptr; }; // Inherits from the abstract listener class, and implements the the file action handler class UpdateListener : public efsw::FileWatchListener { public: + void processQueue(); void handleFileAction(efsw::WatchID, const std::string& dir, const std::string& filename, efsw::Action action, std::string) override; + + private: + struct fileAction + { + efsw::WatchID watchID; + std::string dir; + std::string filename; + efsw::Action action; + std::string oldFilename; + }; + std::mutex actionMutex; + std::vector queue{}; + size_t lastQueueSize = queue.size(); }; } diff --git a/src/State.cpp b/src/State.cpp index 2e40af3ba..c8dec244d 100644 --- a/src/State.cpp +++ b/src/State.cpp @@ -222,6 +222,8 @@ void State::Load(bool a_test) shaderCache.compilationThreadCount = std::clamp(advanced["Compiler Threads"].get(), 1, static_cast(std::thread::hardware_concurrency())); if (advanced["Background Compiler Threads"].is_number_integer()) shaderCache.backgroundCompilationThreadCount = std::clamp(advanced["Background Compiler Threads"].get(), 1, static_cast(std::thread::hardware_concurrency())); + if (advanced["Use FileWatcher"].is_boolean()) + shaderCache.SetFileWatcher(advanced["Use FileWatcher"]); } if (settings["General"].is_object()) { @@ -270,6 +272,7 @@ void State::Save(bool a_test) advanced["Shader Defines"] = shaderDefinesString; advanced["Compiler Threads"] = shaderCache.compilationThreadCount; advanced["Background Compiler Threads"] = shaderCache.backgroundCompilationThreadCount; + advanced["Use FileWatcher"] = shaderCache.UseFileWatcher(); settings["Advanced"] = advanced; json general; diff --git a/src/XSEPlugin.cpp b/src/XSEPlugin.cpp index ebd4bd176..350a9ac8d 100644 --- a/src/XSEPlugin.cpp +++ b/src/XSEPlugin.cpp @@ -96,7 +96,8 @@ void MessageHandler(SKSE::MessagingInterface::Message* message) auto& shaderCache = SIE::ShaderCache::Instance(); shaderCache.ValidateDiskCache(); - shaderCache.StartFileWatcher(); + if (shaderCache.UseFileWatcher()) + shaderCache.StartFileWatcher(); for (auto* feature : Feature::GetFeatureList()) { if (feature->loaded) { feature->PostPostLoad();