From c68a3bf9ec4504252d6855ab8a650e050a1a1893 Mon Sep 17 00:00:00 2001 From: Alan Tse Date: Mon, 11 Mar 2024 12:45:32 -0700 Subject: [PATCH] feat: recompile automatically for changed shaders (#206) Adds feature where changes to the shader .hlsl files will automatically recompile shaders. Changes to RE::BSShader::Type files in Data\Shaders (e.g., Lighting.hlsl) will result in a type specific recompile. All other modification to hlsl/hlsli files will recompile all files since there isn't a way to detect what may be impacted. closes #205 --- CMakeLists.txt | 7 ++- src/ShaderCache.cpp | 124 +++++++++++++++++++++++++++++++++++++++++--- src/ShaderCache.h | 36 ++++++++++++- src/XSEPlugin.cpp | 2 +- vcpkg.json | 3 +- 5 files changed, 157 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d341f8ab4..7971c5d6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ find_path(CLIB_UTIL_INCLUDE_DIRS "ClibUtil/utils.hpp") find_package(pystring CONFIG REQUIRED) find_package(cppwinrt CONFIG REQUIRED) find_package(unordered_dense CONFIG REQUIRED) - +find_package(efsw CONFIG REQUIRED) target_include_directories( ${PROJECT_NAME} PRIVATE @@ -62,6 +62,7 @@ target_link_libraries( Microsoft::DirectXTex pystring::pystring unordered_dense::unordered_dense + efsw::efsw ) # https://gitlab.kitware.com/cmake/cmake/-/issues/24922#note_1371990 @@ -87,7 +88,6 @@ endif() # ####################################################################################################################### # # Feature version detection # ####################################################################################################################### - file(GLOB_RECURSE FEATURE_CONFIG_FILES LIST_DIRECTORIES false CONFIGURE_DEPENDS @@ -106,7 +106,7 @@ endforeach() set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${FEATURE_CONFIG_FILES}") -string (REPLACE ";" ",\n" FEATURE_VERSIONS "${FEATURE_VERSIONS}") +string(REPLACE ";" ",\n" FEATURE_VERSIONS "${FEATURE_VERSIONS}") configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/FeatureVersions.h.in @@ -189,7 +189,6 @@ endif() # Create a AIO zip for easier testing if(AIO_ZIP_TO_DIST) - if(NOT ZIP_TO_DIST) add_custom_target(build-time-make-directory ALL COMMAND ${CMAKE_COMMAND} -E remove_directory "${ZIP_DIR}" ${CMAKE_SOURCE_DIR}/dist diff --git a/src/ShaderCache.cpp b/src/ShaderCache.cpp index 1cdfb73b4..1962b360f 100644 --- a/src/ShaderCache.cpp +++ b/src/ShaderCache.cpp @@ -16,6 +16,12 @@ namespace SIE { static void GetShaderDefines(RE::BSShader::Type, uint32_t, D3D_SHADER_MACRO*); static std::string GetShaderString(ShaderClass, const RE::BSShader&, uint32_t, bool = false); + /** + @brief Get the BSShader::Type from the ShaderString + @param a_key The key generated from GetShaderString + @return A string with a valid BSShader::Type + */ + static std::string GetTypeFromShaderString(std::string); constexpr const char* VertexShaderProfile = "vs_5_0"; constexpr const char* PixelShaderProfile = "ps_5_0"; constexpr const char* ComputeShaderProfile = "cs_5_0"; @@ -952,6 +958,15 @@ namespace SIE return result; } + std::string GetTypeFromShaderString(std::string a_key) + { + std::string type = ""; + std::string::size_type pos = a_key.find(':'); + if (pos != std::string::npos) + type = a_key.substr(0, pos); + return type; + } + static ID3DBlob* CompileShader(ShaderClass shaderClass, const RE::BSShader& shader, uint32_t descriptor, bool useDiskCache) { ID3DBlob* shaderBlob = nullptr; @@ -971,7 +986,11 @@ namespace SIE if (!shaderBlob && useDiskCache && std::filesystem::exists(diskPath)) { shaderBlob = nullptr; - if (FAILED(D3DReadFileToBlob(diskPath.c_str(), &shaderBlob))) { + // check build time of cache + auto diskCacheTime = std::chrono::clock_cast(std::filesystem::last_write_time(diskPath)); + if (cache.ShaderModifiedSince(shader.fxpFilename, diskCacheTime)) { + logger::debug("Diskcached shader {} older than {}", SIE::SShaderCache::GetShaderString(shaderClass, shader, descriptor, true), std::format("{:%Y%m%d%H%M}", diskCacheTime)); + } else if (FAILED(D3DReadFileToBlob(diskPath.c_str(), &shaderBlob))) { logger::error("Failed to load {} shader {}::{}", magic_enum::enum_name(shaderClass), magic_enum::enum_name(type), descriptor); if (shaderBlob != nullptr) { @@ -1010,10 +1029,19 @@ namespace SIE defines[lastIndex] = { nullptr, nullptr }; // do final entry GetShaderDefines(type, descriptor, &defines[lastIndex]); - logger::debug("Defines set for {}:{}:{:X} to {}", magic_enum::enum_name(type), magic_enum::enum_name(shaderClass), descriptor, MergeDefinesString(defines)); + const std::wstring path = GetShaderPath(shader.fxpFilename); + + // Set timestamp based on file timestamp + auto shaderSourceTime = std::chrono::clock_cast(std::filesystem::last_write_time(path)); + cache.modifiedShaderMap.insert_or_assign(std::string(shader.fxpFilename), shaderSourceTime); + + std::string strPath; + std::transform(path.begin(), path.end(), std::back_inserter(strPath), [](wchar_t c) { + return (char)c; + }); + logger::debug("Compiling {} {}:{}:{:X} to {}", strPath, magic_enum::enum_name(type), magic_enum::enum_name(shaderClass), descriptor, MergeDefinesString(defines)); // compile shaders - const std::wstring path = GetShaderPath(shader.fxpFilename); ID3DBlob* errorBlob = nullptr; const uint32_t flags = D3DCOMPILE_OPTIMIZATION_LEVEL3; const HRESULT compileResult = D3DCompileFromFile(path.c_str(), defines.data(), D3D_COMPILE_STANDARD_FILE_INCLUDE, "main", @@ -1286,6 +1314,7 @@ namespace SIE ShaderCache::~ShaderCache() { Clear(); + fileWatcher->removeWatch(watchID); } void ShaderCache::Clear() @@ -1308,23 +1337,48 @@ namespace SIE shaderMap.clear(); } + void ShaderCache::Clear(RE::BSShader::Type a_type) + { + logger::debug("Clearing cache for {}", magic_enum::enum_name(a_type)); + std::lock_guard lockGuardV(vertexShadersMutex); + { + for (auto& [id, shader] : vertexShaders[static_cast(a_type)]) { + shader->shader->Release(); + } + vertexShaders[static_cast(a_type)].clear(); + } + std::lock_guard lockGuardP(pixelShadersMutex); + { + for (auto& [id, shader] : pixelShaders[static_cast(a_type)]) { + shader->shader->Release(); + } + pixelShaders[static_cast(a_type)].clear(); + } + compilationSet.Clear(); + } + bool ShaderCache::AddCompletedShader(ShaderClass shaderClass, const RE::BSShader& shader, uint32_t descriptor, ID3DBlob* a_blob) { auto key = SIE::SShaderCache::GetShaderString(shaderClass, shader, descriptor, true); auto status = a_blob ? ShaderCompilationTask::Status::Completed : ShaderCompilationTask::Status::Failed; std::unique_lock lock{ mapMutex }; logger::debug("Adding {} shader to map: {}", magic_enum ::enum_name(status), key); - shaderMap.insert_or_assign(key, std::pair(a_blob, status)); + shaderMap.insert_or_assign(key, ShaderCacheResult{ a_blob, status, system_clock::now() }); return (bool)a_blob; } ID3DBlob* ShaderCache::GetCompletedShader(const std::string a_key) { + std::string type = SIE::SShaderCache::GetTypeFromShaderString(a_key); std::scoped_lock lock{ mapMutex }; if (!shaderMap.empty() && shaderMap.contains(a_key)) { - auto status = shaderMap.at(a_key).second; + if (ShaderModifiedSince(type, shaderMap.at(a_key).compileTime)) { + logger::debug("Shader {} compiled {} before changes at {}", a_key, std::format("{:%Y%m%d%H%M}", shaderMap.at(a_key).compileTime), std::format("{:%Y%m%d%H%M}", modifiedShaderMap.at(type))); + return nullptr; + } + auto status = shaderMap.at(a_key).status; if (status != ShaderCompilationTask::Status::Pending) - return shaderMap.at(a_key).first; + return shaderMap.at(a_key).blob; } return nullptr; } @@ -1346,7 +1400,7 @@ namespace SIE { std::scoped_lock lock{ mapMutex }; if (!shaderMap.empty() && shaderMap.contains(a_key)) { - return shaderMap.at(a_key).second; + return shaderMap.at(a_key).status; } return ShaderCompilationTask::Status::Pending; } @@ -1452,6 +1506,29 @@ namespace SIE compilationPool.push_task(&ShaderCache::ManageCompilationSet, this, ssource.get_token()); } + void ShaderCache::StartFileWatcher() + { + fileWatcher = new efsw::FileWatcher(); + listener = new UpdateListener(); + // Add a folder to watch, and get the efsw::WatchID + // Reporting the files and directories changes to the instance of the listener + watchID = fileWatcher->addWatch("Data\\Shaders", listener, true); + // Start watching asynchronously the directories + fileWatcher->watch(); + std::string pathStr = ""; + for (auto path : fileWatcher->directories()) { + pathStr += std::format("{}; ", path); + } + logger::debug("ShaderCache watching for changes in {}", pathStr); + } + + bool ShaderCache::ShaderModifiedSince(std::string a_type, system_clock::time_point a_current) + { + return !a_type.empty() && magic_enum::enum_cast(a_type, magic_enum::case_insensitive).has_value() // type is valid + && !modifiedShaderMap.empty() && modifiedShaderMap.contains(a_type) // map has Type + && modifiedShaderMap.at(a_type) > a_current; //modification time is older than a_current + } + RE::BSGraphics::VertexShader* ShaderCache::MakeAndAddVertexShader(const RE::BSShader& shader, uint32_t descriptor) { @@ -1726,4 +1803,37 @@ namespace SIE GetHumanTime(totalMs), GetHumanTime(GetEta() + totalMs)); } + + void UpdateListener::handleFileAction(efsw::WatchID, const std::string& dir, const std::string& filename, efsw::Action action, std::string) + { + auto& cache = SIE::ShaderCache::Instance(); + const std::filesystem::path filePath = std::filesystem::path(std::format("{}\\{}", dir, filename)); + auto modifiedTime = std::chrono::clock_cast(std::filesystem::last_write_time(filePath)); + std::string extension = filePath.extension().string(); + std::string parentDir = filePath.parent_path().string(); + std::string shaderTypeString = filePath.stem().string(); + auto shaderType = magic_enum::enum_cast(shaderTypeString, magic_enum::case_insensitive); + switch (action) { + case efsw::Actions::Add: + break; + case efsw::Actions::Delete: + break; + case efsw::Actions::Modified: + logger::info("Detected changed file {}", filePath.string()); + if (extension.starts_with(".hlsl") && parentDir.ends_with("Shaders") && shaderType.has_value()) { // TODO: Case insensitive checks + // Shader types, so only invalidate specific shader type (e.g,. Lighting) + cache.modifiedShaderMap.insert_or_assign(shaderTypeString, modifiedTime); + cache.Clear(shaderType.value()); + } else if (extension.starts_with(".hlsl")) { // TODO: Case insensitive checks + // all other shaders, since we don't know what is using it, clear everything + cache.DeleteDiskCache(); + cache.Clear(); + } + break; + case efsw::Actions::Moved: + break; + default: + logger::error("Filewatcher received invalid action {}", magic_enum::enum_name(action)); + } + } } \ No newline at end of file diff --git a/src/ShaderCache.h b/src/ShaderCache.h index 9a122c74a..0e77efc3f 100644 --- a/src/ShaderCache.h +++ b/src/ShaderCache.h @@ -3,6 +3,7 @@ #include #include "BS_thread_pool.hpp" +#include "efsw/efsw.hpp" #include #include #include @@ -84,6 +85,15 @@ namespace SIE double totalMs = (double)duration_cast(lastReset - lastReset).count(); }; + struct ShaderCacheResult + { + ID3DBlob* blob; + ShaderCompilationTask::Status status; + system_clock::time_point compileTime = system_clock::now(); + }; + + class UpdateListener; + class ShaderCache { public: @@ -126,7 +136,16 @@ namespace SIE void DeleteDiskCache(); void ValidateDiskCache(); void WriteDiskCacheInfo(); + void StartFileWatcher(); + /** @brief Whether the ShaderFile for RE::BSShader::Type has been modified since the timestamp. + @param a_type Case insensitive string for the type of shader. E.g., Lighting + @param a_current The current time in system_clock::time_point. + @return True if the shader for the type (i.e., Lighting.hlsl) has been modified since the timestamp + */ + bool ShaderModifiedSince(std::string a_type, system_clock::time_point a_current); + void Clear(); + void Clear(RE::BSShader::Type a_type); bool AddCompletedShader(ShaderClass shaderClass, const RE::BSShader& shader, uint32_t descriptor, ID3DBlob* a_blob); ID3DBlob* GetCompletedShader(const std::string a_key); @@ -263,7 +282,8 @@ namespace SIE uint blockedKeyIndex = (uint)-1; // index in shaderMap; negative value indicates disabled std::string blockedKey = ""; - std::vector blockedIDs; // more than one descriptor could be blocked based on shader hash + std::vector blockedIDs; // more than one descriptor could be blocked based on shader hash + std::unordered_map modifiedShaderMap{}; // hashmap when a shader source file last modified private: ShaderCache(); @@ -289,7 +309,19 @@ namespace SIE std::mutex vertexShadersMutex; std::mutex pixelShadersMutex; CompilationSet compilationSet; - std::unordered_map> shaderMap{}; + std::unordered_map shaderMap{}; std::mutex mapMutex; + + // efsw file watcher + efsw::FileWatcher* fileWatcher; + efsw::WatchID watchID; + UpdateListener* listener; + }; + + // Inherits from the abstract listener class, and implements the the file action handler + class UpdateListener : public efsw::FileWatchListener + { + public: + void handleFileAction(efsw::WatchID, const std::string& dir, const std::string& filename, efsw::Action action, std::string) override; }; } diff --git a/src/XSEPlugin.cpp b/src/XSEPlugin.cpp index 072fc56b8..ebd4bd176 100644 --- a/src/XSEPlugin.cpp +++ b/src/XSEPlugin.cpp @@ -96,7 +96,7 @@ void MessageHandler(SKSE::MessagingInterface::Message* message) auto& shaderCache = SIE::ShaderCache::Instance(); shaderCache.ValidateDiskCache(); - + shaderCache.StartFileWatcher(); for (auto* feature : Feature::GetFeatureList()) { if (feature->loaded) { feature->PostPostLoad(); diff --git a/vcpkg.json b/vcpkg.json index 7d9f6c19b..9eeeb399a 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -24,7 +24,8 @@ }, "eastl", "clib-util", - "unordered-dense" + "unordered-dense", + "efsw" ], "overrides": [ {