Skip to content

Commit

Permalink
feat: recompile automatically for changed shaders (#206)
Browse files Browse the repository at this point in the history 
Adds feature where changes to the shader .hlsl files will automatically recompile shaders.
Changes to RE::BSShader::Type files in Data\Shaders (e.g., Lighting.hlsl) will result in a type specific recompile. All other modification to hlsl/hlsli files will recompile all files since there isn't a way to detect what may be impacted.

closes #205
  • Loading branch information
@alandtse
alandtse authored Mar 11, 2024
1 parent 8fa27ac commit c68a3bf
Show file tree
Hide file tree
Showing 5 changed files with 157 additions and 15 deletions.
7 changes: 3 additions & 4 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ find_path(CLIB_UTIL_INCLUDE_DIRS "ClibUtil/utils.hpp")
find_package(pystring CONFIG REQUIRED)
find_package(cppwinrt CONFIG REQUIRED)
find_package(unordered_dense CONFIG REQUIRED)

find_package(efsw CONFIG REQUIRED)
target_include_directories(
${PROJECT_NAME}
PRIVATE
Expand All @@ -62,6 +62,7 @@ target_link_libraries(
Microsoft::DirectXTex
pystring::pystring
unordered_dense::unordered_dense
efsw::efsw
)

# https://gitlab.kitware.com/cmake/cmake/-/issues/24922#note_1371990
Expand All @@ -87,7 +88,6 @@ endif()
# #######################################################################################################################
# # Feature version detection
# #######################################################################################################################

file(GLOB_RECURSE FEATURE_CONFIG_FILES
LIST_DIRECTORIES false
CONFIGURE_DEPENDS
Expand All @@ -106,7 +106,7 @@ endforeach()

set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${FEATURE_CONFIG_FILES}")

string (REPLACE ";" ",\n" FEATURE_VERSIONS "${FEATURE_VERSIONS}")
string(REPLACE ";" ",\n" FEATURE_VERSIONS "${FEATURE_VERSIONS}")

configure_file(
${CMAKE_CURRENT_SOURCE_DIR}/cmake/FeatureVersions.h.in
Expand Down Expand Up @@ -189,7 +189,6 @@ endif()

# Create a AIO zip for easier testing
if(AIO_ZIP_TO_DIST)

if(NOT ZIP_TO_DIST)
add_custom_target(build-time-make-directory ALL
COMMAND ${CMAKE_COMMAND} -E remove_directory "${ZIP_DIR}" ${CMAKE_SOURCE_DIR}/dist
Expand Down
124 changes: 117 additions & 7 deletions src/ShaderCache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@ namespace SIE
{
static void GetShaderDefines(RE::BSShader::Type, uint32_t, D3D_SHADER_MACRO*);
static std::string GetShaderString(ShaderClass, const RE::BSShader&, uint32_t, bool = false);
/**
@brief Get the BSShader::Type from the ShaderString
@param a_key The key generated from GetShaderString
@return A string with a valid BSShader::Type
*/
static std::string GetTypeFromShaderString(std::string);
constexpr const char* VertexShaderProfile = "vs_5_0";
constexpr const char* PixelShaderProfile = "ps_5_0";
constexpr const char* ComputeShaderProfile = "cs_5_0";
Expand Down Expand Up @@ -952,6 +958,15 @@ namespace SIE
return result;
}

std::string GetTypeFromShaderString(std::string a_key)
{
std::string type = "";
std::string::size_type pos = a_key.find(':');
if (pos != std::string::npos)
type = a_key.substr(0, pos);
return type;
}

static ID3DBlob* CompileShader(ShaderClass shaderClass, const RE::BSShader& shader, uint32_t descriptor, bool useDiskCache)
{
ID3DBlob* shaderBlob = nullptr;
Expand All @@ -971,7 +986,11 @@ namespace SIE

if (!shaderBlob && useDiskCache && std::filesystem::exists(diskPath)) {
shaderBlob = nullptr;
if (FAILED(D3DReadFileToBlob(diskPath.c_str(), &shaderBlob))) {
// check build time of cache
auto diskCacheTime = std::chrono::clock_cast<std::chrono::system_clock>(std::filesystem::last_write_time(diskPath));
if (cache.ShaderModifiedSince(shader.fxpFilename, diskCacheTime)) {
logger::debug("Diskcached shader {} older than {}", SIE::SShaderCache::GetShaderString(shaderClass, shader, descriptor, true), std::format("{:%Y%m%d%H%M}", diskCacheTime));
} else if (FAILED(D3DReadFileToBlob(diskPath.c_str(), &shaderBlob))) {
logger::error("Failed to load {} shader {}::{}", magic_enum::enum_name(shaderClass), magic_enum::enum_name(type), descriptor);

if (shaderBlob != nullptr) {
Expand Down Expand Up @@ -1010,10 +1029,19 @@ namespace SIE
defines[lastIndex] = { nullptr, nullptr }; // do final entry
GetShaderDefines(type, descriptor, &defines[lastIndex]);

logger::debug("Defines set for {}:{}:{:X} to {}", magic_enum::enum_name(type), magic_enum::enum_name(shaderClass), descriptor, MergeDefinesString(defines));
const std::wstring path = GetShaderPath(shader.fxpFilename);

// Set timestamp based on file timestamp
auto shaderSourceTime = std::chrono::clock_cast<std::chrono::system_clock>(std::filesystem::last_write_time(path));
cache.modifiedShaderMap.insert_or_assign(std::string(shader.fxpFilename), shaderSourceTime);

std::string strPath;
std::transform(path.begin(), path.end(), std::back_inserter(strPath), [](wchar_t c) {
return (char)c;
});
logger::debug("Compiling {} {}:{}:{:X} to {}", strPath, magic_enum::enum_name(type), magic_enum::enum_name(shaderClass), descriptor, MergeDefinesString(defines));

// compile shaders
const std::wstring path = GetShaderPath(shader.fxpFilename);
ID3DBlob* errorBlob = nullptr;
const uint32_t flags = D3DCOMPILE_OPTIMIZATION_LEVEL3;
const HRESULT compileResult = D3DCompileFromFile(path.c_str(), defines.data(), D3D_COMPILE_STANDARD_FILE_INCLUDE, "main",
Expand Down Expand Up @@ -1286,6 +1314,7 @@ namespace SIE
ShaderCache::~ShaderCache()
{
Clear();
fileWatcher->removeWatch(watchID);
}

void ShaderCache::Clear()
Expand All @@ -1308,23 +1337,48 @@ namespace SIE
shaderMap.clear();
}

void ShaderCache::Clear(RE::BSShader::Type a_type)
{
logger::debug("Clearing cache for {}", magic_enum::enum_name(a_type));
std::lock_guard lockGuardV(vertexShadersMutex);
{
for (auto& [id, shader] : vertexShaders[static_cast<size_t>(a_type)]) {
shader->shader->Release();
}
vertexShaders[static_cast<size_t>(a_type)].clear();
}
std::lock_guard lockGuardP(pixelShadersMutex);
{
for (auto& [id, shader] : pixelShaders[static_cast<size_t>(a_type)]) {
shader->shader->Release();
}
pixelShaders[static_cast<size_t>(a_type)].clear();
}
compilationSet.Clear();
}

bool ShaderCache::AddCompletedShader(ShaderClass shaderClass, const RE::BSShader& shader, uint32_t descriptor, ID3DBlob* a_blob)
{
auto key = SIE::SShaderCache::GetShaderString(shaderClass, shader, descriptor, true);
auto status = a_blob ? ShaderCompilationTask::Status::Completed : ShaderCompilationTask::Status::Failed;
std::unique_lock lock{ mapMutex };
logger::debug("Adding {} shader to map: {}", magic_enum ::enum_name(status), key);
shaderMap.insert_or_assign(key, std::pair(a_blob, status));
shaderMap.insert_or_assign(key, ShaderCacheResult{ a_blob, status, system_clock::now() });
return (bool)a_blob;
}

ID3DBlob* ShaderCache::GetCompletedShader(const std::string a_key)
{
std::string type = SIE::SShaderCache::GetTypeFromShaderString(a_key);
std::scoped_lock lock{ mapMutex };
if (!shaderMap.empty() && shaderMap.contains(a_key)) {
auto status = shaderMap.at(a_key).second;
if (ShaderModifiedSince(type, shaderMap.at(a_key).compileTime)) {
logger::debug("Shader {} compiled {} before changes at {}", a_key, std::format("{:%Y%m%d%H%M}", shaderMap.at(a_key).compileTime), std::format("{:%Y%m%d%H%M}", modifiedShaderMap.at(type)));
return nullptr;
}
auto status = shaderMap.at(a_key).status;
if (status != ShaderCompilationTask::Status::Pending)
return shaderMap.at(a_key).first;
return shaderMap.at(a_key).blob;
}
return nullptr;
}
Expand All @@ -1346,7 +1400,7 @@ namespace SIE
{
std::scoped_lock lock{ mapMutex };
if (!shaderMap.empty() && shaderMap.contains(a_key)) {
return shaderMap.at(a_key).second;
return shaderMap.at(a_key).status;
}
return ShaderCompilationTask::Status::Pending;
}
Expand Down Expand Up @@ -1452,6 +1506,29 @@ namespace SIE
compilationPool.push_task(&ShaderCache::ManageCompilationSet, this, ssource.get_token());
}

void ShaderCache::StartFileWatcher()
{
fileWatcher = new efsw::FileWatcher();
listener = new UpdateListener();
// Add a folder to watch, and get the efsw::WatchID
// Reporting the files and directories changes to the instance of the listener
watchID = fileWatcher->addWatch("Data\\Shaders", listener, true);
// Start watching asynchronously the directories
fileWatcher->watch();
std::string pathStr = "";
for (auto path : fileWatcher->directories()) {
pathStr += std::format("{}; ", path);
}
logger::debug("ShaderCache watching for changes in {}", pathStr);
}

bool ShaderCache::ShaderModifiedSince(std::string a_type, system_clock::time_point a_current)
{
return !a_type.empty() && magic_enum::enum_cast<RE::BSShader::Type>(a_type, magic_enum::case_insensitive).has_value() // type is valid
&& !modifiedShaderMap.empty() && modifiedShaderMap.contains(a_type) // map has Type
&& modifiedShaderMap.at(a_type) > a_current; //modification time is older than a_current
}

RE::BSGraphics::VertexShader* ShaderCache::MakeAndAddVertexShader(const RE::BSShader& shader,
uint32_t descriptor)
{
Expand Down Expand Up @@ -1726,4 +1803,37 @@ namespace SIE
GetHumanTime(totalMs),
GetHumanTime(GetEta() + totalMs));
}

void UpdateListener::handleFileAction(efsw::WatchID, const std::string& dir, const std::string& filename, efsw::Action action, std::string)
{
auto& cache = SIE::ShaderCache::Instance();
const std::filesystem::path filePath = std::filesystem::path(std::format("{}\\{}", dir, filename));
auto modifiedTime = std::chrono::clock_cast<std::chrono::system_clock>(std::filesystem::last_write_time(filePath));
std::string extension = filePath.extension().string();
std::string parentDir = filePath.parent_path().string();
std::string shaderTypeString = filePath.stem().string();
auto shaderType = magic_enum::enum_cast<RE::BSShader::Type>(shaderTypeString, magic_enum::case_insensitive);
switch (action) {
case efsw::Actions::Add:
break;
case efsw::Actions::Delete:
break;
case efsw::Actions::Modified:
logger::info("Detected changed file {}", filePath.string());
if (extension.starts_with(".hlsl") && parentDir.ends_with("Shaders") && shaderType.has_value()) { // TODO: Case insensitive checks
// Shader types, so only invalidate specific shader type (e.g,. Lighting)
cache.modifiedShaderMap.insert_or_assign(shaderTypeString, modifiedTime);
cache.Clear(shaderType.value());
} else if (extension.starts_with(".hlsl")) { // TODO: Case insensitive checks
// all other shaders, since we don't know what is using it, clear everything
cache.DeleteDiskCache();
cache.Clear();
}
break;
case efsw::Actions::Moved:
break;
default:
logger::error("Filewatcher received invalid action {}", magic_enum::enum_name(action));
}
}
}
36 changes: 34 additions & 2 deletions src/ShaderCache.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <RE/B/BSShader.h>

#include "BS_thread_pool.hpp"
#include "efsw/efsw.hpp"
#include <chrono>
#include <condition_variable>
#include <unordered_map>
Expand Down Expand Up @@ -84,6 +85,15 @@ namespace SIE
double totalMs = (double)duration_cast<std::chrono::milliseconds>(lastReset - lastReset).count();
};

struct ShaderCacheResult
{
ID3DBlob* blob;
ShaderCompilationTask::Status status;
system_clock::time_point compileTime = system_clock::now();
};

class UpdateListener;

class ShaderCache
{
public:
Expand Down Expand Up @@ -126,7 +136,16 @@ namespace SIE
void DeleteDiskCache();
void ValidateDiskCache();
void WriteDiskCacheInfo();
void StartFileWatcher();
/** @brief Whether the ShaderFile for RE::BSShader::Type has been modified since the timestamp.
@param a_type Case insensitive string for the type of shader. E.g., Lighting
@param a_current The current time in system_clock::time_point.
@return True if the shader for the type (i.e., Lighting.hlsl) has been modified since the timestamp
*/
bool ShaderModifiedSince(std::string a_type, system_clock::time_point a_current);

void Clear();
void Clear(RE::BSShader::Type a_type);

bool AddCompletedShader(ShaderClass shaderClass, const RE::BSShader& shader, uint32_t descriptor, ID3DBlob* a_blob);
ID3DBlob* GetCompletedShader(const std::string a_key);
Expand Down Expand Up @@ -263,7 +282,8 @@ namespace SIE

uint blockedKeyIndex = (uint)-1; // index in shaderMap; negative value indicates disabled
std::string blockedKey = "";
std::vector<uint32_t> blockedIDs; // more than one descriptor could be blocked based on shader hash
std::vector<uint32_t> blockedIDs; // more than one descriptor could be blocked based on shader hash
std::unordered_map<std::string, system_clock::time_point> modifiedShaderMap{}; // hashmap when a shader source file last modified

private:
ShaderCache();
Expand All @@ -289,7 +309,19 @@ namespace SIE
std::mutex vertexShadersMutex;
std::mutex pixelShadersMutex;
CompilationSet compilationSet;
std::unordered_map<std::string, std::pair<ID3DBlob*, ShaderCompilationTask::Status>> shaderMap{};
std::unordered_map<std::string, ShaderCacheResult> shaderMap{};
std::mutex mapMutex;

// efsw file watcher
efsw::FileWatcher* fileWatcher;
efsw::WatchID watchID;
UpdateListener* listener;
};

// Inherits from the abstract listener class, and implements the the file action handler
class UpdateListener : public efsw::FileWatchListener
{
public:
void handleFileAction(efsw::WatchID, const std::string& dir, const std::string& filename, efsw::Action action, std::string) override;
};
}
2 changes: 1 addition & 1 deletion src/XSEPlugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ void MessageHandler(SKSE::MessagingInterface::Message* message)
auto& shaderCache = SIE::ShaderCache::Instance();

shaderCache.ValidateDiskCache();

shaderCache.StartFileWatcher();
for (auto* feature : Feature::GetFeatureList()) {
if (feature->loaded) {
feature->PostPostLoad();
Expand Down
3 changes: 2 additions & 1 deletion vcpkg.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@
},
"eastl",
"clib-util",
"unordered-dense"
"unordered-dense",
"efsw"
],
"overrides": [
{
Expand Down

0 comments on commit c68a3bf

Please sign in to comment.