From 9643518d59e05a0030b6e1a1614d9bb06091318a Mon Sep 17 00:00:00 2001 From: quadpixels Date: Thu, 20 Jan 2022 19:44:34 -0800 Subject: [PATCH] [dx11] Add underlying DX11 device, memory allocation, and some tests (#3971) * [dx11] Add ID3D11Device and ID3D11DeviceContext This change adds the actual D3D11 device class (ID3D11Device and ID3D11DeviceContext) from . Tested: Tested with unit test and saw the Dx11Device and the D3D11 device it creates can be successfully created. Enabled Graphics Debug and confirmed there are no remaining live objects when the test is finished. * [dx11] Add a class for getting info from the D3D11 info queue This class fetches information from the D3D11 info queue. This enables us to count the number of live D3D11 objects. This is most meant for usage with unit tests and during development. Tested: Tested with unit tests * [dx11] Allocate and deallocate memory This change adds Dx11Device::allocate_memory and Dx11Device::dealloc_memory. Tested: Tested with unit tests. The D3D11 objects (Buffer and UAV) are created and released as expected. * Auto Format * Don't build DX11 test when the backend is not built * Update prtags.json * [dx11] Make kD3d11DebugEnabled a compile-time constant This change makes kD3d11DebugEnabled a constant. Tested: rebuilt-and ran unit test with either kD3d11DebugEnabled set to ON or OFF. * Update taichi/backends/dx/dx_info_queue.h Co-authored-by: Ye Kuang * Auto Format Co-authored-by: Taichi Gardener Co-authored-by: Bob Cao Co-authored-by: Ye Kuang --- cmake/TaichiCore.cmake | 1 - cmake/TaichiTests.cmake | 1 + misc/prtags.json | 2 + taichi/backends/dx/dx_api.h | 1 + taichi/backends/dx/dx_device.cpp | 190 +++++++++++++++++++++++- taichi/backends/dx/dx_device.h | 30 ++++ taichi/backends/dx/dx_info_queue.cpp | 140 +++++++++++++++++ taichi/backends/dx/dx_info_queue.h | 36 +++++ tests/cpp/backends/dx11_device_test.cpp | 94 ++++++++++++ 9 files changed, 492 insertions(+), 3 deletions(-) create mode 100644 taichi/backends/dx/dx_info_queue.cpp create mode 100644 taichi/backends/dx/dx_info_queue.h create mode 100644 tests/cpp/backends/dx11_device_test.cpp diff --git a/cmake/TaichiCore.cmake b/cmake/TaichiCore.cmake index 9fe1dafbbb724..b9f2f32576b69 100644 --- a/cmake/TaichiCore.cmake +++ b/cmake/TaichiCore.cmake @@ -8,7 +8,6 @@ option(TI_WITH_CC "Build with the C backend" ON) option(TI_WITH_VULKAN "Build with the Vulkan backend" OFF) option(TI_WITH_DX11 "Build with the DX11 backend" OFF) - if(UNIX AND NOT APPLE) # Handy helper for Linux # https://stackoverflow.com/a/32259072/12003165 diff --git a/cmake/TaichiTests.cmake b/cmake/TaichiTests.cmake index 8b5661aadf48e..2b6da207c82ff 100644 --- a/cmake/TaichiTests.cmake +++ b/cmake/TaichiTests.cmake @@ -13,6 +13,7 @@ endif() file(GLOB_RECURSE TAICHI_TESTS_SOURCE "tests/cpp/analysis/*.cpp" "tests/cpp/aot/*.cpp" + "tests/cpp/backends/*.cpp" "tests/cpp/codegen/*.cpp" "tests/cpp/common/*.cpp" "tests/cpp/ir/*.cpp" diff --git a/misc/prtags.json b/misc/prtags.json index b9719f513945f..b3c33f517c7bd 100644 --- a/misc/prtags.json +++ b/misc/prtags.json @@ -12,6 +12,8 @@ "metal" : "Metal backend", "opengl" : "OpenGL backend", "vulkan" : "Vulkan backend", + "dx11" : "DirectX 11 backend", + "spirv" : "SPIR-V common codegen", "wasm" : "WebAssembly backend", "misc" : "Miscellaneous", "std" : "Standard library", diff --git a/taichi/backends/dx/dx_api.h b/taichi/backends/dx/dx_api.h index 6f1c370c7f0a2..753299a3df15e 100644 --- a/taichi/backends/dx/dx_api.h +++ b/taichi/backends/dx/dx_api.h @@ -1,6 +1,7 @@ #pragma once #pragma comment(lib, "d3d11.lib") #pragma comment(lib, "d3dcompiler.lib") +#pragma comment(lib, "dxguid.lib") #include "taichi/common/core.h" diff --git a/taichi/backends/dx/dx_device.cpp b/taichi/backends/dx/dx_device.cpp index eadd023c07ce2..4060d246ded5f 100644 --- a/taichi/backends/dx/dx_device.cpp +++ b/taichi/backends/dx/dx_device.cpp @@ -4,6 +4,12 @@ namespace taichi { namespace lang { namespace directx11 { +void check_dx_error(HRESULT hr, const char *msg) { + if (!SUCCEEDED(hr)) { + TI_ERROR("Error in {}: {}", msg, hr); + } +} + Dx11ResourceBinder::~Dx11ResourceBinder() { } @@ -19,19 +25,199 @@ ResourceBinder *Dx11Pipeline::resource_binder() { return nullptr; } +namespace { +HRESULT create_compute_device(ID3D11Device **out_device, + ID3D11DeviceContext **out_context, + bool force_ref, + bool debug_enabled) { + const D3D_FEATURE_LEVEL levels[] = { + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + D3D_FEATURE_LEVEL_10_1, + D3D_FEATURE_LEVEL_10_0, + }; + + UINT flags = 0; + if (debug_enabled) + flags |= D3D11_CREATE_DEVICE_DEBUG; + + ID3D11Device *device = nullptr; + ID3D11DeviceContext *context = nullptr; + HRESULT hr; + + D3D_DRIVER_TYPE driver_types[] = { + D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_SOFTWARE, + D3D_DRIVER_TYPE_REFERENCE, D3D_DRIVER_TYPE_WARP}; + const char *driver_type_names[] = { + "D3D_DRIVER_TYPE_HARDWARE", "D3D_DRIVER_TYPE_SOFTWARE", + "D3D_DRIVER_TYPE_REFERENCE", "D3D_DRIVER_TYPE_WARP"}; + + const int num_types = sizeof(driver_types) / sizeof(driver_types[0]); + + int attempt_idx = 0; + if (force_ref) { + attempt_idx = 2; + } + + for (; attempt_idx < num_types; attempt_idx++) { + D3D_DRIVER_TYPE driver_type = driver_types[attempt_idx]; + hr = D3D11CreateDevice(nullptr, driver_type, nullptr, flags, levels, + _countof(levels), D3D11_SDK_VERSION, &device, + nullptr, &context); + + if (FAILED(hr) || device == nullptr) { + TI_WARN("Failed to create D3D11 device with type {}: {}\n", driver_type, + driver_type_names[attempt_idx]); + continue; + } + + if (device->GetFeatureLevel() < D3D_FEATURE_LEVEL_11_0) { + D3D11_FEATURE_DATA_D3D10_X_HARDWARE_OPTIONS hwopts = {0}; + device->CheckFeatureSupport(D3D11_FEATURE_D3D10_X_HARDWARE_OPTIONS, + &hwopts, sizeof(hwopts)); + if (!hwopts.ComputeShaders_Plus_RawAndStructuredBuffers_Via_Shader_4_x) { + device->Release(); + TI_WARN( + "DirectCompute not supported via " + "ComputeShaders_Plus_RawAndStructuredBuffers_Via_Shader_4"); + } + continue; + } + + TI_INFO("Successfully created DX11 device with type {}", + driver_type_names[attempt_idx]); + *out_device = device; + *out_context = context; + break; + } + + if (*out_device == nullptr || *out_context == nullptr) { + TI_ERROR("Failed to create DX11 device using all {} driver types", + num_types); + } + + return hr; +} + +HRESULT create_raw_buffer(ID3D11Device *device, + UINT size, + void *init_data, + ID3D11Buffer **out_buf) { + *out_buf = nullptr; + D3D11_BUFFER_DESC desc = {}; + desc.BindFlags = D3D11_BIND_UNORDERED_ACCESS | D3D11_BIND_SHADER_RESOURCE; + desc.ByteWidth = size; + desc.MiscFlags = D3D11_RESOURCE_MISC_BUFFER_ALLOW_RAW_VIEWS; + if (init_data) { + D3D11_SUBRESOURCE_DATA data; + data.pSysMem = init_data; + return device->CreateBuffer(&desc, &data, out_buf); + } else { + return device->CreateBuffer(&desc, nullptr, out_buf); + } +} + +HRESULT create_buffer_uav(ID3D11Device *device, + ID3D11Buffer *buffer, + ID3D11UnorderedAccessView **out_uav) { + D3D11_BUFFER_DESC buf_desc = {}; + buffer->GetDesc(&buf_desc); + D3D11_UNORDERED_ACCESS_VIEW_DESC uav_desc = {}; + uav_desc.ViewDimension = D3D11_UAV_DIMENSION_BUFFER; + uav_desc.Buffer.FirstElement = 0; + if (buf_desc.MiscFlags & D3D11_RESOURCE_MISC_BUFFER_ALLOW_RAW_VIEWS) { + uav_desc.Format = DXGI_FORMAT_R32_TYPELESS; + uav_desc.Buffer.Flags = D3D11_BUFFER_UAV_FLAG_RAW; + uav_desc.Buffer.NumElements = buf_desc.ByteWidth / 4; + } else if (buf_desc.MiscFlags & D3D11_RESOURCE_MISC_BUFFER_STRUCTURED) { + uav_desc.Format = DXGI_FORMAT_UNKNOWN; + uav_desc.Buffer.NumElements = + buf_desc.ByteWidth / buf_desc.StructureByteStride; + } else + return E_INVALIDARG; + return device->CreateUnorderedAccessView(buffer, &uav_desc, out_uav); +} + +} // namespace + Dx11Device::Dx11Device() { + create_dx11_device(); + if (kD3d11DebugEnabled) { + info_queue_ = std::make_unique(device_); + } set_cap(DeviceCapability::spirv_version, 0x10300); } Dx11Device::~Dx11Device() { + destroy_dx11_device(); +} + +void Dx11Device::create_dx11_device() { + if (device_ != nullptr && context_ != nullptr) { + TI_TRACE("D3D11 device has already been created."); + return; + } + TI_TRACE("Creating D3D11 device"); + create_compute_device(&device_, &context_, kD3d11ForceRef, + kD3d11DebugEnabled); +} + +void Dx11Device::destroy_dx11_device() { + if (device_ != nullptr) { + device_->Release(); + device_ = nullptr; + } + if (context_ != nullptr) { + context_->Release(); + context_ = nullptr; + } +} + +int Dx11Device::live_dx11_object_count() { + TI_ASSERT(info_queue_ != nullptr); + return info_queue_->live_object_count(); } DeviceAllocation Dx11Device::allocate_memory(const AllocParams ¶ms) { - TI_NOT_IMPLEMENTED; + ID3D11Buffer *buf; + HRESULT hr; + hr = create_raw_buffer(device_, params.size, nullptr, &buf); + check_dx_error(hr, "create raw buffer"); + alloc_id_to_buffer_[alloc_serial_] = buf; + + ID3D11UnorderedAccessView *uav; + hr = create_buffer_uav(device_, buf, &uav); + check_dx_error(hr, "create UAV for buffer"); + alloc_id_to_uav_[alloc_serial_] = uav; + + // Set debug names + std::string buf_name = "buffer alloc#" + std::to_string(alloc_serial_) + + " size=" + std::to_string(params.size) + '\0'; + hr = buf->SetPrivateData(WKPDID_D3DDebugObjectName, buf_name.size(), + buf_name.c_str()); + check_dx_error(hr, "set name for buffer"); + + std::string uav_name = "UAV of " + buf_name; + hr = uav->SetPrivateData(WKPDID_D3DDebugObjectName, uav_name.size(), + uav_name.c_str()); + check_dx_error(hr, "set name for UAV"); + + DeviceAllocation alloc; + alloc.device = this; + alloc.alloc_id = alloc_serial_; + ++alloc_serial_; + + return alloc; } void Dx11Device::dealloc_memory(DeviceAllocation handle) { - TI_NOT_IMPLEMENTED; + uint32_t alloc_id = handle.alloc_id; + ID3D11Buffer *buf = alloc_id_to_buffer_[alloc_id]; + buf->Release(); + alloc_id_to_buffer_.erase(alloc_id); + ID3D11UnorderedAccessView *uav = alloc_id_to_uav_[alloc_id]; + uav->Release(); + alloc_id_to_uav_.erase(alloc_id); } std::unique_ptr Dx11Device::create_pipeline( diff --git a/taichi/backends/dx/dx_device.h b/taichi/backends/dx/dx_device.h index 73c0d7397550e..d22c985fd18d1 100644 --- a/taichi/backends/dx/dx_device.h +++ b/taichi/backends/dx/dx_device.h @@ -1,11 +1,22 @@ #pragma once #include "taichi/backends/device.h" +#include "taichi/backends/dx/dx_info_queue.h" +#include namespace taichi { namespace lang { namespace directx11 { +// Only enable debug layer when the corresponding testing facility is enabled +constexpr bool kD3d11DebugEnabled = true; +constexpr bool kD3d11ForceRef = false; // Force REF device. May be used to + // force software rendering. + +void debug_enabled(bool); +void force_ref(bool); +void check_dx_error(HRESULT hr, const char *msg); + class Dx11ResourceBinder : public ResourceBinder { ~Dx11ResourceBinder() override; }; @@ -55,6 +66,25 @@ class Dx11Device : public GraphicsDevice { DeviceAllocation src_img, ImageLayout img_layout, const BufferImageCopyParams ¶ms) override; + + int live_dx11_object_count(); + + private: + void create_dx11_device(); + void destroy_dx11_device(); + ID3D11Buffer *alloc_id_to_buffer(uint32_t alloc_id); + ID3D11Buffer *alloc_id_to_buffer_cpu_copy(uint32_t alloc_id); + ID3D11UnorderedAccessView *alloc_id_to_uav(uint32_t alloc_id); + ID3D11Device *device_{}; + ID3D11DeviceContext *context_{}; + std::unique_ptr info_queue_{}; + std::unordered_map + alloc_id_to_buffer_; // binding ID to buffer + std::unordered_map + alloc_id_to_cpucopy_; // binding ID to CPU copy of buffer + std::unordered_map + alloc_id_to_uav_; // binding ID to UAV + int alloc_serial_; }; } // namespace directx11 diff --git a/taichi/backends/dx/dx_info_queue.cpp b/taichi/backends/dx/dx_info_queue.cpp new file mode 100644 index 0000000000000..c9a9b936ac075 --- /dev/null +++ b/taichi/backends/dx/dx_info_queue.cpp @@ -0,0 +1,140 @@ +#include "taichi/backends/dx/dx_info_queue.h" + +namespace taichi { +namespace lang { +namespace directx11 { + +void check_dx_error(HRESULT hr, const char *msg); + +namespace { +inline std::string trim_string(const std::string &s) { + int begin = 0, end = (int)s.size(); + while (begin < end && std::isspace(s[begin])) { + begin++; + } + while (begin < end && std::isspace(s[end - 1])) { + end--; + } + return std::string(s.begin() + begin, s.begin() + end); +} +} // namespace + +std::string munch_token(std::string &s) { + if (s.empty()) { + return ""; + } + size_t idx = s.find(' '); + std::string ret; + ret = trim_string(s.substr(0, idx)); + s = s.substr(idx + 1); + if (ret.empty() == false && ret.back() == ',') + ret.pop_back(); + return ret; +} + +std::vector Dx11InfoQueue::parse_reference_count( + const std::vector messages) { + std::vector ret; + for (std::string line : messages) { + Dx11InfoQueue::Entry entry; + line = trim_string(line); + std::string x; + x = munch_token(line); + + // Example 1: "Live ID3D11Query at 0x0000018F64E81DA0, Refcount: 0, IntRef: + // 1" Example 2: "Live ID3D11Buffer at 0x000001F8AF284370, Name: buffer + // alloc#0 size=1048576, Refcount: 1, IntRef: 1" + + if (x != "Live") + continue; + + x = munch_token(line); + entry.type = x; + + x = munch_token(line); + if (x != "at") + continue; + + x = munch_token(line); + if (x.empty()) + continue; + + entry.addr = reinterpret_cast(std::atoll(x.c_str())); + + while (true) { + x = munch_token(line); + if (x == "Refcount:") { + x = munch_token(line); + entry.refcount = std::atoi(x.c_str()); + } else if (x == "IntRef:") { + x = munch_token(line); + entry.intref = std::atoi(x.c_str()); + } else + break; + } + ret.push_back(entry); + } + return ret; +} + +Dx11InfoQueue::Dx11InfoQueue(ID3D11Device *device) + : device_(device), last_message_count_(0) { + init(); +} + +void Dx11InfoQueue::init() { + typedef HRESULT(WINAPI * DXGIGetDebugInterface)(REFIID, void **); + + HRESULT hr; + hr = device_->QueryInterface(__uuidof(ID3D11InfoQueue), + reinterpret_cast(&info_queue_)); + check_dx_error(hr, "Query ID3D11InfoQueue interface from the DX11 device"); + hr = device_->QueryInterface(__uuidof(ID3D11Debug), + reinterpret_cast(&debug_)); + check_dx_error(hr, "Query ID3D11Debug interface from the DX11 device"); +} + +std::vector Dx11InfoQueue::get_updated_messages() { + std::vector ret; + if (!info_queue_) { + return ret; + } + const int num_messages = info_queue_->GetNumStoredMessages(); + const int n = num_messages - last_message_count_; + ret.resize(n); + for (int i = 0; i < n; i++) { + D3D11_MESSAGE *msg; + size_t len = 0; + HRESULT hr = + info_queue_->GetMessageW(i + last_message_count_, nullptr, &len); + check_dx_error(hr, "Check D3D11 info queue message length"); + msg = (D3D11_MESSAGE *)malloc(len); + hr = info_queue_->GetMessageW(i + last_message_count_, msg, &len); + check_dx_error(hr, "Obtain D3D11 info queue message content"); + ret[i] = std::string(msg->pDescription); + free(msg); + } + last_message_count_ = num_messages; + return ret; +} + +bool Dx11InfoQueue::has_updated_messages() { + if (!info_queue_) { + return false; + } + const int n = info_queue_->GetNumStoredMessages(); + return n > last_message_count_; +} + +int Dx11InfoQueue::live_object_count() { + get_updated_messages(); // Drain message queue + debug_->ReportLiveDeviceObjects(D3D11_RLDO_DETAIL); + if (has_updated_messages()) { + live_objects_ = parse_reference_count(get_updated_messages()); + } + return static_cast(live_objects_.size()); +} + +} // namespace directx11 +} // namespace lang +} // namespace taichi diff --git a/taichi/backends/dx/dx_info_queue.h b/taichi/backends/dx/dx_info_queue.h new file mode 100644 index 0000000000000..18cda1043a999 --- /dev/null +++ b/taichi/backends/dx/dx_info_queue.h @@ -0,0 +1,36 @@ +#pragma once + +#include "taichi/backends/device.h" +#include + +namespace taichi { +namespace lang { +namespace directx11 { + +class Dx11InfoQueue { + public: + struct Entry { + std::string type; + void *addr; + int refcount; + int intref; + }; + static std::vector parse_reference_count( + const std::vector &); + explicit Dx11InfoQueue(ID3D11Device *device); + int live_object_count(); + + private: + bool has_updated_messages(); + std::vector get_updated_messages(); + std::vector live_objects_; + void init(); + ID3D11Device *device_{}; + ID3D11Debug *debug_{}; + ID3D11InfoQueue *info_queue_{}; + int last_message_count_; +}; + +} // namespace directx11 +} // namespace lang +} // namespace taichi diff --git a/tests/cpp/backends/dx11_device_test.cpp b/tests/cpp/backends/dx11_device_test.cpp new file mode 100644 index 0000000000000..63549393e62f3 --- /dev/null +++ b/tests/cpp/backends/dx11_device_test.cpp @@ -0,0 +1,94 @@ +#include "gtest/gtest.h" + +#ifdef TI_WITH_DX11 + +#include "taichi/backends/dx/dx_device.h" +#include "taichi/backends/dx/dx_info_queue.h" + +namespace taichi { +namespace lang { +namespace directx11 { + +TEST(Dx11DeviceCreationTest, CreateDeviceAndAllocateMemory) { + std::unique_ptr device = + std::make_unique(); + + // Should not crash + EXPECT_TRUE(device != nullptr); + + // Should have one object of each of the following types: + // ID3D11Device + // ID3D11Context + // ID3DDeviceContextState + // ID3D11BlendState + // ID3D11DepthStencilState + // ID3D11RasterizerState + // ID3D11Sampler + // ID3D11Query + int count0, count1, count2; + if (kD3d11DebugEnabled) { + count0 = device->live_dx11_object_count(); + EXPECT_EQ(count0, 8); + } + + taichi::lang::Device::AllocParams params; + params.size = 1048576; + const taichi::lang::DeviceAllocation device_alloc = + device->allocate_memory(params); + if (kD3d11DebugEnabled) { + count1 = device->live_dx11_object_count(); + // Should have allocated an UAV and a Buffer, so 2 more objects. + EXPECT_EQ(count1 - count0, 2); + } + + // The 2 objects should have been released. + device->dealloc_memory(device_alloc); + if (kD3d11DebugEnabled) { + count2 = device->live_dx11_object_count(); + EXPECT_EQ(count2 - count1, -2); + } +} + +TEST(Dx11InfoQueueTest, ParseReferenceCount) { + const std::vector messages = { + "Create ID3D11Context: Name=\"unnamed\", Addr=0x0000018F6678E080, " + "ExtRef=1, IntRef=0", + "Create ID3DDeviceContextState: Name=\"unnamed\", " + "Addr=0x0000018F6686CE10, " + "ExtRef=1, IntRef=0", + "Create ID3D11BlendState: Name=\"unnamed\", Addr=0x0000018F667F6DB0, " + "ExtRef=1, IntRef=0", + "Create ID3D11DepthStencilState: Name=\"unnamed\", " + "Addr=0x0000018F667F6BC0, ExtRef=1, IntRef=0", + "Create ID3D11RasterizerState: Name=\"unnamed\", " + "Addr=0x0000018F64891420, " + "ExtRef=1, IntRef=0", + "Create ID3D11Sampler: Name=\"unnamed\", Addr=0x0000018F667F6FA0, " + "ExtRef=1, IntRef=0", + "Create ID3D11Query: Name=\"unnamed\", Addr=0x0000018F64E81DA0, " + "ExtRef=1, IntRef=0", + "Create ID3D11Fence: Name=\"unnamed\", Addr=0x0000018F64FF7380, " + "ExtRef=1, IntRef=0", + "Destroy ID3D11Fence: Name=\"unnamed\", Addr=0x0000018F64FF7380", + "Live ID3D11Device at 0x0000018F66782250, Refcount: 5", + "Live ID3D11Context at 0x0000018F6678E080, Refcount: 1, IntRef: 1", + "Live ID3DDeviceContextState at 0x0000018F6686CE10, Refcount: 0, IntRef: " + "1", + "Live ID3D11BlendState at 0x0000018F667F6DB0, Refcount: 0, " + "IntRef: 1", + "Live ID3D11DepthStencilState at 0x0000018F667F6BC0, Refcount: 0, " + "IntRef: 1", + "Live ID3D11RasterizerState at 0x0000018F64891420, Refcount: 0, " + "IntRef: 1", + "Live ID3D11Sampler at 0x0000018F667F6FA0, Refcount: 0, IntRef: 1", + "Live ID3D11Query at 0x0000018F64E81DA0, Refcount: 0, IntRef: 1"}; + std::vector entries = + directx11::Dx11InfoQueue::parse_reference_count(messages); + EXPECT_EQ(entries.size(), 8); +} + +} // namespace directx11 +} // namespace lang +} // namespace taichi + +#endif