diff --git a/cpp-client/deephaven/dhcore/CMakeLists.txt b/cpp-client/deephaven/dhcore/CMakeLists.txt index 90a850253a8..2db271b125f 100644 --- a/cpp-client/deephaven/dhcore/CMakeLists.txt +++ b/cpp-client/deephaven/dhcore/CMakeLists.txt @@ -21,6 +21,8 @@ set(ALL_FILES src/container/row_sequence.cc src/immerutil/abstract_flex_vector.cc src/immerutil/immer_column_source.cc + src/interop/testapi/basic_interop_interactions.cc + src/interop/interop_util.cc src/ticking/barrage_processor.cc src/ticking/immer_table_state.cc src/ticking/index_decoder.cc @@ -47,6 +49,8 @@ set(ALL_FILES include/public/deephaven/dhcore/column/column_source_helpers.h include/public/deephaven/dhcore/column/column_source_utils.h include/public/deephaven/dhcore/container/row_sequence.h + include/public/deephaven/dhcore/interop/testapi/basic_interop_interactions.h + include/public/deephaven/dhcore/interop/interop_util.h include/public/deephaven/dhcore/ticking/barrage_processor.h include/public/deephaven/dhcore/ticking/ticking.h include/public/deephaven/dhcore/utility/cython_support.h diff --git a/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/interop/interop_util.h b/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/interop/interop_util.h new file mode 100644 index 00000000000..19b9cc2fc65 --- /dev/null +++ b/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/interop/interop_util.h @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace deephaven::dhcore::interop { +/** + * This class simply wraps a pointer. It is meant to mirror a similar struct on the C# side, namely + * + * [StructLayout(LayoutKind.Sequential)] + * public struct NativePtr { + * public IntPtr ptr; + * } + * + * The purpose of all of this is to enable a little more type checking on the C# side. + * Rather than just dealing with IntPtr on the C# side (which is effectively equivalent to + * (void*) and has just as little typechecking, we use NativePtr struct instead. + * This prevents us from making mistakes like passing a NativePtr when we meant + * NativePtr. Our protocol is that .NET passes us a NativePtr by value, so + * in our C API, we receive a NativePtr by value. + * + * There is a little cheat we use because the pointers aren't fully typechecked. + * + * On the C++ side, we will have a NativePtr which points to a deephaven::client::Client + * object. The C# side doesn't have access to the *actual* deephaven::client::Client, nor would + * we want it to. Instead, on the .NET side we have a static "proxy" class called "NativeClient" which + * has no instance or data members. It happens to be where we keep our (static) interop method + * definitions but it has no other function. + * + * In summary on the C++ side, we will have a Client*, which we will then wrap as NativePtr + * and return to C#. C# will get this as NativePtr. This is fine because it's an + * opaque pointer. The C# code can't do anything with it except pass it back to C++ at some future + * point, where it will be interpreted again as a NativePtr, and the internal Client* can + * be pulled back out of it. + */ +template +struct NativePtr { + explicit NativePtr(T *ptr) : ptr_(ptr) {} + + [[nodiscard]] + T *Get() const { return ptr_; } + + [[nodiscard]] + operator T*() const { return ptr_; } + + [[nodiscard]] + T *operator ->() const { return ptr_; } + + void Reset(T *new_ptr) { ptr_ = new_ptr; } + +private: + T *ptr_ = nullptr; +}; + +/** + * It is not safe to pass .NET bool over interop. Instead we pass a struct which wraps + * an int8_t. On the C++ side we have an explicit constructor and an explicit conversion + * operator, to make it a bit more pleasant to convert to/from. + */ +class InteropBool { +public: + explicit InteropBool(bool value) : value_(value ? 1 : 0) {} + + explicit operator bool() const { return value_ != 0; } + +private: + int8_t value_ = 0; +}; + +/** + * The following classes support our protocol for getting strings back from C++ to .NET. + * Here is some background: + * + * Getting strings in the forward direction (.NET to C++) is easy: we can just take a + * const char *. This is a UTF-8 formatted string, terminated by NUL, whose memory is owned + * by .NET. (So we can look at the strings or copy them, but we can't hold on to the pointer). + * + * Getting strings in the reverse direction (C++ to .NET) takes a little more effort. + * The technique we use is to pack the (UTF-8) string data to be returned into a "StringPool", + * and then the .NET caller makes a second call to copy over the string data. + * + * The following data structures participate in this protocol: + * StringPoolBuilder - helper class used on the C++ side to build a StringPool + * StringPool - the StringPool (still on the C++ side) that holds final set of packed strings + * StringHandle - Opaque "handle" to a string in the StringPool (implementation: a zero-based index). + * This is passed back to the .NET side. + * StringPoolHandle - A "handle" to the StringPool. Contains a pointer to the StringPool on the C++ + * side, plus the total number of bytes in the packed strings, plus the number of strings + * in the pool. This is passed back to the .NET side. Given this information, the .NET side will + * allocate buffers and then call back into C++ side to populate those buffers. + * + * Rough example from C# side: + * void Doit() { + * InvokeSomeCPlusPlusMethod(args, out StringHandle stringHandle0, + * out StringHandle stringHandle1, out StringPoolHandle stringPoolHandle); + * var packedText = new byte[stringPoolHandle.numBytes_]; + * var stringEnds = new Int32[stringPoolHandle.numStrings_]; + * // This is an interop call that does three things: + * // it copies the "packedText" data into the array we just allocated on the C# side + * // it copies the "stringEnds" data into the array we just allocated on the C# side + * // Then it calls "delete" on the stringPoolHandle.stringPool_ pointer (so we can only + * // call this entry point once). + * var errorCode = deephaven_dhcore_interop_StringPool_ExportAndDestroy( + * stringPoolHandle.stringPool_, + * packedText, packedText.Length, + * stringEnds, stringEnds.Length); + * + * // Now we have our string data: + * // The first byte of string[0] is at offset 0. The (exclusive) end byte is at stringEnds[0]. + * // The first byte of string[1] is at stringEnds[0]. The (exclusive) end byte is stringEnds[1]. + * // ... + * // The first byte of string[N] is at stringEnds[N-1]. The (exclusive) end byte is stringEnds[N]. + * + * // The individual strings that the method wanted to return are referred to by index + * // as "StringHandles". Their start and end text position is calculated by looking inside + * // stringEnds_ + * + * var string0Begin = stringHandle0.index_ == 0 ? 0 : stringEnds[stringHandle0.index_ - 1]; + * var string0End = stringEnds[stringHandle0.index_]; + * + * var string1Begin = stringHandle1.index_ ==0 ? 0 : stringEnds[stringHandle1.index_ - 1]; + * var string1End = stringEnds[stringHandle1.index_]; + * + * var string0 = Encoding.UTF8.GetString(packedText, string0Begin, string0End - string0Begin); + * var string1 = Encoding.UTF8.GetString(packedText, string1Begin, string1End - string1Begin); + * + * deephaven_dhcore_interop_StringPool_ExportAndDestroy should not fail, unless you call it + * with bad arguments. For completeness it returns an error code: 0 for success and nonzero for + * error. Getting a nonzero value should be considered a catastrophic programming error. + * + * In C# we have a class StringPoolHandle that implements the above interaction for us. + * It has the same data layout as the C++ class, but it has a method called ExportAndDestroy + * that does the above interaction and returns a C# StringPool. The C# StringPool does not + * have much to do with the C++ class of the same name, except that they both conceptually + * represent a pool of strings. On C#, it is very simple and contains an array of the digested + * strings: + * + * public sealed class StringPool { + * public readonly string[] Strings; + * public StringPool(string[] strings) => Strings = strings; + * public string Get(StringHandle handle) { + * return Strings[handle.Index]; + * } + * } + * + * On the C++ side, it contains the packed text and the string end positions. + */ + +class StringPool { +public: + static int32_t ExportAndDestroy( + StringPool *self, + uint8_t *bytes, int32_t bytes_length, + int32_t *ends, int32_t ends_length); + + StringPool(std::vector bytes, std::vector ends); + StringPool(const StringPool &other) = delete; + StringPool &operator=(const StringPool &other) = delete; + ~StringPool(); + +private: + std::vector bytes_; + std::vector ends_; +}; + +struct StringHandle { + explicit StringHandle(int32_t index) : index_(index) {} + + int32_t index_ = 0; +}; + +struct StringPoolHandle { + StringPoolHandle() = default; + StringPoolHandle(StringPool *string_pool, int32_t num_bytes, int32_t num_strings) : + stringPool_(string_pool), numBytes_(num_bytes), numStrings_(num_strings) {} + + StringPool *stringPool_ = nullptr; + int32_t numBytes_ = 0; + int32_t numStrings_ = 0; +}; + +class StringPoolBuilder { +public: + StringPoolBuilder(); + StringPoolBuilder(const StringPoolBuilder &other) = delete; + StringPoolBuilder &operator=(const StringPoolBuilder &other) = delete; + ~StringPoolBuilder(); + + [[nodiscard]] + StringHandle Add(std::string_view sv); + [[nodiscard]] + StringPoolHandle Build(); + +private: + std::vector bytes_; + std::vector ends_; +}; + +/** + * ErrorStatus is our "out" struct that returns error information to the caller. We represent + * error returns as strings (basically the stringified version of the C++ exception that was + * thrown). + * + * The ErrorStatus representation is fairly simple: just a handle to a string pool and a + * handle to a string. On success the StringPoolHandle will point to an empty StringPool. + * On failure, the StringPoolHandle will point to a StringPool containing one string, and the + * StringHandle will be a handle to that string. Careful readers will note that if there is + * an error, the StringHandle will always point to the first string, and therefore always + * have index 0. If we cared about saving a few bytes, we could eliminate this field, but + * there is no need to care about saving a few bytes at this point. + */ +class ErrorStatus { +public: + /** + * This is a convenience method used by C++ code to wrap a lambda and catch exceptions. + * If the lambda completes, then the StringPoolBuilder will be empty. If the lambda + * fails with an exception, the exception text is stored in the StringPoolBuilder and + * the StringHandle is set. In either case the (empty or populated) StringPool is built. + */ + template + void Run(const T &callback) { + StringPoolBuilder builder; + try { + // Sanity check for this method's callers to make sure they're not returning a value + // which would be a programming mistake because it is ignored here. + static_assert(std::is_same_v); + callback(); + } catch (const std::exception &e) { + stringHandle_ = builder.Add(e.what()); + } catch (...) { + stringHandle_ = builder.Add("Unknown exception"); + } + stringPoolHandle_ = builder.Build(); + } + +private: + StringHandle stringHandle_; + StringPoolHandle stringPoolHandle_; +}; +} // namespace deephaven::dhcore::interop + +extern "C" { +int32_t deephaven_dhcore_interop_StringPool_ExportAndDestroy( + deephaven::dhcore::interop::NativePtr string_pool, + uint8_t *bytes, int32_t bytes_length, + int32_t *ends, int32_t ends_length); +} // extern "C" diff --git a/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/interop/testapi/basic_interop_interactions.h b/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/interop/testapi/basic_interop_interactions.h new file mode 100644 index 00000000000..be4585f7a54 --- /dev/null +++ b/cpp-client/deephaven/dhcore/include/public/deephaven/dhcore/interop/testapi/basic_interop_interactions.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +#pragma once + +#include +#include "deephaven/dhcore/interop/interop_util.h" + +namespace deephaven::dhcore::interop::testapi { +/** + * A simple struct that we use to confirm we can pass between C++ and C#. + */ +struct BasicStruct { + BasicStruct(int32_t i, double d) : i_(i), d_(d) {} + + [[nodiscard]] + BasicStruct Add(const BasicStruct &other) const { + return BasicStruct(i_ + other.i_, d_ + other.d_); + } + + int32_t i_; + double d_; +}; + +/** + * A nested struct that we use to confirm we can pass between C++ and C#. + */ +struct NestedStruct { + NestedStruct(const BasicStruct &a, const BasicStruct &b) : a_(a), b_(b) {} + + [[nodiscard]] + NestedStruct Add(const NestedStruct &other) const { + return NestedStruct(a_.Add(other.a_), b_.Add(other.b_)); + } + + BasicStruct a_; + BasicStruct b_; +}; +} + +extern "C" { +/** + * Adds 'a' and 'b', stores a result in *result. + * Tests passing int32 back and forth. + * Note: although interop supports returning a value, we typically always + * return void and use "out" parameters for everything. We do this for simplicity + * and consistency: our style is to have zero or more 'out' parameters of simple types + * and typically one more 'out' parameter for the ErrorStatus struct which contains + * our exception/error information. + */ +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_Add( + int32_t a, int32_t b, int32_t *result); + +/** + * Adds the arrays 'a' and 'b' elementwise, stores the results in *result. The caller + * needs to allocate an array of 'length' elements for 'result'. + */ +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_AddArrays( + const int32_t *a, const int32_t *b, int32_t length, int32_t *result); + +/** + * Performs a XOR b, stores result in *result. Demonstrates our use of InteropBool, + * because regular C# bool doesn't interop with C++ bool. + */ +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_Xor( + deephaven::dhcore::interop::InteropBool a, + deephaven::dhcore::interop::InteropBool b, + deephaven::dhcore::interop::InteropBool *result); + +/** + * Performs an elementwise XOR of the a and b arrays, stores result in *result. + * The caller needs to allocate an array of 'length' elements for 'result'. + */ +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_XorArrays( + const deephaven::dhcore::interop::InteropBool *a, + const deephaven::dhcore::interop::InteropBool *b, + int32_t length, + deephaven::dhcore::interop::InteropBool *result); + +/** + * Concats strings a and b, stores the result in string_pool_handle and result_handle. + * Demonstrates our StringPool protocol. + */ +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_Concat( + const char *a, const char *b, + deephaven::dhcore::interop::StringHandle *result_handle, + deephaven::dhcore::interop::StringPoolHandle *string_pool_handle); + +/** + * Concats arrays of strings a and b, elementwise, and stores the results in + * string_pool_handle and result_handles. The caller needs to allocate an array + * of 'length' elements for 'result_handles'. + */ +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_ConcatArrays(const char **a, + const char **b, int32_t length, + deephaven::dhcore::interop::StringHandle *result_handles, + deephaven::dhcore::interop::StringPoolHandle *string_pool_handle); + +/** + * Adds the structs a and b, fieldwise, and stores the result in *result. + */ +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_AddBasicStruct( + const deephaven::dhcore::interop::testapi::BasicStruct *a, + const deephaven::dhcore::interop::testapi::BasicStruct *b, + deephaven::dhcore::interop::testapi::BasicStruct *result); + +/** + * Adds arrays of the structs a and b, elementwise, and then fieldwise, and stores the result in + * *result. The caller needs to allocate an array of 'length' elements for 'result'. + */ +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_AddBasicStructArrays( + const deephaven::dhcore::interop::testapi::BasicStruct *a, + const deephaven::dhcore::interop::testapi::BasicStruct *b, + int32_t length, + deephaven::dhcore::interop::testapi::BasicStruct *result); + +/** + * Adds the structs a and b, fieldwise, and stores the result in *result. + * a and b are "nested structs" containing other structs. + */ +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_AddNestedStruct( + const deephaven::dhcore::interop::testapi::NestedStruct *a, + const deephaven::dhcore::interop::testapi::NestedStruct *b, + deephaven::dhcore::interop::testapi::NestedStruct *result); + +/** + * Adds arrays of the structs a and b, elementwise, and then fieldwise, and stores the result in + * *result. * a and b are "nested structs" containing other structs. The caller needs to + * allocate an array of 'length' elements for 'result'. + */ +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_AddNestedStructArrays( + const deephaven::dhcore::interop::testapi::NestedStruct *a, + const deephaven::dhcore::interop::testapi::NestedStruct *b, + int32_t length, + deephaven::dhcore::interop::testapi::NestedStruct *result); + +/** + * Tests our error protocol. Sets an error if a < b. + */ +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_SetErrorIfLessThan( + int32_t a, int32_t b, + deephaven::dhcore::interop::ErrorStatus *error_status); +} // extern "C" diff --git a/cpp-client/deephaven/dhcore/src/interop/interop_util.cc b/cpp-client/deephaven/dhcore/src/interop/interop_util.cc new file mode 100644 index 00000000000..520a24c177f --- /dev/null +++ b/cpp-client/deephaven/dhcore/src/interop/interop_util.cc @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +#include "deephaven/dhcore/interop/interop_util.h" +#include "deephaven/dhcore/utility/utility.h" + +#include + +using deephaven::dhcore::interop::NativePtr; +using deephaven::dhcore::interop::StringPool; +using deephaven::dhcore::utility::MakeReservedVector; + +namespace deephaven::dhcore::interop { +int32_t StringPool::ExportAndDestroy(StringPool *self, + uint8_t *bytes, int32_t bytes_length, + int32_t *ends, int32_t ends_length) { + // StringPoolBuilder::Build is allowed to return null if there are no strings. + if (self == nullptr) { + if (bytes_length != 0 || ends_length != 0) { + // This arbitrary return code indicates that something has gone wrong. + return 1; + } + return 0; + } + if (bytes_length != static_cast(self->bytes_.size()) || + ends_length != static_cast(self->ends_.size())) { + // This arbitrary return code indicates that something has gone wrong. + return 2; + } + std::copy(self->bytes_.begin(), self->bytes_.end(), bytes); + std::copy(self->ends_.begin(), self->ends_.end(), ends); + delete self; + return 0; +} + +StringPool::StringPool(std::vector bytes, std::vector ends) : + bytes_(std::move(bytes)), ends_(std::move(ends)) {} +StringPool::~StringPool() = default; + +StringPoolBuilder::StringPoolBuilder() = default; +StringPoolBuilder::~StringPoolBuilder() = default; + +StringHandle StringPoolBuilder::Add(std::string_view sv) { + StringHandle result(static_cast(ends_.size())); + bytes_.insert(bytes_.end(), sv.begin(), sv.end()); + ends_.push_back(static_cast(bytes_.size())); + return result; +} + +StringPoolHandle StringPoolBuilder::Build() { + auto num_bytes = bytes_.size(); + auto num_strings = ends_.size(); + if (num_strings == 0) { + return StringPoolHandle(nullptr, 0, 0); + } + auto *sp = new StringPool(std::move(bytes_), std::move(ends_)); + return StringPoolHandle(sp, static_cast(num_bytes), + static_cast(num_strings)); +} +} // namespace deephaven::dhcore::interop + +extern "C" { +int32_t deephaven_dhcore_interop_StringPool_ExportAndDestroy( + NativePtr string_pool, + uint8_t *bytes, int32_t bytes_length, + int32_t *ends, int32_t ends_length) { + return StringPool::ExportAndDestroy(string_pool.Get(), bytes, bytes_length, ends, ends_length); +} +} // extern "C" diff --git a/cpp-client/deephaven/dhcore/src/interop/testapi/basic_interop_interactions.cc b/cpp-client/deephaven/dhcore/src/interop/testapi/basic_interop_interactions.cc new file mode 100644 index 00000000000..9c0c47013d1 --- /dev/null +++ b/cpp-client/deephaven/dhcore/src/interop/testapi/basic_interop_interactions.cc @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +#include "deephaven/dhcore/interop/testapi/basic_interop_interactions.h" + +#include +#include "deephaven/dhcore/utility/utility.h" +#include "deephaven/third_party/fmt/format.h" + +using deephaven::dhcore::interop::ErrorStatus; +using deephaven::dhcore::interop::InteropBool; +using deephaven::dhcore::interop::StringHandle; +using deephaven::dhcore::interop::StringPool; +using deephaven::dhcore::interop::StringPoolHandle; +using deephaven::dhcore::interop::StringPoolBuilder; +using deephaven::dhcore::interop::testapi::BasicStruct; +using deephaven::dhcore::interop::testapi::NestedStruct; +using deephaven::dhcore::utility::MakeReservedVector; + +extern "C" { +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_Add( + int32_t a, int32_t b, int32_t *result) { + *result = a + b; +} + +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_AddArrays( + const int32_t *a, const int32_t *b, int32_t length, int32_t *result) { + for (int32_t i = 0; i != length; ++i) { + result[i] = a[i] + b[i]; + } +} + +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_Xor( + InteropBool a, InteropBool b, InteropBool *result) { + *result = InteropBool((bool)a ^ (bool)b); +} + +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_XorArrays( + const InteropBool *a, const InteropBool *b, int32_t length, InteropBool *result) { + for (int32_t i = 0; i != length; ++i) { + result[i] = InteropBool((bool)a[i] ^ (bool)b[i]); + } +} + +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_Concat( + const char *a, const char *b, + StringHandle *result_handle, StringPoolHandle *string_pool_handle) { + StringPoolBuilder builder; + auto text = std::string(a) + b; + *result_handle = builder.Add(text); + *string_pool_handle = builder.Build(); +} + +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_ConcatArrays( + const char **a, const char **b, int32_t length, + deephaven::dhcore::interop::StringHandle *result_handles, + deephaven::dhcore::interop::StringPoolHandle *string_pool_handle) { + StringPoolBuilder builder; + for (int32_t i = 0; i != length; ++i) { + auto text = std::string(a[i]) + b[i]; + result_handles[i] = builder.Add(text); + } + *string_pool_handle = builder.Build(); +} + +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_AddBasicStruct( + const BasicStruct *a, const BasicStruct *b, BasicStruct *result) { + *result = a->Add(*b); +} + +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_AddBasicStructArrays( + const BasicStruct *a, const BasicStruct *b, int32_t length, BasicStruct *result) { + for (int32_t i = 0; i != length; ++i) { + result[i] = a[i].Add(b[i]); + } +} + +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_AddNestedStruct( + const NestedStruct *a, const NestedStruct *b, NestedStruct *result) { + *result = a->Add(*b); +} + +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_AddNestedStructArrays( + const NestedStruct *a, const NestedStruct *b, int32_t length, NestedStruct *result) { + for (int32_t i = 0; i != length; ++i) { + result[i] = a[i].Add(b[i]); + } +} + +void deephaven_dhcore_interop_testapi_BasicInteropInteractions_SetErrorIfLessThan( + int32_t a, int32_t b, ErrorStatus *error_status) { + error_status->Run([=]() { + if (a < b) { + auto message = fmt::format("{} < {}, which is not allowed", a, b); + throw std::runtime_error(message); + } + }); +} +} // extern "C"