Skip to content

Commit

Permalink
Updated C++ client code, tests, and examples. (#2818)
Browse files Browse the repository at this point in the history 
* Updated C++ client code, tests, and examples.

* Move third-party flatbuffers to a different directory; write a
README.md file; make it clear how I patched flatbuffers.
  • Loading branch information
@kosak
kosak authored Sep 7, 2022
1 parent 1ea9d0c commit 0c33c60
Show file tree
Hide file tree
Showing 92 changed files with 2,105 additions and 2,282 deletions.
26 changes: 23 additions & 3 deletions cpp-client/deephaven/client/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ set(ALL_FILES
include/private/deephaven/client/impl/util.h

include/private/deephaven/client/arrowutil/arrow_visitors.h
include/private/deephaven/client/arrowutil/arrow_value_converter.h

src/columns.cc
src/expressions.cc
Expand All @@ -53,6 +54,7 @@ set(ALL_FILES
src/chunk/chunk.cc
src/chunk/chunk_filler.cc
src/chunk/chunk_maker.cc
src/column/array_column_source.cc
src/column/column_source.cc
src/container/row_sequence.cc
src/table/table.cc
Expand All @@ -65,6 +67,7 @@ set(ALL_FILES
include/public/deephaven/client/chunk/chunk.h
include/public/deephaven/client/chunk/chunk_filler.h
include/public/deephaven/client/chunk/chunk_maker.h
include/public/deephaven/client/column/array_column_source.h
include/public/deephaven/client/column/column_source.h
include/public/deephaven/client/container/row_sequence.h
include/public/deephaven/client/table/table.h
Expand All @@ -87,7 +90,6 @@ set(ALL_FILES
src/subscription/subscribe_thread.cc
src/subscription/update_processor.cc


src/immerutil/abstract_flex_vector.cc
src/immerutil/immer_column_source.cc
include/private/deephaven/client/immerutil/abstract_flex_vector.h
Expand All @@ -108,6 +110,8 @@ set(ALL_FILES
include/public/deephaven/client/utility/table_maker.h
include/public/deephaven/client/utility/utility.h

flatbuf/deephaven/flatbuf/Barrage_generated.h

proto/deephaven/proto/application.grpc.pb.cc
proto/deephaven/proto/application.grpc.pb.h
proto/deephaven/proto/application.pb.cc
Expand Down Expand Up @@ -141,7 +145,23 @@ set(ALL_FILES
proto/deephaven/proto/ticket.pb.cc
proto/deephaven/proto/ticket.pb.h

flatbuf/deephaven/flatbuf/Barrage_generated.h
third_party/flatbuffers/include/flatbuffers/allocator.h
third_party/flatbuffers/include/flatbuffers/array.h
third_party/flatbuffers/include/flatbuffers/base.h
third_party/flatbuffers/include/flatbuffers/buffer.h
third_party/flatbuffers/include/flatbuffers/buffer_ref.h
third_party/flatbuffers/include/flatbuffers/default_allocator.h
third_party/flatbuffers/include/flatbuffers/detached_buffer.h
third_party/flatbuffers/include/flatbuffers/flatbuffer_builder.h
third_party/flatbuffers/include/flatbuffers/flatbuffers.h
third_party/flatbuffers/include/flatbuffers/stl_emulation.h
third_party/flatbuffers/include/flatbuffers/string.h
third_party/flatbuffers/include/flatbuffers/struct.h
third_party/flatbuffers/include/flatbuffers/table.h
third_party/flatbuffers/include/flatbuffers/util.h
third_party/flatbuffers/include/flatbuffers/vector_downward.h
third_party/flatbuffers/include/flatbuffers/vector.h
third_party/flatbuffers/include/flatbuffers/verifier.h
)

add_library(client ${ALL_FILES})
Expand All @@ -152,7 +172,7 @@ target_compile_options(client PRIVATE -Wall -Werror)

target_include_directories(client PRIVATE ${Boost_INCLUDE_DIR})
target_include_directories(client PRIVATE include/private)
target_include_directories(client PRIVATE third_party)
target_include_directories(client PRIVATE third_party/flatbuffers/include)
target_include_directories(client PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include/public>)

# Protos and flatbuf are doing their own thing.
Expand Down
30 changes: 30 additions & 0 deletions ...lient/deephaven/client/include/private/deephaven/client/arrowutil/arrow_value_converter.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/**
* Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending
*/
#pragma once

#include <string>
#include <arrow/type.h>
#include <arrow/util/string_view.h>
#include <deephaven/client/types.h>

namespace deephaven::client::arrowutil {
class ArrowValueConverter {
public:
// "convert" function for anything except string_view is the identity function.
template<typename SRC, typename DEST>
static void convert(SRC src, DEST *dest) {
*dest = src;
}

// "convert" function for string_view is std::string
static void convert(arrow::util::string_view sv, std::string *dest) {
dest->clear();
dest->append(sv.data(), sv.size());
}

static void convert(int64_t src, deephaven::client::DateTime *dest) {
*dest = deephaven::client::DateTime::fromNanos(src);
}
};
} // namespace deephaven::client::arrowutil
52 changes: 49 additions & 3 deletions cpp-client/deephaven/client/include/private/deephaven/client/arrowutil/arrow_visitors.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#pragma once

#include <arrow/type.h>
#include "deephaven/client/types.h"

namespace deephaven::client::arrowutil {

Expand Down Expand Up @@ -33,21 +34,31 @@ class ArrowTypeVisitor final : public arrow::TypeVisitor {
return arrow::Status::OK();
}

arrow::Status Visit(const arrow::FloatType &type) final {
arrow::Status Visit(const arrow::FloatType &) final {
inner_.template operator()<float>();
return arrow::Status::OK();
}

arrow::Status Visit(const arrow::DoubleType &type) final {
arrow::Status Visit(const arrow::DoubleType &) final {
inner_.template operator()<double>();
return arrow::Status::OK();
}

arrow::Status Visit(const arrow::StringType &type) final {
arrow::Status Visit(const arrow::BooleanType &) final {
inner_.template operator()<bool>();
return arrow::Status::OK();
}

arrow::Status Visit(const arrow::StringType &) final {
inner_.template operator()<std::string>();
return arrow::Status::OK();
}

arrow::Status Visit(const arrow::TimestampType &) final {
inner_.template operator()<deephaven::client::DateTime>();
return arrow::Status::OK();
}

Inner &inner() { return inner_; }
const Inner &inner() const { return inner_; }

Expand Down Expand Up @@ -96,10 +107,45 @@ class ArrowArrayTypeVisitor : public arrow::ArrayVisitor {
return arrow::Status::OK();
}

arrow::Status Visit(const arrow::BooleanArray &) final {
inner_.template operator()<bool>();
return arrow::Status::OK();
}

arrow::Status Visit(const arrow::TimestampArray &) final {
inner_.template operator()<deephaven::client::DateTime>();
return arrow::Status::OK();
}

Inner &inner() { return inner_; }
const Inner &inner() const { return inner_; }

private:
Inner inner_;
};

/**
* Returns true iff type T is one of the numeric types.
*/
template<typename T>
constexpr bool isNumericType() {
static_assert(
std::is_same_v<T, int8_t> ||
std::is_same_v<T, int16_t> ||
std::is_same_v<T, int32_t> ||
std::is_same_v<T, int64_t> ||
std::is_same_v<T, float> ||
std::is_same_v<T, double> ||
std::is_same_v<T, bool> ||
std::is_same_v<T, std::string> ||
std::is_same_v<T, deephaven::client::DateTime>,
"T is not one of the supported element types for Deephaven columns");

return std::is_same_v<T, int8_t> ||
std::is_same_v<T, int16_t> ||
std::is_same_v<T, int32_t> ||
std::is_same_v<T, int64_t> ||
std::is_same_v<T, float> ||
std::is_same_v<T, double>;
}
} // namespace deephaven::client::arrowutil
128 changes: 94 additions & 34 deletions ...client/deephaven/client/include/private/deephaven/client/immerutil/abstract_flex_vector.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <immer/flex_vector.hpp>
#include <immer/flex_vector_transient.hpp>
#include <arrow/array.h>
#include "deephaven/client/arrowutil/arrow_value_converter.h"
#include "deephaven/client/column/column_source.h"
#include "deephaven/client/immerutil/immer_column_source.h"
#include "deephaven/client/utility/utility.h"
Expand Down Expand Up @@ -46,38 +47,53 @@ struct CorrespondingArrowArrayType<double> {
typedef arrow::DoubleArray type_t;
};

template<>
struct CorrespondingArrowArrayType<bool> {
typedef arrow::BooleanArray type_t;
};

template<>
struct CorrespondingArrowArrayType<std::string> {
typedef arrow::StringArray type_t;
};

template<>
struct CorrespondingArrowArrayType<deephaven::client::DateTime> {
typedef arrow::TimestampArray type_t;
};

struct FlexVectorAppender {
template<typename ARROW_SRC, typename T>
static void append(const ARROW_SRC &src, immer::flex_vector<T> *dest) {
auto transient = dest->transient();
for (auto element : src) {
if (!element.has_value()) {
throw std::runtime_error("TODO(kosak): we are not dealing with null values yet");
}
transient.push_back(*element);
}
*dest = transient.persistent();
}
static void append(const ARROW_SRC &src, immer::flex_vector<T> *destData,
immer::flex_vector<bool> *optionalDestNulls) {

static void append(const arrow::StringArray &src, immer::flex_vector<std::string> *dest) {
auto transient = dest->transient();
for (auto element : src) {
if (!element.has_value()) {
throw std::runtime_error("TODO(kosak): we are not dealing with null values yet");
auto transientData = destData->transient();
immer::flex_vector_transient<bool> transientNulls;
if (optionalDestNulls != nullptr) {
transientNulls = optionalDestNulls->transient();
}
for (auto optValue : src) {
bool isNull = !optValue.has_value();
T value = T();
if (isNull) {
if constexpr(deephaven::client::arrowutil::isNumericType<T>()) {
value = deephaven::client::DeephavenConstantsForType<T>::NULL_VALUE;
}
} else {
deephaven::client::arrowutil::ArrowValueConverter::convert(*optValue, &value);
}
transient.push_back(std::string(*element));
transientData.push_back(std::move(value));
if (optionalDestNulls != nullptr) {
transientNulls.push_back(isNull);
}
}
*destData = transientData.persistent();
if (optionalDestNulls != nullptr) {
*optionalDestNulls = transientNulls.persistent();
}
*dest = transient.persistent();
}
};
} // namespace internal
template<typename T>
class AbstractFlexVector;

/**
* This class allows us to manipulate an immer::flex_vector without needing to know what type
Expand All @@ -87,9 +103,6 @@ class AbstractFlexVectorBase {
protected:
typedef deephaven::client::column::ColumnSource ColumnSource;
public:
template<typename T>
static std::unique_ptr<AbstractFlexVectorBase> create(immer::flex_vector<T> vec);

virtual ~AbstractFlexVectorBase();

virtual std::unique_ptr<AbstractFlexVectorBase> take(size_t n) = 0;
Expand All @@ -101,12 +114,16 @@ class AbstractFlexVectorBase {
};

template<typename T>
class AbstractFlexVector final : public AbstractFlexVectorBase {
class NumericAbstractFlexVector final : public AbstractFlexVectorBase {
public:
explicit AbstractFlexVector(immer::flex_vector<T> vec) : vec_(std::move(vec)) {}
NumericAbstractFlexVector() = default;

explicit NumericAbstractFlexVector(immer::flex_vector<T> vec) : vec_(std::move(vec)) {}

~NumericAbstractFlexVector() final = default;

std::unique_ptr<AbstractFlexVectorBase> take(size_t n) final {
return create(vec_.take(n));
return std::make_unique<NumericAbstractFlexVector>(vec_.take(n));
}

void inPlaceDrop(size_t n) final {
Expand All @@ -115,29 +132,72 @@ class AbstractFlexVector final : public AbstractFlexVectorBase {
}

void inPlaceAppend(std::unique_ptr<AbstractFlexVectorBase> other) final {
auto *otherVec = deephaven::client::utility::verboseCast<AbstractFlexVector*>(other.get(),
DEEPHAVEN_PRETTY_FUNCTION);
auto *otherVec = deephaven::client::utility::verboseCast<NumericAbstractFlexVector*>(
DEEPHAVEN_EXPR_MSG(other.get()));
auto temp = std::move(vec_) + std::move(otherVec->vec_);
vec_ = std::move(temp);
}

void inPlaceAppendArrow(const arrow::Array &data) final {
typedef typename internal::CorrespondingArrowArrayType<T>::type_t arrowArrayType_t;
auto *typedArrow = deephaven::client::utility::verboseCast<const arrowArrayType_t*>(&data,
DEEPHAVEN_PRETTY_FUNCTION);
internal::FlexVectorAppender::append(*typedArrow, &vec_);
auto *typedArrow = deephaven::client::utility::verboseCast<const arrowArrayType_t*>(
DEEPHAVEN_EXPR_MSG(&data));
internal::FlexVectorAppender::append(*typedArrow, &vec_, nullptr);
}

std::shared_ptr<ColumnSource> makeColumnSource() const final {
return deephaven::client::immerutil::ImmerColumnSource::create(vec_);
return deephaven::client::immerutil::NumericImmerColumnSource<T>::create(vec_);
}

private:
immer::flex_vector<T> vec_;
};

template<typename T>
std::unique_ptr<AbstractFlexVectorBase> AbstractFlexVectorBase::create(immer::flex_vector<T> vec) {
return std::make_unique<AbstractFlexVector<T>>(std::move(vec));
}
class GenericAbstractFlexVector final : public AbstractFlexVectorBase {
public:
GenericAbstractFlexVector() = default;

GenericAbstractFlexVector(immer::flex_vector<T> data, immer::flex_vector<bool> nulls) :
data_(std::move(data)), nulls_(std::move(nulls)) {}

~GenericAbstractFlexVector() final = default;

std::unique_ptr<AbstractFlexVectorBase> take(size_t n) final {
return std::make_unique<GenericAbstractFlexVector>(data_.take(n), nulls_.take(n));
}

void inPlaceDrop(size_t n) final {
auto tempData = std::move(data_).drop(n);
data_ = std::move(tempData);

auto tempNulls = std::move(nulls_).drop(n);
nulls_ = std::move(tempNulls);
}

void inPlaceAppend(std::unique_ptr<AbstractFlexVectorBase> other) final {
auto *otherVec = deephaven::client::utility::verboseCast<GenericAbstractFlexVector*>(
DEEPHAVEN_EXPR_MSG(other.get()));
auto tempData = std::move(data_) + std::move(otherVec->data_);
data_ = std::move(tempData);

auto tempNulls = std::move(nulls_) + std::move(otherVec->nulls_);
nulls_ = std::move(tempNulls);
}

void inPlaceAppendArrow(const arrow::Array &data) final {
typedef typename internal::CorrespondingArrowArrayType<T>::type_t arrowArrayType_t;
auto *typedArrow = deephaven::client::utility::verboseCast<const arrowArrayType_t*>(
DEEPHAVEN_EXPR_MSG(&data));
internal::FlexVectorAppender::append(*typedArrow, &data_, &nulls_);
}

std::shared_ptr<ColumnSource> makeColumnSource() const final {
return deephaven::client::immerutil::GenericImmerColumnSource<T>::create(data_, nulls_);
}

private:
immer::flex_vector<T> data_;
immer::flex_vector<bool> nulls_;
};
} // namespace deephaven::client::immerutil
Loading

0 comments on commit 0c33c60

Please sign in to comment.