Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update ORC statistics API to use C++17 standard library #8241

Merged
merged 3 commits into from
May 18, 2021
Merged

Update ORC statistics API to use C++17 standard library #8241

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ee3a931
use optional and variant
vuule May 13, 2021
1a68d2e
remove now unused proocol buffer support for unique_ptr
vuule May 13, 2021
b82fff4
use the actual monostate
vuule May 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 26 additions & 81 deletions cpp/include/cudf/io/orc_metadata.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@

#include <cudf/io/types.hpp>

#include <optional>
#include <variant>
#include <vector>

namespace cudf {
Expand Down Expand Up @@ -61,21 +63,9 @@ struct raw_orc_statistics {
raw_orc_statistics read_raw_orc_statistics(source_info const& src_info);

/**
* @brief Enumerator for types of column statistics that can be included in `column_statistics`.
*
* The statistics type depends on the column data type.
* @brief Monostate type alias for the statistics variant.
*/
enum class statistics_type {
NONE,
INT,
DOUBLE,
STRING,
BUCKET,
DECIMAL,
DATE,
BINARY,
TIMESTAMP,
};
using no_statistics = std::monostate;

/**
* @brief Base class for column statistics that include optional minimum and maximum.
Expand All @@ -84,13 +74,8 @@ enum class statistics_type {
*/
template <typename T>
struct minmax_statistics {
std::unique_ptr<T> _minimum;
std::unique_ptr<T> _maximum;

auto has_minimum() const { return _minimum != nullptr; }
auto has_maximum() const { return _maximum != nullptr; }
auto minimum() const { return _minimum.get(); }
auto maximum() const { return _maximum.get(); }
std::optional<T> minimum;
std::optional<T> maximum;
};

/**
Expand All @@ -100,24 +85,19 @@ struct minmax_statistics {
*/
template <typename T>
struct sum_statistics {
std::unique_ptr<T> _sum;

auto has_sum() const { return _sum != nullptr; }
auto sum() const { return _sum.get(); }
std::optional<T> sum;
};

/**
* @brief Statistics for integral columns.
*/
struct integer_statistics : minmax_statistics<int64_t>, sum_statistics<int64_t> {
static constexpr statistics_type type = statistics_type::INT;
};

/**
* @brief Statistics for floating point columns.
*/
struct double_statistics : minmax_statistics<double>, sum_statistics<double> {
static constexpr statistics_type type = statistics_type::DOUBLE;
};

/**
Expand All @@ -128,7 +108,6 @@ struct double_statistics : minmax_statistics<double>, sum_statistics<double> {
* Note: According to ORC specs, the sum should be signed, but pyarrow uses unsigned value
*/
struct string_statistics : minmax_statistics<std::string>, sum_statistics<uint64_t> {
static constexpr statistics_type type = statistics_type::STRING;
};

/**
Expand All @@ -137,34 +116,26 @@ struct string_statistics : minmax_statistics<std::string>, sum_statistics<uint64
* The `count` array includes the count of `false` and `true` values.
*/
struct bucket_statistics {
static constexpr statistics_type type = statistics_type::BUCKET;
std::vector<uint64_t> _count;

auto count(size_t index) const { return &_count.at(index); }
std::vector<uint64_t> count;
};

/**
* @brief Statistics for decimal columns.
*/
struct decimal_statistics : minmax_statistics<std::string>, sum_statistics<std::string> {
static constexpr statistics_type type = statistics_type::DECIMAL;
};

/**
* @brief Statistics for date(time) columns.
*/
struct date_statistics : minmax_statistics<int32_t> {
static constexpr statistics_type type = statistics_type::DATE;
};
using date_statistics = minmax_statistics<int32_t>;

/**
* @brief Statistics for binary columns.
*
* The `sum` is the total number of bytes across all elements.
*/
struct binary_statistics : sum_statistics<int64_t> {
static constexpr statistics_type type = statistics_type::BINARY;
};
using binary_statistics = sum_statistics<int64_t>;

/**
* @brief Statistics for timestamp columns.
Expand All @@ -173,14 +144,8 @@ struct binary_statistics : sum_statistics<int64_t> {
* the UNIX epoch. The `minimum_utc` and `maximum_utc` are the same values adjusted to UTC.
*/
struct timestamp_statistics : minmax_statistics<int64_t> {
static constexpr statistics_type type = statistics_type::TIMESTAMP;
std::unique_ptr<int64_t> _minimum_utc;
std::unique_ptr<int64_t> _maximum_utc;

auto has_minimum_utc() const { return _minimum_utc != nullptr; }
auto has_maximum_utc() const { return _maximum_utc != nullptr; }
auto minimum_utc() const { return _minimum_utc.get(); }
auto maximum_utc() const { return _maximum_utc.get(); }
std::optional<int64_t> minimum_utc;
std::optional<int64_t> maximum_utc;
};

namespace orc {
Expand All @@ -196,40 +161,20 @@ struct column_statistics;
* All columns can have the `number_of_values` statistics. Depending on the data type, a column can
* have additional statistics, accessible through `type_specific_stats` accessor.
*/
class column_statistics {
private:
std::unique_ptr<uint64_t> _number_of_values;
statistics_type _type = statistics_type::NONE;
void* _type_specific_stats = nullptr;

public:
column_statistics() = default;
column_statistics(cudf::io::orc::column_statistics&& other);

column_statistics& operator=(column_statistics&&) noexcept;
column_statistics(column_statistics&&) noexcept;

auto has_number_of_values() const { return _number_of_values != nullptr; }
auto number_of_values() const { return _number_of_values.get(); }

auto type() const { return _type; }

/**
* @brief Returns a non-owning pointer to the type-specific statistics of the given type.
*
* Returns null if the requested statistics type does not match the type of the currently held
* type-specific statistics.
*
* @tparam T the statistics type
*/
template <typename T>
T const* type_specific_stats() const
{
if (T::type != _type) return nullptr;
return static_cast<T*>(_type_specific_stats);
}

~column_statistics();
struct column_statistics {
std::optional<uint64_t> number_of_values;
std::variant<no_statistics,
integer_statistics,
double_statistics,
string_statistics,
bucket_statistics,
decimal_statistics,
date_statistics,
binary_statistics,
timestamp_statistics>
type_specific_stats;

column_statistics(cudf::io::orc::column_statistics&& detail_statistics);
};

/**
Expand Down
87 changes: 17 additions & 70 deletions cpp/src/io/functions.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,76 +237,23 @@ raw_orc_statistics read_raw_orc_statistics(source_info const& src_info)

column_statistics::column_statistics(cudf::io::orc::column_statistics&& cs)
{
_number_of_values = std::move(cs.number_of_values);
if (cs.int_stats.get()) {
_type = statistics_type::INT;
_type_specific_stats = cs.int_stats.release();
} else if (cs.double_stats.get()) {
_type = statistics_type::DOUBLE;
_type_specific_stats = cs.double_stats.release();
} else if (cs.string_stats.get()) {
_type = statistics_type::STRING;
_type_specific_stats = cs.string_stats.release();
} else if (cs.bucket_stats.get()) {
_type = statistics_type::BUCKET;
_type_specific_stats = cs.bucket_stats.release();
} else if (cs.decimal_stats.get()) {
_type = statistics_type::DECIMAL;
_type_specific_stats = cs.decimal_stats.release();
} else if (cs.date_stats.get()) {
_type = statistics_type::DATE;
_type_specific_stats = cs.date_stats.release();
} else if (cs.binary_stats.get()) {
_type = statistics_type::BINARY;
_type_specific_stats = cs.binary_stats.release();
} else if (cs.timestamp_stats.get()) {
_type = statistics_type::TIMESTAMP;
_type_specific_stats = cs.timestamp_stats.release();
}
}

column_statistics& column_statistics::operator=(column_statistics&& other) noexcept
{
_number_of_values = std::move(other._number_of_values);
_type = other._type;
_type_specific_stats = other._type_specific_stats;

other._type = statistics_type::NONE;
other._type_specific_stats = nullptr;

return *this;
}

column_statistics::column_statistics(column_statistics&& other) noexcept
{
*this = std::move(other);
}

column_statistics::~column_statistics()
{
switch (_type) {
case statistics_type::NONE: // error state, but can't throw from a destructor.
break;
case statistics_type::INT: delete static_cast<integer_statistics*>(_type_specific_stats); break;
case statistics_type::DOUBLE:
delete static_cast<double_statistics*>(_type_specific_stats);
break;
case statistics_type::STRING:
delete static_cast<string_statistics*>(_type_specific_stats);
break;
case statistics_type::BUCKET:
delete static_cast<bucket_statistics*>(_type_specific_stats);
break;
case statistics_type::DECIMAL:
delete static_cast<decimal_statistics*>(_type_specific_stats);
break;
case statistics_type::DATE: delete static_cast<date_statistics*>(_type_specific_stats); break;
case statistics_type::BINARY:
delete static_cast<binary_statistics*>(_type_specific_stats);
break;
case statistics_type::TIMESTAMP:
delete static_cast<timestamp_statistics*>(_type_specific_stats);
break;
number_of_values = cs.number_of_values;
if (cs.int_stats) {
type_specific_stats = *cs.int_stats;
} else if (cs.double_stats) {
type_specific_stats = *cs.double_stats;
} else if (cs.string_stats) {
type_specific_stats = *cs.string_stats;
} else if (cs.bucket_stats) {
type_specific_stats = *cs.bucket_stats;
} else if (cs.decimal_stats) {
type_specific_stats = *cs.decimal_stats;
} else if (cs.date_stats) {
type_specific_stats = *cs.date_stats;
} else if (cs.binary_stats) {
type_specific_stats = *cs.binary_stats;
} else if (cs.timestamp_stats) {
type_specific_stats = *cs.timestamp_stats;
}
}

Expand Down
34 changes: 15 additions & 19 deletions cpp/src/io/orc/orc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,60 +118,56 @@ void ProtobufReader::read(ColumnEncoding &s, size_t maxlen)

void ProtobufReader::read(integer_statistics &s, size_t maxlen)
{
auto op = std::make_tuple(make_field_reader(1, s._minimum),
make_field_reader(2, s._maximum),
make_field_reader(3, s._sum));
auto op = std::make_tuple(
make_field_reader(1, s.minimum), make_field_reader(2, s.maximum), make_field_reader(3, s.sum));
function_builder(s, maxlen, op);
}

void ProtobufReader::read(double_statistics &s, size_t maxlen)
{
auto op = std::make_tuple(make_field_reader(1, s._minimum),
make_field_reader(2, s._maximum),
make_field_reader(3, s._sum));
auto op = std::make_tuple(
make_field_reader(1, s.minimum), make_field_reader(2, s.maximum), make_field_reader(3, s.sum));
function_builder(s, maxlen, op);
}

void ProtobufReader::read(string_statistics &s, size_t maxlen)
{
auto op = std::make_tuple(make_field_reader(1, s._minimum),
make_field_reader(2, s._maximum),
make_field_reader(3, s._sum));
auto op = std::make_tuple(
make_field_reader(1, s.minimum), make_field_reader(2, s.maximum), make_field_reader(3, s.sum));
function_builder(s, maxlen, op);
}

void ProtobufReader::read(bucket_statistics &s, size_t maxlen)
{
auto op = std::make_tuple(make_packed_field_reader(1, s._count));
auto op = std::make_tuple(make_packed_field_reader(1, s.count));
function_builder(s, maxlen, op);
}

void ProtobufReader::read(decimal_statistics &s, size_t maxlen)
{
auto op = std::make_tuple(make_field_reader(1, s._minimum),
make_field_reader(2, s._maximum),
make_field_reader(3, s._sum));
auto op = std::make_tuple(
make_field_reader(1, s.minimum), make_field_reader(2, s.maximum), make_field_reader(3, s.sum));
function_builder(s, maxlen, op);
}

void ProtobufReader::read(date_statistics &s, size_t maxlen)
{
auto op = std::make_tuple(make_field_reader(1, s._minimum), make_field_reader(2, s._maximum));
auto op = std::make_tuple(make_field_reader(1, s.minimum), make_field_reader(2, s.maximum));
function_builder(s, maxlen, op);
}

void ProtobufReader::read(binary_statistics &s, size_t maxlen)
{
auto op = std::make_tuple(make_field_reader(1, s._sum));
auto op = std::make_tuple(make_field_reader(1, s.sum));
function_builder(s, maxlen, op);
}

void ProtobufReader::read(timestamp_statistics &s, size_t maxlen)
{
auto op = std::make_tuple(make_field_reader(1, s._minimum),
make_field_reader(2, s._maximum),
make_field_reader(3, s._minimum_utc),
make_field_reader(4, s._maximum_utc));
auto op = std::make_tuple(make_field_reader(1, s.minimum),
make_field_reader(2, s.maximum),
make_field_reader(3, s.minimum_utc),
make_field_reader(4, s.maximum_utc));
function_builder(s, maxlen, op);
}

Expand Down
Loading