Skip to content

Commit

Permalink
thirdparty: Upgrade parallel_hashmap
Browse files Browse the repository at this point in the history
  • Loading branch information
@anlambert
anlambert committed Oct 30, 2024
1 parent d7bc8c9 commit e2b5978
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 53 deletions.
105 changes: 65 additions & 40 deletions thirdparty/parallel_hashmap/phmap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,11 +329,17 @@ static_assert(kDeleted == -2,
// A single block of empty control bytes for tables without any slots allocated.
// This enables removing a branch in the hot path of find().
// --------------------------------------------------------------------------
template <class std_alloc_t>
inline ctrl_t* EmptyGroup() {
alignas(16) static constexpr ctrl_t empty_group[] = {
kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty,
kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty};
return const_cast<ctrl_t*>(empty_group);
PHMAP_IF_CONSTEXPR (std_alloc_t::value) {
alignas(16) static constexpr ctrl_t empty_group[] = {
kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty,
kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty};

return const_cast<ctrl_t*>(empty_group);
} else {
return nullptr;
}
}

// --------------------------------------------------------------------------
Expand Down Expand Up @@ -869,6 +875,8 @@ class raw_hash_set
template <class K>
using key_arg = typename KeyArgImpl::template type<K, key_type>;

using std_alloc_t = std::is_same<typename std::decay<Alloc>::type, phmap::priv::Allocator<value_type>>;

private:
// Give an early error when key_type is not hashable/eq.
auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k));
Expand Down Expand Up @@ -989,6 +997,11 @@ class raw_hash_set
iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) {}

void skip_empty_or_deleted() {
PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
// ctrl_ could be nullptr
if (!ctrl_)
return;
}
while (IsEmptyOrDeleted(*ctrl_)) {
// ctrl is not necessarily aligned to Group::kWidth. It is also likely
// to read past the space for ctrl bytes and into slots. This is ok
Expand Down Expand Up @@ -1057,7 +1070,7 @@ class raw_hash_set
explicit raw_hash_set(size_t bucket_cnt, const hasher& hashfn = hasher(),
const key_equal& eq = key_equal(),
const allocator_type& alloc = allocator_type())
: ctrl_(EmptyGroup()), settings_(0, hashfn, eq, alloc) {
: ctrl_(EmptyGroup<std_alloc_t>()), settings_(0, hashfn, eq, alloc) {
if (bucket_cnt) {
size_t new_capacity = NormalizeCapacity(bucket_cnt);
reset_growth_left(new_capacity);
Expand Down Expand Up @@ -1180,7 +1193,7 @@ class raw_hash_set
std::is_nothrow_copy_constructible<hasher>::value&&
std::is_nothrow_copy_constructible<key_equal>::value&&
std::is_nothrow_copy_constructible<allocator_type>::value)
: ctrl_(phmap::exchange(that.ctrl_, EmptyGroup())),
: ctrl_(phmap::exchange(that.ctrl_, EmptyGroup<std_alloc_t>())),
slots_(phmap::exchange(that.slots_, nullptr)),
size_(phmap::exchange(that.size_, 0)),
capacity_(phmap::exchange(that.capacity_, 0)),
Expand All @@ -1194,7 +1207,7 @@ class raw_hash_set
}

raw_hash_set(raw_hash_set&& that, const allocator_type& a)
: ctrl_(EmptyGroup()),
: ctrl_(EmptyGroup<std_alloc_t>()),
slots_(nullptr),
size_(0),
capacity_(0),
Expand Down Expand Up @@ -1615,6 +1628,7 @@ class raw_hash_set
// This overload is necessary because otherwise erase<K>(const K&) would be
// a better match if non-const iterator is passed as an argument.
iterator erase(iterator it) {
assert(it != end());
auto res = it;
++res;
_erase(it);
Expand Down Expand Up @@ -1738,7 +1752,8 @@ class raw_hash_set

template <class K = key_type>
void prefetch(const key_arg<K>& key) const {
prefetch_hash(this->hash(key));
PHMAP_IF_CONSTEXPR (std_alloc_t::value)
prefetch_hash(this->hash(key));
}

// The API of find() has two extensions.
Expand Down Expand Up @@ -1848,6 +1863,11 @@ class raw_hash_set

template <class K = key_type>
bool find_impl(const key_arg<K>& key, size_t hashval, size_t& offset) {
PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
// ctrl_ could be nullptr
if (!ctrl_)
return false;
}
auto seq = probe(hashval);
while (true) {
Group g{ ctrl_ + seq.offset() };
Expand Down Expand Up @@ -2025,7 +2045,7 @@ class raw_hash_set
// Unpoison before returning the memory to the allocator.
SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_);
Deallocate<Layout::Alignment()>(&alloc_ref(), ctrl_, layout.AllocSize());
ctrl_ = EmptyGroup();
ctrl_ = EmptyGroup<std_alloc_t>();
slots_ = nullptr;
size_ = 0;
capacity_ = 0;
Expand Down Expand Up @@ -2135,6 +2155,11 @@ class raw_hash_set
}

bool has_element(const value_type& elem, size_t hashval) const {
PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
// ctrl_ could be nullptr
if (!ctrl_)
return false;
}
auto seq = probe(hashval);
while (true) {
Group g{ctrl_ + seq.offset()};
Expand Down Expand Up @@ -2197,6 +2222,11 @@ class raw_hash_set
protected:
template <class K>
size_t _find_key(const K& key, size_t hashval) {
PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
// ctrl_ could be nullptr
if (!ctrl_)
return (size_t)-1;
}
auto seq = probe(hashval);
while (true) {
Group g{ctrl_ + seq.offset()};
Expand All @@ -2221,7 +2251,12 @@ class raw_hash_set
}

size_t prepare_insert(size_t hashval) PHMAP_ATTRIBUTE_NOINLINE {
auto target = find_first_non_full(hashval);
PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
// ctrl_ could be nullptr
if (!ctrl_)
rehash_and_grow_if_necessary();
}
FindInfo target = find_first_non_full(hashval);
if (PHMAP_PREDICT_FALSE(growth_left() == 0 &&
!IsDeleted(ctrl_[target.offset]))) {
rehash_and_grow_if_necessary();
Expand Down Expand Up @@ -2335,10 +2370,10 @@ class raw_hash_set
// TODO(alkis): Investigate removing some of these fields:
// - ctrl/slots can be derived from each other
// - size can be moved into the slot array
ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t]
slot_type* slots_ = nullptr; // [capacity * slot_type]
size_t size_ = 0; // number of full slots
size_t capacity_ = 0; // total number of slots
ctrl_t* ctrl_ = EmptyGroup<std_alloc_t>(); // [(capacity + 1) * ctrl_t]
slot_type* slots_ = nullptr; // [capacity * slot_type]
size_t size_ = 0; // number of full slots
size_t capacity_ = 0; // total number of slots
HashtablezInfoHandle infoz_;
std::tuple<size_t /* growth_left */, hasher, key_equal, allocator_type>
settings_{0, hasher{}, key_equal{}, allocator_type{}};
Expand Down Expand Up @@ -2582,7 +2617,6 @@ class parallel_hash_set
using UniqueLock = typename Lockable::UniqueLock;
using SharedLock = typename Lockable::SharedLock;
using ReadWriteLock = typename Lockable::ReadWriteLock;


// --------------------------------------------------------------------
struct Inner : public Lockable
Expand Down Expand Up @@ -3144,14 +3178,9 @@ class parallel_hash_set
{
Inner& inner = sets_[subidx(hashval)];
auto& set = inner.set_;
ReadWriteLock m(inner);
UniqueLock m(inner);

size_t offset = set._find_key(key, hashval);
if (offset == (size_t)-1 && m.switch_to_unique()) {
// we did an unlock/lock, and another thread could have inserted the same key, so we need to
// do a find() again.
offset = set._find_key(key, hashval);
}
if (offset == (size_t)-1) {
offset = set.prepare_insert(hashval);
set.emplace_at(offset, std::forward<Args>(args)...);
Expand Down Expand Up @@ -3234,13 +3263,8 @@ class parallel_hash_set
iterator lazy_emplace_with_hash(const key_arg<K>& key, size_t hashval, F&& f) {
Inner& inner = sets_[subidx(hashval)];
auto& set = inner.set_;
ReadWriteLock m(inner);
UniqueLock m(inner);
size_t offset = set._find_key(key, hashval);
if (offset == (size_t)-1 && m.switch_to_unique()) {
// we did an unlock/lock, and another thread could have inserted the same key, so we need to
// do a find() again.
offset = set._find_key(key, hashval);
}
if (offset == (size_t)-1) {
offset = set.prepare_insert(hashval);
set.lazy_emplace_at(offset, std::forward<F>(f));
Expand Down Expand Up @@ -3355,7 +3379,7 @@ class parallel_hash_set
template <class K = key_type, class FExists, class FEmplace>
bool lazy_emplace_l(const key_arg<K>& key, FExists&& fExists, FEmplace&& fEmplace) {
size_t hashval = this->hash(key);
ReadWriteLock m;
UniqueLock m;
auto res = this->find_or_prepare_insert_with_hash(hashval, key, m);
Inner* inner = std::get<0>(res);
if (std::get<2>(res)) {
Expand Down Expand Up @@ -3444,6 +3468,10 @@ class parallel_hash_set
return sets_[idx];
}

const Inner& get_inner(size_t idx) const {
return sets_[idx];
}

// Extension API: support for heterogeneous keys.
//
// std::unordered_set<std::string> s;
Expand Down Expand Up @@ -3805,16 +3833,11 @@ class parallel_hash_set

template <class K>
std::tuple<Inner*, size_t, bool>
find_or_prepare_insert_with_hash(size_t hashval, const K& key, ReadWriteLock &mutexlock) {
find_or_prepare_insert_with_hash(size_t hashval, const K& key, UniqueLock &mutexlock) {
Inner& inner = sets_[subidx(hashval)];
auto& set = inner.set_;
mutexlock = std::move(ReadWriteLock(inner));
mutexlock = std::move(UniqueLock(inner));
size_t offset = set._find_key(key, hashval);
if (offset == (size_t)-1 && mutexlock.switch_to_unique()) {
// we did an unlock/lock, and another thread could have inserted the same key, so we need to
// do a find() again.
offset = set._find_key(key, hashval);
}
if (offset == (size_t)-1) {
offset = set.prepare_insert(hashval);
return std::make_tuple(&inner, offset, true);
Expand All @@ -3824,7 +3847,7 @@ class parallel_hash_set

template <class K>
std::tuple<Inner*, size_t, bool>
find_or_prepare_insert(const K& key, ReadWriteLock &mutexlock) {
find_or_prepare_insert(const K& key, UniqueLock &mutexlock) {
return find_or_prepare_insert_with_hash<K>(this->hash(key), key, mutexlock);
}

Expand Down Expand Up @@ -4046,7 +4069,7 @@ class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash
template <class K = key_type, class F, class... Args>
bool try_emplace_l(K&& k, F&& f, Args&&... args) {
size_t hashval = this->hash(k);
ReadWriteLock m;
UniqueLock m;
auto res = this->find_or_prepare_insert_with_hash(hashval, k, m);
typename Base::Inner *inner = std::get<0>(res);
if (std::get<2>(res)) {
Expand All @@ -4067,7 +4090,7 @@ class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash
template <class K = key_type, class... Args>
std::pair<typename parallel_hash_map::parallel_hash_set::pointer, bool> try_emplace_p(K&& k, Args&&... args) {
size_t hashval = this->hash(k);
ReadWriteLock m;
UniqueLock m;
auto res = this->find_or_prepare_insert_with_hash(hashval, k, m);
typename Base::Inner *inner = std::get<0>(res);
if (std::get<2>(res)) {
Expand Down Expand Up @@ -4097,7 +4120,7 @@ class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash
template <class K, class V>
std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) {
size_t hashval = this->hash(k);
ReadWriteLock m;
UniqueLock m;
auto res = this->find_or_prepare_insert_with_hash(hashval, k, m);
typename Base::Inner *inner = std::get<0>(res);
if (std::get<2>(res)) {
Expand All @@ -4117,7 +4140,7 @@ class parallel_hash_map : public parallel_hash_set<N, RefSet, Mtx_, Policy, Hash

template <class K = key_type, class... Args>
std::pair<iterator, bool> try_emplace_impl_with_hash(size_t hashval, K&& k, Args&&... args) {
ReadWriteLock m;
UniqueLock m;
auto res = this->find_or_prepare_insert_with_hash(hashval, k, m);
typename Base::Inner *inner = std::get<0>(res);
if (std::get<2>(res)) {
Expand Down Expand Up @@ -4572,6 +4595,8 @@ struct HashtableDebugAccess<Set, typename std::enable_if<has_member_type_raw_has

static size_t GetNumProbes(const Set& set,
const typename Set::key_type& key) {
if (!set.ctrl_)
return 0;
size_t num_probes = 0;
size_t hashval = set.hash(key);
auto seq = set.probe(hashval);
Expand Down
29 changes: 16 additions & 13 deletions thirdparty/parallel_hashmap/phmap_config.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
//
// Includes work from abseil-cpp (https://github.com/abseil/abseil-cpp)
// with modifications.
//
//
// Copyright 2018 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
Expand All @@ -35,8 +35,8 @@
// ---------------------------------------------------------------------------

#define PHMAP_VERSION_MAJOR 1
#define PHMAP_VERSION_MINOR 3
#define PHMAP_VERSION_PATCH 12
#define PHMAP_VERSION_MINOR 4
#define PHMAP_VERSION_PATCH 1

// Included for the __GLIBC__ macro (or similar macros on other systems).
#include <limits.h>
Expand Down Expand Up @@ -81,14 +81,14 @@
#error "phmap requires __apple_build_version__ of 4211165 or higher."
#endif

// Enforce C++11 as the minimum.
// Enforce C++11 as the minimum.
#if defined(__cplusplus) && !defined(_MSC_VER)
#if __cplusplus < 201103L
#error "C++ versions less than C++11 are not supported."
#endif
#endif

// We have chosen glibc 2.12 as the minimum
// We have chosen glibc 2.12 as the minimum
#if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
#if !__GLIBC_PREREQ(2, 12)
#error "Minimum required version of glibc is 2.12."
Expand All @@ -103,7 +103,7 @@
#warning "phmap assumes CHAR_BIT == 8."
#endif

// phmap currently assumes that an int is 4 bytes.
// phmap currently assumes that an int is 4 bytes.
#if INT_MAX < 2147483647
#error "phmap assumes that int is at least 4 bytes. "
#endif
Expand Down Expand Up @@ -176,10 +176,10 @@
#undef PHMAP_HAVE_TLS
#undef PHMAP_HAVE_THREAD_LOCAL
#endif
#endif
#endif

// ------------------------------------------------------------
// Checks whether the __int128 compiler extension for a 128-bit
// Checks whether the __int128 compiler extension for a 128-bit
// integral type is supported.
// ------------------------------------------------------------
#ifdef PHMAP_HAVE_INTRINSIC_INT128
Expand All @@ -197,7 +197,7 @@
#endif

// ------------------------------------------------------------------
// Checks whether the compiler both supports and enables exceptions.
// Checks whether the compiler both supports and enables exceptions.
// ------------------------------------------------------------------
#ifdef PHMAP_HAVE_EXCEPTIONS
#error PHMAP_HAVE_EXCEPTIONS cannot be directly set.
Expand Down Expand Up @@ -441,7 +441,9 @@
#define PHMAP_ATTRIBUTE_NONNULL(...)
#endif

#if PHMAP_HAVE_ATTRIBUTE(noreturn) || (defined(__GNUC__) && !defined(__clang__))
#if PHMAP_HAVE_ATTRIBUTE(noreturn)
#define PHMAP_ATTRIBUTE_NORETURN [[noreturn]]
#elif defined(__GNUC__) && !defined(__clang__)
#define PHMAP_ATTRIBUTE_NORETURN __attribute__((noreturn))
#elif defined(_MSC_VER)
#define PHMAP_ATTRIBUTE_NORETURN __declspec(noreturn)
Expand Down Expand Up @@ -646,7 +648,7 @@
// ----------------------------------------------------------------------
#if PHMAP_HAVE_CC17
#define PHMAP_IF_CONSTEXPR(expr) if constexpr ((expr))
#else
#else
#define PHMAP_IF_CONSTEXPR(expr) if ((expr))
#endif

Expand Down Expand Up @@ -680,8 +682,9 @@ namespace macros_internal {
} // namespace macros_internal
} // namespace phmap

// TODO(zhangxy): Use c++17 standard [[fallthrough]] macro, when supported.
#if defined(__clang__) && defined(__has_warning)
#if PHMAP_HAVE_CPP_ATTRIBUTE(fallthrough)
#define PHMAP_FALLTHROUGH [[fallthrough]]
#elif defined(__clang__) && defined(__has_warning)
#if __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough")
#define PHMAP_FALLTHROUGH_INTENDED [[clang::fallthrough]]
#endif
Expand Down
Loading

0 comments on commit e2b5978

Please sign in to comment.