diff --git a/barretenberg/cpp/pil/vm2/README.md b/barretenberg/cpp/pil/vm2/README.md new file mode 100644 index 00000000000..4194d164843 --- /dev/null +++ b/barretenberg/cpp/pil/vm2/README.md @@ -0,0 +1,7 @@ +Compile with: + +``` +~/aztec-packages/bb-pilcom/target/release/bb_pil pil/vm2/execution.pil --name Avm2 -y -o src/barretenberg/vm2/generated && ./format.sh changed +``` + +while on the `barretenberg/cpp` directory. diff --git a/barretenberg/cpp/pil/vm2/addressing.pil b/barretenberg/cpp/pil/vm2/addressing.pil new file mode 100644 index 00000000000..19947f5f661 --- /dev/null +++ b/barretenberg/cpp/pil/vm2/addressing.pil @@ -0,0 +1,21 @@ +// This is a virtual gadget, which is part of the execution trace. +namespace execution(256); + +pol commit stack_pointer_val; +pol commit stack_pointer_tag; +pol commit sel_addressing_error; // true if any error type +pol commit addressing_error_kind; // TODO: might need to be selectors +pol commit addressing_error_idx; // operand index for error, if any + +// whether each operand is an address for the given opcode. +// retrieved from the instruction spec. +pol commit sel_op1_is_address; +pol commit sel_op2_is_address; +pol commit sel_op3_is_address; +pol commit sel_op4_is_address; +// operands after relative resolution +pol commit op1_after_relative; +pol commit op2_after_relative; +pol commit op3_after_relative; +pol commit op4_after_relative; +// operands after indirect resolution are the resolved_operands rop1, ... diff --git a/barretenberg/cpp/pil/vm2/alu.pil b/barretenberg/cpp/pil/vm2/alu.pil new file mode 100644 index 00000000000..ee0de13f9ab --- /dev/null +++ b/barretenberg/cpp/pil/vm2/alu.pil @@ -0,0 +1,16 @@ +namespace alu(256); + +pol commit sel_op_add; +pol commit ia; +pol commit ib; +pol commit ic; +pol commit op; +pol commit ia_addr; +pol commit ib_addr; +pol commit dst_addr; + +#[SEL_ADD_BINARY] +sel_op_add * (1 - sel_op_add) = 0; + +#[ALU_ADD] +ia + ib = ic; \ No newline at end of file diff --git a/barretenberg/cpp/pil/vm2/execution.pil b/barretenberg/cpp/pil/vm2/execution.pil new file mode 100644 index 00000000000..5718674edb5 --- /dev/null +++ b/barretenberg/cpp/pil/vm2/execution.pil @@ -0,0 +1,55 @@ +include "alu.pil"; +include "addressing.pil"; +include "precomputed.pil"; + +namespace execution(256); + +pol commit sel; // subtrace selector + +pol commit ex_opcode; +pol commit indirect; +// operands +pol commit op1; +pol commit op2; +pol commit op3; +pol commit op4; +// resolved operands +pol commit rop1; +pol commit rop2; +pol commit rop3; +pol commit rop4; + +pol commit pc; +pol commit clk; +pol commit last; + +// Selector constraints +sel * (1 - sel) = 0; +last * (1 - last) = 0; + +// If the current row is an execution row, then either +// the next row is an execution row, or the current row is marked as the last row. +// sel => (sel' v last) = 1 iff +// ¬sel v (sel' v last) = 1 iff +// ¬(¬sel v (sel' v last)) = 0 iff +// sel ^ (¬sel' ^ ¬last) = 0 iff +// sel * (1 - sel') * (1 - last) = 0 +#[TRACE_CONTINUITY_1] +sel * (1 - sel') * (1 - last) = 0; +// If the current row is not an execution row, then there are no more execution rows after that. +// (not enforced for the first row) +#[TRACE_CONTINUITY_2] +(1 - precomputed.first_row) * (1 - sel) * sel' = 0; +// If the current row is the last row, then the next row is not an execution row. +#[LAST_IS_LAST] +last * sel' = 0; + +// These are needed to have a non-empty set of columns for each type. +pol public input; +#[LOOKUP_DUMMY_PRECOMPUTED] +sel {/*will be 1=OR*/ sel, clk, clk, clk} in +precomputed.sel_bitwise {precomputed.bitwise_op_id, precomputed.bitwise_input_a, precomputed.bitwise_input_b, precomputed.bitwise_output}; +#[LOOKUP_DUMMY_DYNAMIC] // Just a self-lookup for now, for testing. +sel {op1, op2, op3, op4} in sel {op1, op2, op3, op4}; +#[PERM_DUMMY_DYNAMIC] // Just a self-permutation for now, for testing. +sel {op1, op2, op3, op4} is sel {op1, op2, op3, op4}; \ No newline at end of file diff --git a/barretenberg/cpp/pil/vm2/precomputed.pil b/barretenberg/cpp/pil/vm2/precomputed.pil new file mode 100644 index 00000000000..f5dc8ff2635 --- /dev/null +++ b/barretenberg/cpp/pil/vm2/precomputed.pil @@ -0,0 +1,17 @@ +// General/shared precomputed columns. +namespace precomputed(256); + +// From 0 and incrementing up to the size of the circuit (2^21). +pol constant clk; + +// 1 only at row 0. +pol constant first_row; + +// AND/OR/XOR of all 8-bit numbers. +// The tables are "stacked". First AND, then OR, then XOR. +// Note: think if we can avoid the selector. +pol constant sel_bitwise; // 1 in the first 3 * 256 rows. +pol constant bitwise_op_id; // identifies if operation is AND/OR/XOR. +pol constant bitwise_input_a; // column of all 8-bit numbers. +pol constant bitwise_input_b; // column of all 8-bit numbers. +pol constant bitwise_output; // output = a AND/OR/XOR b. \ No newline at end of file diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index fd992566082..cd0965babb9 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -95,6 +95,7 @@ add_subdirectory(barretenberg/transcript) add_subdirectory(barretenberg/translator_vm) add_subdirectory(barretenberg/ultra_honk) add_subdirectory(barretenberg/vm) +add_subdirectory(barretenberg/vm2) add_subdirectory(barretenberg/wasi) add_subdirectory(barretenberg/world_state) @@ -171,6 +172,7 @@ set(BARRETENBERG_TARGET_OBJECTS if(NOT DISABLE_AZTEC_VM) # enable AVM list(APPEND BARRETENBERG_TARGET_OBJECTS $) + list(APPEND BARRETENBERG_TARGET_OBJECTS $) endif() if(NOT WASM) diff --git a/barretenberg/cpp/src/barretenberg/bb/main.cpp b/barretenberg/cpp/src/barretenberg/bb/main.cpp index cf6f7fefba1..465d9732ac1 100644 --- a/barretenberg/cpp/src/barretenberg/bb/main.cpp +++ b/barretenberg/cpp/src/barretenberg/bb/main.cpp @@ -3,6 +3,7 @@ #include "barretenberg/bb/file_io.hpp" #include "barretenberg/client_ivc/client_ivc.hpp" #include "barretenberg/common/benchmark.hpp" +#include "barretenberg/common/log.hpp" #include "barretenberg/common/map.hpp" #include "barretenberg/common/serialize.hpp" #include "barretenberg/common/timer.hpp" @@ -13,7 +14,6 @@ #include "barretenberg/dsl/acir_format/proof_surgeon.hpp" #include "barretenberg/dsl/acir_proofs/acir_composer.hpp" #include "barretenberg/dsl/acir_proofs/honk_contract.hpp" -#include "barretenberg/flavor/flavor.hpp" #include "barretenberg/honk/proof_system/types/proof.hpp" #include "barretenberg/numeric/bitop/get_msb.hpp" #include "barretenberg/plonk/proof_system/proving_key/serialize.hpp" @@ -24,15 +24,17 @@ #include "barretenberg/stdlib_circuit_builders/ultra_flavor.hpp" #include "barretenberg/stdlib_circuit_builders/ultra_keccak_flavor.hpp" #include "barretenberg/stdlib_circuit_builders/ultra_rollup_flavor.hpp" -#include "barretenberg/vm/avm/trace/public_inputs.hpp" -#include #ifndef DISABLE_AZTEC_VM #include "barretenberg/vm/avm/generated/flavor.hpp" #include "barretenberg/vm/avm/trace/common.hpp" #include "barretenberg/vm/avm/trace/execution.hpp" +#include "barretenberg/vm/avm/trace/public_inputs.hpp" #include "barretenberg/vm/aztec_constants.hpp" #include "barretenberg/vm/stats.hpp" +#include "barretenberg/vm2/avm_api.hpp" +#include "barretenberg/vm2/common/aztec_types.hpp" +#include "barretenberg/vm2/common/constants.hpp" #endif using namespace bb; @@ -671,6 +673,16 @@ void vk_as_fields(const std::string& vk_path, const std::string& output_path) } #ifndef DISABLE_AZTEC_VM +void print_avm_stats() +{ +#ifdef AVM_TRACK_STATS + info("------- STATS -------"); + const auto& stats = avm_trace::Stats::get(); + const int levels = std::getenv("AVM_STATS_DEPTH") != nullptr ? std::stoi(std::getenv("AVM_STATS_DEPTH")) : 2; + info(stats.to_string(levels)); +#endif +} + /** * @brief Writes an avm proof and corresponding (incomplete) verification key to files. * @@ -726,12 +738,34 @@ void avm_prove(const std::filesystem::path& public_inputs_path, write_file(vk_fields_path, { vk_json.begin(), vk_json.end() }); vinfo("vk as fields written to: ", vk_fields_path); -#ifdef AVM_TRACK_STATS - info("------- STATS -------"); - const auto& stats = avm_trace::Stats::get(); - const int levels = std::getenv("AVM_STATS_DEPTH") != nullptr ? std::stoi(std::getenv("AVM_STATS_DEPTH")) : 2; - info(stats.to_string(levels)); -#endif + print_avm_stats(); +} + +void avm2_prove(const std::filesystem::path& inputs_path, const std::filesystem::path& output_path) +{ + avm2::AvmAPI avm; + auto inputs = avm2::AvmAPI::ProvingInputs::from(read_file(inputs_path)); + + // This is bigger than CIRCUIT_SUBGROUP_SIZE because of BB inefficiencies. + init_bn254_crs(avm2::CIRCUIT_SUBGROUP_SIZE * 2); + auto [proof, vk] = avm.prove(inputs); + + // NOTE: As opposed to Avm1 and other proof systems, the public inputs are NOT part of the proof. + write_file(output_path / "proof", to_buffer(proof)); + write_file(output_path / "vk", vk); + + print_avm_stats(); +} + +void avm2_check_circuit(const std::filesystem::path& inputs_path) +{ + avm2::AvmAPI avm; + auto inputs = avm2::AvmAPI::ProvingInputs::from(read_file(inputs_path)); + + bool res = avm.check_circuit(inputs); + info("circuit check: ", res ? "success" : "failure"); + + print_avm_stats(); } /** @@ -783,8 +817,28 @@ bool avm_verify(const std::filesystem::path& proof_path, const std::filesystem:: const bool verified = AVM_TRACK_TIME_V("verify/all", avm_trace::Execution::verify(vk, proof)); vinfo("verified: ", verified); + + print_avm_stats(); return verified; } + +// NOTE: The proof should NOT include the public inputs. +bool avm2_verify(const std::filesystem::path& proof_path, + const std::filesystem::path& public_inputs_path, + const std::filesystem::path& vk_path) +{ + const auto proof = many_from_buffer(read_file(proof_path)); + std::vector vk_bytes = read_file(vk_path); + auto public_inputs = avm2::PublicInputs::from(read_file(public_inputs_path)); + + init_bn254_crs(1); + avm2::AvmAPI avm; + bool res = avm.verify(proof, public_inputs, vk_bytes); + info("verification: ", res ? "success" : "failure"); + + print_avm_stats(); + return res; +} #endif /** @@ -1382,6 +1436,18 @@ int main(int argc, char* argv[]) std::string output_path = get_option(args, "-o", "./target"); write_recursion_inputs_honk(bytecode_path, witness_path, output_path, recursive); #ifndef DISABLE_AZTEC_VM + } else if (command == "avm2_prove") { + std::filesystem::path inputs_path = get_option(args, "--avm-inputs", "./target/avm_inputs.bin"); + // This outputs both files: proof and vk, under the given directory. + std::filesystem::path output_path = get_option(args, "-o", "./proofs"); + avm2_prove(inputs_path, output_path); + } else if (command == "avm2_check_circuit") { + std::filesystem::path inputs_path = get_option(args, "--avm-inputs", "./target/avm_inputs.bin"); + avm2_check_circuit(inputs_path); + } else if (command == "avm2_verify") { + std::filesystem::path public_inputs_path = + get_option(args, "--avm-public-inputs", "./target/avm_public_inputs.bin"); + return avm2_verify(proof_path, public_inputs_path, vk_path) ? 0 : 1; } else if (command == "avm_prove") { std::filesystem::path avm_public_inputs_path = get_option(args, "--avm-public-inputs", "./target/avm_public_inputs.bin"); diff --git a/barretenberg/cpp/src/barretenberg/ecc/groups/affine_element.hpp b/barretenberg/cpp/src/barretenberg/ecc/groups/affine_element.hpp index 883f8bcb017..3372899d730 100644 --- a/barretenberg/cpp/src/barretenberg/ecc/groups/affine_element.hpp +++ b/barretenberg/cpp/src/barretenberg/ecc/groups/affine_element.hpp @@ -170,6 +170,9 @@ template class alignas(64) affine_ } Fq x; Fq y; + + // Note: this serialization from typescript does not support infinity. + MSGPACK_FIELDS(x, y); }; template diff --git a/barretenberg/cpp/src/barretenberg/vm2/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/vm2/CMakeLists.txt new file mode 100644 index 00000000000..8084b7b8306 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/vm2/CMakeLists.txt @@ -0,0 +1,3 @@ +if(NOT DISABLE_AZTEC_VM) + barretenberg_module(vm2 sumcheck stdlib_honk_verifier) +endif() \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/vm2/avm_api.cpp b/barretenberg/cpp/src/barretenberg/vm2/avm_api.cpp new file mode 100644 index 00000000000..8dddc62058d --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/vm2/avm_api.cpp @@ -0,0 +1,58 @@ +#include "barretenberg/vm2/avm_api.hpp" + +#include "barretenberg/vm/stats.hpp" +#include "barretenberg/vm2/proving_helper.hpp" +#include "barretenberg/vm2/simulation_helper.hpp" +#include "barretenberg/vm2/tracegen_helper.hpp" + +namespace bb::avm2 { + +using namespace bb::avm2::simulation; + +std::pair AvmAPI::prove(const AvmAPI::ProvingInputs& inputs) +{ + // Simulate. + info("Simulating..."); + AvmSimulationHelper simulation_helper(inputs); + auto events = AVM_TRACK_TIME_V("simulation/all", simulation_helper.simulate()); + + // Generate trace. + info("Generating trace..."); + AvmTraceGenHelper tracegen_helper; + auto trace = AVM_TRACK_TIME_V("tracegen/all", tracegen_helper.generate_trace(std::move(events))); + + // Prove. + info("Proving..."); + AvmProvingHelper proving_helper; + auto [proof, vk] = AVM_TRACK_TIME_V("proving/all", proving_helper.prove(std::move(trace))); + + info("Done!"); + return { std::move(proof), std::move(vk) }; +} + +bool AvmAPI::check_circuit(const AvmAPI::ProvingInputs& inputs) +{ + // Simulate. + info("Simulating..."); + AvmSimulationHelper simulation_helper(inputs); + auto events = AVM_TRACK_TIME_V("simulation/all", simulation_helper.simulate()); + + // Generate trace. + info("Generating trace..."); + AvmTraceGenHelper tracegen_helper; + auto trace = AVM_TRACK_TIME_V("tracegen/all", tracegen_helper.generate_trace(std::move(events))); + + // Check circuit. + info("Checking circuit..."); + AvmProvingHelper proving_helper; + return proving_helper.check_circuit(std::move(trace)); +} + +bool AvmAPI::verify(const AvmProof& proof, const PublicInputs& pi, const AvmVerificationKey& vk_data) +{ + info("Verifying..."); + AvmProvingHelper proving_helper; + return AVM_TRACK_TIME_V("verifing/all", proving_helper.verify(proof, pi, vk_data)); +} + +} // namespace bb::avm2 \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/vm2/avm_api.hpp b/barretenberg/cpp/src/barretenberg/vm2/avm_api.hpp new file mode 100644 index 00000000000..9e94e922c5a --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/vm2/avm_api.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include + +#include "barretenberg/vm2/common/avm_inputs.hpp" +#include "barretenberg/vm2/proving_helper.hpp" + +namespace bb::avm2 { + +class AvmAPI { + public: + using AvmProof = AvmProvingHelper::Proof; + using AvmVerificationKey = std::vector; + using ProvingInputs = AvmProvingInputs; + + AvmAPI() = default; + + // NOTE: The public inputs are NOT part of the proof. + std::pair prove(const ProvingInputs& inputs); + bool check_circuit(const ProvingInputs& inputs); + bool verify(const AvmProof& proof, const PublicInputs& pi, const AvmVerificationKey& vk_data); +}; + +} // namespace bb::avm2 \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/vm2/common/ankerl_map.hpp b/barretenberg/cpp/src/barretenberg/vm2/common/ankerl_map.hpp new file mode 100644 index 00000000000..6342ce0171a --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/vm2/common/ankerl_map.hpp @@ -0,0 +1,2126 @@ +///////////////////////// ankerl::unordered_dense::{map, set} ///////////////////////// + +// A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion. +// Version 4.5.0 +// https://github.com/martinus/unordered_dense +// +// Licensed under the MIT License . +// SPDX-License-Identifier: MIT +// Copyright (c) 2022-2024 Martin Leitner-Ankerl +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef ANKERL_UNORDERED_DENSE_H +#define ANKERL_UNORDERED_DENSE_H + +// see https://semver.org/spec/v2.0.0.html +#define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 4 // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes +#define ANKERL_UNORDERED_DENSE_VERSION_MINOR \ + 5 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality +#define ANKERL_UNORDERED_DENSE_VERSION_PATCH 0 // NOLINT(cppcoreguidelines-macro-usage) backwards compatible bug fixes + +// API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/ + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT(major, minor, patch) \ + ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) +#define ANKERL_UNORDERED_DENSE_NAMESPACE \ + ANKERL_UNORDERED_DENSE_VERSION_CONCAT(ANKERL_UNORDERED_DENSE_VERSION_MAJOR, \ + ANKERL_UNORDERED_DENSE_VERSION_MINOR, \ + ANKERL_UNORDERED_DENSE_VERSION_PATCH) + +#if defined(_MSVC_LANG) +#define ANKERL_UNORDERED_DENSE_CPP_VERSION _MSVC_LANG +#else +#define ANKERL_UNORDERED_DENSE_CPP_VERSION __cplusplus +#endif + +#if defined(__GNUC__) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define ANKERL_UNORDERED_DENSE_PACK(decl) decl __attribute__((__packed__)) +#elif defined(_MSC_VER) +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define ANKERL_UNORDERED_DENSE_PACK(decl) __pragma(pack(push, 1)) decl __pragma(pack(pop)) +#endif + +// exceptions +#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND) +#define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 1 // NOLINT(cppcoreguidelines-macro-usage) +#else +#define ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() 0 // NOLINT(cppcoreguidelines-macro-usage) +#endif +#ifdef _MSC_VER +#define ANKERL_UNORDERED_DENSE_NOINLINE __declspec(noinline) +#else +#define ANKERL_UNORDERED_DENSE_NOINLINE __attribute__((noinline)) +#endif + +// defined in unordered_dense.cpp +#if !defined(ANKERL_UNORDERED_DENSE_EXPORT) +#define ANKERL_UNORDERED_DENSE_EXPORT +#endif + +#if ANKERL_UNORDERED_DENSE_CPP_VERSION < 201703L +#error ankerl::unordered_dense requires C++17 or higher +#else +#include // for array +#include // for uint64_t, uint32_t, uint8_t, UINT64_C +#include // for size_t, memcpy, memset +#include // for equal_to, hash +#include // for initializer_list +#include // for pair, distance +#include // for numeric_limits +#include // for allocator, allocator_traits, shared_ptr +#include // for optional +#include // for out_of_range +#include // for basic_string +#include // for basic_string_view, hash +#include // for forward_as_tuple +#include // for enable_if_t, declval, conditional_t, ena... +#include // for forward, exchange, pair, as_const, piece... +#include // for vector +#if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() == 0 +#include // for abort +#endif + +#if defined(__has_include) && !defined(ANKERL_UNORDERED_DENSE_DISABLE_PMR) +#if __has_include() +#define ANKERL_UNORDERED_DENSE_PMR std::pmr // NOLINT(cppcoreguidelines-macro-usage) +#include // for polymorphic_allocator +#elif __has_include() +#define ANKERL_UNORDERED_DENSE_PMR std::experimental::pmr // NOLINT(cppcoreguidelines-macro-usage) +#include // for polymorphic_allocator +#endif +#endif + +#if defined(_MSC_VER) && defined(_M_X64) +#include +#pragma intrinsic(_umul128) +#endif + +#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) +#define ANKERL_UNORDERED_DENSE_LIKELY(x) __builtin_expect(x, 1) // NOLINT(cppcoreguidelines-macro-usage) +#define ANKERL_UNORDERED_DENSE_UNLIKELY(x) __builtin_expect(x, 0) // NOLINT(cppcoreguidelines-macro-usage) +#else +#define ANKERL_UNORDERED_DENSE_LIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage) +#define ANKERL_UNORDERED_DENSE_UNLIKELY(x) (x) // NOLINT(cppcoreguidelines-macro-usage) +#endif + +namespace ankerl::unordered_dense { +inline namespace ANKERL_UNORDERED_DENSE_NAMESPACE { + +namespace detail { + +#if ANKERL_UNORDERED_DENSE_HAS_EXCEPTIONS() + +// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other +// inlinings more difficult. Throws are also generally the slow path. +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_key_not_found() +{ + throw std::out_of_range("ankerl::unordered_dense::map::at(): key not found"); +} +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_bucket_overflow() +{ + throw std::overflow_error("ankerl::unordered_dense: reached max bucket size, cannot increase size"); +} +[[noreturn]] inline ANKERL_UNORDERED_DENSE_NOINLINE void on_error_too_many_elements() +{ + throw std::out_of_range("ankerl::unordered_dense::map::replace(): too many elements"); +} + +#else + +[[noreturn]] inline void on_error_key_not_found() +{ + abort(); +} +[[noreturn]] inline void on_error_bucket_overflow() +{ + abort(); +} +[[noreturn]] inline void on_error_too_many_elements() +{ + abort(); +} + +#endif + +} // namespace detail + +// hash /////////////////////////////////////////////////////////////////////// + +// This is a stripped-down implementation of wyhash: https://github.com/wangyi-fudan/wyhash +// No big-endian support (because different values on different machines don't matter), +// hardcodes seed and the secret, reformats the code, and clang-tidy fixes. +namespace detail::wyhash { + +inline void mum(uint64_t* a, uint64_t* b) +{ +#if defined(__SIZEOF_INT128__) + __uint128_t r = *a; + r *= *b; + *a = static_cast(r); + *b = static_cast(r >> 64U); +#elif defined(_MSC_VER) && defined(_M_X64) + *a = _umul128(*a, *b, b); +#else + uint64_t ha = *a >> 32U; + uint64_t hb = *b >> 32U; + uint64_t la = static_cast(*a); + uint64_t lb = static_cast(*b); + uint64_t hi{}; + uint64_t lo{}; + uint64_t rh = ha * hb; + uint64_t rm0 = ha * lb; + uint64_t rm1 = hb * la; + uint64_t rl = la * lb; + uint64_t t = rl + (rm0 << 32U); + auto c = static_cast(t < rl); + lo = t + (rm1 << 32U); + c += static_cast(lo < t); + hi = rh + (rm0 >> 32U) + (rm1 >> 32U) + c; + *a = lo; + *b = hi; +#endif +} + +// multiply and xor mix function, aka MUM +[[nodiscard]] inline auto mix(uint64_t a, uint64_t b) -> uint64_t +{ + mum(&a, &b); + return a ^ b; +} + +// read functions. WARNING: we don't care about endianness, so results are different on big endian! +[[nodiscard]] inline auto r8(const uint8_t* p) -> uint64_t +{ + uint64_t v{}; + std::memcpy(&v, p, 8U); + return v; +} + +[[nodiscard]] inline auto r4(const uint8_t* p) -> uint64_t +{ + uint32_t v{}; + std::memcpy(&v, p, 4); + return v; +} + +// reads 1, 2, or 3 bytes +[[nodiscard]] inline auto r3(const uint8_t* p, size_t k) -> uint64_t +{ + return (static_cast(p[0]) << 16U) | (static_cast(p[k >> 1U]) << 8U) | p[k - 1]; +} + +[[maybe_unused]] [[nodiscard]] inline auto hash(void const* key, size_t len) -> uint64_t +{ + static constexpr auto secret = std::array{ UINT64_C(0xa0761d6478bd642f), + UINT64_C(0xe7037ed1a0b428db), + UINT64_C(0x8ebc6af09c88c6e3), + UINT64_C(0x589965cc75374cc3) }; + + auto const* p = static_cast(key); + uint64_t seed = secret[0]; + uint64_t a{}; + uint64_t b{}; + if (ANKERL_UNORDERED_DENSE_LIKELY(len <= 16)) { + if (ANKERL_UNORDERED_DENSE_LIKELY(len >= 4)) { + a = (r4(p) << 32U) | r4(p + ((len >> 3U) << 2U)); + b = (r4(p + len - 4) << 32U) | r4(p + len - 4 - ((len >> 3U) << 2U)); + } else if (ANKERL_UNORDERED_DENSE_LIKELY(len > 0)) { + a = r3(p, len); + b = 0; + } else { + a = 0; + b = 0; + } + } else { + size_t i = len; + if (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 48)) { + uint64_t see1 = seed; + uint64_t see2 = seed; + do { + seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed); + see1 = mix(r8(p + 16) ^ secret[2], r8(p + 24) ^ see1); + see2 = mix(r8(p + 32) ^ secret[3], r8(p + 40) ^ see2); + p += 48; + i -= 48; + } while (ANKERL_UNORDERED_DENSE_LIKELY(i > 48)); + seed ^= see1 ^ see2; + } + while (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 16)) { + seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed); + i -= 16; + p += 16; + } + a = r8(p + i - 16); + b = r8(p + i - 8); + } + + return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed)); +} + +[[nodiscard]] inline auto hash(uint64_t x) -> uint64_t +{ + return detail::wyhash::mix(x, UINT64_C(0x9E3779B97F4A7C15)); +} + +} // namespace detail::wyhash + +ANKERL_UNORDERED_DENSE_EXPORT template struct hash { + auto operator()(T const& obj) const + noexcept(noexcept(std::declval>().operator()(std::declval()))) -> uint64_t + { + return std::hash{}(obj); + } +}; + +template struct hash::is_avalanching> { + using is_avalanching = void; + auto operator()(T const& obj) const + noexcept(noexcept(std::declval>().operator()(std::declval()))) -> uint64_t + { + return std::hash{}(obj); + } +}; + +template struct hash> { + using is_avalanching = void; + auto operator()(std::basic_string const& str) const noexcept -> uint64_t + { + return detail::wyhash::hash(str.data(), sizeof(CharT) * str.size()); + } +}; + +template struct hash> { + using is_avalanching = void; + auto operator()(std::basic_string_view const& sv) const noexcept -> uint64_t + { + return detail::wyhash::hash(sv.data(), sizeof(CharT) * sv.size()); + } +}; + +template struct hash { + using is_avalanching = void; + auto operator()(T* ptr) const noexcept -> uint64_t + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return detail::wyhash::hash(reinterpret_cast(ptr)); + } +}; + +template struct hash> { + using is_avalanching = void; + auto operator()(std::unique_ptr const& ptr) const noexcept -> uint64_t + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return detail::wyhash::hash(reinterpret_cast(ptr.get())); + } +}; + +template struct hash> { + using is_avalanching = void; + auto operator()(std::shared_ptr const& ptr) const noexcept -> uint64_t + { + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + return detail::wyhash::hash(reinterpret_cast(ptr.get())); + } +}; + +template struct hash::value>::type> { + using is_avalanching = void; + auto operator()(Enum e) const noexcept -> uint64_t + { + using underlying = typename std::underlying_type_t; + return detail::wyhash::hash(static_cast(e)); + } +}; + +template struct tuple_hash_helper { + // Converts the value into 64bit. If it is an integral type, just cast it. Mixing is doing the rest. + // If it isn't an integral we need to hash it. + template [[nodiscard]] constexpr static auto to64(Arg const& arg) -> uint64_t + { + if constexpr (std::is_integral_v || std::is_enum_v) { + return static_cast(arg); + } else { + return hash{}(arg); + } + } + + [[nodiscard]] static auto mix64(uint64_t state, uint64_t v) -> uint64_t + { + return detail::wyhash::mix(state + v, uint64_t{ 0x9ddfea08eb382d69 }); + } + + // Creates a buffer that holds all the data from each element of the tuple. If possible we memcpy the data directly. + // If not, we hash the object and use this for the array. Size of the array is known at compile time, and memcpy is + // optimized away, so filling the buffer is highly efficient. Finally, call wyhash with this buffer. + template + [[nodiscard]] static auto calc_hash(T const& t, std::index_sequence) noexcept -> uint64_t + { + auto h = uint64_t{}; + ((h = mix64(h, to64(std::get(t)))), ...); + return h; + } +}; + +template struct hash> : tuple_hash_helper { + using is_avalanching = void; + auto operator()(std::tuple const& t) const noexcept -> uint64_t + { + return tuple_hash_helper::calc_hash(t, std::index_sequence_for{}); + } +}; + +template struct hash> : tuple_hash_helper { + using is_avalanching = void; + auto operator()(std::pair const& t) const noexcept -> uint64_t + { + return tuple_hash_helper::calc_hash(t, std::index_sequence_for{}); + } +}; + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define ANKERL_UNORDERED_DENSE_HASH_STATICCAST(T) \ + template <> struct hash { \ + using is_avalanching = void; \ + auto operator()(T const& obj) const noexcept -> uint64_t \ + { \ + return detail::wyhash::hash(static_cast(obj)); \ + } \ + } + +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuseless-cast" +#endif +// see https://en.cppreference.com/w/cpp/utility/hash +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(bool); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(signed char); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned char); +#if ANKERL_UNORDERED_DENSE_CPP_VERSION >= 202002L && defined(__cpp_char8_t) +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char8_t); +#endif +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char16_t); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char32_t); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(wchar_t); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(short); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned short); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(int); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned int); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long long); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long); +ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long long); + +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + +// bucket_type ////////////////////////////////////////////////////////// + +namespace bucket_type { + +struct standard { + static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint + static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint + + uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash + uint32_t m_value_idx; // index into the m_values vector. +}; + +ANKERL_UNORDERED_DENSE_PACK(struct big { + static constexpr uint32_t dist_inc = 1U << 8U; // skip 1 byte fingerprint + static constexpr uint32_t fingerprint_mask = dist_inc - 1; // mask for 1 byte of fingerprint + + uint32_t m_dist_and_fingerprint; // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash + size_t m_value_idx; // index into the m_values vector. +}); + +} // namespace bucket_type + +namespace detail { + +struct nonesuch {}; +struct default_container_t {}; + +template class Op, class... Args> struct detector { + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> { + using value_t = std::true_type; + using type = Op; +}; + +template