Skip to content

Commit

Permalink
Merge 4c25787 into merged_master (Bitcoin PR #18021)
Browse files Browse the repository at this point in the history 
Had to change the signature of VectorReader::operator>>. I think this
is a bug upstream that only affects us beacuse we are using this operator
in a couple places to deserialize Bitcoin transactions and Merkle proofs
from pegin witnesses. Upstream does not appear to use this operator at
all. I am a little nervous about PR'ing to fix upstream because I am
afraid they'll take it away, so best we keep this among ourselves.
  • Loading branch information
@apoelstra
apoelstra committed Nov 26, 2020
2 parents 68844be + 4c25787 commit f11b83b
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 89 deletions.
100 changes: 68 additions & 32 deletions src/serialize.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@

static const unsigned int MAX_SIZE = 0x02000000;

/** Maximum amount of memory (in bytes) to allocate at once when deserializing vectors. */
static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;

/**
* Dummy data type to identify deserializing constructors.
*
Expand Down Expand Up @@ -593,6 +596,53 @@ class LimitedString
template<typename I>
BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); }

/** Formatter to serialize/deserialize vector elements using another formatter
*
* Example:
* struct X {
* std::vector<uint64_t> v;
* SERIALIZE_METHODS(X, obj) { READWRITE(Using<VectorFormatter<VarInt>>(obj.v)); }
* };
* will define a struct that contains a vector of uint64_t, which is serialized
* as a vector of VarInt-encoded integers.
*
* V is not required to be an std::vector type. It works for any class that
* exposes a value_type, size, reserve, push_back, and const iterators.
*/
template<class Formatter>
struct VectorFormatter
{
template<typename Stream, typename V>
void Ser(Stream& s, const V& v)
{
WriteCompactSize(s, v.size());
for (const typename V::value_type& elem : v) {
s << Using<Formatter>(elem);
}
}

template<typename Stream, typename V>
void Unser(Stream& s, V& v)
{
v.clear();
size_t size = ReadCompactSize(s);
size_t allocated = 0;
while (allocated < size) {
// For DoS prevention, do not blindly allocate as much as the stream claims to contain.
// Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
// X MiB of data to make us allocate X+5 Mib.
static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
v.reserve(allocated);
while (v.size() < allocated) {
typename V::value_type val;
s >> Using<Formatter>(val);
v.push_back(std::move(val));
}
}
};
};

/**
* Forward declarations
*/
Expand Down Expand Up @@ -673,6 +723,20 @@ inline void Unserialize(Stream& is, T&& a)
a.Unserialize(is);
}

/** Default formatter. Serializes objects as themselves.
*
* The vector/prevector serialization code passes this to VectorFormatter
* to enable reusing that logic. It shouldn't be needed elsewhere.
*/
struct DefaultFormatter
{
template<typename Stream, typename T>
static void Ser(Stream& s, const T& t) { Serialize(s, t); }

template<typename Stream, typename T>
static void Unser(Stream& s, T& t) { Unserialize(s, t); }
};




Expand Down Expand Up @@ -713,9 +777,7 @@ void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&)
template<typename Stream, unsigned int N, typename T, typename V>
void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&)
{
WriteCompactSize(os, v.size());
for (typename prevector<N, T>::const_iterator vi = v.begin(); vi != v.end(); ++vi)
::Serialize(os, (*vi));
Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
}

template<typename Stream, unsigned int N, typename T>
Expand Down Expand Up @@ -744,19 +806,7 @@ void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&)
template<typename Stream, unsigned int N, typename T, typename V>
void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&)
{
v.clear();
unsigned int nSize = ReadCompactSize(is);
unsigned int i = 0;
unsigned int nMid = 0;
while (nMid < nSize)
{
nMid += 5000000 / sizeof(T);
if (nMid > nSize)
nMid = nSize;
v.resize_uninitialized(nMid);
for (; i < nMid; ++i)
Unserialize(is, v[i]);
}
Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
}

template<typename Stream, unsigned int N, typename T>
Expand Down Expand Up @@ -793,9 +843,7 @@ void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&)
template<typename Stream, typename T, typename A, typename V>
void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&)
{
WriteCompactSize(os, v.size());
for (typename std::vector<T, A>::const_iterator vi = v.begin(); vi != v.end(); ++vi)
::Serialize(os, (*vi));
Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
}

template<typename Stream, typename T, typename A>
Expand Down Expand Up @@ -824,19 +872,7 @@ void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&)
template<typename Stream, typename T, typename A, typename V>
void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&)
{
v.clear();
unsigned int nSize = ReadCompactSize(is);
unsigned int i = 0;
unsigned int nMid = 0;
while (nMid < nSize)
{
nMid += 5000000 / sizeof(T);
if (nMid > nSize)
nMid = nSize;
v.resize(nMid);
for (; i < nMid; i++)
Unserialize(is, v[i]);
}
Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
}

template<typename Stream, typename T, typename A>
Expand Down
2 changes: 1 addition & 1 deletion src/streams.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ class VectorReader
}

template<typename T>
VectorReader& operator>>(T& obj)
VectorReader& operator>>(T&& obj)
{
// Unserialize from this stream
::Unserialize(*this, obj);
Expand Down
69 changes: 13 additions & 56 deletions src/undo.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,88 +13,50 @@
#include <serialize.h>
#include <version.h>

/** Undo information for a CTxIn
/** Formatter for undo information for a CTxIn
*
* Contains the prevout's CTxOut being spent, and its metadata as well
* (coinbase or not, height). The serialization contains a dummy value of
* zero. This is compatible with older versions which expect to see
* the transaction version there.
*/
class TxInUndoSerializer
struct TxInUndoFormatter
{
const Coin* txout;

public:
template<typename Stream>
void Serialize(Stream &s) const {
::Serialize(s, VARINT(txout->nHeight * 2 + (txout->fCoinBase ? 1u : 0u)));
if (txout->nHeight > 0) {
void Ser(Stream &s, const Coin& txout) {
::Serialize(s, VARINT(txout.nHeight * 2 + (txout.fCoinBase ? 1u : 0u)));
if (txout.nHeight > 0) {
// Required to maintain compatibility with older undo format.
::Serialize(s, (unsigned char)0);
}
::Serialize(s, Using<TxOutCompression>(REF(txout->out)));
::Serialize(s, Using<TxOutCompression>(txout.out));
}

explicit TxInUndoSerializer(const Coin* coin) : txout(coin) {}
};

class TxInUndoDeserializer
{
Coin* txout;

public:
template<typename Stream>
void Unserialize(Stream &s) {
void Unser(Stream &s, Coin& txout) {
unsigned int nCode = 0;
::Unserialize(s, VARINT(nCode));
txout->nHeight = nCode / 2;
txout->fCoinBase = nCode & 1;
if (txout->nHeight > 0) {
txout.nHeight = nCode / 2;
txout.fCoinBase = nCode & 1;
if (txout.nHeight > 0) {
// Old versions stored the version number for the last spend of
// a transaction's outputs. Non-final spends were indicated with
// height = 0.
unsigned int nVersionDummy;
::Unserialize(s, VARINT(nVersionDummy));
}
::Unserialize(s, Using<TxOutCompression>(REF(txout->out)));
::Unserialize(s, Using<TxOutCompression>(txout.out));
}

explicit TxInUndoDeserializer(Coin* coin) : txout(coin) {}
};

static const size_t MIN_TRANSACTION_INPUT_WEIGHT = WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxIn(), PROTOCOL_VERSION);
static const size_t MAX_INPUTS_PER_BLOCK = MAX_BLOCK_WEIGHT / MIN_TRANSACTION_INPUT_WEIGHT;

/** Undo information for a CTransaction */
class CTxUndo
{
public:
// undo information for all txins
std::vector<Coin> vprevout;

template <typename Stream>
void Serialize(Stream& s) const {
// TODO: avoid reimplementing vector serializer
uint64_t count = vprevout.size();
::Serialize(s, COMPACTSIZE(REF(count)));
for (const auto& prevout : vprevout) {
::Serialize(s, TxInUndoSerializer(&prevout));
}
}

template <typename Stream>
void Unserialize(Stream& s) {
// TODO: avoid reimplementing vector deserializer
uint64_t count = 0;
::Unserialize(s, COMPACTSIZE(count));
if (count > MAX_INPUTS_PER_BLOCK) {
throw std::ios_base::failure("Too many input undo records");
}
vprevout.resize(count);
for (auto& prevout : vprevout) {
::Unserialize(s, TxInUndoDeserializer(&prevout));
}
}
SERIALIZE_METHODS(CTxUndo, obj) { READWRITE(Using<VectorFormatter<TxInUndoFormatter>>(obj.vprevout)); }
};

/** Undo information for a CBlock */
Expand All @@ -103,12 +65,7 @@ class CBlockUndo
public:
std::vector<CTxUndo> vtxundo; // for all but the coinbase

ADD_SERIALIZE_METHODS;

template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(vtxundo);
}
SERIALIZE_METHODS(CBlockUndo, obj) { READWRITE(obj.vtxundo); }
};

#endif // BITCOIN_UNDO_H

0 comments on commit f11b83b

Please sign in to comment.