Skip to content

Commit

Permalink
add guts and bao packages
Browse files Browse the repository at this point in the history
  • Loading branch information
@lukechampine
lukechampine committed May 2, 2024
1 parent 8541013 commit 2d147a7
Show file tree
Hide file tree
Showing 13 changed files with 598 additions and 561 deletions.
150 changes: 84 additions & 66 deletions bao.go → bao/bao.go
View file
Original file line number Diff line number Diff line change
@@ -1,50 +1,69 @@
package blake3
// Package bao implements BLAKE3 verified streaming.
package bao

import (
"bytes"
"encoding/binary"
"errors"
"io"
"math/bits"

"lukechampine.com/blake3/guts"
)

func compressGroup(p []byte, counter uint64) node {
var stack [54 - maxSIMD][8]uint32
func bytesToCV(b []byte) (cv [8]uint32) {
_ = b[31] // bounds check hint
for i := range cv {
cv[i] = binary.LittleEndian.Uint32(b[4*i:])
}
return cv
}

func cvToBytes(cv *[8]uint32) *[32]byte {
var b [32]byte
for i, w := range cv {
binary.LittleEndian.PutUint32(b[4*i:], w)
}
return &b
}

func compressGroup(p []byte, counter uint64) guts.Node {
var stack [54 - guts.MaxSIMD][8]uint32
var sc uint64
pushSubtree := func(cv [8]uint32) {
i := 0
for sc&(1<<i) != 0 {
cv = chainingValue(parentNode(stack[i], cv, iv, 0))
cv = guts.ChainingValue(guts.ParentNode(stack[i], cv, &guts.IV, 0))
i++
}
stack[i] = cv
sc++
}

var buf [maxSIMD * chunkSize]byte
var buf [guts.MaxSIMD * guts.ChunkSize]byte
var buflen int
for len(p) > 0 {
if buflen == len(buf) {
pushSubtree(chainingValue(compressBuffer(&buf, buflen, &iv, counter+(sc*maxSIMD), 0)))
pushSubtree(guts.ChainingValue(guts.CompressBuffer(&buf, buflen, &guts.IV, counter+(sc*guts.MaxSIMD), 0)))
buflen = 0
}
n := copy(buf[buflen:], p)
buflen += n
p = p[n:]
}
n := compressBuffer(&buf, buflen, &iv, counter+(sc*maxSIMD), 0)
n := guts.CompressBuffer(&buf, buflen, &guts.IV, counter+(sc*guts.MaxSIMD), 0)
for i := bits.TrailingZeros64(sc); i < bits.Len64(sc); i++ {
if sc&(1<<i) != 0 {
n = parentNode(stack[i], chainingValue(n), iv, 0)
n = guts.ParentNode(stack[i], guts.ChainingValue(n), &guts.IV, 0)
}
}
return n
}

// BaoEncodedSize returns the size of a Bao encoding for the provided quantity
// EncodedSize returns the size of a Bao encoding for the provided quantity
// of data.
func BaoEncodedSize(dataLen int, group int, outboard bool) int {
groupSize := chunkSize << group
func EncodedSize(dataLen int, group int, outboard bool) int {
groupSize := guts.ChunkSize << group
size := 8
if dataLen > 0 {
chunks := (dataLen + groupSize - 1) / groupSize
Expand All @@ -57,16 +76,16 @@ func BaoEncodedSize(dataLen int, group int, outboard bool) int {
return size
}

// BaoEncode computes the intermediate BLAKE3 tree hashes of data and writes
// them to dst. If outboard is false, the contents of data are also written to
// dst, interleaved with the tree hashes. It also returns the tree root, i.e.
// the 256-bit BLAKE3 hash. The group parameter controls how many chunks are
// hashed per "group," as a power of 2; for standard Bao, use 0.
// Encode computes the intermediate BLAKE3 tree hashes of data and writes them
// to dst. If outboard is false, the contents of data are also written to dst,
// interleaved with the tree hashes. It also returns the tree root, i.e. the
// 256-bit BLAKE3 hash. The group parameter controls how many chunks are hashed
// per "group," as a power of 2; for standard Bao, use 0.
//
// Note that dst is not written sequentially, and therefore must be initialized
// with sufficient capacity to hold the encoding; see BaoEncodedSize.
func BaoEncode(dst io.WriterAt, data io.Reader, dataLen int64, group int, outboard bool) ([32]byte, error) {
groupSize := uint64(chunkSize << group)
// with sufficient capacity to hold the encoding; see EncodedSize.
func Encode(dst io.WriterAt, data io.Reader, dataLen int64, group int, outboard bool) ([32]byte, error) {
groupSize := uint64(guts.ChunkSize << group)
buf := make([]byte, groupSize)
var err error
read := func(p []byte) []byte {
Expand Down Expand Up @@ -97,9 +116,9 @@ func BaoEncode(dst io.WriterAt, data io.Reader, dataLen int64, group int, outboa
write(g, off)
}
n := compressGroup(g, counter)
counter += bufLen / chunkSize
n.flags |= flags
return 0, chainingValue(n)
counter += bufLen / guts.ChunkSize
n.Flags |= flags
return 0, guts.ChainingValue(n)
}
mid := uint64(1) << (bits.Len64(bufLen-1) - 1)
lchildren, l := rec(mid, 0, off+64)
Expand All @@ -110,23 +129,23 @@ func BaoEncode(dst io.WriterAt, data io.Reader, dataLen int64, group int, outboa
rchildren, r := rec(bufLen-mid, 0, off+64+llen)
write(cvToBytes(&l)[:], off)
write(cvToBytes(&r)[:], off+32)
return 2 + lchildren + rchildren, chainingValue(parentNode(l, r, iv, flags))
return 2 + lchildren + rchildren, guts.ChainingValue(guts.ParentNode(l, r, &guts.IV, flags))
}

binary.LittleEndian.PutUint64(buf[:8], uint64(dataLen))
write(buf[:8], 0)
_, root := rec(uint64(dataLen), flagRoot, 8)
_, root := rec(uint64(dataLen), guts.FlagRoot, 8)
return *cvToBytes(&root), err
}

// BaoDecode reads content and tree data from the provided reader(s), and
// Decode reads content and tree data from the provided reader(s), and
// streams the verified content to dst. It returns false if verification fails.
// If the content and tree data are interleaved, outboard should be nil.
func BaoDecode(dst io.Writer, data, outboard io.Reader, group int, root [32]byte) (bool, error) {
func Decode(dst io.Writer, data, outboard io.Reader, group int, root [32]byte) (bool, error) {
if outboard == nil {
outboard = data
}
groupSize := uint64(chunkSize << group)
groupSize := uint64(guts.ChunkSize << group)
buf := make([]byte, groupSize)
var err error
read := func(r io.Reader, p []byte) []byte {
Expand All @@ -151,23 +170,23 @@ func BaoDecode(dst io.Writer, data, outboard io.Reader, group int, root [32]byte
return false
} else if bufLen <= groupSize {
n := compressGroup(read(data, buf[:bufLen]), counter)
counter += bufLen / chunkSize
n.flags |= flags
valid := cv == chainingValue(n)
counter += bufLen / guts.ChunkSize
n.Flags |= flags
valid := cv == guts.ChainingValue(n)
if valid {
write(dst, buf[:bufLen])
}
return valid
}
l, r := readParent()
n := parentNode(l, r, iv, flags)
n := guts.ParentNode(l, r, &guts.IV, flags)
mid := uint64(1) << (bits.Len64(bufLen-1) - 1)
return chainingValue(n) == cv && rec(l, mid, 0) && rec(r, bufLen-mid, 0)
return guts.ChainingValue(n) == cv && rec(l, mid, 0) && rec(r, bufLen-mid, 0)
}

read(outboard, buf[:8])
dataLen := binary.LittleEndian.Uint64(buf[:8])
ok := rec(bytesToCV(root[:]), dataLen, flagRoot)
ok := rec(bytesToCV(root[:]), dataLen, guts.FlagRoot)
return ok, err
}

Expand All @@ -182,34 +201,34 @@ func (b *bufferAt) WriteAt(p []byte, off int64) (int, error) {
return len(p), nil
}

// BaoEncodeBuf returns the Bao encoding and root (i.e. BLAKE3 hash) for data.
func BaoEncodeBuf(data []byte, group int, outboard bool) ([]byte, [32]byte) {
buf := bufferAt{buf: make([]byte, BaoEncodedSize(len(data), group, outboard))}
root, _ := BaoEncode(&buf, bytes.NewReader(data), int64(len(data)), group, outboard)
// EncodeBuf returns the Bao encoding and root (i.e. BLAKE3 hash) for data.
func EncodeBuf(data []byte, group int, outboard bool) ([]byte, [32]byte) {
buf := bufferAt{buf: make([]byte, EncodedSize(len(data), group, outboard))}
root, _ := Encode(&buf, bytes.NewReader(data), int64(len(data)), group, outboard)
return buf.buf, root
}

// BaoVerifyBuf verifies the Bao encoding and root (i.e. BLAKE3 hash) for data.
// VerifyBuf verifies the Bao encoding and root (i.e. BLAKE3 hash) for data.
// If the content and tree data are interleaved, outboard should be nil.
func BaoVerifyBuf(data, outboard []byte, group int, root [32]byte) bool {
func VerifyBuf(data, outboard []byte, group int, root [32]byte) bool {
d, o := bytes.NewBuffer(data), bytes.NewBuffer(outboard)
var or io.Reader = o
if outboard == nil {
or = nil
}
ok, _ := BaoDecode(io.Discard, d, or, group, root)
ok, _ := Decode(io.Discard, d, or, group, root)
return ok && d.Len() == 0 && o.Len() == 0 // check for trailing data
}

// BaoExtractSlice returns the slice encoding for the given offset and length.
// When extracting from an outboard encoding, data should contain only the chunk
// ExtractSlice returns the slice encoding for the given offset and length. When
// extracting from an outboard encoding, data should contain only the chunk
// groups that will be present in the slice.
func BaoExtractSlice(dst io.Writer, data, outboard io.Reader, group int, offset uint64, length uint64) error {
func ExtractSlice(dst io.Writer, data, outboard io.Reader, group int, offset uint64, length uint64) error {
combinedEncoding := outboard == nil
if combinedEncoding {
outboard = data
}
groupSize := uint64(chunkSize << group)
groupSize := uint64(guts.ChunkSize << group)
buf := make([]byte, groupSize)
var err error
read := func(r io.Reader, n uint64, copy bool) {
Expand Down Expand Up @@ -245,11 +264,11 @@ func BaoExtractSlice(dst io.Writer, data, outboard io.Reader, group int, offset
return err
}

// BaoDecodeSlice reads from data, which must contain a slice encoding for the
// DecodeSlice reads from data, which must contain a slice encoding for the
// given offset and length, and streams verified content to dst. It returns
// false if verification fails.
func BaoDecodeSlice(dst io.Writer, data io.Reader, group int, offset, length uint64, root [32]byte) (bool, error) {
groupSize := uint64(chunkSize << group)
func DecodeSlice(dst io.Writer, data io.Reader, group int, offset, length uint64, root [32]byte) (bool, error) {
groupSize := uint64(guts.ChunkSize << group)
buf := make([]byte, groupSize)
var err error
read := func(n uint64) []byte {
Expand All @@ -276,9 +295,9 @@ func BaoDecodeSlice(dst io.Writer, data io.Reader, group int, offset, length uin
if !inSlice {
return true
}
n := compressGroup(read(bufLen), pos/chunkSize)
n.flags |= flags
valid := cv == chainingValue(n)
n := compressGroup(read(bufLen), pos/guts.ChunkSize)
n.Flags |= flags
valid := cv == guts.ChainingValue(n)
if valid {
// only write within range
p := buf[:bufLen]
Expand All @@ -296,36 +315,35 @@ func BaoDecodeSlice(dst io.Writer, data io.Reader, group int, offset, length uin
return true
}
l, r := readParent()
n := parentNode(l, r, iv, flags)
n := guts.ParentNode(l, r, &guts.IV, flags)
mid := uint64(1) << (bits.Len64(bufLen-1) - 1)
return chainingValue(n) == cv && rec(l, pos, mid, 0) && rec(r, pos+mid, bufLen-mid, 0)
return guts.ChainingValue(n) == cv && rec(l, pos, mid, 0) && rec(r, pos+mid, bufLen-mid, 0)
}

dataLen := binary.LittleEndian.Uint64(read(8))
if dataLen < offset+length {
return false, errors.New("invalid slice length")
}
ok := rec(bytesToCV(root[:]), 0, dataLen, flagRoot)
ok := rec(bytesToCV(root[:]), 0, dataLen, guts.FlagRoot)
return ok, err
}

// BaoVerifySlice verifies the Bao slice encoding in data, returning the
// VerifySlice verifies the Bao slice encoding in data, returning the
// verified bytes.
func BaoVerifySlice(data []byte, group int, offset uint64, length uint64, root [32]byte) ([]byte, bool) {
func VerifySlice(data []byte, group int, offset uint64, length uint64, root [32]byte) ([]byte, bool) {
d := bytes.NewBuffer(data)
var buf bytes.Buffer
if ok, _ := BaoDecodeSlice(&buf, d, group, offset, length, root); !ok || d.Len() > 0 {
if ok, _ := DecodeSlice(&buf, d, group, offset, length, root); !ok || d.Len() > 0 {
return nil, false
}
return buf.Bytes(), true
}

// BaoVerifyChunks verifies the provided chunks using the provided outboard
// encoding,
func BaoVerifyChunk(chunks, outboard []byte, group int, offset uint64, root [32]byte) bool {
// VerifyChunks verifies the provided chunks with a full outboard encoding.
func VerifyChunk(chunks, outboard []byte, group int, offset uint64, root [32]byte) bool {
cbuf := bytes.NewBuffer(chunks)
obuf := bytes.NewBuffer(outboard)
groupSize := uint64(chunkSize << group)
groupSize := uint64(guts.ChunkSize << group)
length := uint64(len(chunks))
nodesWithin := func(bufLen uint64) int {
n := int(bufLen / groupSize)
Expand All @@ -342,18 +360,18 @@ func BaoVerifyChunk(chunks, outboard []byte, group int, offset uint64, root [32]
if !inSlice {
return true
}
n := compressGroup(cbuf.Next(int(groupSize)), pos/chunkSize)
n.flags |= flags
return cv == chainingValue(n)
n := compressGroup(cbuf.Next(int(groupSize)), pos/guts.ChunkSize)
n.Flags |= flags
return cv == guts.ChainingValue(n)
}
if !inSlice {
_ = obuf.Next(64 * nodesWithin(bufLen)) // skip
return true
}
l, r := bytesToCV(obuf.Next(32)), bytesToCV(obuf.Next(32))
n := parentNode(l, r, iv, flags)
n := guts.ParentNode(l, r, &guts.IV, flags)
mid := uint64(1) << (bits.Len64(bufLen-1) - 1)
return chainingValue(n) == cv && rec(l, pos, mid, 0) && rec(r, pos+mid, bufLen-mid, 0)
return guts.ChainingValue(n) == cv && rec(l, pos, mid, 0) && rec(r, pos+mid, bufLen-mid, 0)
}

if obuf.Len() < 8 {
Expand All @@ -363,5 +381,5 @@ func BaoVerifyChunk(chunks, outboard []byte, group int, offset uint64, root [32]
if dataLen < offset+length || obuf.Len() != 64*nodesWithin(dataLen) {
return false
}
return rec(bytesToCV(root[:]), 0, dataLen, flagRoot)
return rec(bytesToCV(root[:]), 0, dataLen, guts.FlagRoot)
}
Loading

0 comments on commit 2d147a7

Please sign in to comment.