feat(examples): add haystack package and realm (#3082)

Haystack is a permissionless , immutable, content-addressed,
append-only, fixed-length key-value store for small payloads.

This is an experiment to port over a storage and validation
implementation of a tool I'd written several years ago called
[haystack](https://github.com/nomasters/haystack). My goal in porting
this over to gno is to provide a simple, correct, and test covered
implementation that is composable with other tools I plan to port over
as well.

## Overview

You store a needle in the haystack.

A Needle is 192 bytes. It is composed of 32 bytes for a sha256 hash, and
the 160 byte fixed-length payload.

```
hash     | payload
---------|----------
32 bytes | 160 bytes
```

The Haystack storage server supports two calls, other than Render, you
may "Add" a needle, by its hex-encoded string, or you may "Get" a needle
by its hex encoded hash. The add operation ensures that the needle is
valid and that it has not been added to the storage before. The Get
operation will return the full hex encoded needle if the hash exists in
the database, otherwise it will panic.

The structure is broken down into 2 packages and 1 very simple realm

### packages
- https://gno.land/p/demo/haystack/needle/
- https://gno.land/p/demo/haystack/

### realm
- https://gno.land/r/demo/haystack

## How to Try it out?

You can generate your own synthetic needle from the CLI by using this
magical one-liner.

```shell
➜  ~ (dd if=/dev/urandom bs=160 count=1 2>/dev/null | tee >(sha256sum |  cut -d' ' -f1) | od -An -t x1) | tr -d ' \n'
5d82091003a6749b46a96c38b2597ca96e9b0b272594249099dcf2ade188346679ca753999661820dad7beb351559c89a275ed4935a82245fda290906670ec7535b0b856dfccadd62e5f5399892455d2b524724ffdef8e58be03e9da4762c6ab582ce91c29a9e26ea9cc38b66953fdc425ad37baeb12c712e049ae6d456e682b6b63eea74ebf7a9d506ba486d08c9c54c5161d38a7fbc5fcbb1cdac370682ad6a59579167fd1aa1cd1fc109660a7eba36775d6b06058d72aa57debe63d0144b8
```
This leverages `dd`, `/dev/urandom`, `tee`, `cut`, `od`, and `tr` to
generate a properly formatted hex-encoded needle.

### Getting a needle

I've already stored the above needle in Haystack, so you can read it by
running this command from the CLI

```shell
gnokey maketx call -pkgpath "gno.land/r/demo/haystack" \
    -func "Get" \
    -gas-fee 1000000ugnot \
    -gas-wanted 2000000 \
    -send "" \
    -broadcast \
    -chainid "portal-loop" \
    -args "5d82091003a6749b46a96c38b2597ca96e9b0b272594249099dcf2ade1883466" \
    -remote "https://rpc.gno.land:443" \
    $YOUR_WALLET_ADDRESS
```



<details><summary>Contributors' checklist...</summary>

- [X] Added new tests, or not needed, or not feasible
- [X] Provided an example (e.g. screenshot) to aid review or the PR is
self-explanatory
- [X] Updated the official documentation or not needed
- [X] No breaking changes were made, or a `BREAKING CHANGE: xxx` message
was included in the description
- [X] Added references to related issues and PRs
- [X] Provided any useful hints for running manual tests
</details>

examples/gno.land/p/n2p5/haystack/gno.mod

@@ -0,0 +1,6 @@
+module gno.land/p/n2p5/haystack
+
+require (
+	gno.land/p/demo/avl v0.0.0-latest
+	gno.land/p/n2p5/haystack/needle v0.0.0-latest
+)

examples/gno.land/p/n2p5/haystack/haystack.gno

@@ -0,0 +1,99 @@
+package haystack
+
+import (
+	"encoding/hex"
+	"errors"
+
+	"gno.land/p/demo/avl"
+	"gno.land/p/n2p5/haystack/needle"
+)
+
+var (
+	// ErrorNeedleNotFound is returned when a needle is not found in the haystack.
+	ErrorNeedleNotFound = errors.New("needle not found")
+	// ErrorNeedleLength is returned when a needle is not the correct length.
+	ErrorNeedleLength = errors.New("invalid needle length")
+	// ErrorHashLength is returned when a needle hash is not the correct length.
+	ErrorHashLength = errors.New("invalid hash length")
+	// ErrorDuplicateNeedle is returned when a needle already exists in the haystack.
+	ErrorDuplicateNeedle = errors.New("needle already exists")
+	// ErrorHashMismatch is returned when a needle hash does not match the needle. This should
+	// never happen and indicates a critical internal storage error.
+	ErrorHashMismatch = errors.New("storage error: hash mismatch")
+	// ErrorValueInvalidType is returned when a needle value is not a byte slice. This should
+	// never happen and indicates a critical internal storage error.
+	ErrorValueInvalidType = errors.New("storage error: invalid value type, expected []byte")
+)
+
+const (
+	// EncodedHashLength is the length of the hex-encoded needle hash.
+	EncodedHashLength = needle.HashLength * 2
+	// EncodedPayloadLength is the length of the hex-encoded needle payload.
+	EncodedPayloadLength = needle.PayloadLength * 2
+	// EncodedNeedleLength is the length of the hex-encoded needle.
+	EncodedNeedleLength = EncodedHashLength + EncodedPayloadLength
+)
+
+// Haystack is a permissionless, append-only, content-addressed key-value store for fix
+// length messages known as needles. A needle is a 192 byte byte slice with a 32 byte
+// hash (sha256) and a 160 byte payload.
+type Haystack struct{ internal *avl.Tree }
+
+// New creates a new instance of a Haystack key-value store.
+func New() *Haystack {
+	return &Haystack{
+		internal: avl.NewTree(),
+	}
+}
+
+// Add takes a fixed-length hex-encoded needle bytes and adds it to the haystack key-value
+// store. The key is the first 32 bytes of the needle hash (64 bytes hex-encoded) of the
+// sha256 sum of the payload. The value is the 160 byte byte slice of the needle payload.
+// An error is returned if the needle is found to be invalid.
+func (h *Haystack) Add(needleHex string) error {
+	if len(needleHex) != EncodedNeedleLength {
+		return ErrorNeedleLength
+	}
+	b, err := hex.DecodeString(needleHex)
+	if err != nil {
+		return err
+	}
+	n, err := needle.FromBytes(b)
+	if err != nil {
+		return err
+	}
+	if h.internal.Has(needleHex[:EncodedHashLength]) {
+		return ErrorDuplicateNeedle
+	}
+	h.internal.Set(needleHex[:EncodedHashLength], n.Payload())
+	return nil
+}
+
+// Get takes a hex-encoded needle hash and returns the complete hex-encoded needle bytes
+// and an error. Errors covers errors that span from the needle not being found, internal
+// storage error inconsistencies, and invalid value types.
+func (h *Haystack) Get(hash string) (string, error) {
+	if len(hash) != EncodedHashLength {
+		return "", ErrorHashLength
+	}
+	if _, err := hex.DecodeString(hash); err != nil {
+		return "", err
+	}
+	v, ok := h.internal.Get(hash)
+	if !ok {
+		return "", ErrorNeedleNotFound
+	}
+	b, ok := v.([]byte)
+	if !ok {
+		return "", ErrorValueInvalidType
+	}
+	n, err := needle.New(b)
+	if err != nil {
+		return "", err
+	}
+	needleHash := hex.EncodeToString(n.Hash())
+	if needleHash != hash {
+		return "", ErrorHashMismatch
+	}
+	return hex.EncodeToString(n.Bytes()), nil
+}

examples/gno.land/p/n2p5/haystack/haystack_test.gno

@@ -0,0 +1,94 @@
+package haystack
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"gno.land/p/n2p5/haystack/needle"
+)
+
+func TestHaystack(t *testing.T) {
+	t.Parallel()
+
+	t.Run("New", func(t *testing.T) {
+		t.Parallel()
+		h := New()
+		if h == nil {
+			t.Error("New returned nil")
+		}
+	})
+
+	t.Run("Add", func(t *testing.T) {
+		t.Parallel()
+		h := New()
+		n, _ := needle.New(make([]byte, needle.PayloadLength))
+		validNeedleHex := hex.EncodeToString(n.Bytes())
+
+		testTable := []struct {
+			needleHex string
+			err       error
+		}{
+			{validNeedleHex, nil},
+			{validNeedleHex, ErrorDuplicateNeedle},
+			{"bad" + validNeedleHex[3:], needle.ErrorInvalidHash},
+			{"XXX" + validNeedleHex[3:], hex.InvalidByteError('X')},
+			{validNeedleHex[:len(validNeedleHex)-2], ErrorNeedleLength},
+			{validNeedleHex + "00", ErrorNeedleLength},
+			{"000", ErrorNeedleLength},
+		}
+		for _, tt := range testTable {
+			err := h.Add(tt.needleHex)
+			if err != tt.err {
+				t.Error(tt.needleHex, err.Error(), "!=", tt.err.Error())
+			}
+		}
+	})
+
+	t.Run("Get", func(t *testing.T) {
+		t.Parallel()
+		h := New()
+
+		// genNeedleHex returns a hex-encoded needle and its hash for a given index.
+		genNeedleHex := func(i int) (string, string) {
+			b := make([]byte, needle.PayloadLength)
+			b[0] = byte(i)
+			n, _ := needle.New(b)
+			return hex.EncodeToString(n.Bytes()), hex.EncodeToString(n.Hash())
+		}
+
+		// Add a valid needle to the haystack.
+		validNeedleHex, validHash := genNeedleHex(0)
+		h.Add(validNeedleHex)
+
+		// Add a needle and break the value type.
+		_, brokenHashValueType := genNeedleHex(1)
+		h.internal.Set(brokenHashValueType, 0)
+
+		// Add a needle with invalid hash.
+		_, invalidHash := genNeedleHex(2)
+		h.internal.Set(invalidHash, make([]byte, needle.PayloadLength))
+
+		testTable := []struct {
+			hash     string
+			expected string
+			err      error
+		}{
+			{validHash, validNeedleHex, nil},
+			{validHash[:len(validHash)-2], "", ErrorHashLength},
+			{validHash + "00", "", ErrorHashLength},
+			{"XXX" + validHash[3:], "", hex.InvalidByteError('X')},
+			{"bad" + validHash[3:], "", ErrorNeedleNotFound},
+			{brokenHashValueType, "", ErrorValueInvalidType},
+			{invalidHash, "", ErrorHashMismatch},
+		}
+		for _, tt := range testTable {
+			actual, err := h.Get(tt.hash)
+			if err != tt.err {
+				t.Error(tt.hash, err.Error(), "!=", tt.err.Error())
+			}
+			if actual != tt.expected {
+				t.Error(tt.hash, actual, "!=", tt.expected)
+			}
+		}
+	})
+}

examples/gno.land/p/n2p5/haystack/needle/gno.mod

@@ -0,0 +1 @@
+module gno.land/p/n2p5/haystack/needle

examples/gno.land/p/n2p5/haystack/needle/needle.gno

@@ -0,0 +1,91 @@
+package needle
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"errors"
+)
+
+const (
+	// HashLength is the length in bytes of the hash prefix in any message
+	HashLength = 32
+	// PayloadLength is the length of the remaining bytes of the message.
+	PayloadLength = 160
+	// NeedleLength is the number of bytes required for a valid needle.
+	NeedleLength = HashLength + PayloadLength
+)
+
+// Needle is a container for a 160 byte payload
+// and a 32 byte sha256 hash of the payload.
+type Needle struct {
+	hash    [HashLength]byte
+	payload [PayloadLength]byte
+}
+
+var (
+	// ErrorInvalidHash is an error for in invalid hash
+	ErrorInvalidHash = errors.New("invalid hash")
+	// ErrorByteSliceLength is an error for an invalid byte slice length passed in to New or FromBytes
+	ErrorByteSliceLength = errors.New("invalid byte slice length")
+)
+
+// New creates a Needle used for submitting a payload to a Haystack sever. It takes a Payload
+// byte slice that is 160 bytes in length and returns a reference to a
+// Needle and an error. The purpose of this function is to make it
+// easy to create a new Needle from a payload. This function handles creating a sha256
+// hash of the payload, which is used by the Needle to submit to a haystack server.
+func New(p []byte) (*Needle, error) {
+	if len(p) != PayloadLength {
+		return nil, ErrorByteSliceLength
+	}
+	var n Needle
+	sum := sha256.Sum256(p)
+	copy(n.hash[:], sum[:])
+	copy(n.payload[:], p)
+	return &n, nil
+}
+
+// FromBytes is intended convert raw bytes (from UDP or storage) into a Needle.
+// It takes a byte slice and expects it to be exactly the length of NeedleLength.
+// The byte slice should consist of the first 32 bytes being the sha256 hash of the
+// payload and the payload bytes. This function verifies the length of the byte slice,
+// copies the bytes into a private [192]byte array, and validates the Needle. It returns
+// a reference to a Needle and an error.
+func FromBytes(b []byte) (*Needle, error) {
+	if len(b) != NeedleLength {
+		return nil, ErrorByteSliceLength
+	}
+	var n Needle
+	copy(n.hash[:], b[:HashLength])
+	copy(n.payload[:], b[HashLength:])
+	if err := n.validate(); err != nil {
+		return nil, err
+	}
+	return &n, nil
+}
+
+// Hash returns a copy of the bytes of the sha256 256 hash of the Needle payload.
+func (n *Needle) Hash() []byte {
+	return n.Bytes()[:HashLength]
+}
+
+// Payload returns a byte slice of the Needle payload
+func (n *Needle) Payload() []byte {
+	return n.Bytes()[HashLength:]
+}
+
+// Bytes returns a byte slice of the entire 192 byte hash + payload
+func (n *Needle) Bytes() []byte {
+	b := make([]byte, NeedleLength)
+	copy(b, n.hash[:])
+	copy(b[HashLength:], n.payload[:])
+	return b
+}
+
+// validate checks that a Needle has a valid hash, it returns either nil or an error.
+func (n *Needle) validate() error {
+	if hash := sha256.Sum256(n.Payload()); !bytes.Equal(n.Hash(), hash[:]) {
+		return ErrorInvalidHash
+	}
+	return nil
+}