Merkle db intermediate node key compression

x/merkledb/intermediate_node_db.go

@@ -89,13 +89,13 @@ func (db *intermediateNodeDB) onEviction(key path, n *node) error {
 }
 
 func (db *intermediateNodeDB) addToBatch(b database.Batch, key path, n *node) error {
-	prefixedKey := addPrefixToKey(db.bufferPool, intermediateNodePrefix, key.Bytes())
-	defer db.bufferPool.Put(prefixedKey)
+	dbKey := db.constructDBKey(key)
+	defer db.bufferPool.Put(dbKey)
 	db.metrics.DatabaseNodeWrite()
 	if n == nil {
-		return b.Delete(prefixedKey)
+		return b.Delete(dbKey)
 	}
-	return b.Put(prefixedKey, n.bytes())
+	return b.Put(dbKey, n.bytes())
 }
 
 func (db *intermediateNodeDB) Get(key path) (*node, error) {
@@ -108,17 +108,38 @@ func (db *intermediateNodeDB) Get(key path) (*node, error) {
 	}
 	db.metrics.IntermediateNodeCacheMiss()
 
-	prefixedKey := addPrefixToKey(db.bufferPool, intermediateNodePrefix, key.Bytes())
+	dbKey := db.constructDBKey(key)
 	db.metrics.DatabaseNodeRead()
-	nodeBytes, err := db.baseDB.Get(prefixedKey)
+	nodeBytes, err := db.baseDB.Get(dbKey)
 	if err != nil {
 		return nil, err
 	}
-	db.bufferPool.Put(prefixedKey)
+	db.bufferPool.Put(dbKey)
 
 	return parseNode(key, nodeBytes)
 }
 
+func (db *intermediateNodeDB) constructDBKey(key path) []byte {
+	// We need differentiate between two paths of equal byte length but different token length
+	// so add the modulo remainder as a key suffix
+	dbKey := db.bufferPool.Get().([]byte)
+	defer db.bufferPool.Put(dbKey)
+	compressedKey := key.Serialize().Value
+	keyLen := len(compressedKey) + 1
+	if cap(dbKey) >= keyLen {
+		// The [] byte we got from the pool is big enough to hold the prefixed key
+		dbKey = dbKey[:keyLen]
+	} else {
+		// The []byte from the pool wasn't big enough.
+		// Put it back and allocate a new, bigger one
+		db.bufferPool.Put(dbKey)
+		dbKey = make([]byte, keyLen)
+	}
+	copy(dbKey, compressedKey)
+	dbKey[len(compressedKey)] = byte(len(key) % 2)
+	return addPrefixToKey(db.bufferPool, intermediateNodePrefix, dbKey)
+}
+
 func (db *intermediateNodeDB) Put(key path, n *node) error {
 	return db.nodeCache.Put(key, n)
 }

x/merkledb/intermediate_node_db_test.go

@@ -70,7 +70,7 @@ func TestIntermediateNodeDB(t *testing.T) {
 	added := 0
 	for {
 		key := newPath([]byte{byte(added)})
-		node := newNode(nil, EmptyPath)
+		node := newNode(nil, key)
 		node.setValue(maybe.Some([]byte{byte(added)}))
 		newExpectedSize := expectedSize + cacheEntrySize(key, node)
 		if newExpectedSize > cacheSize {
@@ -90,7 +90,7 @@ func TestIntermediateNodeDB(t *testing.T) {
 	// of all but 2 elements. 2 elements remain rather than 1 element because of
 	// the added key prefix increasing the size tracked by the batch.
 	key := newPath([]byte{byte(added)})
-	node := newNode(nil, EmptyPath)
+	node := newNode(nil, key)
 	node.setValue(maybe.Some([]byte{byte(added)}))
 	require.NoError(db.Put(key, node))