backport: merge bitcoin#27452, #29347, #29356, #29353, #29452, #29483, #30545, #31383 (BIP324 backports: part 5)

test/functional/feature_anchors.py

@@ -7,11 +7,14 @@
 import os
 
 from test_framework.p2p import P2PInterface
+from test_framework.socks5 import Socks5Configuration, Socks5Server
+from test_framework.messages import CAddress, hash256, NODE_NETWORK
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import check_node_connections
+from test_framework.util import check_node_connections, assert_equal, p2p_port
 
 INBOUND_CONNECTIONS = 5
 BLOCK_RELAY_CONNECTIONS = 2
+ONION_ADDR = "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion:8333"
 
 
 class AnchorsTest(BitcoinTestFramework):
@@ -55,7 +58,7 @@ def run_test(self):
             else:
                 inbound_nodes_port.append(hex(int(addr_split[1]))[2:])
 
-        self.log.info("Stop node 0")
+        self.log.debug("Stop node")
         self.stop_node(0)
 
         # It should contain only the block-relay-only addresses
@@ -79,12 +82,64 @@ def run_test(self):
                 tweaked_contents[20:20] = b'1'
                 out_file_handler.write(bytes(tweaked_contents))
 
-            self.log.info("Start node")
+            self.log.debug("Start node")
             self.start_node(0)
 
         self.log.info("When node starts, check if anchors.dat doesn't exist anymore")
         assert not os.path.exists(node_anchors_path)
 
+        self.log.info("Ensure addrv2 support")
+        # Use proxies to catch outbound connections to networks with 256-bit addresses
+        onion_conf = Socks5Configuration()
+        onion_conf.auth = True
+        onion_conf.unauth = True
+        onion_conf.addr = ('127.0.0.1', p2p_port(self.num_nodes))
+        onion_conf.keep_alive = True
+        onion_proxy = Socks5Server(onion_conf)
+        onion_proxy.start()
+        self.restart_node(0, extra_args=[f"-onion={onion_conf.addr[0]}:{onion_conf.addr[1]}"])
+
+        self.log.info("Add 256-bit-address block-relay-only connections to node")
+        self.nodes[0].addconnection(ONION_ADDR, 'block-relay-only')
+
+        self.log.debug("Stop node")
+        with self.nodes[0].assert_debug_log([f"DumpAnchors: Flush 1 outbound block-relay-only peer addresses to anchors.dat"]):
+            self.stop_node(0)
+        # Manually close keep_alive proxy connection
+        onion_proxy.stop()
+
+        self.log.info("Check for addrv2 addresses in anchors.dat")
+        caddr = CAddress()
+        caddr.net = CAddress.NET_TORV3
+        caddr.ip, port_str = ONION_ADDR.split(":")
+        caddr.port = int(port_str)
+        # TorV3 addrv2 serialization:
+        # time(4) | services(1) | networkID(1) | address length(1) | address(32)
+        expected_pubkey = caddr.serialize_v2()[7:39].hex()
+
+        # position of services byte of first addr in anchors.dat
+        # network magic, vector length, version, nTime
+        services_index = 4 + 1 + 4 + 4
+        data = bytes()
+        with open(node_anchors_path, "rb") as file_handler:
+            data = file_handler.read()
+            assert_equal(data[services_index], 0x00)  # services == NONE
+            anchors2 = data.hex()
+            assert expected_pubkey in anchors2
+
+        with open(node_anchors_path, "wb") as file_handler:
+            # Modify service flags for this address even though we never connected to it.
+            # This is necessary because on restart we will not attempt an anchor connection
+            # to a host without our required services, even if its address is in the anchors.dat file
+            new_data = bytearray(data)[:-32]
+            new_data[services_index] = NODE_NETWORK
+            new_data_hash = hash256(new_data)
+            file_handler.write(new_data + new_data_hash)
+
+        self.log.info("Restarting node attempts to reconnect to anchors")
+        with self.nodes[0].assert_debug_log([f"Trying to make an anchor connection to {ONION_ADDR}"]):
+            self.start_node(0, extra_args=[f"-onion={onion_conf.addr[0]}:{onion_conf.addr[1]}"])
+
 
 if __name__ == "__main__":
     AnchorsTest().main()

test/functional/p2p_addrv2_relay.py

@@ -18,6 +18,7 @@
 from test_framework.util import assert_equal
 
 I2P_ADDR = "c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p"
+ONION_ADDR = "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion"
 
 ADDRS: List[CAddress] = []
 
@@ -37,6 +38,16 @@ def on_addrv2(self, message):
     def wait_for_addrv2(self):
         self.wait_until(lambda: "addrv2" in self.last_message)
 
+def calc_addrv2_msg_size(addrs):
+    size = 1  # vector length byte
+    for addr in addrs:
+        size += 4  # time
+        size += 1  # services, COMPACTSIZE(P2P_SERVICES)
+        size += 1  # network id
+        size += 1  # address length byte
+        size += addr.ADDRV2_ADDRESS_LENGTH[addr.net]  # address
+        size += 2  # port
+    return size
 
 class AddrTest(BitcoinTestFramework):
     def set_test_params(self):
@@ -48,14 +59,18 @@ def run_test(self):
         for i in range(10):
             addr = CAddress()
             addr.time = int(self.mocktime) + i
+            addr.port = 8333 + i
             addr.nServices = NODE_NETWORK
-            # Add one I2P address at an arbitrary position.
+            # Add one I2P and one onion V3 address at an arbitrary position.
             if i == 5:
                 addr.net = addr.NET_I2P
                 addr.ip = I2P_ADDR
+                addr.port = 0
+            elif i == 8:
+                addr.net = addr.NET_TORV3
+                addr.ip = ONION_ADDR
             else:
                 addr.ip = f"123.123.123.{i % 256}"
-            addr.port = 8333 + i
             ADDRS.append(addr)
 
         self.log.info('Create connection that sends addrv2 messages')
@@ -73,14 +88,15 @@ def run_test(self):
         addr_source = self.nodes[0].add_p2p_connection(P2PInterface())
         addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver())
         msg.addrs = ADDRS
+        msg_size = calc_addrv2_msg_size(ADDRS)
         with self.nodes[0].assert_debug_log([
-                'received: addrv2 (159 bytes) peer=1',
+                f'received: addrv2 ({msg_size} bytes) peer=1',
         ]):
             addr_source.send_and_ping(msg)
 
         # Wait until "Added ..." before bumping mocktime to make sure addv2 is (almost) fully processed
         with self.nodes[0].assert_debug_log([
-                'sending addrv2 (159 bytes) peer=2',
+                f'sending addrv2 ({msg_size} bytes) peer=2',
         ]):
             self.bump_mocktime(30 * 60)
             addr_receiver.wait_for_addrv2()

test/functional/test_framework/messages.py

@@ -25,6 +25,7 @@
 import socket
 import struct
 import time
+import unittest
 
 from test_framework.crypto.siphash import siphash256
 from test_framework.util import assert_equal
@@ -74,6 +75,9 @@ def sha256(s):
     return hashlib.sha256(s).digest()
 
 
+def sha3(s):
+    return hashlib.sha3_256(s).digest()
+
 
 def hash256(s):
     return sha256(sha256(s))
@@ -249,16 +253,25 @@ class CAddress:
 
     # see https://github.com/bitcoin/bips/blob/master/bip-0155.mediawiki
     NET_IPV4 = 1
+    NET_IPV6 = 2
+    NET_TORV3 = 4
     NET_I2P = 5
+    NET_CJDNS = 6
 
     ADDRV2_NET_NAME = {
         NET_IPV4: "IPv4",
-        NET_I2P: "I2P"
+        NET_IPV6: "IPv6",
+        NET_TORV3: "TorV3",
+        NET_I2P: "I2P",
+        NET_CJDNS: "CJDNS"
     }
 
     ADDRV2_ADDRESS_LENGTH = {
         NET_IPV4: 4,
-        NET_I2P: 32
+        NET_IPV6: 16,
+        NET_TORV3: 32,
+        NET_I2P: 32,
+        NET_CJDNS: 16
     }
 
     I2P_PAD = "===="
@@ -305,33 +318,54 @@ def deserialize_v2(self, f):
         self.nServices = deser_compact_size(f)
 
         self.net = struct.unpack("B", f.read(1))[0]
-        assert self.net in (self.NET_IPV4, self.NET_I2P)
+        assert self.net in self.ADDRV2_NET_NAME
 
         address_length = deser_compact_size(f)
         assert address_length == self.ADDRV2_ADDRESS_LENGTH[self.net]
 
         addr_bytes = f.read(address_length)
         if self.net == self.NET_IPV4:
             self.ip = socket.inet_ntoa(addr_bytes)
-        else:
+        elif self.net == self.NET_IPV6:
+            self.ip = socket.inet_ntop(socket.AF_INET6, addr_bytes)
+        elif self.net == self.NET_TORV3:
+            prefix = b".onion checksum"
+            version = bytes([3])
+            checksum = sha3(prefix + addr_bytes + version)[:2]
+            self.ip = b32encode(addr_bytes + checksum + version).decode("ascii").lower() + ".onion"
+        elif self.net == self.NET_I2P:
             self.ip = b32encode(addr_bytes)[0:-len(self.I2P_PAD)].decode("ascii").lower() + ".b32.i2p"
+        elif self.net == self.NET_CJDNS:
+            self.ip = socket.inet_ntop(socket.AF_INET6, addr_bytes)
+        else:
+            raise Exception(f"Address type not supported")
 
         self.port = struct.unpack(">H", f.read(2))[0]
 
     def serialize_v2(self):
         """Serialize in addrv2 format (BIP155)"""
-        assert self.net in (self.NET_IPV4, self.NET_I2P)
+        assert self.net in self.ADDRV2_NET_NAME
         r = b""
         r += struct.pack("<I", self.time)
         r += ser_compact_size(self.nServices)
         r += struct.pack("B", self.net)
         r += ser_compact_size(self.ADDRV2_ADDRESS_LENGTH[self.net])
         if self.net == self.NET_IPV4:
             r += socket.inet_aton(self.ip)
-        else:
+        elif self.net == self.NET_IPV6:
+            r += socket.inet_pton(socket.AF_INET6, self.ip)
+        elif self.net == self.NET_TORV3:
+            sfx = ".onion"
+            assert self.ip.endswith(sfx)
+            r += b32decode(self.ip[0:-len(sfx)], True)[0:32]
+        elif self.net == self.NET_I2P:
             sfx = ".b32.i2p"
             assert self.ip.endswith(sfx)
             r += b32decode(self.ip[0:-len(sfx)] + self.I2P_PAD, True)
+        elif self.net == self.NET_CJDNS:
+            r += socket.inet_pton(socket.AF_INET6, self.ip)
+        else:
+            raise Exception(f"Address type not supported")
         r += struct.pack(">H", self.port)
         return r
 
@@ -2592,3 +2626,19 @@ def serialize(self):
     def __repr__(self):
         return "msg_sendtxrcncl(version=%lu, salt=%lu)" %\
             (self.version, self.salt)
+
+class TestFrameworkScript(unittest.TestCase):
+    def test_addrv2_encode_decode(self):
+        def check_addrv2(ip, net):
+            addr = CAddress()
+            addr.net, addr.ip = net, ip
+            ser = addr.serialize_v2()
+            actual = CAddress()
+            actual.deserialize_v2(BytesIO(ser))
+            self.assertEqual(actual, addr)
+
+        check_addrv2("1.65.195.98", CAddress.NET_IPV4)
+        check_addrv2("2001:41f0::62:6974:636f:696e", CAddress.NET_IPV6)
+        check_addrv2("2bqghnldu6mcug4pikzprwhtjjnsyederctvci6klcwzepnjd46ikjyd.onion", CAddress.NET_TORV3)
+        check_addrv2("255fhcp6ajvftnyo7bwz3an3t4a4brhopm3bamyh2iu5r3gnr2rq.b32.i2p", CAddress.NET_I2P)
+        check_addrv2("fc32:17ea:e415:c3bf:9808:149d:b5a2:c9aa", CAddress.NET_CJDNS)

test/functional/test_framework/socks5.py

@@ -40,6 +40,7 @@ def __init__(self):
         self.af = socket.AF_INET # Bind address family
         self.unauth = False  # Support unauthenticated
         self.auth = False  # Support authentication
+        self.keep_alive = False  # Do not automatically close connections
 
 class Socks5Command():
     """Information about an incoming socks5 command."""
@@ -115,13 +116,14 @@ def handle(self):
 
             cmdin = Socks5Command(cmd, atyp, addr, port, username, password)
             self.serv.queue.put(cmdin)
-            logger.info('Proxy: %s', cmdin)
+            logger.debug('Proxy: %s', cmdin)
             # Fall through to disconnect
         except Exception as e:
             logger.exception("socks5 request handling failed.")
             self.serv.queue.put(e)
         finally:
-            self.conn.close()
+            if not self.serv.keep_alive:
+                self.conn.close()
 
 class Socks5Server():
     def __init__(self, conf):
@@ -133,6 +135,7 @@ def __init__(self, conf):
         self.running = False
         self.thread = None
         self.queue = queue.Queue() # report connections and exceptions to client
+        self.keep_alive = conf.keep_alive
 
     def run(self):
         while self.running:
@@ -157,4 +160,3 @@ def stop(self):
         s.connect(self.conf.addr)
         s.close()
         self.thread.join()
-

test/functional/test_runner.py

@@ -76,6 +76,7 @@
     "crypto.chacha20",
     "crypto.ellswift",
     "key",
+    "messages",
     "crypto.muhash",
     "crypto.poly1305",
     "crypto.ripemd160",