diff --git a/packages/auto-tls/src/domain-mapper.ts b/packages/auto-tls/src/domain-mapper.ts index 1c87747528..abc783dfe7 100644 --- a/packages/auto-tls/src/domain-mapper.ts +++ b/packages/auto-tls/src/domain-mapper.ts @@ -58,7 +58,10 @@ export class DomainMapper { } updateMappings (): void { - const publicIps = getPublicIps(this.addressManager.getAddresses()) + const publicIps = getPublicIps( + this.addressManager.getAddressesWithMetadata() + .map(({ multiaddr }) => multiaddr) + ) // did our public IPs change? const addedIp4 = [] diff --git a/packages/auto-tls/test/domain-mapper.spec.ts b/packages/auto-tls/test/domain-mapper.spec.ts index 0e8c246046..821f6472db 100644 --- a/packages/auto-tls/test/domain-mapper.spec.ts +++ b/packages/auto-tls/test/domain-mapper.spec.ts @@ -42,13 +42,32 @@ describe('domain-mapper', () => { const ip4 = '81.12.12.9' const ip6 = '2001:4860:4860::8889' - components.addressManager.getAddresses.returns([ - multiaddr('/ip4/127.0.0.1/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), - multiaddr('/ip4/192.168.1.234/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), - multiaddr('/dns4/example.com/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), - multiaddr(`/ip4/${ip4}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`), - multiaddr(`/ip6/${ip6}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`) - ]) + components.addressManager.getAddressesWithMetadata.returns([{ + multiaddr: multiaddr('/ip4/127.0.0.1/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr('/ip4/192.168.1.234/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr('/dns4/example.com/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr(`/ip4/${ip4}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`), + verified: true, + expires: Infinity, + type: 'ip-mapping' + }, { + multiaddr: multiaddr(`/ip6/${ip6}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`), + verified: true, + expires: Infinity, + type: 'ip-mapping' + }]) components.events.safeDispatchEvent('certificate:provision', { detail: { @@ -69,13 +88,32 @@ describe('domain-mapper', () => { const ip4v1 = '81.12.12.9' const ip6v1 = '2001:4860:4860::8889' - components.addressManager.getAddresses.returns([ - multiaddr('/ip4/127.0.0.1/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), - multiaddr('/ip4/192.168.1.234/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), - multiaddr('/dns4/example.com/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), - multiaddr(`/ip4/${ip4v1}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`), - multiaddr(`/ip6/${ip6v1}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`) - ]) + components.addressManager.getAddressesWithMetadata.returns([{ + multiaddr: multiaddr('/ip4/127.0.0.1/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr('/ip4/192.168.1.234/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr('/dns4/example.com/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr(`/ip4/${ip4v1}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`), + verified: true, + expires: Infinity, + type: 'ip-mapping' + }, { + multiaddr: multiaddr(`/ip6/${ip6v1}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`), + verified: true, + expires: Infinity, + type: 'ip-mapping' + }]) components.events.safeDispatchEvent('certificate:provision', { detail: { @@ -94,13 +132,32 @@ describe('domain-mapper', () => { const ip4v2 = '81.12.12.10' const ip6v2 = '2001:4860:4860::8890' - components.addressManager.getAddresses.returns([ - multiaddr('/ip4/127.0.0.1/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), - multiaddr('/ip4/192.168.1.234/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), - multiaddr('/dns4/example.com/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), - multiaddr(`/ip4/${ip4v2}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`), - multiaddr(`/ip6/${ip6v2}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`) - ]) + components.addressManager.getAddressesWithMetadata.returns([{ + multiaddr: multiaddr('/ip4/127.0.0.1/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr('/ip4/192.168.1.234/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr('/dns4/example.com/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr(`/ip4/${ip4v2}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`), + verified: true, + expires: Infinity, + type: 'ip-mapping' + }, { + multiaddr: multiaddr(`/ip6/${ip6v2}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`), + verified: true, + expires: Infinity, + type: 'ip-mapping' + }]) components.events.safeDispatchEvent('self:peer:update', { detail: stubInterface() @@ -121,13 +178,32 @@ describe('domain-mapper', () => { const ip4 = '81.12.12.9' const ip6 = '2001:4860:4860::8889' - components.addressManager.getAddresses.returns([ - multiaddr('/ip4/127.0.0.1/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), - multiaddr('/ip4/192.168.1.234/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), - multiaddr('/dns4/example.com/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), - multiaddr(`/ip4/${ip4}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`), - multiaddr(`/ip6/${ip6}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`) - ]) + components.addressManager.getAddressesWithMetadata.returns([{ + multiaddr: multiaddr('/ip4/127.0.0.1/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr('/ip4/192.168.1.234/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr('/dns4/example.com/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN'), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr(`/ip4/${ip4}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`), + verified: true, + expires: Infinity, + type: 'ip-mapping' + }, { + multiaddr: multiaddr(`/ip6/${ip6}/tcp/1234/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN`), + verified: true, + expires: Infinity, + type: 'ip-mapping' + }]) components.events.safeDispatchEvent('self:peer:update', { detail: stubInterface() diff --git a/packages/auto-tls/test/index.spec.ts b/packages/auto-tls/test/index.spec.ts index 4c3eb57b22..40c5429ef5 100644 --- a/packages/auto-tls/test/index.spec.ts +++ b/packages/auto-tls/test/index.spec.ts @@ -16,7 +16,7 @@ import { DEFAULT_CERTIFICATE_DATASTORE_KEY, DEFAULT_CERTIFICATE_PRIVATE_KEY_NAME import { importFromPem } from '../src/utils.js' import { CERT, CERT_FOR_OTHER_KEY, EXPIRED_CERT, INVALID_CERT, PRIVATE_KEY_PEM } from './fixtures/cert.js' import type { ComponentLogger, Libp2pEvents, Peer, PeerId, PrivateKey, RSAPrivateKey, TypedEventTarget } from '@libp2p/interface' -import type { AddressManager } from '@libp2p/interface-internal' +import type { AddressManager, NodeAddress } from '@libp2p/interface-internal' import type { Keychain } from '@libp2p/keychain' import type { StubbedInstance } from 'sinon-ts' @@ -49,12 +49,26 @@ describe('auto-tls', () => { datastore: new MemoryDatastore() } - // mixture of LAN and public addresses - components.addressManager.getAddresses.returns([ - multiaddr(`/ip4/127.0.0.1/tcp/1235/p2p/${components.peerId}`), - multiaddr(`/ip4/192.168.0.100/tcp/1235/p2p/${components.peerId}`), - multiaddr(`/ip4/82.32.57.46/tcp/2345/p2p/${components.peerId}`) - ]) + // a mixture of LAN and public addresses + const addresses: NodeAddress[] = [{ + multiaddr: multiaddr(`/ip4/127.0.0.1/tcp/1235/p2p/${components.peerId}`), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr(`/ip4/192.168.0.100/tcp/1235/p2p/${components.peerId}`), + verified: true, + expires: Infinity, + type: 'transport' + }, { + multiaddr: multiaddr(`/ip4/82.32.57.46/tcp/2345/p2p/${components.peerId}`), + verified: true, + expires: Infinity, + type: 'ip-mapping' + }] + + components.addressManager.getAddressesWithMetadata.returns(addresses) + components.addressManager.getAddresses.returns(addresses.map(({ multiaddr }) => multiaddr)) }) afterEach(async () => { diff --git a/packages/integration-tests/test/addresses.spec.ts b/packages/integration-tests/test/addresses.spec.ts deleted file mode 100644 index 26123516f1..0000000000 --- a/packages/integration-tests/test/addresses.spec.ts +++ /dev/null @@ -1,159 +0,0 @@ -/* eslint-env mocha */ - -import { memory } from '@libp2p/memory' -import { multiaddr } from '@multiformats/multiaddr' -import { expect } from 'aegir/chai' -import { createLibp2p } from 'libp2p' -import { pEvent } from 'p-event' -import type { Libp2p, PeerUpdate } from '@libp2p/interface' -import type { AddressManager } from '@libp2p/interface-internal' -import type { Multiaddr } from '@multiformats/multiaddr' - -const listenAddresses = ['/memory/address-1', '/memory/address-2'] -const announceAddresses = ['/dns4/peer.io/tcp/433/p2p/12D3KooWNvSZnPi3RrhrTwEY4LuuBeB6K6facKUCJcyWG1aoDd2p'] - -describe('addresses', () => { - let libp2p: Libp2p - - afterEach(async () => { - await libp2p?.stop() - }) - - it('should return transport listen addresses if announce addresses are not provided', async () => { - libp2p = await createLibp2p({ - addresses: { - listen: listenAddresses - }, - transports: [ - memory() - ] - }) - - expect(libp2p.getMultiaddrs().map(ma => ma.decapsulate('/p2p').toString())).to.deep.equal(listenAddresses) - }) - - it('should override listen addresses with announce addresses when provided', async () => { - libp2p = await createLibp2p({ - addresses: { - listen: listenAddresses, - announce: announceAddresses - }, - transports: [ - memory() - ] - }) - - expect(libp2p.getMultiaddrs().map(ma => ma.decapsulate('/p2p').toString())).to.deep.equal(announceAddresses) - }) - - it('should filter listen addresses filtered by the announce filter', async () => { - libp2p = await createLibp2p({ - addresses: { - listen: listenAddresses, - announceFilter: (multiaddrs: Multiaddr[]) => multiaddrs.slice(1) - }, - transports: [ - memory() - ] - }) - - expect(libp2p.getMultiaddrs().map(ma => ma.decapsulate('/p2p').toString())).to.deep.equal([listenAddresses[1]]) - }) - - it('should filter announce addresses filtered by the announce filter', async () => { - libp2p = await createLibp2p({ - addresses: { - listen: listenAddresses, - announce: announceAddresses, - announceFilter: () => [] - }, - transports: [ - memory() - ] - }) - - expect(libp2p.getMultiaddrs().map(ma => ma.decapsulate('/p2p').toString())).to.have.lengthOf(0) - }) - - it('should include observed addresses in returned multiaddrs', async () => { - const ma = '/ip4/83.32.123.53/tcp/43928' - - libp2p = await createLibp2p({ - start: false, - addresses: { - listen: listenAddresses - }, - transports: [ - memory() - ], - services: { - observer: (components: { addressManager: AddressManager }) => { - components.addressManager.confirmObservedAddr(multiaddr(ma)) - } - } - }) - - expect(libp2p.getMultiaddrs().map(ma => ma.decapsulate('/p2p').toString())).to.include(ma) - }) - - it('should update our peer record with announce addresses on startup', async () => { - libp2p = await createLibp2p({ - start: false, - addresses: { - listen: listenAddresses, - announce: announceAddresses - }, - transports: [ - memory() - ] - }) - - const eventPromise = pEvent<'self:peer:update', CustomEvent>(libp2p, 'self:peer:update', { - filter: (event) => { - return event.detail.peer.addresses.map(({ multiaddr }) => multiaddr.toString()) - .includes(announceAddresses[0]) - } - }) - - await libp2p.start() - - const event = await eventPromise - - expect(event.detail.peer.addresses.map(({ multiaddr }) => multiaddr.toString())) - .to.include.members(announceAddresses, 'peer info did not include announce addresses') - }) - - it('should only include confirmed observed addresses in peer record', async () => { - const unconfirmedAddress = '/ip4/127.0.0.1/tcp/4010/ws' - const confirmedAddress = '/ip4/127.0.0.1/tcp/4011/ws' - - libp2p = await createLibp2p({ - start: false, - addresses: { - listen: listenAddresses, - announce: announceAddresses - }, - transports: [ - memory() - ], - services: { - observer: (components: { addressManager: AddressManager }) => { - components.addressManager.confirmObservedAddr(multiaddr(confirmedAddress)) - components.addressManager.addObservedAddr(multiaddr(unconfirmedAddress)) - } - } - }) - - await libp2p.start() - - const eventPromise = pEvent<'self:peer:update', CustomEvent>(libp2p, 'self:peer:update') - - const event = await eventPromise - - expect(event.detail.peer.addresses.map(({ multiaddr }) => multiaddr.toString())) - .to.not.include(unconfirmedAddress, 'peer info included unconfirmed observed address') - - expect(event.detail.peer.addresses.map(({ multiaddr }) => multiaddr.toString())) - .to.include(confirmedAddress, 'peer info did not include confirmed observed address') - }) -}) diff --git a/packages/interface-internal/src/address-manager/index.ts b/packages/interface-internal/src/address-manager/index.ts index 34cb48825e..a21687c27d 100644 --- a/packages/interface-internal/src/address-manager/index.ts +++ b/packages/interface-internal/src/address-manager/index.ts @@ -1,5 +1,45 @@ import type { Multiaddr } from '@multiformats/multiaddr' +/** + * The type of address: + * + * - 'transport' a listen address supplied by a transport + * - 'announce' a pre-configured announce address + * - 'observed' a peer reported this as a public address + * - 'dns-mapping' a DNS address dynamically mapped to one or more public addresses + * - 'ip-mapping' an external IP address dynamically mapped to a LAN address + */ +export type AddressType = 'transport' | 'announce' | 'observed' | 'dns-mapping' | 'ip-mapping' + +/** + * An address that has been configured or detected + */ +export interface NodeAddress { + /** + * The multiaddr that represents the address + */ + multiaddr: Multiaddr + + /** + * Dynamically configured addresses such as observed or IP/DNS mapped ones + * must be verified as valid by AutoNAT or some other means before the current + * node will add them to it's peer record and share them with peers. + * + * When this value is true, it's safe to share the address. + */ + verified: boolean + + /** + * A millisecond timestamp after which this address should be reverified + */ + expires: number + + /** + * The source of this address + */ + type: AddressType +} + export interface AddressManager { /** * Get peer listen multiaddrs @@ -41,6 +81,11 @@ export interface AddressManager { */ getAddresses(): Multiaddr[] + /** + * Return all known addresses with metadata + */ + getAddressesWithMetadata(): NodeAddress[] + /** * Adds a mapping between one or more IP addresses and a domain name - when * `getAddresses` is invoked, where the IP addresses are present in a diff --git a/packages/libp2p/src/address-manager.ts b/packages/libp2p/src/address-manager.ts deleted file mode 100644 index b14e2b19d2..0000000000 --- a/packages/libp2p/src/address-manager.ts +++ /dev/null @@ -1,454 +0,0 @@ -import { isIPv4 } from '@chainsafe/is-ip' -import { peerIdFromString } from '@libp2p/peer-id' -import { debounce } from '@libp2p/utils/debounce' -import { multiaddr, protocols } from '@multiformats/multiaddr' -import type { ComponentLogger, Libp2pEvents, Logger, TypedEventTarget, PeerId, PeerStore } from '@libp2p/interface' -import type { AddressManager as AddressManagerInterface, TransportManager } from '@libp2p/interface-internal' -import type { Multiaddr } from '@multiformats/multiaddr' - -export const defaultValues = { - maxObservedAddresses: 10 -} - -export interface AddressManagerInit { - /** - * Pass an function in this field to override the list of addresses - * that are announced to the network - */ - announceFilter?: AddressFilter - - /** - * A list of string multiaddrs to listen on - */ - listen?: string[] - - /** - * A list of string multiaddrs to use instead of those reported by transports - */ - announce?: string[] - - /** - * A list of string multiaddrs string to never announce - */ - noAnnounce?: string[] - - /** - * A list of string multiaddrs to add to the list of announced addresses - */ - appendAnnounce?: string[] - - /** - * Limits the number of observed addresses we will store - */ - maxObservedAddresses?: number -} - -export interface AddressManagerComponents { - peerId: PeerId - transportManager: TransportManager - peerStore: PeerStore - events: TypedEventTarget - logger: ComponentLogger -} - -/** - * A function that takes a list of multiaddrs and returns a list - * to announce - */ -export interface AddressFilter { - (addrs: Multiaddr[]): Multiaddr[] -} - -const defaultAddressFilter = (addrs: Multiaddr[]): Multiaddr[] => addrs - -interface ObservedAddressMetadata { - confident: boolean -} - -/** - * If the passed multiaddr contains the passed peer id, remove it - */ -function stripPeerId (ma: Multiaddr, peerId: PeerId): Multiaddr { - const observedPeerIdStr = ma.getPeerId() - - // strip our peer id if it has been passed - if (observedPeerIdStr != null) { - const observedPeerId = peerIdFromString(observedPeerIdStr) - - // use same encoding for comparison - if (observedPeerId.equals(peerId)) { - ma = ma.decapsulate(multiaddr(`/p2p/${peerId.toString()}`)) - } - } - - return ma -} - -const CODEC_IP4 = 0x04 -const CODEC_IP6 = 0x29 -const CODEC_DNS4 = 0x36 -const CODEC_DNS6 = 0x37 -const CODEC_TCP = 0x06 -const CODEC_UDP = 0x0111 - -interface PublicAddressMapping { - externalIp: string - externalPort: number -} - -interface DNSMapping { - domain: string - confident: boolean -} - -export class AddressManager implements AddressManagerInterface { - private readonly log: Logger - private readonly components: AddressManagerComponents - // this is an array to allow for duplicates, e.g. multiples of `/ip4/0.0.0.0/tcp/0` - private readonly listen: string[] - private readonly announce: Set - private readonly appendAnnounce: Set - private readonly observed: Map - private readonly announceFilter: AddressFilter - private readonly ipDomainMappings: Map - private readonly publicAddressMappings: Map - private readonly maxObservedAddresses: number - - /** - * Responsible for managing the peer addresses. - * Peers can specify their listen and announce addresses. - * The listen addresses will be used by the libp2p transports to listen for new connections, - * while the announce addresses will be used for the peer addresses' to other peers in the network. - */ - constructor (components: AddressManagerComponents, init: AddressManagerInit = {}) { - const { listen = [], announce = [], appendAnnounce = [] } = init - - this.components = components - this.log = components.logger.forComponent('libp2p:address-manager') - this.listen = listen.map(ma => ma.toString()) - this.announce = new Set(announce.map(ma => ma.toString())) - this.appendAnnounce = new Set(appendAnnounce.map(ma => ma.toString())) - this.observed = new Map() - this.ipDomainMappings = new Map() - this.publicAddressMappings = new Map() - this.announceFilter = init.announceFilter ?? defaultAddressFilter - this.maxObservedAddresses = init.maxObservedAddresses ?? defaultValues.maxObservedAddresses - - // this method gets called repeatedly on startup when transports start listening so - // debounce it so we don't cause multiple self:peer:update events to be emitted - this._updatePeerStoreAddresses = debounce(this._updatePeerStoreAddresses.bind(this), 1000) - - // update our stored addresses when new transports listen - components.events.addEventListener('transport:listening', () => { - this._updatePeerStoreAddresses() - }) - // update our stored addresses when existing transports stop listening - components.events.addEventListener('transport:close', () => { - this._updatePeerStoreAddresses() - }) - } - - readonly [Symbol.toStringTag] = '@libp2p/address-manager' - - _updatePeerStoreAddresses (): void { - // if announce addresses have been configured, ensure they make it into our peer - // record for things like identify - const addrs = this.getAddresses() - .map(ma => { - // strip our peer id if it is present - if (ma.getPeerId() === this.components.peerId.toString()) { - return ma.decapsulate(`/p2p/${this.components.peerId.toString()}`) - } - - return ma - }) - - this.components.peerStore.patch(this.components.peerId, { - multiaddrs: addrs - }) - .catch(err => { - this.log.error('error updating addresses', err) - }) - } - - /** - * Get peer listen multiaddrs - */ - getListenAddrs (): Multiaddr[] { - return Array.from(this.listen).map((a) => multiaddr(a)) - } - - /** - * Get peer announcing multiaddrs - */ - getAnnounceAddrs (): Multiaddr[] { - return Array.from(this.announce).map((a) => multiaddr(a)) - } - - /** - * Get peer announcing multiaddrs - */ - getAppendAnnounceAddrs (): Multiaddr[] { - return Array.from(this.appendAnnounce).map((a) => multiaddr(a)) - } - - /** - * Get observed multiaddrs - */ - getObservedAddrs (): Multiaddr[] { - return Array.from(this.observed).map(([a]) => multiaddr(a)) - } - - /** - * Add peer observed addresses - */ - addObservedAddr (addr: Multiaddr): void { - if (this.observed.size === this.maxObservedAddresses) { - return - } - - addr = stripPeerId(addr, this.components.peerId) - const addrString = addr.toString() - - // do not trigger the change:addresses event if we already know about this address - if (this.observed.has(addrString)) { - return - } - - this.observed.set(addrString, { - confident: false - }) - } - - confirmObservedAddr (addr: Multiaddr): void { - addr = stripPeerId(addr, this.components.peerId) - const addrString = addr.toString() - - const metadata = this.observed.get(addrString) ?? { - confident: false - } - - const startingConfidence = metadata.confident - - this.observed.set(addrString, { - confident: true - }) - - // only trigger the 'self:peer:update' event if our confidence in an address has changed - if (!startingConfidence) { - this._updatePeerStoreAddresses() - } - } - - removeObservedAddr (addr: Multiaddr): void { - addr = stripPeerId(addr, this.components.peerId) - const addrString = addr.toString() - - this.observed.delete(addrString) - } - - getAddresses (): Multiaddr[] { - let multiaddrs = this.getAnnounceAddrs() - - if (multiaddrs.length === 0) { - // no configured announce addrs, add configured listen addresses - multiaddrs = this.components.transportManager.getAddrs() - } - - multiaddrs = multiaddrs - .concat( - // add additional announce addresses - ...this.getAppendAnnounceAddrs(), - - // add observed addresses we are confident in - Array.from(this.observed) - .filter(([ma, metadata]) => metadata.confident) - .map(([ma]) => multiaddr(ma)) - ) - - // add public addresses - const ipMappedMultiaddrs: Multiaddr[] = [] - multiaddrs.forEach(ma => { - const tuples = ma.stringTuples() - let tuple: string | undefined - - // see if the internal host/port/protocol tuple has been mapped externally - if ((tuples[0][0] === CODEC_IP4 || tuples[0][0] === CODEC_IP6) && tuples[1][0] === CODEC_TCP) { - tuple = `${tuples[0][1]}-${tuples[1][1]}-tcp` - } else if ((tuples[0][0] === CODEC_IP4 || tuples[0][0] === CODEC_IP6) && tuples[1][0] === CODEC_UDP) { - tuple = `${tuples[0][1]}-${tuples[1][1]}-udp` - } - - if (tuple == null) { - return - } - - const mappings = this.publicAddressMappings.get(tuple) - - if (mappings == null) { - return - } - - for (const mapping of mappings) { - tuples[0][0] = isIPv4(mapping.externalIp) ? CODEC_IP4 : CODEC_IP6 - tuples[0][1] = mapping.externalIp - tuples[1][1] = `${mapping.externalPort}` - - ipMappedMultiaddrs.push( - multiaddr(`/${ - tuples.map(tuple => { - return [ - protocols(tuple[0]).name, - tuple[1] - ].join('/') - }).join('/') - }`) - ) - } - }) - multiaddrs = multiaddrs.concat(ipMappedMultiaddrs) - - // add ip->domain mappings - const dnsMappedMultiaddrs: Multiaddr[] = [] - for (const ma of multiaddrs) { - const tuples = ma.stringTuples() - let mappedIp = false - - for (const [ip, mapping] of this.ipDomainMappings.entries()) { - if (!mapping.confident) { - continue - } - - for (let i = 0; i < tuples.length; i++) { - if (tuples[i][1] !== ip) { - continue - } - - if (tuples[i][0] === CODEC_IP4) { - tuples[i][0] = CODEC_DNS4 - tuples[i][1] = mapping.domain - mappedIp = true - } - - if (tuples[i][0] === CODEC_IP6) { - tuples[i][0] = CODEC_DNS6 - tuples[i][1] = mapping.domain - mappedIp = true - } - } - } - - if (mappedIp) { - dnsMappedMultiaddrs.push( - multiaddr(`/${ - tuples.map(tuple => { - return [ - protocols(tuple[0]).name, - tuple[1] - ].join('/') - }).join('/') - }`) - ) - } - } - multiaddrs = multiaddrs.concat(dnsMappedMultiaddrs) - - // dedupe multiaddrs - const addrSet = new Set() - multiaddrs = multiaddrs.filter(ma => { - const maStr = ma.toString() - - if (addrSet.has(maStr)) { - return false - } - - addrSet.add(maStr) - - return true - }) - - // Create advertising list - return this.announceFilter( - Array.from(addrSet) - .map(str => { - const ma = multiaddr(str) - - // do not append our peer id to a path multiaddr as it will become invalid - if (ma.protos().pop()?.path === true) { - return ma - } - - if (ma.getPeerId() === this.components.peerId.toString()) { - return ma - } - - return ma.encapsulate(`/p2p/${this.components.peerId.toString()}`) - }) - ) - } - - addDNSMapping (domain: string, addresses: string[]): void { - addresses.forEach(ip => { - this.log('add DNS mapping %s to %s', ip, domain) - - // check ip/public ip mappings to see if we think we are contactable - const confident = [...this.publicAddressMappings.entries()].some(([key, mappings]) => { - return mappings.some(mapping => mapping.externalIp === ip) - }) - - this.ipDomainMappings.set(ip, { - domain, - confident - }) - }) - this._updatePeerStoreAddresses() - } - - removeDNSMapping (domain: string): void { - for (const [key, mapping] of this.ipDomainMappings.entries()) { - if (mapping.domain === domain) { - this.log('remove DNS mapping for %s', domain) - this.ipDomainMappings.delete(key) - } - } - this._updatePeerStoreAddresses() - } - - addPublicAddressMapping (internalIp: string, internalPort: number, externalIp: string, externalPort: number = internalPort, protocol: 'tcp' | 'udp' = 'tcp'): void { - const key = `${internalIp}-${internalPort}-${protocol}` - const mappings = this.publicAddressMappings.get(key) ?? [] - mappings.push({ - externalIp, - externalPort - }) - - this.publicAddressMappings.set(key, mappings) - - // update domain mappings to indicate we are now confident that any matching - // ip/domain combination can now be resolved externally - for (const [key, mapping] of this.ipDomainMappings.entries()) { - if (key === externalIp) { - mapping.confident = true - - this.ipDomainMappings.set(key, mapping) - } - } - - this._updatePeerStoreAddresses() - } - - removePublicAddressMapping (internalIp: string, internalPort: number, externalIp: string, externalPort: number = internalPort, protocol: 'tcp' | 'udp' = 'tcp'): void { - const key = `${internalIp}-${internalPort}-${protocol}` - const mappings = (this.publicAddressMappings.get(key) ?? []).filter(mapping => { - return mapping.externalIp !== externalIp && mapping.externalPort !== externalPort - }) - - if (mappings.length === 0) { - this.publicAddressMappings.delete(key) - } else { - this.publicAddressMappings.set(key, mappings) - } - - this._updatePeerStoreAddresses() - } -} diff --git a/packages/libp2p/src/address-manager/dns-mappings.ts b/packages/libp2p/src/address-manager/dns-mappings.ts new file mode 100644 index 0000000000..154914ca58 --- /dev/null +++ b/packages/libp2p/src/address-manager/dns-mappings.ts @@ -0,0 +1,159 @@ +import { isPrivateIp } from '@libp2p/utils/private-ip' +import { multiaddr, protocols } from '@multiformats/multiaddr' +import type { AddressManagerComponents, AddressManagerInit } from './index.js' +import type { Logger } from '@libp2p/interface' +import type { NodeAddress } from '@libp2p/interface-internal' +import type { Multiaddr, StringTuple } from '@multiformats/multiaddr' + +export const defaultValues = { + maxObservedAddresses: 10 +} + +interface DNSMapping { + domain: string + verified: boolean + expires: number +} + +const CODEC_TLS = 0x01c0 +const CODEC_SNI = 0x01c1 +const CODEC_DNS = 0x35 +const CODEC_DNS4 = 0x36 +const CODEC_DNS6 = 0x37 +const CODEC_DNSADDR = 0x38 + +export class DNSMappings { + private readonly log: Logger + private readonly mappings: Map + + constructor (components: AddressManagerComponents, init: AddressManagerInit = {}) { + this.log = components.logger.forComponent('libp2p:address-manager:dns-mappings') + this.mappings = new Map() + } + + has (ma: Multiaddr): boolean { + const host = this.findHost(ma) + + for (const mapping of this.mappings.values()) { + if (mapping.domain === host) { + return true + } + } + + return false + } + + add (domain: string, addresses: string[]): void { + addresses.forEach(ip => { + this.log('add DNS mapping %s to %s', ip, domain) + + this.mappings.set(ip, { + domain, + // we are only confident if this is an local domain mapping, otherwise + // we will require external validation + verified: isPrivateIp(ip) === true, + expires: 0 + }) + }) + } + + remove (ma: Multiaddr): boolean { + const host = this.findHost(ma) + let wasConfident = false + + for (const [ip, mapping] of this.mappings.entries()) { + if (mapping.domain === host) { + this.log('removing %s to %s DNS mapping %e', ip, mapping.domain, new Error('where')) + this.mappings.delete(ip) + wasConfident = wasConfident || mapping.verified + } + } + + return wasConfident + } + + getAll (addresses: NodeAddress[]): NodeAddress[] { + const dnsMappedAddresses: NodeAddress[] = [] + + for (let i = 0; i < addresses.length; i++) { + const address = addresses[i] + const tuples = address.multiaddr.stringTuples() + const host = tuples[0][1] + + if (host == null) { + continue + } + + for (const [ip, mapping] of this.mappings.entries()) { + if (host !== ip) { + continue + } + + // insert SNI tuple after TLS tuple, if one is present + const mappedIp = this.maybeAddSNITuple(tuples, mapping.domain) + + if (mappedIp) { + // remove the address and replace it with the version that includes + // the SNI tuple + addresses.splice(i, 1) + i-- + + dnsMappedAddresses.push({ + multiaddr: multiaddr(`/${ + tuples.map(tuple => { + return [ + protocols(tuple[0]).name, + tuple[1] + ].join('/') + }).join('/') + }`), + verified: mapping.verified, + type: 'dns-mapping', + expires: mapping.expires + }) + } + } + } + + return dnsMappedAddresses + } + + private maybeAddSNITuple (tuples: StringTuple[], domain: string): boolean { + for (let j = 0; j < tuples.length; j++) { + if (tuples[j][0] === CODEC_TLS && tuples[j + 1]?.[0] !== CODEC_SNI) { + tuples.splice(j + 1, 0, [CODEC_SNI, domain]) + return true + } + } + + return false + } + + confirm (ma: Multiaddr, ttl: number): boolean { + const host = this.findHost(ma) + let startingConfidence = false + + for (const [ip, mapping] of this.mappings.entries()) { + if (mapping.domain === host) { + this.log('marking %s to %s DNS mapping as verified', ip, mapping.domain) + startingConfidence = mapping.verified + mapping.verified = true + mapping.expires = Date.now() + ttl + } + } + + return startingConfidence + } + + private findHost (ma: Multiaddr): string | undefined { + for (const tuple of ma.stringTuples()) { + if (tuple[0] === CODEC_SNI) { + return tuple[1] + } + + if (tuple[0] === CODEC_DNS || tuple[0] === CODEC_DNS4 || tuple[0] === CODEC_DNS6 || tuple[0] === CODEC_DNSADDR) { + return tuple[1] + } + } + } +} diff --git a/packages/libp2p/src/address-manager/index.ts b/packages/libp2p/src/address-manager/index.ts new file mode 100644 index 0000000000..66394d8df0 --- /dev/null +++ b/packages/libp2p/src/address-manager/index.ts @@ -0,0 +1,368 @@ +/* eslint-disable complexity */ +import { isIPv4 } from '@chainsafe/is-ip' +import { peerIdFromString } from '@libp2p/peer-id' +import { debounce } from '@libp2p/utils/debounce' +import { createScalableCuckooFilter } from '@libp2p/utils/filters' +import { multiaddr } from '@multiformats/multiaddr' +import { DNSMappings } from './dns-mappings.js' +import { IPMappings } from './ip-mappings.js' +import { ObservedAddresses } from './observed-addresses.js' +import type { ComponentLogger, Libp2pEvents, Logger, TypedEventTarget, PeerId, PeerStore } from '@libp2p/interface' +import type { AddressManager as AddressManagerInterface, TransportManager, NodeAddress } from '@libp2p/interface-internal' +import type { Filter } from '@libp2p/utils/filters' +import type { Multiaddr } from '@multiformats/multiaddr' + +export const defaultValues = { + maxObservedAddresses: 10, + observedAddressTTL: 60_000 * 10 +} + +export interface AddressManagerInit { + /** + * Pass an function in this field to override the list of addresses + * that are announced to the network + */ + announceFilter?: AddressFilter + + /** + * A list of string multiaddrs to listen on + */ + listen?: string[] + + /** + * A list of string multiaddrs to use instead of those reported by transports + */ + announce?: string[] + + /** + * A list of string multiaddrs string to never announce + */ + noAnnounce?: string[] + + /** + * A list of string multiaddrs to add to the list of announced addresses + */ + appendAnnounce?: string[] + + /** + * Limits the number of observed addresses we will store + */ + maxObservedAddresses?: number + + /** + * How long before each observed address should be reverified + */ + observedAddressTTL?: number +} + +export interface AddressManagerComponents { + peerId: PeerId + transportManager: TransportManager + peerStore: PeerStore + events: TypedEventTarget + logger: ComponentLogger +} + +/** + * A function that takes a list of multiaddrs and returns a list + * to announce + */ +export interface AddressFilter { + (addrs: Multiaddr[]): Multiaddr[] +} + +const defaultAddressFilter = (addrs: Multiaddr[]): Multiaddr[] => addrs + +/** + * If the passed multiaddr contains the passed peer id, remove it + */ +function stripPeerId (ma: Multiaddr, peerId: PeerId): Multiaddr { + const observedPeerIdStr = ma.getPeerId() + + // strip our peer id if it has been passed + if (observedPeerIdStr != null) { + const observedPeerId = peerIdFromString(observedPeerIdStr) + + // use same encoding for comparison + if (observedPeerId.equals(peerId)) { + ma = ma.decapsulate(multiaddr(`/p2p/${peerId.toString()}`)) + } + } + + return ma +} + +export class AddressManager implements AddressManagerInterface { + private readonly log: Logger + private readonly components: AddressManagerComponents + // this is an array to allow for duplicates, e.g. multiples of `/ip4/0.0.0.0/tcp/0` + private readonly listen: string[] + private readonly announce: Set + private readonly appendAnnounce: Set + private readonly announceFilter: AddressFilter + private readonly observed: ObservedAddresses + private readonly dnsMappings: DNSMappings + private readonly ipMappings: IPMappings + private readonly observedAddressFilter: Filter + private readonly observedAddressTTL: number + + /** + * Responsible for managing the peer addresses. + * Peers can specify their listen and announce addresses. + * The listen addresses will be used by the libp2p transports to listen for new connections, + * while the announce addresses will be used for the peer addresses' to other peers in the network. + */ + constructor (components: AddressManagerComponents, init: AddressManagerInit = {}) { + const { listen = [], announce = [], appendAnnounce = [] } = init + + this.components = components + this.log = components.logger.forComponent('libp2p:address-manager') + this.listen = listen.map(ma => ma.toString()) + this.announce = new Set(announce.map(ma => ma.toString())) + this.appendAnnounce = new Set(appendAnnounce.map(ma => ma.toString())) + this.observed = new ObservedAddresses(components, init) + this.dnsMappings = new DNSMappings(components, init) + this.ipMappings = new IPMappings(components, init) + this.announceFilter = init.announceFilter ?? defaultAddressFilter + this.observedAddressFilter = createScalableCuckooFilter(1024) + this.observedAddressTTL = init.observedAddressTTL ?? defaultValues.observedAddressTTL + + // this method gets called repeatedly on startup when transports start listening so + // debounce it so we don't cause multiple self:peer:update events to be emitted + this._updatePeerStoreAddresses = debounce(this._updatePeerStoreAddresses.bind(this), 1000) + + // update our stored addresses when new transports listen + components.events.addEventListener('transport:listening', () => { + this._updatePeerStoreAddresses() + }) + // update our stored addresses when existing transports stop listening + components.events.addEventListener('transport:close', () => { + this._updatePeerStoreAddresses() + }) + } + + readonly [Symbol.toStringTag] = '@libp2p/address-manager' + + _updatePeerStoreAddresses (): void { + // if announce addresses have been configured, ensure they make it into our peer + // record for things like identify + const addrs = this.getAddresses() + .map(ma => { + // strip our peer id if it is present + if (ma.getPeerId() === this.components.peerId.toString()) { + return ma.decapsulate(`/p2p/${this.components.peerId.toString()}`) + } + + return ma + }) + + this.components.peerStore.patch(this.components.peerId, { + multiaddrs: addrs + }) + .catch(err => { + this.log.error('error updating addresses', err) + }) + } + + /** + * Get peer listen multiaddrs + */ + getListenAddrs (): Multiaddr[] { + return Array.from(this.listen).map((a) => multiaddr(a)) + } + + /** + * Get peer announcing multiaddrs + */ + getAnnounceAddrs (): Multiaddr[] { + return Array.from(this.announce).map((a) => multiaddr(a)) + } + + /** + * Get peer announcing multiaddrs + */ + getAppendAnnounceAddrs (): Multiaddr[] { + return Array.from(this.appendAnnounce).map((a) => multiaddr(a)) + } + + /** + * Get observed multiaddrs + */ + getObservedAddrs (): Multiaddr[] { + return this.observed.getAll().map(addr => addr.multiaddr) + } + + /** + * Add peer observed addresses + */ + addObservedAddr (addr: Multiaddr): void { + const tuples = addr.stringTuples() + const socketAddress = `${tuples[0][1]}:${tuples[1][1]}` + + // ignore if this address if it's been observed before + if (this.observedAddressFilter.has(socketAddress)) { + return + } + + this.observedAddressFilter.add(socketAddress) + + addr = stripPeerId(addr, this.components.peerId) + + // ignore observed address if it is an IP mapping + if (this.ipMappings.has(addr)) { + return + } + + // ignore observed address if it is a DNS mapping + if (this.dnsMappings.has(addr)) { + return + } + + this.observed.add(addr) + } + + confirmObservedAddr (addr: Multiaddr): void { + addr = stripPeerId(addr, this.components.peerId) + let startingConfidence = true + + if (this.observed.has(addr)) { + startingConfidence = this.observed.confirm(addr, this.observedAddressTTL) + } + + if (this.dnsMappings.has(addr)) { + startingConfidence = this.dnsMappings.confirm(addr, this.observedAddressTTL) + } + + if (this.ipMappings.has(addr)) { + startingConfidence = this.ipMappings.confirm(addr, this.observedAddressTTL) + } + + // only trigger the 'self:peer:update' event if our confidence in an address has changed + if (!startingConfidence) { + this._updatePeerStoreAddresses() + } + } + + removeObservedAddr (addr: Multiaddr): void { + addr = stripPeerId(addr, this.components.peerId) + + this.observed.remove(addr) + this.dnsMappings.remove(addr) + this.ipMappings.remove(addr) + } + + getAddresses (): Multiaddr[] { + const addresses = new Set() + + const multiaddrs = this.getAddressesWithMetadata() + .filter(addr => { + if (!addr.verified) { + return false + } + + const maStr = addr.multiaddr.toString() + + if (addresses.has(maStr)) { + return false + } + + addresses.add(maStr) + + return true + }) + .map(address => address.multiaddr) + + // filter addressees before returning + return this.announceFilter( + multiaddrs.map(str => { + const ma = multiaddr(str) + + // do not append our peer id to a path multiaddr as it will become invalid + if (ma.protos().pop()?.path === true) { + return ma + } + + if (ma.getPeerId() === this.components.peerId.toString()) { + return ma + } + + return ma.encapsulate(`/p2p/${this.components.peerId.toString()}`) + }) + ) + } + + getAddressesWithMetadata (): NodeAddress[] { + const announceMultiaddrs = this.getAnnounceAddrs() + + if (announceMultiaddrs.length > 0) { + return announceMultiaddrs.map(multiaddr => ({ + multiaddr, + verified: true, + type: 'announce', + expires: Date.now() + this.observedAddressTTL + })) + } + + let addresses: NodeAddress[] = [] + + // add transport addresses + addresses = addresses.concat( + this.components.transportManager.getAddrs().map(multiaddr => ({ + multiaddr, + verified: true, + type: 'transport', + expires: Date.now() + this.observedAddressTTL + })) + ) + + // add append announce addresses + addresses = addresses.concat( + this.getAppendAnnounceAddrs().map(multiaddr => ({ + multiaddr, + verified: true, + type: 'announce', + expires: Date.now() + this.observedAddressTTL + })) + ) + + // add observed addresses + addresses = addresses.concat( + this.observed.getAll() + ) + + // add ip mapped addresses + addresses = addresses.concat( + this.ipMappings.getAll(addresses) + ) + + // add ip->domain mappings, must be done after IP mappings + addresses = addresses.concat( + this.dnsMappings.getAll(addresses) + ) + + return addresses + } + + addDNSMapping (domain: string, addresses: string[]): void { + this.dnsMappings.add(domain, addresses) + } + + removeDNSMapping (domain: string): void { + if (this.dnsMappings.remove(multiaddr(`/dns/${domain}`))) { + this._updatePeerStoreAddresses() + } + } + + addPublicAddressMapping (internalIp: string, internalPort: number, externalIp: string, externalPort: number = internalPort, protocol: 'tcp' | 'udp' = 'tcp'): void { + this.ipMappings.add(internalIp, internalPort, externalIp, externalPort, protocol) + + // remove duplicate observed addresses + this.observed.removePrefixed(`/ip${isIPv4(externalIp) ? 4 : 6}/${externalIp}/${protocol}/${externalPort}`) + } + + removePublicAddressMapping (internalIp: string, internalPort: number, externalIp: string, externalPort: number = internalPort, protocol: 'tcp' | 'udp' = 'tcp'): void { + if (this.ipMappings.remove(multiaddr(`/ip${isIPv4(externalIp) ? 4 : 6}/${externalIp}/${protocol}/${externalPort}`))) { + this._updatePeerStoreAddresses() + } + } +} diff --git a/packages/libp2p/src/address-manager/ip-mappings.ts b/packages/libp2p/src/address-manager/ip-mappings.ts new file mode 100644 index 0000000000..518f1f235b --- /dev/null +++ b/packages/libp2p/src/address-manager/ip-mappings.ts @@ -0,0 +1,164 @@ +import { isIPv4 } from '@chainsafe/is-ip' +import { multiaddr, protocols } from '@multiformats/multiaddr' +import type { AddressManagerComponents, AddressManagerInit } from './index.js' +import type { Logger } from '@libp2p/interface' +import type { NodeAddress } from '@libp2p/interface-internal' +import type { Multiaddr } from '@multiformats/multiaddr' + +export const defaultValues = { + maxObservedAddresses: 10 +} + +interface PublicAddressMapping { + internalIp: string + internalPort: number + externalIp: string + externalPort: number + externalFamily: 4 | 6 + protocol: 'tcp' | 'udp' + verified: boolean + expires: number +} + +const CODEC_IP4 = 0x04 +const CODEC_IP6 = 0x29 +const CODEC_TCP = 0x06 +const CODEC_UDP = 0x0111 + +export class IPMappings { + private readonly log: Logger + private readonly mappings: Map + + constructor (components: AddressManagerComponents, init: AddressManagerInit = {}) { + this.log = components.logger.forComponent('libp2p:address-manager:ip-mappings') + this.mappings = new Map() + } + + has (ma: Multiaddr): boolean { + const tuples = ma.stringTuples() + + for (const mappings of this.mappings.values()) { + for (const mapping of mappings) { + if (mapping.externalIp === tuples[0][1]) { + return true + } + } + } + + return false + } + + add (internalIp: string, internalPort: number, externalIp: string, externalPort: number = internalPort, protocol: 'tcp' | 'udp' = 'tcp'): void { + const key = `${internalIp}-${internalPort}-${protocol}` + const mappings = this.mappings.get(key) ?? [] + const mapping: PublicAddressMapping = { + internalIp, + internalPort, + externalIp, + externalPort, + externalFamily: isIPv4(externalIp) ? 4 : 6, + protocol, + verified: false, + expires: 0 + } + mappings.push(mapping) + + this.mappings.set(key, mappings) + } + + remove (ma: Multiaddr): boolean { + const tuples = ma.stringTuples() + const host = tuples[0][1] ?? '' + const protocol = tuples[1][0] === CODEC_TCP ? 'tcp' : 'udp' + const port = parseInt(tuples[1][1] ?? '0') + let wasConfident = false + + for (const [key, mappings] of this.mappings.entries()) { + for (let i = 0; i < mappings.length; i++) { + const mapping = mappings[i] + + if (mapping.externalIp === host && mapping.externalPort === port && mapping.protocol === protocol) { + this.log('removing %s:%s to %s:%s %s IP mapping', mapping.externalIp, mapping.externalPort, host, port, protocol) + + wasConfident = wasConfident || mapping.verified + mappings.splice(i, 1) + i-- + } + } + + if (mappings.length === 0) { + this.mappings.delete(key) + } + } + + return wasConfident + } + + getAll (addresses: NodeAddress[]): NodeAddress[] { + const ipMappedAddresses: NodeAddress[] = [] + + for (const { multiaddr: ma } of addresses) { + const tuples = ma.stringTuples() + let tuple: string | undefined + + // see if the internal host/port/protocol tuple has been mapped externally + if ((tuples[0][0] === CODEC_IP4 || tuples[0][0] === CODEC_IP6) && tuples[1][0] === CODEC_TCP) { + tuple = `${tuples[0][1]}-${tuples[1][1]}-tcp` + } else if ((tuples[0][0] === CODEC_IP4 || tuples[0][0] === CODEC_IP6) && tuples[1][0] === CODEC_UDP) { + tuple = `${tuples[0][1]}-${tuples[1][1]}-udp` + } + + if (tuple == null) { + continue + } + + const mappings = this.mappings.get(tuple) + + if (mappings == null) { + continue + } + + for (const mapping of mappings) { + tuples[0][0] = mapping.externalFamily === 4 ? CODEC_IP4 : CODEC_IP6 + tuples[0][1] = mapping.externalIp + tuples[1][1] = `${mapping.externalPort}` + + ipMappedAddresses.push({ + multiaddr: multiaddr(`/${ + tuples.map(tuple => { + return [ + protocols(tuple[0]).name, + tuple[1] + ].join('/') + }).join('/') + }`), + verified: mapping.verified, + type: 'ip-mapping', + expires: mapping.expires + }) + } + } + + return ipMappedAddresses + } + + confirm (ma: Multiaddr, ttl: number): boolean { + const tuples = ma.stringTuples() + const host = tuples[0][1] + let startingConfidence = false + + for (const mappings of this.mappings.values()) { + for (const mapping of mappings) { + // eslint-disable-next-line max-depth + if (mapping.externalIp === host) { + this.log('marking %s to %s IP mapping as verified', mapping.internalIp, mapping.externalIp) + startingConfidence = mapping.verified + mapping.verified = true + mapping.expires = Date.now() + ttl + } + } + } + + return startingConfidence + } +} diff --git a/packages/libp2p/src/address-manager/observed-addresses.ts b/packages/libp2p/src/address-manager/observed-addresses.ts new file mode 100644 index 0000000000..5b45b7efc8 --- /dev/null +++ b/packages/libp2p/src/address-manager/observed-addresses.ts @@ -0,0 +1,86 @@ +import { isLinkLocal } from '@libp2p/utils/multiaddr/is-link-local' +import { isPrivate } from '@libp2p/utils/multiaddr/is-private' +import { multiaddr } from '@multiformats/multiaddr' +import type { AddressManagerComponents, AddressManagerInit } from './index.js' +import type { Logger } from '@libp2p/interface' +import type { NodeAddress } from '@libp2p/interface-internal' +import type { Multiaddr } from '@multiformats/multiaddr' + +export const defaultValues = { + maxObservedAddresses: 10 +} + +interface ObservedAddressMetadata { + verified: boolean + expires: number +} + +export class ObservedAddresses { + private readonly log: Logger + private readonly addresses: Map + private readonly maxObservedAddresses: number + + constructor (components: AddressManagerComponents, init: AddressManagerInit = {}) { + this.log = components.logger.forComponent('libp2p:address-manager:observed-addresses') + this.addresses = new Map() + this.maxObservedAddresses = init.maxObservedAddresses ?? defaultValues.maxObservedAddresses + } + + has (ma: Multiaddr): boolean { + return this.addresses.has(ma.toString()) + } + + removePrefixed (prefix: string): void { + for (const key of this.addresses.keys()) { + if (key.toString().startsWith(prefix)) { + this.addresses.delete(key) + } + } + } + + add (ma: Multiaddr): void { + if (this.addresses.size === this.maxObservedAddresses) { + return + } + + if (isPrivate(ma) || isLinkLocal(ma)) { + return + } + + this.log('adding observed address %a', ma) + this.addresses.set(ma.toString(), { + verified: false, + expires: 0 + }) + } + + getAll (): NodeAddress[] { + return Array.from(this.addresses) + .map(([ma, metadata]) => ({ + multiaddr: multiaddr(ma), + verified: metadata.verified, + type: 'observed', + expires: metadata.expires + })) + } + + remove (ma: Multiaddr): void { + this.log('removing observed address %a', ma) + this.addresses.delete(ma.toString()) + } + + confirm (ma: Multiaddr, ttl: number): boolean { + const addrString = ma.toString() + const metadata = this.addresses.get(addrString) ?? { + verified: false, + expires: Date.now() + ttl + } + const startingConfidence = metadata.verified + metadata.verified = true + + this.log('marking observed address %a as verified', addrString) + this.addresses.set(addrString, metadata) + + return startingConfidence + } +} diff --git a/packages/libp2p/src/index.ts b/packages/libp2p/src/index.ts index d05b2883c6..fc68bc0bdf 100644 --- a/packages/libp2p/src/index.ts +++ b/packages/libp2p/src/index.ts @@ -18,7 +18,7 @@ import { generateKeyPair } from '@libp2p/crypto/keys' import { peerIdFromPrivateKey } from '@libp2p/peer-id' import { validateConfig } from './config.js' import { Libp2p as Libp2pClass } from './libp2p.js' -import type { AddressManagerInit, AddressFilter } from './address-manager.js' +import type { AddressManagerInit, AddressFilter } from './address-manager/index.js' import type { Components } from './components.js' import type { ConnectionManagerInit } from './connection-manager/index.js' import type { ConnectionMonitorInit } from './connection-monitor.js' diff --git a/packages/libp2p/src/libp2p.ts b/packages/libp2p/src/libp2p.ts index b88e29dab9..46470e1c27 100644 --- a/packages/libp2p/src/libp2p.ts +++ b/packages/libp2p/src/libp2p.ts @@ -8,7 +8,7 @@ import { isMultiaddr, type Multiaddr } from '@multiformats/multiaddr' import { MemoryDatastore } from 'datastore-core/memory' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { AddressManager } from './address-manager.js' +import { AddressManager } from './address-manager/index.js' import { checkServiceDependencies, defaultComponents } from './components.js' import { connectionGater } from './config/connection-gater.js' import { DefaultConnectionManager } from './connection-manager/index.js' diff --git a/packages/libp2p/test/addresses/address-manager.spec.ts b/packages/libp2p/test/addresses/address-manager.spec.ts index 2cbe8a0427..112329e38e 100644 --- a/packages/libp2p/test/addresses/address-manager.spec.ts +++ b/packages/libp2p/test/addresses/address-manager.spec.ts @@ -9,7 +9,7 @@ import { expect } from 'aegir/chai' import delay from 'delay' import Sinon from 'sinon' import { type StubbedInstance, stubInterface } from 'sinon-ts' -import { type AddressFilter, AddressManager } from '../../src/address-manager.js' +import { type AddressFilter, AddressManager } from '../../src/address-manager/index.js' import type { TransportManager } from '@libp2p/interface-internal' const listenAddresses = ['/ip4/127.0.0.1/tcp/15006/ws', '/ip4/127.0.0.1/tcp/15008/ws'] @@ -196,6 +196,7 @@ describe('Address Manager', () => { it('should only set addresses once', async () => { const ma = '/ip4/123.123.123.123/tcp/39201' + const ma2 = `${ma.toString()}/p2p/${peerId.toString()}` const am = new AddressManager({ peerId, transportManager: stubInterface({ @@ -206,10 +207,13 @@ describe('Address Manager', () => { logger: defaultLogger() }) + am.addObservedAddr(multiaddr(ma)) + am.addObservedAddr(multiaddr(ma2)) + am.confirmObservedAddr(multiaddr(ma)) am.confirmObservedAddr(multiaddr(ma)) am.confirmObservedAddr(multiaddr(ma)) - am.confirmObservedAddr(multiaddr(`${ma.toString()}/p2p/${peerId.toString()}`)) + am.confirmObservedAddr(multiaddr(ma2)) // wait for address manager _updatePeerStoreAddresses debounce await delay(1500) @@ -291,15 +295,17 @@ describe('Address Manager', () => { const internalPort = 1234 const protocol = 'tcp' - // one loopback, one LAN address + // one loopback, one LAN, one TLS address transportManager.getAddrs.returns([ multiaddr(`/ip4/127.0.0.1/${protocol}/${internalPort}`), - multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}`) + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}`), + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/tls/ws`) ]) expect(am.getAddresses()).to.deep.equal([ multiaddr(`/ip4/127.0.0.1/${protocol}/${internalPort}/p2p/${peerId}`), - multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId}`) + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId}`), + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/tls/ws/p2p/${peerId}`) ]) const domain = 'example.com' @@ -307,21 +313,25 @@ describe('Address Manager', () => { const externalPort = 4566 am.addDNSMapping(domain, [externalIp]) + am.addPublicAddressMapping(internalIp, internalPort, externalIp, externalPort, 'tcp') // have not verified DNS mapping so it is not included expect(am.getAddresses()).to.deep.equal([ multiaddr(`/ip4/127.0.0.1/${protocol}/${internalPort}/p2p/${peerId}`), - multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId}`) + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId}`), + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/tls/ws/p2p/${peerId}`) ]) - // public address mapping confirms DNS mapping - am.addPublicAddressMapping(internalIp, internalPort, externalIp, externalPort, protocol) + // confirm public IP and DNS mapping + am.confirmObservedAddr(multiaddr(`/ip4/${externalIp}/tcp/${externalPort}`)) + am.confirmObservedAddr(multiaddr(`/dns4/${domain}/tcp/${externalPort}`)) expect(am.getAddresses()).to.deep.equal([ multiaddr(`/ip4/127.0.0.1/${protocol}/${internalPort}/p2p/${peerId}`), multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId}`), + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/tls/ws/p2p/${peerId}`), multiaddr(`/ip4/${externalIp}/tcp/${externalPort}/p2p/${peerId}`), - multiaddr(`/dns4/${domain}/tcp/${externalPort}/p2p/${peerId}`) + multiaddr(`/ip4/${externalIp}/tcp/${externalPort}/tls/sni/${domain}/ws/p2p/${peerId}`) ]) }) @@ -340,15 +350,17 @@ describe('Address Manager', () => { const internalPort = 1234 const protocol = 'tcp' - // one loopback, one LAN address + // one loopback, one LAN, one TLS address transportManager.getAddrs.returns([ multiaddr(`/ip4/127.0.0.1/${protocol}/${internalPort}`), - multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}`) + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}`), + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/tls/ws`) ]) expect(am.getAddresses()).to.deep.equal([ multiaddr(`/ip4/127.0.0.1/${protocol}/${internalPort}/p2p/${peerId}`), - multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId}`) + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId}`), + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/tls/ws/p2p/${peerId}`) ]) const domain = 'example.com' @@ -356,21 +368,25 @@ describe('Address Manager', () => { const externalPort = 4566 am.addDNSMapping(domain, [externalIp]) + am.addPublicAddressMapping(internalIp, internalPort, externalIp, externalPort, 'tcp') // have not verified DNS mapping so it is not included expect(am.getAddresses()).to.deep.equal([ multiaddr(`/ip4/127.0.0.1/${protocol}/${internalPort}/p2p/${peerId}`), - multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId}`) + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId}`), + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/tls/ws/p2p/${peerId}`) ]) - // public address mapping confirms DNS mapping - am.addPublicAddressMapping(internalIp, internalPort, externalIp, externalPort, protocol) + // confirm public IP and DNS mapping + am.confirmObservedAddr(multiaddr(`/ip6/${externalIp}/tcp/${externalPort}`)) + am.confirmObservedAddr(multiaddr(`/dns6/${domain}/tcp/${externalPort}`)) expect(am.getAddresses()).to.deep.equal([ multiaddr(`/ip4/127.0.0.1/${protocol}/${internalPort}/p2p/${peerId}`), multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId}`), + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/tls/ws/p2p/${peerId}`), multiaddr(`/ip6/${externalIp}/tcp/${externalPort}/p2p/${peerId}`), - multiaddr(`/dns6/${domain}/tcp/${externalPort}/p2p/${peerId}`) + multiaddr(`/ip6/${externalIp}/tcp/${externalPort}/tls/sni/${domain}/ws/p2p/${peerId}`) ]) }) @@ -389,10 +405,11 @@ describe('Address Manager', () => { const internalPort = 1234 const protocol = 'tcp' - // one loopback, one LAN address + // one loopback, one LAN, one TLS address transportManager.getAddrs.returns([ multiaddr(`/ip4/127.0.0.1/${protocol}/${internalPort}`), - multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}`) + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}`), + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/tls/ws`) ]) const domain = 'example.com' @@ -402,20 +419,27 @@ describe('Address Manager', () => { am.addDNSMapping(domain, [externalIp]) am.addPublicAddressMapping(internalIp, internalPort, externalIp, externalPort, protocol) + // confirm public IP and DNS mapping + am.confirmObservedAddr(multiaddr(`/ip4/${externalIp}/tcp/${externalPort}`)) + am.confirmObservedAddr(multiaddr(`/dns4/${domain}/tcp/${externalPort}`)) + expect(am.getAddresses()).to.deep.equal([ multiaddr(`/ip4/127.0.0.1/${protocol}/${internalPort}/p2p/${peerId}`), multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId}`), + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/tls/ws/p2p/${peerId}`), multiaddr(`/ip4/${externalIp}/tcp/${externalPort}/p2p/${peerId}`), - multiaddr(`/dns4/${domain}/tcp/${externalPort}/p2p/${peerId}`) + multiaddr(`/ip4/${externalIp}/tcp/${externalPort}/tls/sni/${domain}/ws/p2p/${peerId}`) ]) - // public address mapping confirms DNS mapping + // remove DNS mapping am.removeDNSMapping(domain) expect(am.getAddresses()).to.deep.equal([ multiaddr(`/ip4/127.0.0.1/${protocol}/${internalPort}/p2p/${peerId}`), multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId}`), - multiaddr(`/ip4/${externalIp}/tcp/${externalPort}/p2p/${peerId}`) + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/tls/ws/p2p/${peerId}`), + multiaddr(`/ip4/${externalIp}/tcp/${externalPort}/p2p/${peerId}`), + multiaddr(`/ip4/${externalIp}/tcp/${externalPort}/tls/ws/p2p/${peerId}`) ]) }) @@ -443,6 +467,15 @@ describe('Address Manager', () => { multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}`) ]) + // not confirmed the mapping yet + expect(am.getAddresses()).to.deep.equal([ + multiaddr(`/ip4/127.0.0.1/tcp/1234/p2p/${peerId.toString()}`), + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId.toString()}`) + ]) + + // confirm IP mapping + am.confirmObservedAddr(multiaddr(`/ip4/${externalIp}/${protocol}/${externalPort}`)) + // should have mapped the LAN address to the external IP expect(am.getAddresses()).to.deep.equal([ multiaddr(`/ip4/127.0.0.1/tcp/1234/p2p/${peerId.toString()}`), @@ -482,6 +515,15 @@ describe('Address Manager', () => { multiaddr(`/ip6/${internalIp}/${protocol}/${internalPort}`) ]) + // not confirmed the mapping yet + expect(am.getAddresses()).to.deep.equal([ + multiaddr(`/ip6/::1/tcp/1234/p2p/${peerId.toString()}`), + multiaddr(`/ip6/${internalIp}/${protocol}/${internalPort}/p2p/${peerId.toString()}`) + ]) + + // confirm IP mapping + am.confirmObservedAddr(multiaddr(`/ip6/${externalIp}/${protocol}/${externalPort}`)) + // should have mapped the LAN address to the external IP expect(am.getAddresses()).to.deep.equal([ multiaddr(`/ip6/::1/tcp/1234/p2p/${peerId.toString()}`), @@ -520,6 +562,9 @@ describe('Address Manager', () => { multiaddr(`/ip6/${internalIp}/${protocol}/${internalPort}`) ]) + // confirm IP mapping + am.confirmObservedAddr(multiaddr(`/ip4/${externalIp}/${protocol}/${externalPort}`)) + // should have mapped the LAN address to the external IP expect(am.getAddresses()).to.deep.equal([ multiaddr(`/ip6/${internalIp}/${protocol}/${internalPort}/p2p/${peerId.toString()}`), @@ -556,6 +601,9 @@ describe('Address Manager', () => { multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}`) ]) + // confirm IP mapping + am.confirmObservedAddr(multiaddr(`/ip6/${externalIp}/${protocol}/${externalPort}`)) + // should have mapped the LAN address to the external IP expect(am.getAddresses()).to.deep.equal([ multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId.toString()}`), @@ -568,4 +616,34 @@ describe('Address Manager', () => { multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId.toString()}`) ]) }) + + it('should not confirm unknown observed addresses', () => { + const transportManager = stubInterface() + const am = new AddressManager({ + peerId, + transportManager, + peerStore, + events, + logger: defaultLogger() + }) + + const internalIp = '192.168.1.123' + const internalPort = 4567 + const externalIp = '2a00:23c6:14b1:7e00:28b8:30d:944e:27f3' + const externalPort = 8910 + const protocol = 'tcp' + + // one loopback, one LAN address + transportManager.getAddrs.returns([ + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}`) + ]) + + // confirm address we have not observed + am.confirmObservedAddr(multiaddr(`/ip6/${externalIp}/${protocol}/${externalPort}`)) + + // should not have changed the address list + expect(am.getAddresses()).to.deep.equal([ + multiaddr(`/ip4/${internalIp}/${protocol}/${internalPort}/p2p/${peerId.toString()}`) + ]) + }) }) diff --git a/packages/libp2p/test/core/consume-peer-record.spec.ts b/packages/libp2p/test/core/consume-peer-record.spec.ts deleted file mode 100644 index f3da3952bd..0000000000 --- a/packages/libp2p/test/core/consume-peer-record.spec.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* eslint-env mocha */ - -import { multiaddr } from '@multiformats/multiaddr' -import { createLibp2p } from '../../src/index.js' -import type { Libp2p } from '@libp2p/interface' - -describe('Consume peer record', () => { - let libp2p: Libp2p - - beforeEach(async () => { - libp2p = await createLibp2p() - }) - - afterEach(async () => { - await libp2p.stop() - }) - - it('should update addresses when observed addrs are confirmed', async () => { - let done: () => void - - libp2p.peerStore.patch = async () => { - done() - return {} as any - } - - const p = new Promise(resolve => { - done = resolve - }) - - await libp2p.start() - - // @ts-expect-error components field is private - libp2p.components.addressManager.confirmObservedAddr(multiaddr('/ip4/123.123.123.123/tcp/3983')) - - await p - - await libp2p.stop() - }) -}) diff --git a/packages/libp2p/test/transports/transport-manager.spec.ts b/packages/libp2p/test/transports/transport-manager.spec.ts index 2ee34f1b69..010aec0366 100644 --- a/packages/libp2p/test/transports/transport-manager.spec.ts +++ b/packages/libp2p/test/transports/transport-manager.spec.ts @@ -12,7 +12,7 @@ import { pEvent } from 'p-event' import pWaitFor from 'p-wait-for' import Sinon from 'sinon' import { stubInterface } from 'sinon-ts' -import { AddressManager } from '../../src/address-manager.js' +import { AddressManager } from '../../src/address-manager/index.js' import { DefaultTransportManager } from '../../src/transport-manager.js' import type { Components } from '../../src/components.js' import type { Connection, Transport, Upgrader, Listener } from '@libp2p/interface' diff --git a/packages/protocol-autonat/package.json b/packages/protocol-autonat/package.json index 98e2e353de..934c29f0d0 100644 --- a/packages/protocol-autonat/package.json +++ b/packages/protocol-autonat/package.json @@ -54,14 +54,12 @@ "dependencies": { "@libp2p/interface": "^2.2.1", "@libp2p/interface-internal": "^2.1.1", + "@libp2p/peer-collections": "^6.0.12", "@libp2p/peer-id": "^5.0.8", "@libp2p/utils": "^6.2.1", "@multiformats/multiaddr": "^12.3.3", - "it-first": "^3.0.6", - "it-length-prefixed": "^9.1.0", - "it-map": "^3.1.1", - "it-parallel": "^3.0.8", - "it-pipe": "^3.0.1", + "any-signal": "^4.1.1", + "it-protobuf-stream": "^1.1.5", "multiformats": "^13.3.1", "protons-runtime": "^5.5.0", "uint8arraylist": "^2.4.8" @@ -71,7 +69,11 @@ "@libp2p/logger": "^5.1.4", "aegir": "^45.0.5", "it-all": "^3.0.6", + "it-drain": "^3.0.7", + "it-length-prefixed": "^9.1.0", + "it-pipe": "^3.0.1", "it-pushable": "^3.2.3", + "p-retry": "^6.2.1", "protons": "^7.6.0", "sinon": "^19.0.2", "sinon-ts": "^2.0.0" diff --git a/packages/protocol-autonat/src/autonat.ts b/packages/protocol-autonat/src/autonat.ts index ef456be702..6e1d9436dc 100644 --- a/packages/protocol-autonat/src/autonat.ts +++ b/packages/protocol-autonat/src/autonat.ts @@ -1,39 +1,86 @@ -import { AbortError, serviceCapabilities, setMaxListeners } from '@libp2p/interface' +import { serviceCapabilities, serviceDependencies, setMaxListeners } from '@libp2p/interface' +import { peerSet } from '@libp2p/peer-collections' import { peerIdFromMultihash } from '@libp2p/peer-id' -import { isPrivateIp } from '@libp2p/utils/private-ip' -import { multiaddr, protocols } from '@multiformats/multiaddr' -import first from 'it-first' -import * as lp from 'it-length-prefixed' -import map from 'it-map' -import parallel from 'it-parallel' -import { pipe } from 'it-pipe' +import { createScalableCuckooFilter } from '@libp2p/utils/filters' +import { isPrivate } from '@libp2p/utils/multiaddr/is-private' +import { PeerQueue } from '@libp2p/utils/peer-queue' +import { repeatingTask } from '@libp2p/utils/repeating-task' +import { multiaddr, protocols, type Multiaddr } from '@multiformats/multiaddr' +import { anySignal } from 'any-signal' +import { pbStream } from 'it-protobuf-stream' import * as Digest from 'multiformats/hashes/digest' import { - MAX_INBOUND_STREAMS, - MAX_OUTBOUND_STREAMS, - PROTOCOL_NAME, PROTOCOL_PREFIX, PROTOCOL_VERSION, REFRESH_INTERVAL, STARTUP_DELAY, TIMEOUT + MAX_INBOUND_STREAMS, MAX_OUTBOUND_STREAMS, PROTOCOL_NAME, PROTOCOL_PREFIX, PROTOCOL_VERSION, TIMEOUT } from './constants.js' import { Message } from './pb/index.js' import type { AutoNATComponents, AutoNATServiceInit } from './index.js' -import type { Logger, Connection, PeerId, PeerInfo, Startable, AbortOptions } from '@libp2p/interface' +import type { Logger, Connection, PeerId, Startable, AbortOptions } from '@libp2p/interface' import type { IncomingStreamData } from '@libp2p/interface-internal' +import type { PeerSet } from '@libp2p/peer-collections' +import type { Filter } from '@libp2p/utils/filters' +import type { RepeatingTask } from '@libp2p/utils/repeating-task' // if more than 3 peers manage to dial us on what we believe to be our external // IP then we are convinced that it is, in fact, our external IP -// https://github.com/libp2p/specs/blob/master/autonat/README.md#autonat-protocol +// https://github.com/libp2p/specs/blob/master/autonat/autonat-v1.md#autonat-protocol const REQUIRED_SUCCESSFUL_DIALS = 4 +interface TestAddressOptions extends AbortOptions { + multiaddr: Multiaddr + peerId: PeerId +} + +interface DialResults { + /** + * The address being tested + */ + multiaddr: Multiaddr + + /** + * The number of successful dials from peers + */ + success: number + + /** + * The number of dial failures from peers + */ + failure: number + + /** + * For the multiaddr corresponding the the string key of the `dialResults` + * map, these are the IP segments that a successful dial result has been + * received from + */ + networkSegments: string[] + + /** + * Ensure that the same peer id can't verify multiple times + */ + verifyingPeers: PeerSet + + /** + * The number of peers currently verifying this address + */ + queue: PeerQueue + + /** + * Updated when this address is verified or failed + */ + result?: boolean +} + export class AutoNATService implements Startable { private readonly components: AutoNATComponents - private readonly startupDelay: number - private readonly refreshInterval: number private readonly protocol: string private readonly timeout: number private readonly maxInboundStreams: number private readonly maxOutboundStreams: number - private verifyAddressTimeout?: ReturnType private started: boolean private readonly log: Logger + private topologyId?: string + private readonly dialResults: Map + private readonly findPeers: RepeatingTask + private readonly addressFilter: Filter constructor (components: AutoNATComponents, init: AutoNATServiceInit) { this.components = components @@ -43,9 +90,9 @@ export class AutoNATService implements Startable { this.timeout = init.timeout ?? TIMEOUT this.maxInboundStreams = init.maxInboundStreams ?? MAX_INBOUND_STREAMS this.maxOutboundStreams = init.maxOutboundStreams ?? MAX_OUTBOUND_STREAMS - this.startupDelay = init.startupDelay ?? STARTUP_DELAY - this.refreshInterval = init.refreshInterval ?? REFRESH_INTERVAL - this._verifyExternalAddresses = this._verifyExternalAddresses.bind(this) + this.dialResults = new Map() + this.findPeers = repeatingTask(this.findRandomPeers.bind(this), 60_000) + this.addressFilter = createScalableCuckooFilter(1024) } readonly [Symbol.toStringTag] = '@libp2p/autonat' @@ -54,6 +101,12 @@ export class AutoNATService implements Startable { '@libp2p/autonat' ] + get [serviceDependencies] (): string[] { + return [ + '@libp2p/identify' + ] + } + isStarted (): boolean { return this.started } @@ -66,100 +119,107 @@ export class AutoNATService implements Startable { await this.components.registrar.handle(this.protocol, (data) => { void this.handleIncomingAutonatStream(data) .catch(err => { - this.log.error('error handling incoming autonat stream', err) + this.log.error('error handling incoming autonat stream - %e', err) }) }, { maxInboundStreams: this.maxInboundStreams, maxOutboundStreams: this.maxOutboundStreams }) - this.verifyAddressTimeout = setTimeout(this._verifyExternalAddresses, this.startupDelay) + this.topologyId = await this.components.registrar.register(this.protocol, { + onConnect: (peerId, connection) => { + this.verifyExternalAddresses(connection) + .catch(err => { + this.log.error('could not verify addresses - %e', err) + }) + } + }) + this.findPeers.start() this.started = true } async stop (): Promise { await this.components.registrar.unhandle(this.protocol) - clearTimeout(this.verifyAddressTimeout) + if (this.topologyId != null) { + await this.components.registrar.unhandle(this.topologyId) + } + + this.dialResults.clear() + this.findPeers.stop() this.started = false } - /** - * Handle an incoming AutoNAT request - */ - async handleIncomingAutonatStream (data: IncomingStreamData): Promise { - const signal = AbortSignal.timeout(this.timeout) + private allAddressesAreVerified (): boolean { + return this.components.addressManager.getAddressesWithMetadata().every(addr => addr.verified) + } - const onAbort = (): void => { - data.stream.abort(new AbortError()) + async findRandomPeers (options?: AbortOptions): Promise { + // skip if all addresses are verified + if (this.allAddressesAreVerified()) { + return } - signal.addEventListener('abort', onAbort, { once: true }) - - // this controller may be used while dialing lots of peers so prevent MaxListenersExceededWarning - // appearing in the console - setMaxListeners(Infinity, signal) + const signal = anySignal([ + AbortSignal.timeout(10_000), + options?.signal + ]) + // spend a few seconds finding random peers - dial them which will run + // identify to trigger the topology callbacks and run AutoNAT try { - const self = this - - await pipe( - data.stream, - (source) => lp.decode(source), - async function * (stream) { - const buf = await first(stream) - - if (buf == null) { - self.log('no message received') - yield Message.encode({ - type: Message.MessageType.DIAL_RESPONSE, - dialResponse: { - status: Message.ResponseStatus.E_BAD_REQUEST, - statusText: 'No message was sent' - } - }) + this.log('starting random walk to find peers to run AutoNAT') - return - } + for await (const peer of this.components.randomWalk.walk({ signal })) { + if (!(await this.components.connectionManager.isDialable(peer.multiaddrs))) { + this.log.trace('random peer %p was not dialable %s', peer.id, peer.multiaddrs.map(ma => ma.toString()).join(', ')) - let request: Message + // skip peers we can't dial + continue + } - try { - request = Message.decode(buf) - } catch (err) { - self.log.error('could not decode message', err) + try { + this.log.trace('dial random peer %p', peer.id) + await this.components.connectionManager.openConnection(peer.multiaddrs, { + signal + }) + } catch {} - yield Message.encode({ - type: Message.MessageType.DIAL_RESPONSE, - dialResponse: { - status: Message.ResponseStatus.E_BAD_REQUEST, - statusText: 'Could not decode message' - } - }) + if (this.allAddressesAreVerified()) { + this.log('stopping random walk, all addresses are verified') + return + } + } + } catch {} + } - return - } + /** + * Handle an incoming AutoNAT request + */ + async handleIncomingAutonatStream (data: IncomingStreamData): Promise { + const signal = AbortSignal.timeout(this.timeout) + setMaxListeners(Infinity, signal) - yield Message.encode(await self.handleAutonatMessage(request, data.connection, { - signal - })) - }, - (source) => lp.encode(source), - data.stream - ) - } catch (err) { - this.log.error('error handling incoming autonat stream', err) - } finally { - signal.removeEventListener('abort', onAbort) - } - } + const messages = pbStream(data.stream).pb(Message) - _verifyExternalAddresses (): void { - void this.verifyExternalAddresses() - .catch(err => { - this.log.error('error verifying external address', err) + try { + const request = await messages.read({ + signal + }) + const response = await this.handleAutonatMessage(request, data.connection, { + signal }) + await messages.write(response, { + signal + }) + await messages.unwrap().unwrap().close({ + signal + }) + } catch (err: any) { + this.log.error('error handling incoming autonat stream - %e', err) + data.stream.abort(err) + } } private async handleAutonatMessage (message: Message, connection: Connection, options?: AbortOptions): Promise { @@ -199,7 +259,7 @@ export class AutoNATService implements Startable { const digest = Digest.decode(peer.id) peerId = peerIdFromMultihash(digest) } catch (err) { - this.log.error('invalid PeerId', err) + this.log.error('invalid PeerId - %e', err) return { type: Message.MessageType.DIAL_RESPONSE, @@ -236,10 +296,9 @@ export class AutoNATService implements Startable { return isFromSameHost }) .filter(ma => { - const host = ma.toOptions().host - const isPublicIp = !(isPrivateIp(host) ?? false) + const isPublicIp = !isPrivate(ma) - this.log.trace('host %s was public %s', host, isPublicIp) + this.log.trace('%a was public %s', ma, isPublicIp) // don't try to dial private addresses return isPublicIp }) @@ -297,7 +356,7 @@ export class AutoNATService implements Startable { throw new Error('Unexpected remote address') } - this.log('Success %p', peerId) + this.log('successfully dialed %p via %a', peerId, multiaddr) return { type: Message.MessageType.DIAL_RESPONSE, @@ -307,7 +366,7 @@ export class AutoNATService implements Startable { } } } catch (err: any) { - this.log('could not dial %p', peerId, err) + this.log('could not dial %p - %e', peerId, err) errorMessage = err.message } finally { if (connection != null) { @@ -327,191 +386,277 @@ export class AutoNATService implements Startable { } /** - * Our multicodec topology noticed a new peer that supports autonat + * The AutoNAT v1 server is only required to send us the address that it + * dialed successfully. + * + * When addresses fail, it can be because they are NATed, or because the peer + * did't support the transport, we have no way of knowing, so just send them + * one address so we can treat the response as: + * + * - OK - the dial request worked and the address is not NATed + * - E_DIAL_ERROR - the dial request failed and the address may be NATed + * - E_DIAL_REFUSED/E_BAD_REQUEST/E_INTERNAL_ERROR - the remote didn't dial the address */ - async verifyExternalAddresses (): Promise { - clearTimeout(this.verifyAddressTimeout) + private getFirstUnverifiedMultiaddr (segment: string): DialResults | undefined { + const addrs = this.components.addressManager.getAddressesWithMetadata() + .sort((a, b) => { + // sort addresses, prioritize DNS/IP mapped addresses over observed ones + if (a.type === 'dns-mapping') { + return -1 + } - // Do not try to push if we are not running - if (!this.isStarted()) { - return - } + if (b.type === 'dns-mapping') { + return 1 + } - const addressManager = this.components.addressManager + if (a.type === 'ip-mapping') { + return -1 + } - const multiaddrs = addressManager.getObservedAddrs() - .filter(ma => { - const options = ma.toOptions() + if (b.type === 'ip-mapping') { + return 1 + } - return !(isPrivateIp(options.host) ?? false) + return 0 }) + .filter(addr => { + if (addr.verified && addr.expires > Date.now()) { + // skip verified addresses within their TTL + return false + } - if (multiaddrs.length === 0) { - this.log('no public addresses found, not requesting verification') - this.verifyAddressTimeout = setTimeout(this._verifyExternalAddresses, this.refreshInterval) - - return - } + if (isPrivate(addr.multiaddr)) { + // skip private addresses + return false + } - const signal = AbortSignal.timeout(this.timeout) + return true + }) - // this controller may be used while dialing lots of peers so prevent MaxListenersExceededWarning - // appearing in the console - setMaxListeners(Infinity, signal) + for (const addr of addrs) { + const addrString = addr.multiaddr.toString() + let results = this.dialResults.get(addrString) - const self = this + if (results != null) { + if (results.networkSegments.includes(segment)) { + this.log.trace('%a already has a network segment result from %s', results.multiaddr, segment) + // skip this address if we already have a dial result from the + // network segment the peer is in + continue + } - try { - this.log('verify multiaddrs %s', multiaddrs.map(ma => ma.toString()).join(', ')) - - const request = Message.encode({ - type: Message.MessageType.DIAL, - dial: { - peer: { - id: this.components.peerId.toMultihash().bytes, - addrs: multiaddrs.map(map => map.bytes) - } + if (results.queue.size > 10) { + this.log.trace('%a already has enough peers queued', results.multiaddr) + // already have enough peers verifying this address, skip on to the + // next one + continue } - }) + } - const results: Record = {} - const networkSegments: string[] = [] + // will include this multiaddr, ensure we have a results object + if (results == null) { + const needsRevalidating = addr.expires < Date.now() - const verifyAddress = async (peer: PeerInfo): Promise => { - let onAbort = (): void => {} + // allow re-validating addresses that worked previously + if (needsRevalidating) { + this.addressFilter.remove?.(addrString) + } - try { - this.log('asking %p to verify multiaddr', peer.id) + if (this.addressFilter.has(addrString)) { + continue + } - const connection = await self.components.connectionManager.openConnection(peer.id, { - signal + // only try to validate the address once + this.addressFilter.add(addrString) + + this.log.trace('creating dial result %s %s', needsRevalidating ? 'to revalidate' : 'for', addrString) + results = { + multiaddr: addr.multiaddr, + success: 0, + failure: 0, + networkSegments: [], + verifyingPeers: peerSet(), + queue: new PeerQueue({ + concurrency: 3, + maxSize: 50 }) + } - const stream = await connection.newStream(this.protocol, { - signal - }) + this.dialResults.set(addrString, results) + } - onAbort = () => { stream.abort(new AbortError()) } + return results + } + } - signal.addEventListener('abort', onAbort, { once: true }) + /** + * Removes any multiaddr result objects created for old multiaddrs that we are + * no longer waiting on + */ + private removeOutdatedMultiaddrResults (): void { + const unverifiedMultiaddrs = new Set(this.components.addressManager.getAddressesWithMetadata() + .filter(({ verified }) => !verified) + .map(({ multiaddr }) => multiaddr.toString()) + ) + + for (const multiaddr of this.dialResults.keys()) { + if (!unverifiedMultiaddrs.has(multiaddr)) { + this.log.trace('remove results for %a', multiaddr) + this.dialResults.delete(multiaddr) + } + } + } - const buf = await pipe( - [request], - (source) => lp.encode(source), - stream, - (source) => lp.decode(source), - async (stream) => first(stream) - ) - if (buf == null) { - this.log('no response received from %p', connection.remotePeer) - return undefined - } - const response = Message.decode(buf) + /** + * Our multicodec topology noticed a new peer that supports autonat + */ + async verifyExternalAddresses (connection: Connection): Promise { + // do nothing if we are not running + if (!this.isStarted()) { + return + } - if (response.type !== Message.MessageType.DIAL_RESPONSE || response.dialResponse == null) { - this.log('invalid autonat response from %p', connection.remotePeer) - return undefined - } + // perform cleanup + this.removeOutdatedMultiaddrResults() - if (response.dialResponse.status === Message.ResponseStatus.OK) { - // make sure we use different network segments - const options = connection.remoteAddr.toOptions() - let segment: string - - if (options.family === 4) { - const octets = options.host.split('.') - segment = octets[0] - } else if (options.family === 6) { - const octets = options.host.split(':') - segment = octets[0] - } else { - this.log('remote address "%s" was not IP4 or IP6?', options.host) - return undefined - } + // get multiaddrs this peer is eligible to verify + const segment = this.getNetworkSegment(connection.remoteAddr) + let results = this.getFirstUnverifiedMultiaddr(segment) - if (networkSegments.includes(segment)) { - this.log('already have response from network segment %d - %s', segment, options.host) - return undefined - } + if (results == null) { + this.log.trace('no unverified public addresses found for peer %p to verify, not requesting verification', connection.remotePeer) + return + } - networkSegments.push(segment) - } + results.queue.add(async (options: TestAddressOptions) => { + results = this.dialResults.get(options.multiaddr.toString()) - return response.dialResponse - } catch (err) { - this.log.error('error asking remote to verify multiaddr', err) - } finally { - signal.removeEventListener('abort', onAbort) - } + if (results == null) { + this.log('%a was verified while %p was queued', options.multiaddr, connection.remotePeer) + return } - // find some random peers - for await (const dialResponse of parallel(map(this.components.randomWalk.walk({ + const signal = AbortSignal.timeout(this.timeout) + setMaxListeners(Infinity, signal) + + this.log.trace('asking %p to verify multiaddr %s', connection.remotePeer, options.multiaddr) + + const stream = await connection.newStream(this.protocol, { signal - }), (peer) => async () => verifyAddress(peer)), { - concurrency: REQUIRED_SUCCESSFUL_DIALS - })) { - try { - if (dialResponse == null) { - continue - } + }) - // they either told us which address worked/didn't work, or we only sent them one address - const addr = dialResponse.addr == null ? multiaddrs[0] : multiaddr(dialResponse.addr) + try { + const messages = pbStream(stream).pb(Message) + const [, response] = await Promise.all([ + messages.write({ + type: Message.MessageType.DIAL, + dial: { + peer: { + id: this.components.peerId.toMultihash().bytes, + addrs: [options.multiaddr.bytes] + } + } + }, { signal }), + messages.read({ signal }) + ]) - this.log('autonat response for %a is %s', addr, dialResponse.status) + if (response.type !== Message.MessageType.DIAL_RESPONSE || response.dialResponse == null) { + this.log('invalid autonat response from %p - %j', connection.remotePeer, response) + return + } - if (dialResponse.status === Message.ResponseStatus.E_BAD_REQUEST) { - // the remote could not parse our request - continue - } + const status = response.dialResponse.status - if (dialResponse.status === Message.ResponseStatus.E_DIAL_REFUSED) { - // the remote could not honour our request - continue - } + this.log.trace('autonat response from %p for %a is %s', connection.remotePeer, options.multiaddr, status) - if (dialResponse.addr == null && multiaddrs.length > 1) { - // we sent the remote multiple addrs but they didn't tell us which ones worked/didn't work - continue - } + if (status !== Message.ResponseStatus.OK && status !== Message.ResponseStatus.E_DIAL_ERROR) { + return + } - if (!multiaddrs.some(ma => ma.equals(addr))) { - this.log('peer reported %a as %s but it was not in our observed address list', addr, dialResponse.status) - continue - } + results = this.dialResults.get(options.multiaddr.toString()) - const addrStr = addr.toString() + if (results == null) { + this.log.trace('peer reported %a as %s but there is no result object', options.multiaddr, response.dialResponse.status) + return + } - if (results[addrStr] == null) { - results[addrStr] = { success: 0, failure: 0 } - } + if (results.networkSegments.includes(segment)) { + this.log.trace('%a results included network segment %s', options.multiaddr, segment) + return + } - if (dialResponse.status === Message.ResponseStatus.OK) { - results[addrStr].success++ - } else if (dialResponse.status === Message.ResponseStatus.E_DIAL_ERROR) { - results[addrStr].failure++ - } + if (results.result != null) { + this.log.trace('already resolved result for %a, ignoring response from', options.multiaddr, connection.remotePeer) + return + } - if (results[addrStr].success === REQUIRED_SUCCESSFUL_DIALS) { - // we are now convinced - this.log('%a is externally dialable', addr) - addressManager.confirmObservedAddr(addr) - return - } + if (results.verifyingPeers.has(connection.remotePeer)) { + this.log.trace('peer %p has already verified %a, ignoring response', connection.remotePeer, options.multiaddr) + return + } - if (results[addrStr].failure === REQUIRED_SUCCESSFUL_DIALS) { - // we are now unconvinced - this.log('%a is not externally dialable', addr) - addressManager.removeObservedAddr(addr) - return - } - } catch (err) { - this.log.error('could not verify external address', err) + results.verifyingPeers.add(connection.remotePeer) + results.networkSegments.push(segment) + + if (status === Message.ResponseStatus.OK) { + results.success++ + } else if (status === Message.ResponseStatus.E_DIAL_ERROR) { + results.failure++ + } + + this.log('%a success %d failure %d', results.multiaddr, results.success, results.failure) + + if (results.success === REQUIRED_SUCCESSFUL_DIALS) { + // we are now convinced + this.log('%a is externally dialable', results.multiaddr) + this.components.addressManager.confirmObservedAddr(results.multiaddr) + this.dialResults.delete(results.multiaddr.toString()) + + // abort & remove any outstanding verification jobs for this multiaddr + results.result = true + results.queue.abort() + } + + if (results.failure === REQUIRED_SUCCESSFUL_DIALS) { + // we are now unconvinced + this.log('%a is not externally dialable', results.multiaddr) + this.components.addressManager.removeObservedAddr(results.multiaddr) + this.dialResults.delete(results.multiaddr.toString()) + + // abort & remove any outstanding verification jobs for this multiaddr + results.result = false + results.queue.abort() + } + } finally { + try { + await stream.close({ + signal + }) + } catch (err: any) { + stream.abort(err) } } - } finally { - this.verifyAddressTimeout = setTimeout(this._verifyExternalAddresses, this.refreshInterval) + }, { + peerId: connection.remotePeer, + multiaddr: results.multiaddr + }) + .catch(err => { + if (results?.result == null) { + this.log.error('error from %p verifying address %a - %e', connection.remotePeer, results?.multiaddr, err) + } + }) + } + + private getNetworkSegment (ma: Multiaddr): string { + // make sure we use different network segments + const options = ma.toOptions() + + if (options.family === 4) { + const octets = options.host.split('.') + return octets[0].padStart(3, '0') } + + const octets = options.host.split(':') + return octets[0].padStart(4, '0') } } diff --git a/packages/protocol-autonat/src/constants.ts b/packages/protocol-autonat/src/constants.ts index 2b15ab68d1..7237b87508 100644 --- a/packages/protocol-autonat/src/constants.ts +++ b/packages/protocol-autonat/src/constants.ts @@ -13,7 +13,5 @@ export const PROTOCOL_NAME = 'autonat' */ export const PROTOCOL_VERSION = '1.0.0' export const TIMEOUT = 30000 -export const STARTUP_DELAY = 5000 -export const REFRESH_INTERVAL = 60000 export const MAX_INBOUND_STREAMS = 1 export const MAX_OUTBOUND_STREAMS = 1 diff --git a/packages/protocol-autonat/src/index.ts b/packages/protocol-autonat/src/index.ts index 254a076d47..a4bf1b2271 100644 --- a/packages/protocol-autonat/src/index.ts +++ b/packages/protocol-autonat/src/index.ts @@ -31,7 +31,7 @@ */ import { AutoNATService } from './autonat.js' -import type { ComponentLogger, PeerId } from '@libp2p/interface' +import type { ComponentLogger, Libp2pEvents, PeerId, TypedEventTarget } from '@libp2p/interface' import type { AddressManager, ConnectionManager, RandomWalk, Registrar, TransportManager } from '@libp2p/interface-internal' export interface AutoNATServiceInit { @@ -74,6 +74,7 @@ export interface AutoNATComponents { connectionManager: ConnectionManager logger: ComponentLogger randomWalk: RandomWalk + events: TypedEventTarget } export function autoNAT (init: AutoNATServiceInit = {}): (components: AutoNATComponents) => unknown { diff --git a/packages/protocol-autonat/test/index.spec.ts b/packages/protocol-autonat/test/index.spec.ts index 07af351912..f90874c941 100644 --- a/packages/protocol-autonat/test/index.spec.ts +++ b/packages/protocol-autonat/test/index.spec.ts @@ -2,15 +2,17 @@ /* eslint max-nested-callbacks: ["error", 5] */ import { generateKeyPair } from '@libp2p/crypto/keys' -import { start, stop } from '@libp2p/interface' +import { TypedEventEmitter, start, stop } from '@libp2p/interface' import { defaultLogger } from '@libp2p/logger' import { peerIdFromPrivateKey } from '@libp2p/peer-id' import { multiaddr } from '@multiformats/multiaddr' import { expect } from 'aegir/chai' import all from 'it-all' +import drain from 'it-drain' import * as lp from 'it-length-prefixed' import { pipe } from 'it-pipe' import { pushable } from 'it-pushable' +import pRetry from 'p-retry' import sinon from 'sinon' import { stubInterface } from 'sinon-ts' import { Uint8ArrayList } from 'uint8arraylist' @@ -18,7 +20,7 @@ import { AutoNATService } from '../src/autonat.js' import { PROTOCOL_NAME, PROTOCOL_PREFIX, PROTOCOL_VERSION } from '../src/constants.js' import { Message } from '../src/pb/index.js' import type { AutoNATComponents, AutoNATServiceInit } from '../src/index.js' -import type { Connection, Stream, PeerId, PeerInfo, Transport } from '@libp2p/interface' +import type { Connection, Stream, PeerId, Transport, Libp2pEvents } from '@libp2p/interface' import type { AddressManager, ConnectionManager, RandomWalk, Registrar, TransportManager } from '@libp2p/interface-internal' import type { Multiaddr } from '@multiformats/multiaddr' import type { StubbedInstance } from 'sinon-ts' @@ -57,7 +59,8 @@ describe('autonat', () => { registrar, addressManager, connectionManager, - transportManager + transportManager, + events: new TypedEventEmitter() } service = new AutoNATService(components, defaultInit) @@ -74,7 +77,7 @@ describe('autonat', () => { }) describe('verify our observed addresses', () => { - async function stubPeerResponse (host: string, dialResponse: Message.DialResponse, peerId?: PeerId): Promise { + async function stubPeerResponse (host: string, dialResponse: Message.DialResponse, peerId?: PeerId): Promise { // stub random peer lookup const peer = { id: peerId ?? peerIdFromPrivateKey(await generateKeyPair('Ed25519')), @@ -85,34 +88,43 @@ describe('autonat', () => { // stub connection to remote peer const connection = stubInterface() connection.remoteAddr = multiaddr(`/ip4/${host}/tcp/28319/p2p/${peer.id.toString()}`) + connection.remotePeer = peer.id connectionManager.openConnection.withArgs(peer.id).resolves(connection) connection.newStream.withArgs(`/${PROTOCOL_PREFIX}/${PROTOCOL_NAME}/${PROTOCOL_VERSION}`).callsFake(async () => { - // stub autonat protocol stream - const stream = stubInterface() - // stub autonat response const response = Message.encode({ type: Message.MessageType.DIAL_RESPONSE, dialResponse }) - stream.source = (async function * () { - yield lp.encode.single(response) - }()) - stream.sink.returns(Promise.resolve()) + + // stub autonat protocol stream + const stream = stubInterface({ + source: (async function * () { + yield lp.encode.single(response) + }()), + sink: async (source) => { + await drain(source) + } + }) return stream }) - return peer + return connection } it('should request peers verify our observed address', async () => { const observedAddress = multiaddr('/ip4/123.123.123.123/tcp/28319') - addressManager.getObservedAddrs.returns([observedAddress]) + addressManager.getAddressesWithMetadata.returns([{ + multiaddr: observedAddress, + verified: false, + type: 'observed', + expires: Date.now() + 1000 + }]) // The network says OK - const peers = [ + const connections = [ await stubPeerResponse('124.124.124.124', { status: Message.ResponseStatus.OK }), @@ -127,11 +139,50 @@ describe('autonat', () => { }) ] - randomWalk.walk.returns(async function * () { - yield * peers - }()) + for (const conn of connections) { + await service.verifyExternalAddresses(conn) + } - await service.verifyExternalAddresses() + await pRetry(() => { + expect(addressManager.confirmObservedAddr).to.have.property('called', true) + }) + + expect(addressManager.confirmObservedAddr.calledWith(observedAddress)) + .to.be.true('Did not confirm observed multiaddr') + }) + + it('should request peers re-verify our observed address', async () => { + const observedAddress = multiaddr('/ip4/123.123.123.123/tcp/28319') + addressManager.getAddressesWithMetadata.returns([{ + multiaddr: observedAddress, + verified: true, + type: 'observed', + expires: Date.now() - 1000 + }]) + + // The network says OK + const connections = [ + await stubPeerResponse('124.124.124.124', { + status: Message.ResponseStatus.OK + }), + await stubPeerResponse('125.124.124.124', { + status: Message.ResponseStatus.OK + }), + await stubPeerResponse('126.124.124.124', { + status: Message.ResponseStatus.OK + }), + await stubPeerResponse('127.124.124.124', { + status: Message.ResponseStatus.OK + }) + ] + + for (const conn of connections) { + await service.verifyExternalAddresses(conn) + } + + await pRetry(() => { + expect(addressManager.confirmObservedAddr).to.have.property('called', true) + }) expect(addressManager.confirmObservedAddr.calledWith(observedAddress)) .to.be.true('Did not confirm observed multiaddr') @@ -139,10 +190,15 @@ describe('autonat', () => { it('should mark observed address as low confidence when dialing fails', async () => { const observedAddress = multiaddr('/ip4/123.123.123.123/tcp/28319') - addressManager.getObservedAddrs.returns([observedAddress]) + addressManager.getAddressesWithMetadata.returns([{ + multiaddr: observedAddress, + verified: false, + type: 'observed', + expires: Date.now() + 1000 + }]) // The network says ERROR - const peers = [ + const connections = [ await stubPeerResponse('124.124.124.124', { status: Message.ResponseStatus.E_DIAL_ERROR }), @@ -157,11 +213,13 @@ describe('autonat', () => { }) ] - randomWalk.walk.returns(async function * () { - yield * peers - }()) + for (const conn of connections) { + await service.verifyExternalAddresses(conn) + } - await service.verifyExternalAddresses() + await pRetry(() => { + expect(addressManager.removeObservedAddr).to.have.property('called', true) + }) expect(addressManager.removeObservedAddr.calledWith(observedAddress)) .to.be.true('Did not verify external multiaddr') @@ -169,10 +227,15 @@ describe('autonat', () => { it('should ignore non error or success statuses', async () => { const observedAddress = multiaddr('/ip4/123.123.123.123/tcp/28319') - addressManager.getObservedAddrs.returns([observedAddress]) + addressManager.getAddressesWithMetadata.returns([{ + multiaddr: observedAddress, + verified: false, + type: 'observed', + expires: Date.now() + 1000 + }]) // Mix of responses, mostly OK - const peers = [ + const connections = [ await stubPeerResponse('124.124.124.124', { status: Message.ResponseStatus.OK }), @@ -196,25 +259,29 @@ describe('autonat', () => { }) ] - randomWalk.walk.returns(async function * () { - yield * peers - }()) + for (const conn of connections) { + await service.verifyExternalAddresses(conn) + } - await service.verifyExternalAddresses() + await pRetry(() => { + expect(addressManager.confirmObservedAddr).to.have.property('called', true) + }) expect(addressManager.confirmObservedAddr.calledWith(observedAddress)) .to.be.true('Did not confirm external multiaddr') - - expect(connectionManager.openConnection.callCount) - .to.equal(peers.length, 'Did not open connections to all peers') }) it('should require confirmation from diverse networks', async () => { const observedAddress = multiaddr('/ip4/123.123.123.123/tcp/28319') - addressManager.getObservedAddrs.returns([observedAddress]) + addressManager.getAddressesWithMetadata.returns([{ + multiaddr: observedAddress, + verified: false, + type: 'observed', + expires: Date.now() + 1000 + }]) // an attacker says OK, the rest of the network says ERROR - const peers = [ + const connections = [ await stubPeerResponse('124.124.124.124', { status: Message.ResponseStatus.OK }), @@ -241,27 +308,31 @@ describe('autonat', () => { }) ] - randomWalk.walk.returns(async function * () { - yield * peers - }()) + for (const conn of connections) { + await service.verifyExternalAddresses(conn) + } - await service.verifyExternalAddresses() + await pRetry(() => { + expect(addressManager.removeObservedAddr).to.have.property('called', true) + }) expect(addressManager.removeObservedAddr.calledWith(observedAddress)) .to.be.true('Did not verify external multiaddr') - - expect(connectionManager.openConnection.callCount) - .to.equal(peers.length, 'Did not open connections to all peers') }) it('should require confirmation from diverse peers', async () => { const observedAddress = multiaddr('/ip4/123.123.123.123/tcp/28319') - addressManager.getObservedAddrs.returns([observedAddress]) + addressManager.getAddressesWithMetadata.returns([{ + multiaddr: observedAddress, + verified: false, + type: 'observed', + expires: Date.now() + 1000 + }]) const peerId = peerIdFromPrivateKey(await generateKeyPair('Ed25519')) // an attacker says OK, the rest of the network says ERROR - const peers = [ + const connections = [ await stubPeerResponse('124.124.124.124', { status: Message.ResponseStatus.OK }, peerId), @@ -288,77 +359,29 @@ describe('autonat', () => { }) ] - randomWalk.walk.returns(async function * () { - yield * peers - }()) - - await service.verifyExternalAddresses() - - expect(addressManager.removeObservedAddr.calledWith(observedAddress)) - .to.be.true('Did not verify external multiaddr') - - expect(connectionManager.openConnection.callCount) - .to.equal(peers.length, 'Did not open connections to all peers') - }) - - it('should only accept observed addresses', async () => { - const observedAddress = multiaddr('/ip4/123.123.123.123/tcp/28319') - const reportedAddress = multiaddr('/ip4/100.100.100.100/tcp/28319') - - // our observed addresses - addressManager.getObservedAddrs.returns([observedAddress]) - - // an attacker says OK, the rest of the network says ERROR - const peers = [ - await stubPeerResponse('124.124.124.124', { - status: Message.ResponseStatus.OK, - addr: reportedAddress.bytes - }), - await stubPeerResponse('125.124.124.125', { - status: Message.ResponseStatus.OK, - addr: reportedAddress.bytes - }), - await stubPeerResponse('126.124.124.126', { - status: Message.ResponseStatus.OK, - addr: reportedAddress.bytes - }), - await stubPeerResponse('127.124.124.127', { - status: Message.ResponseStatus.OK, - addr: reportedAddress.bytes - }), - await stubPeerResponse('128.124.124.124', { - status: Message.ResponseStatus.E_DIAL_ERROR - }), - await stubPeerResponse('129.124.124.124', { - status: Message.ResponseStatus.E_DIAL_ERROR - }), - await stubPeerResponse('130.124.124.124', { - status: Message.ResponseStatus.E_DIAL_ERROR - }), - await stubPeerResponse('131.124.124.124', { - status: Message.ResponseStatus.E_DIAL_ERROR - }) - ] - - randomWalk.walk.returns(async function * () { - yield * peers - }()) + for (const conn of connections) { + await service.verifyExternalAddresses(conn) + } - await service.verifyExternalAddresses() + await pRetry(() => { + expect(addressManager.removeObservedAddr).to.have.property('called', true) + }) expect(addressManager.removeObservedAddr.calledWith(observedAddress)) .to.be.true('Did not verify external multiaddr') - - expect(connectionManager.openConnection.callCount) - .to.equal(peers.length, 'Did not open connections to all peers') }) it('should time out when verifying an observed address', async () => { const observedAddress = multiaddr('/ip4/123.123.123.123/tcp/28319') - addressManager.getObservedAddrs.returns([observedAddress]) + addressManager.getAddressesWithMetadata.returns([{ + multiaddr: observedAddress, + verified: false, + type: 'observed', + expires: Date.now() + 1000 + }]) // The network says OK - const peers = [ + const connections = [ await stubPeerResponse('124.124.124.124', { status: Message.ResponseStatus.OK }), @@ -373,28 +396,9 @@ describe('autonat', () => { }) ] - randomWalk.walk.returns(async function * () { - yield * peers - }()) - - connectionManager.openConnection.reset() - connectionManager.openConnection.callsFake(async (peer, options = {}) => { - return Promise.race([ - new Promise((resolve, reject) => { - options.signal?.addEventListener('abort', () => { - reject(new Error('Dial aborted!')) - }) - }), - new Promise((resolve, reject) => { - // longer than the timeout - setTimeout(() => { - reject(new Error('Dial Timeout!')) - }, 1000) - }) - ]) - }) - - await service.verifyExternalAddresses() + for (const conn of connections) { + await service.verifyExternalAddresses(conn) + } expect(addressManager.addObservedAddr.called) .to.be.false('Verify external multiaddr when we should have timed out') @@ -424,11 +428,13 @@ describe('autonat', () => { for await (const buf of stream) { sink.push(new Uint8ArrayList(buf)) } - sink.end() }, abort: (err) => { void stream.source.throw(err) + }, + close: async () => { + sink.end() } } const connection = { @@ -511,26 +517,6 @@ describe('autonat', () => { expect(message).to.have.nested.property('dialResponse.status', Message.ResponseStatus.OK) }) - it('should expect a message', async () => { - const message = await stubIncomingStream({ - message: false - }) - - expect(message).to.have.property('type', Message.MessageType.DIAL_RESPONSE) - expect(message).to.have.nested.property('dialResponse.status', Message.ResponseStatus.E_BAD_REQUEST) - expect(message).to.have.nested.property('dialResponse.statusText', 'No message was sent') - }) - - it('should expect a valid message', async () => { - const message = await stubIncomingStream({ - message: Uint8Array.from([3, 2, 1, 0]) - }) - - expect(message).to.have.property('type', Message.MessageType.DIAL_RESPONSE) - expect(message).to.have.nested.property('dialResponse.status', Message.ResponseStatus.E_BAD_REQUEST) - expect(message).to.have.nested.property('dialResponse.statusText', 'Could not decode message') - }) - it('should expect a dial message', async () => { const message = await stubIncomingStream({ message: {} @@ -634,31 +620,5 @@ describe('autonat', () => { expect(message).to.have.nested.property('dialResponse.status', Message.ResponseStatus.E_DIAL_ERROR) expect(message).to.have.nested.property('dialResponse.statusText', 'Could not dial') }) - - it('should time out when dialing a requested address', async () => { - connectionManager.openConnection.callsFake(async function (ma, options = {}) { - return Promise.race([ - new Promise((resolve, reject) => { - options.signal?.addEventListener('abort', () => { - reject(new Error('Dial aborted!')) - }) - }), - new Promise((resolve, reject) => { - // longer than the timeout - setTimeout(() => { - reject(new Error('Dial Timeout!')) - }, 1000) - }) - ]) - }) - - const message = await stubIncomingStream({ - canDial: undefined - }) - - expect(message).to.have.property('type', Message.MessageType.DIAL_RESPONSE) - expect(message).to.have.nested.property('dialResponse.status', Message.ResponseStatus.E_DIAL_ERROR) - expect(message).to.have.nested.property('dialResponse.statusText', 'Dial aborted!') - }) }) }) diff --git a/packages/upnp-nat/src/index.ts b/packages/upnp-nat/src/index.ts index 0f224750ab..5f7baadf5b 100644 --- a/packages/upnp-nat/src/index.ts +++ b/packages/upnp-nat/src/index.ts @@ -98,6 +98,17 @@ export interface UPnPNATInit { * otherwise one will be created */ portMappingClient?: UPnPNATClient + + /** + * Any mapped addresses are added to the observed address list. These + * addresses require additional verification by the `@libp2p/autonat` protocol + * or similar before they are trusted. + * + * To skip this verification and trust them immediately pass `true` here + * + * @default false + */ + autoConfirmAddress?: boolean } export interface UPnPNATComponents { diff --git a/packages/upnp-nat/src/upnp-nat.ts b/packages/upnp-nat/src/upnp-nat.ts index 7439e1a399..cf917f401c 100644 --- a/packages/upnp-nat/src/upnp-nat.ts +++ b/packages/upnp-nat/src/upnp-nat.ts @@ -1,5 +1,5 @@ import { upnpNat } from '@achingbrain/nat-port-mapper' -import { serviceCapabilities, setMaxListeners, start, stop } from '@libp2p/interface' +import { serviceCapabilities, serviceDependencies, setMaxListeners, start, stop } from '@libp2p/interface' import { debounce } from '@libp2p/utils/debounce' import { GatewayFinder } from './gateway-finder.js' import { UPnPPortMapper } from './upnp-port-mapper.js' @@ -18,6 +18,7 @@ export class UPnPNAT implements Startable, UPnPNATInterface { private readonly mapIpAddressesDebounced: DebouncedFunction private readonly gatewayFinder: GatewayFinder private readonly portMappers: UPnPPortMapper[] + private readonly autoConfirmAddress: boolean constructor (components: UPnPNATComponents, init: UPnPNATInit) { this.log = components.logger.forComponent('libp2p:upnp-nat') @@ -25,6 +26,7 @@ export class UPnPNAT implements Startable, UPnPNATInterface { this.init = init this.started = false this.portMappers = [] + this.autoConfirmAddress = init.autoConfirmAddress ?? false this.portMappingClient = init.portMappingClient ?? upnpNat({ description: init.portMappingDescription ?? `${components.nodeInfo.name}@${components.nodeInfo.version} ${components.peerId.toString()}`, @@ -56,6 +58,16 @@ export class UPnPNAT implements Startable, UPnPNATInterface { '@libp2p/nat-traversal' ] + get [serviceDependencies] (): string[] { + if (!this.autoConfirmAddress) { + return [ + '@libp2p/autonat' + ] + } + + return [] + } + isStarted (): boolean { return this.started } @@ -102,7 +114,9 @@ export class UPnPNAT implements Startable, UPnPNATInterface { async mapIpAddresses (): Promise { try { await Promise.all( - this.portMappers.map(async mapper => mapper.mapIpAddresses()) + this.portMappers.map(async mapper => mapper.mapIpAddresses({ + autoConfirmAddress: this.autoConfirmAddress + })) ) } catch (err: any) { this.log.error('error mapping IP addresses - %e', err) diff --git a/packages/upnp-nat/src/upnp-port-mapper.ts b/packages/upnp-nat/src/upnp-port-mapper.ts index c1b2289bb7..abec7dcbd9 100644 --- a/packages/upnp-nat/src/upnp-port-mapper.ts +++ b/packages/upnp-nat/src/upnp-port-mapper.ts @@ -4,6 +4,7 @@ import { isLinkLocal } from '@libp2p/utils/multiaddr/is-link-local' import { isLoopback } from '@libp2p/utils/multiaddr/is-loopback' import { isPrivate } from '@libp2p/utils/multiaddr/is-private' import { isPrivateIp } from '@libp2p/utils/private-ip' +import { multiaddr } from '@multiformats/multiaddr' import { QUICV1, TCP, WebSockets, WebSocketsSecure, WebTransport } from '@multiformats/multiaddr-matcher' import { dynamicExternalAddress } from './check-external-address.js' import { DoubleNATError } from './errors.js' @@ -29,6 +30,10 @@ interface PortMapping { externalPort: number } +export interface MapPortsOptions { + autoConfirmAddress?: boolean +} + export class UPnPPortMapper { private readonly gateway: Gateway private readonly externalAddress: ExternalAddress @@ -149,7 +154,7 @@ export class UPnPPortMapper { return output } - async mapIpAddresses (): Promise { + async mapIpAddresses (options?: MapPortsOptions): Promise { try { const externalHost = await this.externalAddress.getPublicIp() @@ -194,6 +199,12 @@ export class UPnPPortMapper { this.mappedPorts.set(key, mapping) this.addressManager.addPublicAddressMapping(mapping.internalHost, mapping.internalPort, mapping.externalHost, mapping.externalPort, transport === 'tcp' ? 'tcp' : 'udp') this.log('created mapping of %s:%s to %s:%s for protocol %s', mapping.internalHost, mapping.internalPort, mapping.externalHost, mapping.externalPort, transport) + + if (options?.autoConfirmAddress === true) { + const ma = multiaddr(`/ip${family}/${host}/${transport}/${port}`) + this.log('auto-confirming IP address %a', ma) + this.addressManager.confirmObservedAddr(ma) + } } catch (err) { this.log.error('failed to create mapping for %s:%d for protocol - %e', host, port, transport, err) }