From 786590d5894984ed9b59b53f840e544f56ed6b07 Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:58:56 +0200 Subject: [PATCH 1/5] feat: filter bitswap providers in ipni client --- daemon.go | 2 +- go.mod | 2 +- go.sum | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/daemon.go b/daemon.go index dcda73f..f01aee2 100644 --- a/daemon.go +++ b/daemon.go @@ -149,7 +149,7 @@ type providerOutput struct { // concurrently. A check of connectivity and Bitswap availability is performed // for each provider found. func (d *daemon) runCidCheck(ctx context.Context, cidKey cid.Cid, ipniURL string) (cidCheckOutput, error) { - crClient, err := client.New(ipniURL, client.WithStreamResultsRequired()) + crClient, err := client.New(ipniURL, client.WithStreamResultsRequired(), client.WithProtocolFilter([]string{"transport-bitswap", "unknown"})) if err != nil { return nil, fmt.Errorf("failed to create content router client: %w", err) } diff --git a/go.mod b/go.mod index b8b5971..4f845e5 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( github.com/gavv/httpexpect/v2 v2.16.0 github.com/ipfs-shipyard/vole v0.0.0-20240801195547-d7b80a461193 - github.com/ipfs/boxo v0.22.0 + github.com/ipfs/boxo v0.23.1-0.20241001084405-d45e9df5b0a9 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index d14f0a8..691d02b 100644 --- a/go.sum +++ b/go.sum @@ -212,6 +212,10 @@ github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.22.0 h1:QTC+P5uhsBNq6HzX728nsLyFW6rYDeR/5hggf9YZX78= github.com/ipfs/boxo v0.22.0/go.mod h1:yp1loimX0BDYOR0cyjtcXHv15muEh5V1FqO2QLlzykw= +github.com/ipfs/boxo v0.23.1-0.20241001083525-6fb3a274ac9f h1:blWZqIUp4OZQSdbp7nl5wiplHv0kCC2UqnxchX3E7a4= +github.com/ipfs/boxo v0.23.1-0.20241001083525-6fb3a274ac9f/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= +github.com/ipfs/boxo v0.23.1-0.20241001084405-d45e9df5b0a9 h1:y53+abIiXzb9SnnVtECvSMjF+o4duUnS8lsP3Hh5VII= +github.com/ipfs/boxo v0.23.1-0.20241001084405-d45e9df5b0a9/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= From 872a727d5cfa9250f0e13950f2c80f3cf5da279f Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:16:05 +0200 Subject: [PATCH 2/5] chore: move default protocol filter to variable --- daemon.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/daemon.go b/daemon.go index f01aee2..ce96f88 100644 --- a/daemon.go +++ b/daemon.go @@ -49,6 +49,8 @@ const ( dhtSource = "Amino DHT" ) +var defaultProtocolFilter = []string{"transport-bitswap", "unknown"} + func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) { rm, err := NewResourceManager() if err != nil { @@ -149,7 +151,7 @@ type providerOutput struct { // concurrently. A check of connectivity and Bitswap availability is performed // for each provider found. func (d *daemon) runCidCheck(ctx context.Context, cidKey cid.Cid, ipniURL string) (cidCheckOutput, error) { - crClient, err := client.New(ipniURL, client.WithStreamResultsRequired(), client.WithProtocolFilter([]string{"transport-bitswap", "unknown"})) + crClient, err := client.New(ipniURL, client.WithStreamResultsRequired(), client.WithProtocolFilter(defaultProtocolFilter)) if err != nil { return nil, fmt.Errorf("failed to create content router client: %w", err) } From 5e52a52479e95c3eeee9502498febf932f1f55d3 Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Tue, 1 Oct 2024 12:18:21 +0200 Subject: [PATCH 3/5] chore: go mod tidy --- go.sum | 6 ------ 1 file changed, 6 deletions(-) diff --git a/go.sum b/go.sum index 691d02b..f455e86 100644 --- a/go.sum +++ b/go.sum @@ -210,10 +210,6 @@ github.com/ipfs-shipyard/vole v0.0.0-20240801195547-d7b80a461193 h1:5HPfcUkFXM5K github.com/ipfs-shipyard/vole v0.0.0-20240801195547-d7b80a461193/go.mod h1:ibnGHr4b6P1OYWIR2HLKT1ONUbmd1ZGe9gzRWb/Kcto= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.22.0 h1:QTC+P5uhsBNq6HzX728nsLyFW6rYDeR/5hggf9YZX78= -github.com/ipfs/boxo v0.22.0/go.mod h1:yp1loimX0BDYOR0cyjtcXHv15muEh5V1FqO2QLlzykw= -github.com/ipfs/boxo v0.23.1-0.20241001083525-6fb3a274ac9f h1:blWZqIUp4OZQSdbp7nl5wiplHv0kCC2UqnxchX3E7a4= -github.com/ipfs/boxo v0.23.1-0.20241001083525-6fb3a274ac9f/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= github.com/ipfs/boxo v0.23.1-0.20241001084405-d45e9df5b0a9 h1:y53+abIiXzb9SnnVtECvSMjF+o4duUnS8lsP3Hh5VII= github.com/ipfs/boxo v0.23.1-0.20241001084405-d45e9df5b0a9/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= @@ -229,8 +225,6 @@ github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= -github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= -github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= From b1d6d3f8cbdb910243b8a8b814864c0807a6f9fa Mon Sep 17 00:00:00 2001 From: Daniel Norman <1992255+2color@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:48:49 +0200 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Marcin Rataj --- daemon.go | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon.go b/daemon.go index ce96f88..79c288d 100644 --- a/daemon.go +++ b/daemon.go @@ -49,6 +49,7 @@ const ( dhtSource = "Amino DHT" ) +// TODO: make this configurable, and add support and trustless retrieval probe for transport-ipfs-gateway-http var defaultProtocolFilter = []string{"transport-bitswap", "unknown"} func newDaemon(ctx context.Context, acceleratedDHT bool) (*daemon, error) { From 1dcd8b7914499ae9b4747a532453bf45ee59a931 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 4 Oct 2024 20:27:40 +0200 Subject: [PATCH 5/5] chore: boxo v0.24.0 https://github.com/ipfs/boxo/releases/tag/v0.24.0 --- daemon.go | 6 +++++- go.mod | 6 +++--- go.sum | 14 ++++++-------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/daemon.go b/daemon.go index 79c288d..4778345 100644 --- a/daemon.go +++ b/daemon.go @@ -152,7 +152,11 @@ type providerOutput struct { // concurrently. A check of connectivity and Bitswap availability is performed // for each provider found. func (d *daemon) runCidCheck(ctx context.Context, cidKey cid.Cid, ipniURL string) (cidCheckOutput, error) { - crClient, err := client.New(ipniURL, client.WithStreamResultsRequired(), client.WithProtocolFilter(defaultProtocolFilter)) + crClient, err := client.New(ipniURL, + client.WithStreamResultsRequired(), // // https://specs.ipfs.tech/routing/http-routing-v1/#streaming + client.WithProtocolFilter(defaultProtocolFilter), // IPIP-484 + client.WithDisabledLocalFiltering(false), // force local filtering in case remote server does not support IPIP-484 + ) if err != nil { return nil, fmt.Errorf("failed to create content router client: %w", err) } diff --git a/go.mod b/go.mod index 4f845e5..16021e6 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,11 @@ go 1.22 require ( github.com/gavv/httpexpect/v2 v2.16.0 github.com/ipfs-shipyard/vole v0.0.0-20240801195547-d7b80a461193 - github.com/ipfs/boxo v0.23.1-0.20241001084405-d45e9df5b0a9 + github.com/ipfs/boxo v0.24.0 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 - github.com/libp2p/go-libp2p v0.36.3 + github.com/libp2p/go-libp2p v0.36.5 github.com/libp2p/go-libp2p-kad-dht v0.26.1 github.com/libp2p/go-libp2p-mplex v0.9.0 github.com/libp2p/go-libp2p-record v0.2.0 @@ -104,7 +104,7 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect + github.com/multiformats/go-multiaddr-dns v0.4.0 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect diff --git a/go.sum b/go.sum index f455e86..aca9601 100644 --- a/go.sum +++ b/go.sum @@ -210,8 +210,8 @@ github.com/ipfs-shipyard/vole v0.0.0-20240801195547-d7b80a461193 h1:5HPfcUkFXM5K github.com/ipfs-shipyard/vole v0.0.0-20240801195547-d7b80a461193/go.mod h1:ibnGHr4b6P1OYWIR2HLKT1ONUbmd1ZGe9gzRWb/Kcto= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.23.1-0.20241001084405-d45e9df5b0a9 h1:y53+abIiXzb9SnnVtECvSMjF+o4duUnS8lsP3Hh5VII= -github.com/ipfs/boxo v0.23.1-0.20241001084405-d45e9df5b0a9/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= +github.com/ipfs/boxo v0.24.0 h1:D9gTU3QdxyjPMlJ6QfqhHTG3TIJPplKzjXLO2J30h9U= +github.com/ipfs/boxo v0.24.0/go.mod h1:iP7xUPpHq2QAmVAjwtQvsNBTxTwLpFuy6ZpiRFwmzDA= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -304,8 +304,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.36.3 h1:NHz30+G7D8Y8YmznrVZZla0ofVANrvBl2c+oARfMeDQ= -github.com/libp2p/go-libp2p v0.36.3/go.mod h1:4Y5vFyCUiJuluEPmpnKYf6WFx5ViKPUYs/ixe9ANFZ8= +github.com/libp2p/go-libp2p v0.36.5 h1:DoABsaHO0VXwH6pwCs2F6XKAXWYjFMO4HFBoVxTnF9g= +github.com/libp2p/go-libp2p v0.36.5/go.mod h1:CpszAtXxHYOcyvB7K8rSHgnNlh21eKjYbEfLoMerbEI= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= @@ -362,7 +362,6 @@ github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= @@ -398,8 +397,8 @@ github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ= github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII= -github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= -github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= +github.com/multiformats/go-multiaddr-dns v0.4.0 h1:P76EJ3qzBXpUXZ3twdCDx/kvagMsNo0LMFXpyms/zgU= +github.com/multiformats/go-multiaddr-dns v0.4.0/go.mod h1:7hfthtB4E4pQwirrz+J0CcDUfbWzTqEzVyYKKIKpgkc= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= @@ -775,7 +774,6 @@ golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=