Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Daemon providers flag #309

Merged
merged 3 commits into from
Jun 14, 2023
Merged

Daemon providers flag #309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2307543
feat: add -providers flag to the daemon command
guanzo Jun 14, 2023
98d2fda
add env var to providers flag
guanzo Jun 14, 2023
110046c
fix(itest): add integration test for fixed peer through startup option
hannahhoward Jun 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions cmd/lassie/daemon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"time"

"github.com/filecoin-project/lassie/pkg/lassie"
"github.com/filecoin-project/lassie/pkg/net/host"
"github.com/filecoin-project/lassie/pkg/retriever"
httpserver "github.com/filecoin-project/lassie/pkg/server/http"
"github.com/libp2p/go-libp2p"
"github.com/libp2p/go-libp2p/p2p/net/connmgr"
Expand Down Expand Up @@ -80,6 +82,7 @@ var daemonFlags = []cli.Flag{
FlagVerbose,
FlagVeryVerbose,
FlagProtocols,
FlagAllowProviders,
FlagExcludeProviders,
FlagTempDir,
FlagBitswapConcurrency,
Expand Down Expand Up @@ -128,6 +131,14 @@ func daemonCommand(cctx *cli.Context) error {
if len(protocols) > 0 {
lassieOpts = append(lassieOpts, lassie.WithProtocols(protocols))
}
if len(fetchProviderAddrInfos) > 0 {
host, err := host.InitHost(ctx, []libp2p.Option{})
if err != nil {
return err
}
finderOpt := lassie.WithFinder(retriever.NewDirectCandidateFinder(host, fetchProviderAddrInfos))
lassieOpts = append(lassieOpts, finderOpt)
}
if len(providerBlockList) > 0 {
lassieOpts = append(lassieOpts, lassie.WithProviderBlockList(providerBlockList))
}
Expand Down
20 changes: 1 addition & 19 deletions cmd/lassie/fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,9 @@ import (
"github.com/ipfs/go-cid"
cidlink "github.com/ipld/go-ipld-prime/linking/cid"
"github.com/libp2p/go-libp2p"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/urfave/cli/v2"
)

var fetchProviderAddrInfos []peer.AddrInfo

var fetchCmd = &cli.Command{
Name: "fetch",
Usage: "Fetches content from the IPFS and Filecoin network",
Expand Down Expand Up @@ -71,22 +68,6 @@ var fetchCmd = &cli.Command{
return nil
},
},
&cli.StringFlag{
Name: "providers",
Aliases: []string{"provider"},
DefaultText: "Providers will be discovered automatically",
Usage: "Addresses of providers, including peer IDs, to use instead of automatic discovery, seperated by a comma. All protocols will be attempted when connecting to these providers. Example: /ip4/1.2.3.4/tcp/1234/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4",
Action: func(cctx *cli.Context, v string) error {
// Do nothing if given an empty string
if v == "" {
return nil
}

var err error
fetchProviderAddrInfos, err = types.ParseProviderStrings(v)
return err
},
},
&cli.StringFlag{
Name: "ipni-endpoint",
Aliases: []string{"ipni"},
Expand All @@ -99,6 +80,7 @@ var fetchCmd = &cli.Command{
FlagVerbose,
FlagVeryVerbose,
FlagProtocols,
FlagAllowProviders,
FlagExcludeProviders,
FlagTempDir,
FlagBitswapConcurrency,
Expand Down
20 changes: 20 additions & 0 deletions cmd/lassie/flags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,26 @@ var FlagExcludeProviders = &cli.StringFlag{
},
}

var fetchProviderAddrInfos []peer.AddrInfo

var FlagAllowProviders = &cli.StringFlag{
Name: "providers",
Aliases: []string{"provider"},
DefaultText: "Providers will be discovered automatically",
Usage: "Addresses of providers, including peer IDs, to use instead of automatic discovery, seperated by a comma. All protocols will be attempted when connecting to these providers. Example: /ip4/1.2.3.4/tcp/1234/p2p/12D3KooWBSTEYMLSu5FnQjshEVah9LFGEZoQt26eacCEVYfedWA4",
EnvVars: []string{"LASSIE_ALLOW_PROVIDERS"},
Action: func(cctx *cli.Context, v string) error {
// Do nothing if given an empty string
if v == "" {
return nil
}

var err error
fetchProviderAddrInfos, err = types.ParseProviderStrings(v)
return err
},
}

var protocols []multicodec.Code
var FlagProtocols = &cli.StringFlag{
Name: "protocols",
Expand Down
45 changes: 40 additions & 5 deletions pkg/internal/itest/http_fetch_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/filecoin-project/lassie/pkg/internal/itest/testpeer"
"github.com/filecoin-project/lassie/pkg/internal/testutil"
"github.com/filecoin-project/lassie/pkg/lassie"
"github.com/filecoin-project/lassie/pkg/retriever"
httpserver "github.com/filecoin-project/lassie/pkg/server/http"
"github.com/filecoin-project/lassie/pkg/verifiedcar"
"github.com/google/uuid"
Expand Down Expand Up @@ -59,7 +60,7 @@ func TestHttpFetch(t *testing.T) {
type headerSetter func(http.Header)
type queryModifier func(url.Values, []testpeer.TestPeer)
type bodyValidator func(*testing.T, unixfs.DirEntry, []byte)

type lassieOptsGen func(*testing.T, *mocknet.MockRetrievalNet) []lassie.LassieOption
wrapPath := "/want2/want1/want0"

testCases := []struct {
Expand All @@ -76,7 +77,7 @@ func TestHttpFetch(t *testing.T) {
setHeader headerSetter
modifyQueries []queryModifier
validateBodies []bodyValidator
lassieOpts []lassie.LassieOption
lassieOpts lassieOptsGen
}{
{
name: "graphsync large sharded file",
Expand Down Expand Up @@ -230,7 +231,9 @@ func TestHttpFetch(t *testing.T) {
name: "bitswap block timeout from missing block",
bitswapRemotes: 1,
expectUncleanEnd: true,
lassieOpts: []lassie.LassieOption{lassie.WithProviderTimeout(500 * time.Millisecond)},
lassieOpts: func(t *testing.T, mrn *mocknet.MockRetrievalNet) []lassie.LassieOption {
return []lassie.LassieOption{lassie.WithProviderTimeout(500 * time.Millisecond)}
},
generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
file := unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)
remotes[0].Blockstore().DeleteBlock(context.Background(), file.SelfCids[2])
Expand All @@ -250,7 +253,9 @@ func TestHttpFetch(t *testing.T) {
name: "same content, http missing block, bitswap completes",
bitswapRemotes: 1,
httpRemotes: 1,
lassieOpts: []lassie.LassieOption{lassie.WithProviderTimeout(500 * time.Millisecond)},
lassieOpts: func(t *testing.T, mrn *mocknet.MockRetrievalNet) []lassie.LassieOption {
return []lassie.LassieOption{lassie.WithProviderTimeout(500 * time.Millisecond)}
},
generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
file := unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)
for _, c := range file.SelfCids {
Expand Down Expand Up @@ -832,6 +837,19 @@ func TestHttpFetch(t *testing.T) {
v.Set("providers", strings.Join(maStrings, ","))
}},
},
{
name: "graphsync large sharded file, fixedPeer through startup opts",
graphsyncRemotes: 1,
generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
fileEntry := unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)
// wipe content routing information for remote
remotes[0].Cids = make(map[cid.Cid]struct{})
return []unixfs.DirEntry{fileEntry}
},
lassieOpts: func(t *testing.T, mrn *mocknet.MockRetrievalNet) []lassie.LassieOption {
return []lassie.LassieOption{lassie.WithFinder(retriever.NewDirectCandidateFinder(mrn.Self, []peer.AddrInfo{*mrn.Remotes[0].AddrInfo()}))}
},
},
{
name: "bitswap large sharded file, fixedPeer",
bitswapRemotes: 1,
Expand All @@ -850,6 +868,19 @@ func TestHttpFetch(t *testing.T) {
v.Set("providers", strings.Join(maStrings, ","))
}},
},
{
name: "bitswap large sharded file, fixedPeer through startup opts",
bitswapRemotes: 1,
generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
fileEntry := unixfs.GenerateFile(t, remotes[0].LinkSystem, rndReader, 4<<20)
// wipe content routing information for remote
remotes[0].Cids = make(map[cid.Cid]struct{})
return []unixfs.DirEntry{fileEntry}
},
lassieOpts: func(t *testing.T, mrn *mocknet.MockRetrievalNet) []lassie.LassieOption {
return []lassie.LassieOption{lassie.WithFinder(retriever.NewDirectCandidateFinder(mrn.Self, []peer.AddrInfo{*mrn.Remotes[0].AddrInfo()}))}
},
},
{
name: "http large sharded file with dups",
httpRemotes: 1,
Expand Down Expand Up @@ -915,10 +946,14 @@ func TestHttpFetch(t *testing.T) {

// Setup a new lassie
req := require.New(t)
var customOpts []lassie.LassieOption
if testCase.lassieOpts != nil {
customOpts = testCase.lassieOpts(t, mrn)
}
opts := append([]lassie.LassieOption{lassie.WithProviderTimeout(20 * time.Second),
lassie.WithHost(mrn.Self),
lassie.WithFinder(mrn.Finder),
}, testCase.lassieOpts...)
}, customOpts...)
if testCase.disableGraphsync {
opts = append(opts, lassie.WithProtocols([]multicodec.Code{multicodec.TransportBitswap}))
}
Expand Down