Merge pull request #6892 from ipfs/feat/graphsync-server

feat(graphsync): mount the graphsync libp2p protocol

core/core.go

@@ -17,6 +17,7 @@ import (
 	"github.com/ipfs/go-ipfs-pinner"
 
 	bserv "github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-graphsync"
 	bstore "github.com/ipfs/go-ipfs-blockstore"
 	exchange "github.com/ipfs/go-ipfs-exchange-interface"
 	"github.com/ipfs/go-ipfs-provider"
@@ -81,13 +82,14 @@ type IpfsNode struct {
 	RecordValidator record.Validator
 
 	// Online
-	PeerHost     p2phost.Host        `optional:"true"` // the network host (server+client)
-	Bootstrapper io.Closer           `optional:"true"` // the periodic bootstrapper
-	Routing      routing.Routing     `optional:"true"` // the routing system. recommend ipfs-dht
-	Exchange     exchange.Interface  // the block exchange + strategy (bitswap)
-	Namesys      namesys.NameSystem  // the name system, resolves paths to hashes
-	Provider     provider.System     // the value provider system
-	IpnsRepub    *ipnsrp.Republisher `optional:"true"`
+	PeerHost      p2phost.Host            `optional:"true"` // the network host (server+client)
+	Bootstrapper  io.Closer               `optional:"true"` // the periodic bootstrapper
+	Routing       routing.Routing         `optional:"true"` // the routing system. recommend ipfs-dht
+	Exchange      exchange.Interface      // the block exchange + strategy (bitswap)
+	Namesys       namesys.NameSystem      // the name system, resolves paths to hashes
+	Provider      provider.System         // the value provider system
+	IpnsRepub     *ipnsrp.Republisher     `optional:"true"`
+	GraphExchange graphsync.GraphExchange `optional:"true"`
 
 	AutoNAT  *autonat.AutoNATService    `optional:"true"`
 	PubSub   *pubsub.PubSub             `optional:"true"`

core/node/graphsync.go

@@ -0,0 +1,27 @@
+package node
+
+import (
+	"github.com/ipfs/go-graphsync"
+	gsimpl "github.com/ipfs/go-graphsync/impl"
+	"github.com/ipfs/go-graphsync/ipldbridge"
+	"github.com/ipfs/go-graphsync/network"
+	"github.com/ipfs/go-graphsync/storeutil"
+	"github.com/ipfs/go-ipfs-blockstore"
+	libp2p "github.com/libp2p/go-libp2p-core"
+	"go.uber.org/fx"
+
+	"github.com/ipfs/go-ipfs/core/node/helpers"
+)
+
+// Graphsync constructs a graphsync
+func Graphsync(lc fx.Lifecycle, mctx helpers.MetricsCtx, host libp2p.Host, bs blockstore.GCBlockstore) graphsync.GraphExchange {
+	ctx := helpers.LifecycleCtx(mctx, lc)
+
+	network := network.NewFromLibp2pHost(host)
+	ipldBridge := ipldbridge.NewIPLDBridge()
+	return gsimpl.New(ctx,
+		network, ipldBridge,
+		storeutil.LoaderForBlockstore(bs),
+		storeutil.StorerForBlockstore(bs),
+	)
+}

core/node/groups.go

@@ -231,6 +231,7 @@ func Online(bcfg *BuildCfg, cfg *config.Config) fx.Option {
 
 	return fx.Options(
 		fx.Provide(OnlineExchange(shouldBitswapProvide)),
+		maybeProvide(Graphsync, cfg.Experimental.GraphsyncEnabled),
 		fx.Provide(Namesys(ipnsCacheSize)),
 
 		fx.Invoke(IpnsRepublisher(repubPeriod, recordLifetime)),

docs/experimental-features.md

@@ -30,6 +30,7 @@ the above issue.
 - [AutoRelay](#autorelay)
 - [TLS 1.3 Handshake](#tls-13-as-default-handshake-protocol)
 - [Strategic Providing](#strategic-providing)
+- [Graphsync](graphsync)
 
 ---
 
@@ -705,3 +706,30 @@ ipfs config --json Experimental.StrategicProviding true
     - [ ] provide roots
     - [ ] provide all
     - [ ] provide strategic
+
+## GraphSync
+
+### State
+
+Experimental, disabled by default.
+
+[GraphSync](https://github.com/ipfs/go-graphsync) is the next-gen graph exchange
+protocol for IPFS.
+
+When this feature is enabled, IPFS will make files available over the graphsync
+protocol. However, IPFS will not currently use this protocol to _fetch_ files.
+
+### How to enable
+
+Modify your ipfs config:
+
+```
+ipfs config --json Experimental.GraphsyncEnabled true
+```
+
+### Road to being a real feature
+
+- [ ] We need to confirm that it can't be used to DoS a node. The server-side
+      logic for GraphSync is quite complex and, if we're not careful, the server
+      might end up performing unbounded work when responding to a malicious
+      request.

go.mod

@@ -27,10 +27,11 @@ require (
 	github.com/ipfs/go-ds-measure v0.1.0
 	github.com/ipfs/go-filestore v0.0.3
 	github.com/ipfs/go-fs-lock v0.0.4
+	github.com/ipfs/go-graphsync v0.0.4
 	github.com/ipfs/go-ipfs-blockstore v0.1.4
 	github.com/ipfs/go-ipfs-chunker v0.0.4
 	github.com/ipfs/go-ipfs-cmds v0.1.1
-	github.com/ipfs/go-ipfs-config v0.2.0
+	github.com/ipfs/go-ipfs-config v0.2.1
 	github.com/ipfs/go-ipfs-ds-help v0.1.1
 	github.com/ipfs/go-ipfs-exchange-interface v0.0.1
 	github.com/ipfs/go-ipfs-exchange-offline v0.0.1

test/bin/Rules.mk

@@ -18,6 +18,11 @@ $(d)/go-sleep: github.com/ipfs/go-ipfs/test/dependencies/go-sleep
 	$(go-build-testdep)
 TGTS_$(d) += $(d)/go-sleep
 
+.PHONY: github.com/ipfs/go-ipfs/test/dependencies/graphsync-get
+$(d)/graphsync-get: github.com/ipfs/go-ipfs/test/dependencies/graphsync-get
+	$(go-build-testdep)
+TGTS_$(d) += $(d)/graphsync-get
+
 .PHONY: github.com/ipfs/go-ipfs/test/dependencies/go-timeout
 $(d)/go-timeout: github.com/ipfs/go-ipfs/test/dependencies/go-timeout
 	$(go-build-testdep)

test/dependencies/go.mod

@@ -5,17 +5,26 @@ go 1.13
 require (
 	github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd
 	github.com/golangci/golangci-lint v1.18.0
+	github.com/ipfs/go-blockservice v0.1.2
+	github.com/ipfs/go-cid v0.0.5
 	github.com/ipfs/go-cidutil v0.0.2
-	github.com/ipfs/go-log v0.0.1
+	github.com/ipfs/go-datastore v0.4.4
+	github.com/ipfs/go-graphsync v0.0.4
+	github.com/ipfs/go-ipfs-blockstore v1.0.0
+	github.com/ipfs/go-ipfs-exchange-offline v0.0.1
+	github.com/ipfs/go-log v1.0.2
+	github.com/ipfs/go-merkledag v0.3.1
+	github.com/ipfs/go-unixfs v0.2.4
 	github.com/ipfs/hang-fds v0.0.1
 	github.com/ipfs/iptb v1.4.0
-	github.com/ipfs/iptb-plugins v0.2.0
+	github.com/ipfs/iptb-plugins v0.2.1
+	github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
 	github.com/jbenet/go-random-files v0.0.0-20190219210431-31b3f20ebded
-	github.com/multiformats/go-multiaddr v0.0.4
-	github.com/multiformats/go-multiaddr-net v0.0.1
-	github.com/multiformats/go-multihash v0.0.7
-	github.com/ultraware/funlen v0.0.2 // indirect
-	golang.org/x/tools v0.0.0-20190912185636-87d9f09c5d89 // indirect
+	github.com/libp2p/go-libp2p v0.5.2
+	github.com/libp2p/go-libp2p-core v0.3.1
+	github.com/multiformats/go-multiaddr v0.2.0
+	github.com/multiformats/go-multiaddr-net v0.1.2
+	github.com/multiformats/go-multihash v0.0.13
 	gotest.tools/gotestsum v0.3.5
 )