Merge branch 'master' into feat/tracing

.circleci/main.yml

@@ -154,7 +154,7 @@ jobs:
     working_directory: ~/ipfs/go-ipfs
     environment:
         <<: *default_environment
-        TEST_NO_DOCKER: 1
+        TEST_NO_DOCKER: 0
         TEST_NO_FUSE: 1
         TEST_VERBOSE: 1
     steps:
@@ -292,7 +292,7 @@ jobs:
             - v1-go-api-{{ checksum "~/ipfs/go-ipfs/go-ipfs-api/go.sum" }}
             - v1-go-api-
       - run:
-          command: go test -v ./...
+          command: go test -count=1 -v ./...
           working_directory: ~/ipfs/go-ipfs/go-ipfs-api
       - save_cache:
           key: v1-go-api-{{ checksum "~/ipfs/go-ipfs/go-ipfs-api/go.sum" }}
@@ -318,10 +318,10 @@ jobs:
             - v1-http-client-{{ checksum "~/ipfs/go-ipfs/go-ipfs-http-client/go.sum" }}
             - v1-http-client-
       - run:
-          name: go test -v ./...
+          name: go test -count=1 -v ./...
           command: |
             export PATH=/tmp/circleci-workspace/bin:$PATH
-            go test -v ./...
+            go test -count=1 -v ./...
           working_directory: ~/ipfs/go-ipfs/go-ipfs-http-client
       - save_cache:
           key: v1-http-client-{{ checksum "~/ipfs/go-ipfs/go-ipfs-http-client/go.sum" }}

.dockerignore

@@ -5,6 +5,8 @@ Dockerfile.fast
 !.git/refs/
 !.git/packed-refs
 test/sharness/lib/sharness/
+test/sharness/trash*
+rb-pinning-service-api/
 
 # The Docker client might not be running on Linux
 # so delete any compiled binaries

bin/gencmdref

@@ -3,7 +3,6 @@
 import os
 import sys
 import datetime
-import subprocess
 
 from subprocess import check_output
 

cmd/ipfs/main.go

@@ -27,7 +27,6 @@ import (
 	"github.com/ipfs/go-ipfs-cmds/cli"
 	cmdhttp "github.com/ipfs/go-ipfs-cmds/http"
 	u "github.com/ipfs/go-ipfs-util"
-	config "github.com/ipfs/go-ipfs/config"
 	logging "github.com/ipfs/go-log"
 	loggables "github.com/libp2p/go-libp2p-loggables"
 	ma "github.com/multiformats/go-multiaddr"
@@ -149,7 +148,6 @@ func mainRet() (exitCode int) {
 		// this is so that we can construct the node lazily.
 		return &oldcmds.Context{
 			ConfigRoot: repoPath,
-			LoadConfig: loadConfig,
 			ReqLog:     &oldcmds.ReqLog{},
 			Plugins:    plugins,
 			ConstructNode: func() (n *core.IpfsNode, err error) {
@@ -317,10 +315,6 @@ func getRepoPath(req *cmds.Request) (string, error) {
 	return repoPath, nil
 }
 
-func loadConfig(path string) (*config.Config, error) {
-	return fsrepo.ConfigAt(path)
-}
-
 // startProfiling begins CPU profiling and returns a `stop` function to be
 // executed as late as possible. The stop function captures the memprofile.
 func startProfiling() (func(), error) {

cmd/ipfs/pinmfs.go

@@ -36,7 +36,7 @@ const defaultRepinInterval = 5 * time.Minute
 
 type pinMFSContext interface {
 	Context() context.Context
-	GetConfigNoCache() (*config.Config, error)
+	GetConfig() (*config.Config, error)
 }
 
 type pinMFSNode interface {
@@ -104,7 +104,7 @@ func pinMFSOnChange(configPollInterval time.Duration, cctx pinMFSContext, node p
 		}
 
 		// reread the config, which may have changed in the meantime
-		cfg, err := cctx.GetConfigNoCache()
+		cfg, err := cctx.GetConfig()
 		if err != nil {
 			select {
 			case errCh <- fmt.Errorf("pinning reading config (%v)", err):

cmd/ipfs/pinmfs_test.go

@@ -24,7 +24,7 @@ func (x *testPinMFSContext) Context() context.Context {
 	return x.ctx
 }
 
-func (x *testPinMFSContext) GetConfigNoCache() (*config.Config, error) {
+func (x *testPinMFSContext) GetConfig() (*config.Config, error) {
 	return x.cfg, x.err
 }
 

cmd/ipfswatch/main.go

@@ -20,7 +20,6 @@ import (
 
 	fsnotify "github.com/fsnotify/fsnotify"
 	files "github.com/ipfs/go-ipfs-files"
-	config "github.com/ipfs/go-ipfs/config"
 	process "github.com/jbenet/goprocess"
 	homedir "github.com/mitchellh/go-homedir"
 )
@@ -217,9 +216,6 @@ func IsHidden(path string) bool {
 func cmdCtx(node *core.IpfsNode, repoPath string) commands.Context {
 	return commands.Context{
 		ConfigRoot: repoPath,
-		LoadConfig: func(path string) (*config.Config, error) {
-			return node.Repo.Config()
-		},
 		ConstructNode: func() (*core.IpfsNode, error) {
 			return node, nil
 		},

commands/context.go

@@ -26,30 +26,18 @@ type Context struct {
 
 	Plugins *loader.PluginLoader
 
-	config     *config.Config
-	LoadConfig func(path string) (*config.Config, error)
-
 	Gateway       bool
 	api           coreiface.CoreAPI
 	node          *core.IpfsNode
 	ConstructNode func() (*core.IpfsNode, error)
 }
 
-// GetConfig returns the config of the current Command execution
-// context. It may load it with the provided function.
 func (c *Context) GetConfig() (*config.Config, error) {
-	var err error
-	if c.config == nil {
-		if c.LoadConfig == nil {
-			return nil, errors.New("nil LoadConfig function")
-		}
-		c.config, err = c.LoadConfig(c.ConfigRoot)
+	node, err := c.GetNode()
+	if err != nil {
+		return nil, err
 	}
-	return c.config, err
-}
-
-func (c *Context) GetConfigNoCache() (*config.Config, error) {
-	return c.LoadConfig(c.ConfigRoot)
+	return node.Repo.Config()
 }
 
 // GetNode returns the node of the current Command execution
@@ -61,12 +49,6 @@ func (c *Context) GetNode() (*core.IpfsNode, error) {
 			return nil, errors.New("nil ConstructNode function")
 		}
 		c.node, err = c.ConstructNode()
-		if err == nil {
-			// Pre-load the config from the repo to avoid re-parsing it from disk.
-			if cfg, err := c.node.Repo.Config(); err != nil {
-				c.config = cfg
-			}
-		}
 	}
 	return c.node, err
 }

core/commands/cmdenv/env.go

@@ -9,7 +9,6 @@ import (
 	"github.com/ipfs/go-ipfs/core"
 
 	cmds "github.com/ipfs/go-ipfs-cmds"
-	config "github.com/ipfs/go-ipfs/config"
 	logging "github.com/ipfs/go-log"
 	coreiface "github.com/ipfs/interface-go-ipfs-core"
 	options "github.com/ipfs/interface-go-ipfs-core/options"
@@ -52,16 +51,6 @@ func GetApi(env cmds.Environment, req *cmds.Request) (coreiface.CoreAPI, error)
 	return api, nil
 }
 
-// GetConfig extracts the config from the environment.
-func GetConfig(env cmds.Environment) (*config.Config, error) {
-	ctx, ok := env.(*commands.Context)
-	if !ok {
-		return nil, fmt.Errorf("expected env to be of type %T, got %T", ctx, env)
-	}
-
-	return ctx.GetConfig()
-}
-
 // GetConfigRoot extracts the config root from the environment
 func GetConfigRoot(env cmds.Environment) (string, error) {
 	ctx, ok := env.(*commands.Context)

core/commands/mount_unix.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 
+	oldcmds "github.com/ipfs/go-ipfs/commands"
 	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
 	nodeMount "github.com/ipfs/go-ipfs/fuse/node"
 
@@ -81,7 +82,7 @@ baz
 		cmds.StringOption(mountIPNSPathOptionName, "n", "The path where IPNS should be mounted."),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
-		cfg, err := cmdenv.GetConfig(env)
+		cfg, err := env.(*oldcmds.Context).GetConfig()
 		if err != nil {
 			return err
 		}

core/corehttp/gateway_handler.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"html/template"
 	"io"
+	"mime"
 	"net/http"
 	"net/url"
 	"os"
@@ -350,7 +351,11 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 	}
 
 	// Detect when explicit Accept header or ?format parameter are present
-	responseFormat := customResponseFormat(r)
+	responseFormat, formatParams, err := customResponseFormat(r)
+	if err != nil {
+		webError(w, "error while processing the Accept header", err, http.StatusBadRequest)
+		return
+	}
 	trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResponseFormat", responseFormat))
 	trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResolvedPath", resolvedPath.String()))
 
@@ -393,9 +398,10 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 		logger.Debugw("serving raw block", "path", contentPath)
 		i.serveRawBlock(w, r, resolvedPath, contentPath, begin)
 		return
-	case "application/vnd.ipld.car", "application/vnd.ipld.car; version=1":
+	case "application/vnd.ipld.car":
 		logger.Debugw("serving car stream", "path", contentPath)
-		i.serveCar(w, r, resolvedPath, contentPath, begin)
+		carVersion := formatParams["version"]
+    i.serveCar(w, r, resolvedPath, contentPath, carVersion, begin)
 		return
 	default: // catch-all for unsuported application/vnd.*
 		err := fmt.Errorf("unsupported format %q", responseFormat)
@@ -765,8 +771,8 @@ func getFilename(contentPath ipath.Path) string {
 func getEtag(r *http.Request, cid cid.Cid) string {
 	prefix := `"`
 	suffix := `"`
-	responseFormat := customResponseFormat(r)
-	if responseFormat != "" {
+	responseFormat, _, err := customResponseFormat(r)
+	if err == nil && responseFormat != "" {
 		// application/vnd.ipld.foo → foo
 		f := responseFormat[strings.LastIndex(responseFormat, ".")+1:]
 		// Etag: "cid.foo" (gives us nice compression together with Content-Disposition in block (raw) and car responses)
@@ -777,14 +783,14 @@ func getEtag(r *http.Request, cid cid.Cid) string {
 }
 
 // return explicit response format if specified in request as query parameter or via Accept HTTP header
-func customResponseFormat(r *http.Request) string {
+func customResponseFormat(r *http.Request) (mediaType string, params map[string]string, err error) {
 	if formatParam := r.URL.Query().Get("format"); formatParam != "" {
 		// translate query param to a content type
 		switch formatParam {
 		case "raw":
-			return "application/vnd.ipld.raw"
+			return "application/vnd.ipld.raw", nil, nil
 		case "car":
-			return "application/vnd.ipld.car"
+			return "application/vnd.ipld.car", nil, nil
 		}
 	}
 	// Browsers and other user agents will send Accept header with generic types like:
@@ -793,10 +799,14 @@ func customResponseFormat(r *http.Request) string {
 	for _, accept := range r.Header.Values("Accept") {
 		// respond to the very first ipld content type
 		if strings.HasPrefix(accept, "application/vnd.ipld") {
-			return accept
+			mediatype, params, err := mime.ParseMediaType(accept)
+			if err != nil {
+				return "", nil, err
+			}
+			return mediatype, params, nil
 		}
 	}
-	return ""
+	return "", nil, nil
 }
 
 func (i *gatewayHandler) searchUpTreeFor404(r *http.Request, contentPath ipath.Path) (ipath.Resolved, string, error) {

core/corehttp/gateway_handler_car.go

@@ -2,6 +2,7 @@ package corehttp
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"time"
 
@@ -17,13 +18,20 @@ import (
 )
 
 // serveCar returns a CAR stream for specific DAG+selector
-func (i *gatewayHandler) serveCar(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time) {
+func (i *gatewayHandler) serveCar(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, carVersion string, begin time.Time) {
 	ctx, span := tracing.Span(r.Context(), "Gateway", "ServeCar", trace.WithAttributes(attribute.String("path", resolvedPath.String())))
 	defer span.End()
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	rootCid := resolvedPath.Cid()
+	switch carVersion {
+	case "": // noop, client does not care about version
+	case "1": // noop, we support this
+	default:
+		err := fmt.Errorf("only version=1 is supported")
+		webError(w, "unsupported CAR version", err, http.StatusBadRequest)
+		return
+	}
 
 	// Set Content-Disposition
 	name := rootCid.String() + ".car"

core/mock/mock.go

@@ -69,9 +69,6 @@ func MockCmdsCtx() (commands.Context, error) {
 
 	return commands.Context{
 		ConfigRoot: "/tmp/.mockipfsconfig",
-		LoadConfig: func(path string) (*config.Config, error) {
-			return &conf, nil
-		},
 		ConstructNode: func() (*core.IpfsNode, error) {
 			return node, nil
 		},

docs/gateway.md

@@ -2,19 +2,30 @@
 
 An IPFS Gateway acts as a bridge between traditional web browsers and IPFS.
 Through the gateway, users can browse files and websites stored in IPFS as if
-they were stored in a traditional web server.
+they were stored in a traditional web server. 
 
-By default, go-ipfs nodes run a gateway at `http://127.0.0.1:8080/`.
+[More about Gateways](https://docs.ipfs.io/concepts/ipfs-gateway/) and [addressing IPFS on the web](https://docs.ipfs.io/how-to/address-ipfs-on-web/).
 
-We also provide a public gateway at `https://ipfs.io`. If you've ever seen a
-link in the form `https://ipfs.io/ipfs/Qm...`, that's being served from *our*
-gateway.
+### Local gateway
+
+By default, go-ipfs nodes run
+a [path gateway](https://docs.ipfs.io/how-to/address-ipfs-on-web/#path-gateway) at `http://127.0.0.1:8080/`
+and a [subdomain gateway](https://docs.ipfs.io/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://localhost:8080/`
+
+Additional listening addresses and gateway behaviors can be set in the [config](#configuration) file.
+
+### Public gateways
+
+Protocol Labs provides a public gateway at `https://ipfs.io` (path) and `https://dweb.link` (subdomain).
+If you've ever seen a link in the form `https://ipfs.io/ipfs/Qm...`, that's being served from *our* gateway.
+
+There is a list of third-party public gateways provided by the IPFS community at https://ipfs.github.io/public-gateway-checker/
 
 ## Configuration
 
-The gateway's configuration options are (briefly) described in the
+The `Gateway.*` configuration options are (briefly) described in the
 [config](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#gateway)
-documentation.
+documentation, including a list of common [gateway recipes](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#gateway-recipes).
 
 ### Debug
 The gateway's log level can be changed with this command:
@@ -40,7 +51,7 @@ for details</sub>
 ## Static Websites
 
 You can use an IPFS gateway to serve static websites at a custom domain using
-[DNSLink](https://dnslink.io). See [Example: IPFS
+[DNSLink](https://docs.ipfs.io/concepts/glossary#dnslink). See [Example: IPFS
 Gateway](https://dnslink.io/#example-ipfs-gateway) for instructions.
 
 ## Filenames

go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/ipfs/go-graphsync v0.11.0
 	github.com/ipfs/go-ipfs-blockstore v1.2.0
 	github.com/ipfs/go-ipfs-chunker v0.0.5
-	github.com/ipfs/go-ipfs-cmds v0.7.0
+	github.com/ipfs/go-ipfs-cmds v0.8.0
 	github.com/ipfs/go-ipfs-exchange-interface v0.1.0
 	github.com/ipfs/go-ipfs-exchange-offline v0.2.0
 	github.com/ipfs/go-ipfs-files v0.0.9

go.sum

@@ -483,8 +483,8 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtL
 github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw=
 github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8=
 github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8=
-github.com/ipfs/go-ipfs-cmds v0.7.0 h1:0lEldmB7C83RxIOer38Sv1ob6wIoCAIEOaxiYgcv7wA=
-github.com/ipfs/go-ipfs-cmds v0.7.0/go.mod h1:y0bflH6m4g6ary4HniYt98UqbrVnRxmRarzeMdLIUn0=
+github.com/ipfs/go-ipfs-cmds v0.8.0 h1:M7apkPxhGe7I3rcKuQ8xRJLIPdaEqaZhJz0uPZEE8EU=
+github.com/ipfs/go-ipfs-cmds v0.8.0/go.mod h1:y0bflH6m4g6ary4HniYt98UqbrVnRxmRarzeMdLIUn0=
 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
 github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ=
 github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=