Merge pull request #2039 from ipfs/fast-add-stuff

Fast add stuff

blocks/blockstore/blockstore.go

@@ -5,6 +5,7 @@ package blockstore
 import (
 	"errors"
 	"sync"
+	"sync/atomic"
 
 	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
 	dsns "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace"
@@ -49,6 +50,10 @@ type GCBlockstore interface {
 	// at the same time, but no GC should not happen simulatenously.
 	// Reading during Pinning is safe, and requires no lock.
 	PinLock() func()
+
+	// GcRequested returns true if GCLock has been called and is waiting to
+	// take the lock
+	GCRequested() bool
 }
 
 func NewBlockstore(d ds.Batching) *blockstore {
@@ -63,7 +68,9 @@ func NewBlockstore(d ds.Batching) *blockstore {
 type blockstore struct {
 	datastore ds.Batching
 
-	lk sync.RWMutex
+	lk      sync.RWMutex
+	gcreq   int32
+	gcreqlk sync.Mutex
 }
 
 func (bs *blockstore) Get(k key.Key) (*blocks.Block, error) {
@@ -192,11 +199,17 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan key.Key, error) {
 }
 
 func (bs *blockstore) GCLock() func() {
+	atomic.AddInt32(&bs.gcreq, 1)
 	bs.lk.Lock()
+	atomic.AddInt32(&bs.gcreq, -1)
 	return bs.lk.Unlock
 }
 
 func (bs *blockstore) PinLock() func() {
 	bs.lk.RLock()
 	return bs.lk.RUnlock
 }
+
+func (bs *blockstore) GCRequested() bool {
+	return atomic.LoadInt32(&bs.gcreq) > 0
+}

blocks/blockstore/write_cache.go

@@ -66,3 +66,7 @@ func (w *writecache) GCLock() func() {
 func (w *writecache) PinLock() func() {
 	return w.blockstore.(GCBlockstore).PinLock()
 }
+
+func (w *writecache) GCRequested() bool {
+	return w.blockstore.(GCBlockstore).GCRequested()
+}

commands/cli/parse.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path"
 	"runtime"
+	"sort"
 	"strings"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
@@ -259,8 +260,8 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi
 	}
 
 	stringArgs := make([]string, 0, numInputs)
-	fileArgs := make([]files.File, 0, numInputs)
 
+	fileArgs := make(map[string]files.File)
 	argDefIndex := 0 // the index of the current argument definition
 	for i := 0; i < numInputs; i++ {
 		argDef := getArgDef(argDefIndex, argDefs)
@@ -295,18 +296,21 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi
 		} else if argDef.Type == cmds.ArgFile {
 			if stdin == nil || !argDef.SupportsStdin {
 				// treat stringArg values as file paths
-				fileArgs, inputs, err = appendFile(fileArgs, inputs, argDef, recursive)
+				fpath := inputs[0]
+				inputs = inputs[1:]
+				file, err := appendFile(fpath, argDef, recursive)
 				if err != nil {
 					return nil, nil, err
 				}
 
+				fileArgs[fpath] = file
 			} else {
 				if len(inputs) > 0 {
 					// don't use stdin if we have inputs
 					stdin = nil
 				} else {
 					// if we have a stdin, create a file from it
-					fileArgs, stdin = appendStdinAsFile(fileArgs, stdin)
+					fileArgs[""] = files.NewReaderFile("", "", stdin, nil)
 				}
 			}
 		}
@@ -323,7 +327,23 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi
 		}
 	}
 
-	return stringArgs, fileArgs, nil
+	return stringArgs, filesMapToSortedArr(fileArgs), nil
+}
+
+func filesMapToSortedArr(fs map[string]files.File) []files.File {
+	var names []string
+	for name, _ := range fs {
+		names = append(names, name)
+	}
+
+	sort.Strings(names)
+
+	var out []files.File
+	for _, f := range names {
+		out = append(out, fs[f])
+	}
+
+	return out
 }
 
 func getArgDef(i int, argDefs []cmds.Argument) *cmds.Argument {
@@ -356,44 +376,34 @@ func appendStdinAsString(args []string, stdin *os.File) ([]string, *os.File, err
 	return append(args, strings.Split(input, "\n")...), nil, nil
 }
 
-func appendFile(args []files.File, inputs []string, argDef *cmds.Argument, recursive bool) ([]files.File, []string, error) {
-	fpath := inputs[0]
+const notRecursiveFmtStr = "'%s' is a directory, use the '-%s' flag to specify directories"
+const dirNotSupportedFmtStr = "Invalid path '%s', argument '%s' does not support directories"
+
+func appendFile(fpath string, argDef *cmds.Argument, recursive bool) (files.File, error) {
 
 	if fpath == "." {
 		cwd, err := os.Getwd()
 		if err != nil {
-			return nil, nil, err
+			return nil, err
 		}
 		fpath = cwd
 	}
+
 	stat, err := os.Lstat(fpath)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	if stat.IsDir() {
 		if !argDef.Recursive {
-			err = fmt.Errorf("Invalid path '%s', argument '%s' does not support directories",
-				fpath, argDef.Name)
-			return nil, nil, err
+			return nil, fmt.Errorf(dirNotSupportedFmtStr, fpath, argDef.Name)
 		}
 		if !recursive {
-			err = fmt.Errorf("'%s' is a directory, use the '-%s' flag to specify directories",
-				fpath, cmds.RecShort)
-			return nil, nil, err
+			return nil, fmt.Errorf(notRecursiveFmtStr, fpath, cmds.RecShort)
 		}
 	}
 
-	arg, err := files.NewSerialFile(path.Base(fpath), fpath, stat)
-	if err != nil {
-		return nil, nil, err
-	}
-	return append(args, arg), inputs[1:], nil
-}
-
-func appendStdinAsFile(args []files.File, stdin *os.File) ([]files.File, *os.File) {
-	arg := files.NewReaderFile("", "", stdin, nil)
-	return append(args, arg), nil
+	return files.NewSerialFile(path.Base(fpath), fpath, stat)
 }
 
 // isTerminal returns true if stdin is a Stdin pipe (e.g. `cat file | ipfs`),