From a49e020ce974b491d33b662a0775fdd44321b451 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 2 Dec 2015 00:22:37 -0800 Subject: [PATCH 01/13] add option to disable flushing files structure on writes License: MIT Signed-off-by: Jeromy --- core/commands/files/files.go | 29 +++++++- mfs/dir.go | 121 +++++++++++++++++++++++-------- test/sharness/t0250-files-api.sh | 15 +++- 3 files changed, 130 insertions(+), 35 deletions(-) diff --git a/core/commands/files/files.go b/core/commands/files/files.go index bc788fb6069..94b5fd8916d 100644 --- a/core/commands/files/files.go +++ b/core/commands/files/files.go @@ -68,7 +68,7 @@ var FilesStatCmd = &cmds.Command{ return } - o, err := statNode(fsn) + o, err := statNode(node.DAG, fsn) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -90,13 +90,14 @@ var FilesStatCmd = &cmds.Command{ Type: Object{}, } -func statNode(fsn mfs.FSNode) (*Object, error) { +func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) { nd, err := fsn.GetNode() if err != nil { return nil, err } - k, err := nd.Key() + // add to dagserv to ensure its available + k, err := ds.Add(nd) if err != nil { return nil, err } @@ -434,10 +435,20 @@ a beginning offset to write to. The entire length of the input will be written. If the '--create' option is specified, the file will be created if it does not exist. Nonexistant intermediate directories will not be created. +If the '--flush' option is set to false, changes will not be propogated to the +merkledag root. This can make operations much faster when doing a large number +of writes to a deeper directory structure. + Example: echo "hello world" | ipfs files write --create /myfs/a/b/file echo "hello world" | ipfs files write --truncate /myfs/a/b/file + +Warning: + + Usage of the '--flush=false' option does not guarantee data durability until + the tree has been flushed. This can be accomplished by running 'ipfs files stat' + on the file or any of its ancestors. `, }, Arguments: []cmds.Argument{ @@ -449,6 +460,7 @@ Example: cmds.BoolOption("e", "create", "create the file if it does not exist"), cmds.BoolOption("t", "truncate", "truncate the file before writing"), cmds.IntOption("n", "count", "maximum number of bytes to read"), + cmds.BoolOption("f", "flush", "flush file and ancestors after write (default: true)"), }, Run: func(req cmds.Request, res cmds.Response) { path, err := checkPath(req.Arguments()[0]) @@ -459,6 +471,10 @@ Example: create, _, _ := req.Option("create").Bool() trunc, _, _ := req.Option("truncate").Bool() + flush, set, _ := req.Option("flush").Bool() + if !set { + flush = true + } nd, err := req.InvocContext().GetNode() if err != nil { @@ -471,7 +487,12 @@ Example: res.SetError(err, cmds.ErrNormal) return } - defer fi.Close() + + if flush { + defer fi.Close() + } else { + defer fi.Sync() + } if trunc { if err := fi.Truncate(0); err != nil { diff --git a/mfs/dir.go b/mfs/dir.go index 264dea4a0d7..b86c98d77a3 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -53,7 +53,16 @@ func (d *Directory) closeChild(name string, nd *dag.Node) error { d.lock.Lock() defer d.lock.Unlock() - err = d.node.RemoveNodeLink(name) + err = d.updateChild(name, nd) + if err != nil { + return err + } + + return d.parent.closeChild(d.name, d.node) +} + +func (d *Directory) updateChild(name string, nd *dag.Node) error { + err := d.node.RemoveNodeLink(name) if err != nil && err != dag.ErrNotFound { return err } @@ -63,7 +72,7 @@ func (d *Directory) closeChild(name string, nd *dag.Node) error { return err } - return d.parent.closeChild(d.name, d.node) + return nil } func (d *Directory) Type() NodeType { @@ -77,30 +86,16 @@ func (d *Directory) childFile(name string) (*File, error) { return fi, nil } - nd, err := d.childFromDag(name) - if err != nil { - return nil, err - } - i, err := ft.FromBytes(nd.Data) + fsn, err := d.childNode(name) if err != nil { return nil, err } - switch i.GetType() { - case ufspb.Data_Directory: - return nil, ErrIsDirectory - case ufspb.Data_File: - nfi, err := NewFile(name, nd, d, d.dserv) - if err != nil { - return nil, err - } - d.files[name] = nfi - return nfi, nil - case ufspb.Data_Metadata: - return nil, ErrNotYetImplemented - default: - return nil, ErrInvalidChild + if fi, ok := fsn.(*File); ok { + return fi, nil } + + return nil, fmt.Errorf("%s is not a file", name) } // childDir returns a directory under this directory by the given name if it @@ -111,6 +106,21 @@ func (d *Directory) childDir(name string) (*Directory, error) { return dir, nil } + fsn, err := d.childNode(name) + if err != nil { + return nil, err + } + + if dir, ok := fsn.(*Directory); ok { + return dir, nil + } + + return nil, fmt.Errorf("%s is not a directory", name) +} + +// childNode returns a FSNode under this directory by the given name if it exists. +// it does *not* check the cached dirs and files +func (d *Directory) childNode(name string) (FSNode, error) { nd, err := d.childFromDag(name) if err != nil { return nil, err @@ -127,7 +137,12 @@ func (d *Directory) childDir(name string) (*Directory, error) { d.childDirs[name] = ndir return ndir, nil case ufspb.Data_File: - return nil, fmt.Errorf("%s is not a directory", name) + nfi, err := NewFile(name, nd, d, d.dserv) + if err != nil { + return nil, err + } + d.files[name] = nfi + return nfi, nil case ufspb.Data_Metadata: return nil, ErrNotYetImplemented default: @@ -157,17 +172,17 @@ func (d *Directory) Child(name string) (FSNode, error) { // childUnsync returns the child under this directory by the given name // without locking, useful for operations which already hold a lock func (d *Directory) childUnsync(name string) (FSNode, error) { - - dir, err := d.childDir(name) - if err == nil { - return dir, nil + cdir, ok := d.childDirs[name] + if ok { + return cdir, nil } - fi, err := d.childFile(name) - if err == nil { - return fi, nil + + cfile, ok := d.files[name] + if ok { + return cfile, nil } - return nil, os.ErrNotExist + return d.childNode(name) } type NodeListing struct { @@ -305,7 +320,53 @@ func (d *Directory) AddChild(name string, nd *dag.Node) error { return d.parent.closeChild(d.name, d.node) } +func (d *Directory) sync() error { + for name, dir := range d.childDirs { + nd, err := dir.GetNode() + if err != nil { + return err + } + + _, err = d.dserv.Add(nd) + if err != nil { + return err + } + + err = d.updateChild(name, nd) + if err != nil { + return err + } + } + + for name, file := range d.files { + nd, err := file.GetNode() + if err != nil { + return err + } + + _, err = d.dserv.Add(nd) + if err != nil { + return err + } + + err = d.updateChild(name, nd) + if err != nil { + return err + } + } + + return nil +} + func (d *Directory) GetNode() (*dag.Node, error) { + d.Lock() + defer d.Unlock() + + err := d.sync() + if err != nil { + return nil, err + } + return d.node, nil } diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index b011a8bd57a..e04f0884569 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -316,13 +316,26 @@ test_files_api() { verify_dir_contents /cats file1 ipfs this ' + test_expect_success "write 'no-flush' succeeds" ' + echo "testing" | ipfs files write -f -e /cats/walrus + ' + + test_expect_success "changes bubbled up to root on inspection" ' + ipfs files stat / | head -n1 > root_hash + ' + + test_expect_success "root hash looks good" ' + echo "QmcwKfTMCT7AaeiD92hWjnZn9b6eh9NxnhfSzN5x2vnDpt" > root_hash_exp && + test_cmp root_hash_exp root_hash + ' + # test mv test_expect_success "can mv dir" ' ipfs files mv /cats/this/is /cats/ ' test_expect_success "mv worked" ' - verify_dir_contents /cats file1 ipfs this is && + verify_dir_contents /cats file1 ipfs this is walrus && verify_dir_contents /cats/this ' From a1dca8c04287f395bc73fdfb52dde863c33df49b Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 4 Dec 2015 13:01:39 -0800 Subject: [PATCH 02/13] compute add size in background to not stall add operation License: MIT Signed-off-by: Jeromy --- core/commands/add.go | 104 +++++++++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 43 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 1232f1db6ab..a73396d1a3e 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -65,14 +65,19 @@ remains to be implemented. return nil } - size, err := sizeFile.Size() - if err != nil { - // see comment above - return nil - } + sizeCh := make(chan int64, 1) + req.Values()["size"] = sizeCh - log.Debugf("Total size of file being added: %v\n", size) - req.Values()["size"] = size + go func() { + size, err := sizeFile.Size() + if err != nil { + // see comment above + return + } + + log.Debugf("Total size of file being added: %v\n", size) + sizeCh <- size + }() return nil }, @@ -189,17 +194,12 @@ remains to be implemented. return } - size := int64(0) - s, found := req.Values()["size"] - if found { - size = s.(int64) - } - showProgressBar := !quiet && size >= progressBarMinSize + showProgressBar := !quiet var bar *pb.ProgressBar var terminalWidth int if showProgressBar { - bar = pb.New64(size).SetUnits(pb.U_BYTES) + bar = pb.New64(0).SetUnits(pb.U_BYTES) bar.ManualUpdate = true bar.Start() @@ -215,43 +215,61 @@ remains to be implemented. bar.Update() } + var sizeChan chan int64 + s, found := req.Values()["size"] + if found { + sizeChan = s.(chan int64) + } + lastFile := "" var totalProgress, prevFiles, lastBytes int64 - for out := range outChan { - output := out.(*coreunix.AddedObject) - if len(output.Hash) > 0 { - if showProgressBar { - // clear progress bar line before we print "added x" output - fmt.Fprintf(res.Stderr(), "\033[2K\r") - } - if quiet { - fmt.Fprintf(res.Stdout(), "%s\n", output.Hash) - } else { - fmt.Fprintf(res.Stdout(), "added %s %s\n", output.Hash, output.Name) + LOOP: + for { + select { + case out, ok := <-outChan: + if !ok { + break LOOP } + output := out.(*coreunix.AddedObject) + if len(output.Hash) > 0 { + if showProgressBar { + // clear progress bar line before we print "added x" output + fmt.Fprintf(res.Stderr(), "\033[2K\r") + } + if quiet { + fmt.Fprintf(res.Stdout(), "%s\n", output.Hash) + } else { + fmt.Fprintf(res.Stdout(), "added %s %s\n", output.Hash, output.Name) + } - } else { - log.Debugf("add progress: %v %v\n", output.Name, output.Bytes) - - if !showProgressBar { - continue + } else { + log.Debugf("add progress: %v %v\n", output.Name, output.Bytes) + + if !showProgressBar { + continue + } + + if len(lastFile) == 0 { + lastFile = output.Name + } + if output.Name != lastFile || output.Bytes < lastBytes { + prevFiles += lastBytes + lastFile = output.Name + } + lastBytes = output.Bytes + delta := prevFiles + lastBytes - totalProgress + totalProgress = bar.Add64(delta) } - if len(lastFile) == 0 { - lastFile = output.Name - } - if output.Name != lastFile || output.Bytes < lastBytes { - prevFiles += lastBytes - lastFile = output.Name + if showProgressBar { + bar.Update() } - lastBytes = output.Bytes - delta := prevFiles + lastBytes - totalProgress - totalProgress = bar.Add64(delta) - } - - if showProgressBar { - bar.Update() + case size := <-sizeChan: + bar.Total = size + bar.ShowPercent = true + bar.ShowBar = true + bar.ShowTimeLeft = true } } }, From e5c27e17b7b7f96f0b69901f3f0274fd44b11beb Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 4 Dec 2015 14:25:13 -0800 Subject: [PATCH 03/13] use mfs for adds License: MIT Signed-off-by: Jeromy --- core/commands/add.go | 26 +++++-- core/coreunix/add.go | 146 ++++++++++++++++++++---------------- exchange/bitswap/workers.go | 2 +- merkledag/merkledag.go | 9 +++ mfs/system.go | 21 ++++-- 5 files changed, 129 insertions(+), 75 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index a73396d1a3e..9d323a5d752 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -18,6 +18,7 @@ var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded") const ( quietOptionName = "quiet" + silentOptionName = "silent" progressOptionName = "progress" trickleOptionName = "trickle" wrapOptionName = "wrap-with-directory" @@ -44,6 +45,7 @@ remains to be implemented. Options: []cmds.Option{ cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive) cmds.BoolOption(quietOptionName, "q", "Write minimal output"), + cmds.BoolOption(silentOptionName, "x", "Write no output"), cmds.BoolOption(progressOptionName, "p", "Stream progress data"), cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation"), cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk"), @@ -59,6 +61,9 @@ remains to be implemented. req.SetOption(progressOptionName, true) + log.Error("SKIPPING SIZE") + return nil + sizeFile, ok := req.Files().(files.SizeFile) if !ok { // we don't need to error, the progress bar just won't know how big the files are @@ -100,6 +105,7 @@ remains to be implemented. wrap, _, _ := req.Option(wrapOptionName).Bool() hash, _, _ := req.Option(onlyHashOptionName).Bool() hidden, _, _ := req.Option(hiddenOptionName).Bool() + silent, _, _ := req.Option(silentOptionName).Bool() chunker, _, _ := req.Option(chunkerOptionName).String() dopin, pin_found, _ := req.Option(pinOptionName).Bool() @@ -123,13 +129,18 @@ remains to be implemented. outChan := make(chan interface{}, 8) res.SetOutput((<-chan interface{})(outChan)) - fileAdder := coreunix.NewAdder(req.Context(), n, outChan) + fileAdder, err := coreunix.NewAdder(req.Context(), n, outChan) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } fileAdder.Chunker = chunker fileAdder.Progress = progress fileAdder.Hidden = hidden fileAdder.Trickle = trickle fileAdder.Wrap = wrap fileAdder.Pin = dopin + fileAdder.Silent = silent // addAllFiles loops over a convenience slice file to // add each file individually. e.g. 'ipfs add a b c' @@ -143,7 +154,7 @@ remains to be implemented. return nil // done } - if _, err := fileAdder.AddFile(file); err != nil { + if err := fileAdder.AddFile(file); err != nil { return err } } @@ -159,9 +170,8 @@ remains to be implemented. } // copy intermediary nodes from editor to our actual dagservice - _, err := fileAdder.Finalize(n.DAG) + _, err := fileAdder.Finalize() if err != nil { - log.Error("WRITE OUT: ", err) return err } @@ -194,7 +204,13 @@ remains to be implemented. return } - showProgressBar := !quiet + progress, _, err := req.Option(progressOptionName).Bool() + if err != nil { + res.SetError(u.ErrCast(), cmds.ErrNormal) + return + } + + showProgressBar := !quiet || progress var bar *pb.ProgressBar var terminalWidth int diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 3070e874461..59cf7ada70d 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -15,7 +15,7 @@ import ( "github.com/ipfs/go-ipfs/exchange/offline" importer "github.com/ipfs/go-ipfs/importer" "github.com/ipfs/go-ipfs/importer/chunk" - dagutils "github.com/ipfs/go-ipfs/merkledag/utils" + mfs "github.com/ipfs/go-ipfs/mfs" "github.com/ipfs/go-ipfs/pin" "github.com/ipfs/go-ipfs/commands/files" @@ -62,12 +62,16 @@ type AddedObject struct { Bytes int64 `json:",omitempty"` } -func NewAdder(ctx context.Context, n *core.IpfsNode, out chan interface{}) *Adder { - e := dagutils.NewDagEditor(newDirNode(), nil) +func NewAdder(ctx context.Context, n *core.IpfsNode, out chan interface{}) (*Adder, error) { + mr, err := mfs.NewRoot(ctx, n.DAG, newDirNode(), nil) + if err != nil { + return nil, err + } + return &Adder{ + mr: mr, ctx: ctx, node: n, - editor: e, out: out, Progress: false, Hidden: true, @@ -75,22 +79,23 @@ func NewAdder(ctx context.Context, n *core.IpfsNode, out chan interface{}) *Adde Trickle: false, Wrap: false, Chunker: "", - } + }, nil } // Internal structure for holding the switches passed to the `add` call type Adder struct { ctx context.Context node *core.IpfsNode - editor *dagutils.Editor out chan interface{} Progress bool Hidden bool Pin bool Trickle bool + Silent bool Wrap bool Chunker string root *dag.Node + mr *mfs.Root } // Perform the actual add & pin locally, outputting results to reader @@ -113,26 +118,29 @@ func (params Adder) add(reader io.Reader) (*dag.Node, error) { } func (params *Adder) RootNode() (*dag.Node, error) { - // for memoizing - if params.root != nil { - return params.root, nil - } + return params.mr.GetValue().GetNode() + /* + // for memoizing + if params.root != nil { + return params.root, nil + } - root := params.editor.GetNode() + root := params.editor.GetNode() - // if not wrapping, AND one root file, use that hash as root. - if !params.Wrap && len(root.Links) == 1 { - var err error - root, err = root.Links[0].GetNode(params.ctx, params.editor.GetDagService()) + // if not wrapping, AND one root file, use that hash as root. + if !params.Wrap && len(root.Links) == 1 { + var err error + root, err = root.Links[0].GetNode(params.ctx, params.editor.GetDagService()) + params.root = root + // no need to output, as we've already done so. + return root, err + } + + // otherwise need to output, as we have not. + err := outputDagnode(params.out, "", root) params.root = root - // no need to output, as we've already done so. return root, err - } - - // otherwise need to output, as we have not. - err := outputDagnode(params.out, "", root) - params.root = root - return root, err + */ } func (params *Adder) PinRoot() error { @@ -153,8 +161,8 @@ func (params *Adder) PinRoot() error { return params.node.Pinning.Flush() } -func (params *Adder) Finalize(DAG dag.DAGService) (*dag.Node, error) { - return params.editor.Finalize(DAG) +func (params *Adder) Finalize() (*dag.Node, error) { + return params.mr.GetValue().GetNode() } // Add builds a merkledag from the a reader, pinning all objects to the local @@ -163,7 +171,10 @@ func Add(n *core.IpfsNode, r io.Reader) (string, error) { unlock := n.Blockstore.PinLock() defer unlock() - fileAdder := NewAdder(n.Context(), n, nil) + fileAdder, err := NewAdder(n.Context(), n, nil) + if err != nil { + return "", err + } node, err := fileAdder.add(r) if err != nil { @@ -193,14 +204,22 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { } defer f.Close() - fileAdder := NewAdder(n.Context(), n, nil) + fileAdder, err := NewAdder(n.Context(), n, nil) + if err != nil { + return "", err + } - dagnode, err := fileAdder.AddFile(f) + err = fileAdder.AddFile(f) if err != nil { return "", err } - k, err := dagnode.Key() + nd, err := fileAdder.Finalize() + if err != nil { + return "", err + } + + k, err := nd.Key() if err != nil { return "", err } @@ -215,18 +234,29 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.Node, error) { file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil) dir := files.NewSliceFile("", "", []files.File{file}) - fileAdder := NewAdder(n.Context(), n, nil) + fileAdder, err := NewAdder(n.Context(), n, nil) + if err != nil { + return "", nil, err + } unlock := n.Blockstore.PinLock() defer unlock() - dagnode, err := fileAdder.addDir(dir) + + err = fileAdder.addDir(dir) + if err != nil { + return "", nil, err + } + + dagnode, err := fileAdder.Finalize() if err != nil { return "", nil, err } + k, err := dagnode.Key() if err != nil { return "", nil, err } + return gopath.Join(k.String(), filename), dagnode, nil } @@ -241,19 +271,22 @@ func (params *Adder) addNode(node *dag.Node, path string) error { path = key.Pretty() } - if err := params.editor.InsertNodeAtPath(params.ctx, path, node, newDirNode); err != nil { + if err := mfs.PutNode(params.mr, path, node); err != nil { return err } - return outputDagnode(params.out, path, node) + if !params.Silent { + return outputDagnode(params.out, path, node) + } + return nil } // Add the given file while respecting the params. -func (params *Adder) AddFile(file files.File) (*dag.Node, error) { +func (params *Adder) AddFile(file files.File) error { switch { case files.IsHidden(file) && !params.Hidden: log.Debugf("%s is hidden, skipping", file.FileName()) - return nil, &hiddenFileError{file.FileName()} + return &hiddenFileError{file.FileName()} case file.IsDirectory(): return params.addDir(file) } @@ -262,17 +295,16 @@ func (params *Adder) AddFile(file files.File) (*dag.Node, error) { if s, ok := file.(*files.Symlink); ok { sdata, err := unixfs.SymlinkData(s.Target) if err != nil { - return nil, err + return err } dagnode := &dag.Node{Data: sdata} _, err = params.node.DAG.Add(dagnode) if err != nil { - return nil, err + return err } - err = params.addNode(dagnode, s.FileName()) - return dagnode, err + return params.addNode(dagnode, s.FileName()) } // case for regular file @@ -285,52 +317,40 @@ func (params *Adder) AddFile(file files.File) (*dag.Node, error) { dagnode, err := params.add(reader) if err != nil { - return nil, err + return err } // patch it into the root - log.Infof("adding file: %s", file.FileName()) - err = params.addNode(dagnode, file.FileName()) - return dagnode, err + return params.addNode(dagnode, file.FileName()) } -func (params *Adder) addDir(dir files.File) (*dag.Node, error) { - tree := newDirNode() +func (params *Adder) addDir(dir files.File) error { log.Infof("adding directory: %s", dir.FileName()) + err := mfs.Mkdir(params.mr, dir.FileName(), true) + if err != nil { + return err + } + for { file, err := dir.NextFile() if err != nil && err != io.EOF { - return nil, err + return err } if file == nil { break } - node, err := params.AddFile(file) + err = params.AddFile(file) if _, ok := err.(*hiddenFileError); ok { // hidden file error, skip file continue } else if err != nil { - return nil, err - } - - _, name := gopath.Split(file.FileName()) - - if err := tree.AddNodeLinkClean(name, node); err != nil { - return nil, err + return err } } - if err := params.addNode(tree, dir.FileName()); err != nil { - return nil, err - } - - if _, err := params.node.DAG.Add(tree); err != nil { - return nil, err - } - - return tree, nil + return nil } // outputDagnode sends dagnode info over the output channel @@ -379,7 +399,7 @@ func getOutput(dagnode *dag.Node) (*Object, error) { for i, link := range dagnode.Links { output.Links[i] = Link{ Name: link.Name, - Hash: link.Hash.B58String(), + //Hash: link.Hash.B58String(), Size: link.Size, } } diff --git a/exchange/bitswap/workers.go b/exchange/bitswap/workers.go index 04d9fc2d29f..fbf0d20db73 100644 --- a/exchange/bitswap/workers.go +++ b/exchange/bitswap/workers.go @@ -89,7 +89,7 @@ func (bs *Bitswap) provideWorker(px process.Process) { defer cancel() if err := bs.network.Provide(ctx, k); err != nil { - log.Error(err) + //log.Error(err) } } diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index b84327dfdf3..0486e3321fc 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -3,6 +3,7 @@ package merkledag import ( "fmt" + "time" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" blocks "github.com/ipfs/go-ipfs/blocks" @@ -48,6 +49,14 @@ func (n *dagService) Add(nd *Node) (key.Key, error) { if n == nil { // FIXME remove this assertion. protect with constructor invariant return "", fmt.Errorf("dagService is nil") } + /* + start := time.Now() + defer func() { + took := time.Now().Sub(start) + log.Error("add took: %s", took) + }() + */ + _ = time.Saturday d, err := nd.Encoded(false) if err != nil { diff --git a/mfs/system.go b/mfs/system.go index 22ef63cd4a2..a7aeb2b20f6 100644 --- a/mfs/system.go +++ b/mfs/system.go @@ -71,15 +71,19 @@ func NewRoot(parent context.Context, ds dag.DAGService, node *dag.Node, pf PubFu return nil, err } + var repub *Republisher + if pf != nil { + repub = NewRepublisher(parent, pf, time.Millisecond*300, time.Second*3) + repub.setVal(ndk) + go repub.Run() + } + root := &Root{ node: node, - repub: NewRepublisher(parent, pf, time.Millisecond*300, time.Second*3), + repub: repub, dserv: ds, } - root.repub.setVal(ndk) - go root.repub.Run() - pbn, err := ft.FromBytes(node.Data) if err != nil { log.Error("IPNS pointer was not unixfs node") @@ -113,12 +117,17 @@ func (kr *Root) closeChild(name string, nd *dag.Node) error { return err } - kr.repub.Update(k) + if kr.repub != nil { + kr.repub.Update(k) + } return nil } func (kr *Root) Close() error { - return kr.repub.Close() + if kr.repub != nil { + return kr.repub.Close() + } + return nil } // Republisher manages when to publish a given entry From 07e20d28cf66cf3a2d71a5bff12405bbbfa59c4f Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 4 Dec 2015 15:17:31 -0800 Subject: [PATCH 04/13] enfastify mfs License: MIT Signed-off-by: Jeromy --- core/commands/add.go | 3 -- core/coreunix/add.go | 84 +++++++++++++++++++++++++++++++++----------- mfs/dir.go | 27 +++++--------- 3 files changed, 73 insertions(+), 41 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 9d323a5d752..df8c124ba76 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -61,9 +61,6 @@ remains to be implemented. req.SetOption(progressOptionName, true) - log.Error("SKIPPING SIZE") - return nil - sizeFile, ok := req.Files().(files.SizeFile) if !ok { // we don't need to error, the progress bar just won't know how big the files are diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 59cf7ada70d..2bc2e37fea2 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -1,6 +1,7 @@ package coreunix import ( + "bytes" "fmt" "io" "io/ioutil" @@ -27,6 +28,8 @@ import ( var log = logging.Logger("coreunix") +var folderData = unixfs.FolderPBData() + // how many bytes of progress to wait before sending a progress update message const progressReaderIncrement = 1024 * 256 @@ -118,29 +121,26 @@ func (params Adder) add(reader io.Reader) (*dag.Node, error) { } func (params *Adder) RootNode() (*dag.Node, error) { - return params.mr.GetValue().GetNode() - /* - // for memoizing - if params.root != nil { - return params.root, nil - } + // for memoizing + if params.root != nil { + return params.root, nil + } - root := params.editor.GetNode() + root, err := params.mr.GetValue().GetNode() + if err != nil { + return nil, err + } - // if not wrapping, AND one root file, use that hash as root. - if !params.Wrap && len(root.Links) == 1 { - var err error - root, err = root.Links[0].GetNode(params.ctx, params.editor.GetDagService()) - params.root = root - // no need to output, as we've already done so. - return root, err + // if not wrapping, AND one root file, use that hash as root. + if !params.Wrap && len(root.Links) == 1 { + root, err = root.Links[0].GetNode(params.ctx, params.node.DAG) + if err != nil { + return nil, err } + } - // otherwise need to output, as we have not. - err := outputDagnode(params.out, "", root) - params.root = root - return root, err - */ + params.root = root + return root, err } func (params *Adder) PinRoot() error { @@ -162,7 +162,51 @@ func (params *Adder) PinRoot() error { } func (params *Adder) Finalize() (*dag.Node, error) { - return params.mr.GetValue().GetNode() + root, err := params.mr.GetValue().GetNode() + if err != nil { + return nil, err + } + + params.RootNode() + + var name string + if !params.Wrap { + name = root.Links[0].Name + child, err := root.Links[0].GetNode(params.ctx, params.node.DAG) + if err != nil { + return nil, err + } + root = child + } + + err = params.outputDirs(name, root) + if err != nil { + return nil, err + } + + err = params.mr.Close() + if err != nil { + return nil, err + } + + return root, nil +} + +func (params *Adder) outputDirs(path string, nd *dag.Node) error { + for _, l := range nd.Links { + child, err := l.GetNode(params.ctx, params.node.DAG) + if err != nil { + return err + } + + if bytes.Equal(child.Data, folderData) { + err := params.outputDirs(gopath.Join(path, l.Name), child) + if err != nil { + return err + } + } + } + return outputDagnode(params.out, path, nd) } // Add builds a merkledag from the a reader, pinning all objects to the local diff --git a/mfs/dir.go b/mfs/dir.go index b86c98d77a3..ece79adeb05 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "sync" + "time" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" @@ -28,6 +29,8 @@ type Directory struct { node *dag.Node ctx context.Context + modTime time.Time + name string } @@ -40,6 +43,7 @@ func NewDirectory(ctx context.Context, name string, node *dag.Node, parent child parent: parent, childDirs: make(map[string]*Directory), files: make(map[string]*File), + modTime: time.Now(), } } @@ -72,6 +76,8 @@ func (d *Directory) updateChild(name string, nd *dag.Node) error { return err } + d.modTime = time.Now() + return nil } @@ -285,12 +291,7 @@ func (d *Directory) AddChild(name string, nd *dag.Node) error { d.Lock() defer d.Unlock() - pbn, err := ft.FromBytes(nd.Data) - if err != nil { - return err - } - - _, err = d.childUnsync(name) + _, err := d.childUnsync(name) if err == nil { return ErrDirExists } @@ -305,18 +306,8 @@ func (d *Directory) AddChild(name string, nd *dag.Node) error { return err } - switch pbn.GetType() { - case ft.TDirectory: - d.childDirs[name] = NewDirectory(d.ctx, name, nd, d, d.dserv) - case ft.TFile, ft.TMetadata, ft.TRaw: - nfi, err := NewFile(name, nd, d, d.dserv) - if err != nil { - return err - } - d.files[name] = nfi - default: - return ErrInvalidChild - } + d.modTime = time.Now() + return d.parent.closeChild(d.name, d.node) } From e81235d37375abe20cf9f77a607af628522147ee Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 4 Dec 2015 17:18:16 -0800 Subject: [PATCH 05/13] fix some tests License: MIT Signed-off-by: Jeromy --- .../src/github.com/cheggaaa/pb/pb.go | 6 +- commands/cli/parse.go | 58 +++++++++++-------- core/commands/add.go | 27 ++++++--- core/coreunix/add.go | 13 +++-- mfs/dir.go | 3 +- test/sharness/t0042-add-skip.sh | 2 +- test/sharness/t0043-add-w.sh | 10 ++-- test/sharness/t0045-ls.sh | 4 +- test/sharness/t0080-repo.sh | 5 -- 9 files changed, 75 insertions(+), 53 deletions(-) diff --git a/Godeps/_workspace/src/github.com/cheggaaa/pb/pb.go b/Godeps/_workspace/src/github.com/cheggaaa/pb/pb.go index 104bd4a60e2..d58fb8e943a 100644 --- a/Godeps/_workspace/src/github.com/cheggaaa/pb/pb.go +++ b/Godeps/_workspace/src/github.com/cheggaaa/pb/pb.go @@ -100,7 +100,7 @@ func (pb *ProgressBar) Start() *ProgressBar { pb.ShowBar = false pb.ShowTimeLeft = false pb.ShowPercent = false - } + } if !pb.ManualUpdate { go pb.writer() } @@ -233,7 +233,7 @@ func (pb *ProgressBar) write(current int64) { percent := float64(current) / (float64(pb.Total) / float64(100)) percentBox = fmt.Sprintf(" %#.02f %% ", percent) } - + // counters if pb.ShowCounters { if pb.Total > 0 { @@ -271,7 +271,7 @@ func (pb *ProgressBar) write(current int64) { // bar if pb.ShowBar { size := width - len(countersBox+pb.BarStart+pb.BarEnd+percentBox+timeLeftBox+speedBox+pb.prefix+pb.postfix) - if size > 0 { + if size > 0 && pb.Total > 0 { curCount := int(math.Ceil((float64(current) / float64(pb.Total)) * float64(size))) emptCount := size - curCount barBox = pb.BarStart diff --git a/commands/cli/parse.go b/commands/cli/parse.go index 59d283707ab..6de79e6035b 100644 --- a/commands/cli/parse.go +++ b/commands/cli/parse.go @@ -6,6 +6,7 @@ import ( "os" "path" "runtime" + "sort" "strings" cmds "github.com/ipfs/go-ipfs/commands" @@ -259,8 +260,8 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi } stringArgs := make([]string, 0, numInputs) - fileArgs := make([]files.File, 0, numInputs) + fileArgs := make(map[string]files.File) argDefIndex := 0 // the index of the current argument definition for i := 0; i < numInputs; i++ { argDef := getArgDef(argDefIndex, argDefs) @@ -295,18 +296,21 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi } else if argDef.Type == cmds.ArgFile { if stdin == nil || !argDef.SupportsStdin { // treat stringArg values as file paths - fileArgs, inputs, err = appendFile(fileArgs, inputs, argDef, recursive) + fpath := inputs[0] + inputs = inputs[1:] + file, err := appendFile(fpath, argDef, recursive) if err != nil { return nil, nil, err } + fileArgs[fpath] = file } else { if len(inputs) > 0 { // don't use stdin if we have inputs stdin = nil } else { // if we have a stdin, create a file from it - fileArgs, stdin = appendStdinAsFile(fileArgs, stdin) + fileArgs[""] = files.NewReaderFile("", "", stdin, nil) } } } @@ -323,7 +327,23 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi } } - return stringArgs, fileArgs, nil + return stringArgs, filesMapToSortedArr(fileArgs), nil +} + +func filesMapToSortedArr(fs map[string]files.File) []files.File { + var names []string + for name, _ := range fs { + names = append(names, name) + } + + sort.Strings(names) + + var out []files.File + for _, f := range names { + out = append(out, fs[f]) + } + + return out } func getArgDef(i int, argDefs []cmds.Argument) *cmds.Argument { @@ -356,44 +376,34 @@ func appendStdinAsString(args []string, stdin *os.File) ([]string, *os.File, err return append(args, strings.Split(input, "\n")...), nil, nil } -func appendFile(args []files.File, inputs []string, argDef *cmds.Argument, recursive bool) ([]files.File, []string, error) { - fpath := inputs[0] +const notRecursiveFmtStr = "'%s' is a directory, use the '-%s' flag to specify directories" +const dirNotSupportedFmtStr = "Invalid path '%s', argument '%s' does not support directories" + +func appendFile(fpath string, argDef *cmds.Argument, recursive bool) (files.File, error) { if fpath == "." { cwd, err := os.Getwd() if err != nil { - return nil, nil, err + return nil, err } fpath = cwd } + stat, err := os.Lstat(fpath) if err != nil { - return nil, nil, err + return nil, err } if stat.IsDir() { if !argDef.Recursive { - err = fmt.Errorf("Invalid path '%s', argument '%s' does not support directories", - fpath, argDef.Name) - return nil, nil, err + return nil, fmt.Errorf(dirNotSupportedFmtStr, fpath, argDef.Name) } if !recursive { - err = fmt.Errorf("'%s' is a directory, use the '-%s' flag to specify directories", - fpath, cmds.RecShort) - return nil, nil, err + return nil, fmt.Errorf(notRecursiveFmtStr, fpath, cmds.RecShort) } } - arg, err := files.NewSerialFile(path.Base(fpath), fpath, stat) - if err != nil { - return nil, nil, err - } - return append(args, arg), inputs[1:], nil -} - -func appendStdinAsFile(args []files.File, stdin *os.File) ([]files.File, *os.File) { - arg := files.NewReaderFile("", "", stdin, nil) - return append(args, arg), nil + return files.NewSerialFile(path.Base(fpath), fpath, stat) } // isTerminal returns true if stdin is a Stdin pipe (e.g. `cat file | ipfs`), diff --git a/core/commands/add.go b/core/commands/add.go index df8c124ba76..3ec912b71f0 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -59,7 +59,13 @@ remains to be implemented. return nil } - req.SetOption(progressOptionName, true) + // ipfs cli progress bar defaults to true + progress, found, _ := req.Option(progressOptionName).Bool() + if !found { + progress = true + } + + req.SetOption(progressOptionName, progress) sizeFile, ok := req.Files().(files.SizeFile) if !ok { @@ -201,13 +207,18 @@ remains to be implemented. return } - progress, _, err := req.Option(progressOptionName).Bool() + progress, prgFound, err := req.Option(progressOptionName).Bool() if err != nil { res.SetError(u.ErrCast(), cmds.ErrNormal) return } - showProgressBar := !quiet || progress + var showProgressBar bool + if prgFound { + showProgressBar = progress + } else if !quiet { + showProgressBar = true + } var bar *pb.ProgressBar var terminalWidth int @@ -279,10 +290,12 @@ remains to be implemented. bar.Update() } case size := <-sizeChan: - bar.Total = size - bar.ShowPercent = true - bar.ShowBar = true - bar.ShowTimeLeft = true + if showProgressBar { + bar.Total = size + bar.ShowPercent = true + bar.ShowBar = true + bar.ShowTimeLeft = true + } } } }, diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 2bc2e37fea2..8180b5bef33 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -193,19 +193,22 @@ func (params *Adder) Finalize() (*dag.Node, error) { } func (params *Adder) outputDirs(path string, nd *dag.Node) error { + if !bytes.Equal(nd.Data, folderData) { + return nil + } + for _, l := range nd.Links { child, err := l.GetNode(params.ctx, params.node.DAG) if err != nil { return err } - if bytes.Equal(child.Data, folderData) { - err := params.outputDirs(gopath.Join(path, l.Name), child) - if err != nil { - return err - } + err = params.outputDirs(gopath.Join(path, l.Name), child) + if err != nil { + return err } } + return outputDagnode(params.out, path, nd) } diff --git a/mfs/dir.go b/mfs/dir.go index ece79adeb05..43271fe490f 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -308,7 +308,8 @@ func (d *Directory) AddChild(name string, nd *dag.Node) error { d.modTime = time.Now() - return d.parent.closeChild(d.name, d.node) + //return d.parent.closeChild(d.name, d.node) + return nil } func (d *Directory) sync() error { diff --git a/test/sharness/t0042-add-skip.sh b/test/sharness/t0042-add-skip.sh index f0d4c6fd253..d5f7997984f 100755 --- a/test/sharness/t0042-add-skip.sh +++ b/test/sharness/t0042-add-skip.sh @@ -38,11 +38,11 @@ test_add_skip() { cat >expected <<-\EOF && added QmcAREBcjgnUpKfyFmUGnfajA1NQS5ydqRp7WfqZ6JF8Dx planets/.asteroids/ceres.txt added QmZ5eaLybJ5GUZBNwy24AA9EEDTDpA4B8qXnuN3cGxu2uF planets/.asteroids/pallas.txt - added Qmf6rbs5GF85anDuoxpSAdtuZPM9D2Yt3HngzjUVSQ7kDV planets/.asteroids added QmaowqjedBkUrMUXgzt9c2ZnAJncM9jpJtkFfgdFstGr5a planets/.charon.txt added QmU4zFD5eJtRBsWC63AvpozM9Atiadg9kPVTuTrnCYJiNF planets/.pluto.txt added QmZy3khu7qf696i5HtkgL2NotsCZ8wzvNZJ1eUdA5n8KaV planets/mars.txt added QmQnv4m3Q5512zgVtpbJ9z85osQrzZzGRn934AGh6iVEXz planets/venus.txt + added Qmf6rbs5GF85anDuoxpSAdtuZPM9D2Yt3HngzjUVSQ7kDV planets/.asteroids added QmetajtFdmzhWYodAsZoVZSiqpeJDAiaw2NwbM3xcWcpDj planets EOF test_cmp expected actual diff --git a/test/sharness/t0043-add-w.sh b/test/sharness/t0043-add-w.sh index d4f7decaa12..f875f21b681 100755 --- a/test/sharness/t0043-add-w.sh +++ b/test/sharness/t0043-add-w.sh @@ -15,8 +15,8 @@ add_w_12='added Qme987pqNBhZZXy4ckeXiR7zaRQwBabB7fTgHurW2yJfNu 4r93 added QmVb4ntSZZnT2J2zvCmXKMJc52cmZYH6AB37MzeYewnkjs 4u6ead added QmZPASVB6EsADrLN8S2sak34zEHL8mx4TAVsPJU9cNnQQJ ' -add_w_21='added QmVb4ntSZZnT2J2zvCmXKMJc52cmZYH6AB37MzeYewnkjs 4u6ead -added Qme987pqNBhZZXy4ckeXiR7zaRQwBabB7fTgHurW2yJfNu 4r93 +add_w_21='added Qme987pqNBhZZXy4ckeXiR7zaRQwBabB7fTgHurW2yJfNu 4r93 +added QmVb4ntSZZnT2J2zvCmXKMJc52cmZYH6AB37MzeYewnkjs 4u6ead added QmZPASVB6EsADrLN8S2sak34zEHL8mx4TAVsPJU9cNnQQJ ' add_w_d1='added QmPcaX84tDiTfzdTn8GQxexodgeWH6mHjSss5Zfr5ojssb _jo7/-s782qgs @@ -27,20 +27,20 @@ added QmYC3u5jGWuyFwvTxtvLYm2K3SpWZ31tg3NjpVVvh9cJaJ _jo7/wzvsihy added QmQkib3f9XNX5sj6WEahLUPFpheTcwSRJwUCSvjcv8b9by _jo7 added QmNQoesMj1qp8ApE51NbtTjFYksyzkezPD4cat7V2kzbKN ' -add_w_d2='added QmVaKAt2eVftNKFfKhiBV7Mu5HjCugffuLqWqobSSFgiA7 h3qpecj0 +add_w_d2='added Qme987pqNBhZZXy4ckeXiR7zaRQwBabB7fTgHurW2yJfNu 4r93 added QmU9Jqks8TPu4vFr6t7EKkAKQrSJuEujNj1AkzoCeTEDFJ gnz66h/1k0xpx34 added QmSLYZycXAufRw3ePMVH2brbtYWCcWsmksGLbHcT8ia9Ke gnz66h/9cwudvacx added QmfYmpCCAMU9nLe7xbrYsHf5z2R2GxeQnsm4zavUhX9vq2 gnz66h/9ximv51cbo8 added QmWgEE4e2kfx3b8HZcBk5cLrfhoi8kTMQP2MipgPhykuV3 gnz66h/b54ygh6gs added QmcLbqEqhREGednc6mrVtanee4WHKp5JnUfiwTTHCJwuDf gnz66h/lbl5 -added QmVPwNy8pZegpsNmsjjZvdTQn4uCeuZgtzhgWhRSQWjK9x gnz66h added QmPcaX84tDiTfzdTn8GQxexodgeWH6mHjSss5Zfr5ojssb _jo7/-s782qgs added QmaVBqquUuXKjkyWHXaXfsaQUxAnsCKS95VRDHU8PzGA4K _jo7/15totauzkak- added QmaAHFG8cmhW3WLjofx5siSp44VV25ETN6ThzrU8iAqpkR _jo7/galecuirrj4r added QmeuSfhJNKwBESp1W9H8cfoMdBfW3AeHQDWXbNXQJYWp53 _jo7/mzo50r-1xidf5zx added QmYC3u5jGWuyFwvTxtvLYm2K3SpWZ31tg3NjpVVvh9cJaJ _jo7/wzvsihy +added QmVaKAt2eVftNKFfKhiBV7Mu5HjCugffuLqWqobSSFgiA7 h3qpecj0 +added QmVPwNy8pZegpsNmsjjZvdTQn4uCeuZgtzhgWhRSQWjK9x gnz66h added QmQkib3f9XNX5sj6WEahLUPFpheTcwSRJwUCSvjcv8b9by _jo7 -added Qme987pqNBhZZXy4ckeXiR7zaRQwBabB7fTgHurW2yJfNu 4r93 added QmTmc46fhKC8Liuh5soy1VotdnHcqLu3r6HpPGwDZCnqL1 ' add_w_r='QmcCksBMDuuyuyfAMMNzEAx6Z7jTrdRy9a23WpufAhG9ji' diff --git a/test/sharness/t0045-ls.sh b/test/sharness/t0045-ls.sh index 4ad0acf89fc..8ba9e8ccdde 100755 --- a/test/sharness/t0045-ls.sh +++ b/test/sharness/t0045-ls.sh @@ -27,12 +27,12 @@ test_ls_cmd() { cat <<-\EOF >expected_add && added QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe testData/d1/128 added QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN testData/d1/a - added QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss testData/d1 added QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd testData/d2/1024 added QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL testData/d2/a - added QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy testData/d2 added QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH testData/f1 added QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M testData/f2 + added QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss testData/d1 + added QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy testData/d2 added QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj testData EOF test_cmp expected_add actual_add diff --git a/test/sharness/t0080-repo.sh b/test/sharness/t0080-repo.sh index 1ab6238096d..dc7c49e0ca3 100755 --- a/test/sharness/t0080-repo.sh +++ b/test/sharness/t0080-repo.sh @@ -29,11 +29,6 @@ test_expect_success "'ipfs repo gc' succeeds" ' ipfs repo gc >gc_out_actual ' -test_expect_success "'ipfs repo gc' looks good (patch root)" ' - PATCH_ROOT=QmQXirSbubiySKnqaFyfs5YzziXRB5JEVQVjU6xsd7innr && - grep "removed $PATCH_ROOT" gc_out_actual -' - test_expect_success "'ipfs repo gc' doesnt remove file" ' ipfs cat "$HASH" >out && test_cmp out afile From 559860c79fdc4965162244b84df9c5b9baaf1e10 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 4 Dec 2015 17:44:08 -0800 Subject: [PATCH 06/13] slight cleanup License: MIT Signed-off-by: Jeromy --- merkledag/merkledag.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 0486e3321fc..b84327dfdf3 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -3,7 +3,6 @@ package merkledag import ( "fmt" - "time" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" blocks "github.com/ipfs/go-ipfs/blocks" @@ -49,14 +48,6 @@ func (n *dagService) Add(nd *Node) (key.Key, error) { if n == nil { // FIXME remove this assertion. protect with constructor invariant return "", fmt.Errorf("dagService is nil") } - /* - start := time.Now() - defer func() { - took := time.Now().Sub(start) - log.Error("add took: %s", took) - }() - */ - _ = time.Saturday d, err := nd.Encoded(false) if err != nil { From 742f6da453bd9b0cc0638417049042297edd0e30 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Fri, 4 Dec 2015 21:09:26 -0800 Subject: [PATCH 07/13] fixify tests License: MIT Signed-off-by: Jeromy --- core/coreunix/add.go | 4 ++-- core/coreunix/add_test.go | 10 ++-------- mfs/ops.go | 3 +++ mfs/system.go | 12 ++++++++++++ test/sharness/t0200-unixfs-ls.sh | 4 ++-- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 8180b5bef33..6ca989b2649 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -280,16 +280,16 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { // the directory, and and error if any. func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.Node, error) { file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil) - dir := files.NewSliceFile("", "", []files.File{file}) fileAdder, err := NewAdder(n.Context(), n, nil) if err != nil { return "", nil, err } + fileAdder.Wrap = true unlock := n.Blockstore.PinLock() defer unlock() - err = fileAdder.addDir(dir) + err = fileAdder.AddFile(file) if err != nil { return "", nil, err } diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index 6d4bfb17656..279d7ce74d5 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -1,8 +1,6 @@ package coreunix import ( - "os" - "path" "testing" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" @@ -13,10 +11,6 @@ import ( ) func TestAddRecursive(t *testing.T) { - here, err := os.Getwd() - if err != nil { - t.Fatal(err) - } r := &repo.Mock{ C: config.Config{ Identity: config.Identity{ @@ -29,9 +23,9 @@ func TestAddRecursive(t *testing.T) { if err != nil { t.Fatal(err) } - if k, err := AddR(node, path.Join(here, "test_data")); err != nil { + if k, err := AddR(node, "test_data"); err != nil { t.Fatal(err) } else if k != "QmWCCga8AbTyfAQ7pTnGT6JgmRMAB3Qp8ZmTEFi5q5o8jC" { - t.Fatal("keys do not match") + t.Fatal("keys do not match: ", k) } } diff --git a/mfs/ops.go b/mfs/ops.go index c7309a31d9d..ebb1932edeb 100644 --- a/mfs/ops.go +++ b/mfs/ops.go @@ -101,6 +101,9 @@ func PutNode(r *Root, path string, nd *dag.Node) error { // Mkdir creates a directory at 'path' under the directory 'd', creating // intermediary directories as needed if 'parents' is set to true func Mkdir(r *Root, pth string, parents bool) error { + if pth == "" { + panic("empty path") + } parts := path.SplitList(pth) if parts[0] == "" { parts = parts[1:] diff --git a/mfs/system.go b/mfs/system.go index a7aeb2b20f6..2cfc4e201fd 100644 --- a/mfs/system.go +++ b/mfs/system.go @@ -124,9 +124,21 @@ func (kr *Root) closeChild(name string, nd *dag.Node) error { } func (kr *Root) Close() error { + nd, err := kr.GetValue().GetNode() + if err != nil { + return err + } + + k, err := kr.dserv.Add(nd) + if err != nil { + return err + } + if kr.repub != nil { + kr.repub.Update(k) return kr.repub.Close() } + return nil } diff --git a/test/sharness/t0200-unixfs-ls.sh b/test/sharness/t0200-unixfs-ls.sh index 1b889987d4d..ea386b98ce5 100755 --- a/test/sharness/t0200-unixfs-ls.sh +++ b/test/sharness/t0200-unixfs-ls.sh @@ -27,12 +27,12 @@ test_ls_cmd() { cat <<-\EOF >expected_add && added QmQNd6ubRXaNG6Prov8o6vk3bn6eWsj9FxLGrAVDUAGkGe testData/d1/128 added QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN testData/d1/a - added QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss testData/d1 added QmbQBUSRL9raZtNXfpTDeaxQapibJEG6qEY8WqAN22aUzd testData/d2/1024 added QmaRGe7bVmVaLmxbrMiVNXqW4pRNNp3xq7hFtyRKA3mtJL testData/d2/a - added QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy testData/d2 added QmeomffUNfmQy76CQGy9NdmqEnnHU9soCexBnGU3ezPHVH testData/f1 added QmNtocSs7MoDkJMc1RkyisCSKvLadujPsfJfSdJ3e1eA1M testData/f2 + added QmSix55yz8CzWXf5ZVM9vgEvijnEeeXiTSarVtsqiiCJss testData/d1 + added QmR3jhV4XpxxPjPT3Y8vNnWvWNvakdcT3H6vqpRBsX1MLy testData/d2 added QmfNy183bXiRVyrhyWtq3TwHn79yHEkiAGFr18P7YNzESj testData EOF test_cmp expected_add actual_add From 7341486391579f9b55a6b6b8eec13735531ce98b Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 5 Dec 2015 20:31:25 -0800 Subject: [PATCH 08/13] Allow for gc during adds License: MIT Signed-off-by: Jeromy --- blocks/blockstore/blockstore.go | 15 ++++++++- blocks/blockstore/write_cache.go | 4 +++ core/coreunix/add.go | 58 ++++++++++++++++++++++++++------ pin/gc/gc.go | 2 +- 4 files changed, 66 insertions(+), 13 deletions(-) diff --git a/blocks/blockstore/blockstore.go b/blocks/blockstore/blockstore.go index bc000df932a..59f0f2c72ce 100644 --- a/blocks/blockstore/blockstore.go +++ b/blocks/blockstore/blockstore.go @@ -5,6 +5,7 @@ package blockstore import ( "errors" "sync" + "sync/atomic" ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" dsns "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace" @@ -49,6 +50,10 @@ type GCBlockstore interface { // at the same time, but no GC should not happen simulatenously. // Reading during Pinning is safe, and requires no lock. PinLock() func() + + // GcRequested returns true if GCLock has been called and is waiting to + // take the lock + GCRequested() bool } func NewBlockstore(d ds.Batching) *blockstore { @@ -63,7 +68,9 @@ func NewBlockstore(d ds.Batching) *blockstore { type blockstore struct { datastore ds.Batching - lk sync.RWMutex + lk sync.RWMutex + gcreq int32 + gcreqlk sync.Mutex } func (bs *blockstore) Get(k key.Key) (*blocks.Block, error) { @@ -192,7 +199,9 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan key.Key, error) { } func (bs *blockstore) GCLock() func() { + atomic.AddInt32(&bs.gcreq, 1) bs.lk.Lock() + atomic.AddInt32(&bs.gcreq, -1) return bs.lk.Unlock } @@ -200,3 +209,7 @@ func (bs *blockstore) PinLock() func() { bs.lk.RLock() return bs.lk.RUnlock } + +func (bs *blockstore) GCRequested() bool { + return atomic.LoadInt32(&bs.gcreq) > 0 +} diff --git a/blocks/blockstore/write_cache.go b/blocks/blockstore/write_cache.go index 52af696e4ae..73a7813f5ae 100644 --- a/blocks/blockstore/write_cache.go +++ b/blocks/blockstore/write_cache.go @@ -66,3 +66,7 @@ func (w *writecache) GCLock() func() { func (w *writecache) PinLock() func() { return w.blockstore.(GCBlockstore).PinLock() } + +func (w *writecache) GCRequested() bool { + return w.blockstore.(GCBlockstore).GCRequested() +} diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 6ca989b2649..80a4bb6adb2 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -12,6 +12,7 @@ import ( syncds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync" context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" bstore "github.com/ipfs/go-ipfs/blocks/blockstore" + key "github.com/ipfs/go-ipfs/blocks/key" bserv "github.com/ipfs/go-ipfs/blockservice" "github.com/ipfs/go-ipfs/exchange/offline" importer "github.com/ipfs/go-ipfs/importer" @@ -99,6 +100,8 @@ type Adder struct { Chunker string root *dag.Node mr *mfs.Root + unlock func() + tempRoot key.Key } // Perform the actual add & pin locally, outputting results to reader @@ -157,6 +160,14 @@ func (params *Adder) PinRoot() error { return err } + if params.tempRoot != "" { + err := params.node.Pinning.Unpin(params.ctx, params.tempRoot, true) + if err != nil { + return err + } + params.tempRoot = rnk + } + params.node.Pinning.PinWithMode(rnk, pin.Recursive) return params.node.Pinning.Flush() } @@ -256,7 +267,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { return "", err } - err = fileAdder.AddFile(f) + err = fileAdder.addFile(f) if err != nil { return "", err } @@ -289,7 +300,7 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.No unlock := n.Blockstore.PinLock() defer unlock() - err = fileAdder.AddFile(file) + err = fileAdder.addFile(file) if err != nil { return "", nil, err } @@ -330,12 +341,24 @@ func (params *Adder) addNode(node *dag.Node, path string) error { // Add the given file while respecting the params. func (params *Adder) AddFile(file files.File) error { + params.unlock = params.node.Blockstore.PinLock() + defer params.unlock() + + return params.addFile(file) +} + +func (adder *Adder) addFile(file files.File) error { + err := adder.maybePauseForGC() + if err != nil { + return err + } + switch { - case files.IsHidden(file) && !params.Hidden: + case files.IsHidden(file) && !adder.Hidden: log.Debugf("%s is hidden, skipping", file.FileName()) return &hiddenFileError{file.FileName()} case file.IsDirectory(): - return params.addDir(file) + return adder.addDir(file) } // case for symlink @@ -346,29 +369,29 @@ func (params *Adder) AddFile(file files.File) error { } dagnode := &dag.Node{Data: sdata} - _, err = params.node.DAG.Add(dagnode) + _, err = adder.node.DAG.Add(dagnode) if err != nil { return err } - return params.addNode(dagnode, s.FileName()) + return adder.addNode(dagnode, s.FileName()) } // case for regular file // if the progress flag was specified, wrap the file so that we can send // progress updates to the client (over the output channel) var reader io.Reader = file - if params.Progress { - reader = &progressReader{file: file, out: params.out} + if adder.Progress { + reader = &progressReader{file: file, out: adder.out} } - dagnode, err := params.add(reader) + dagnode, err := adder.add(reader) if err != nil { return err } // patch it into the root - return params.addNode(dagnode, file.FileName()) + return adder.addNode(dagnode, file.FileName()) } func (params *Adder) addDir(dir files.File) error { @@ -388,7 +411,7 @@ func (params *Adder) addDir(dir files.File) error { break } - err = params.AddFile(file) + err = params.addFile(file) if _, ok := err.(*hiddenFileError); ok { // hidden file error, skip file continue @@ -400,6 +423,19 @@ func (params *Adder) addDir(dir files.File) error { return nil } +func (adder *Adder) maybePauseForGC() error { + if adder.node.Blockstore.GCRequested() { + err := adder.PinRoot() + if err != nil { + return err + } + + adder.unlock() + adder.unlock = adder.node.Blockstore.PinLock() + } + return nil +} + // outputDagnode sends dagnode info over the output channel func outputDagnode(out chan interface{}, name string, dn *dag.Node) error { if out == nil { diff --git a/pin/gc/gc.go b/pin/gc/gc.go index ec61f816a44..df9ddedc6b2 100644 --- a/pin/gc/gc.go +++ b/pin/gc/gc.go @@ -24,7 +24,6 @@ var log = logging.Logger("gc") // deletes any block that is not found in the marked set. func GC(ctx context.Context, bs bstore.GCBlockstore, pn pin.Pinner) (<-chan key.Key, error) { unlock := bs.GCLock() - defer unlock() bsrv := bserv.New(bs, offline.Exchange(bs)) ds := dag.NewDAGService(bsrv) @@ -42,6 +41,7 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, pn pin.Pinner) (<-chan key. output := make(chan key.Key) go func() { defer close(output) + defer unlock() for { select { case k, ok := <-keychan: From 32cbdaccdb2d93271e70d7d985586496a831fd96 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 5 Dec 2015 23:42:34 -0800 Subject: [PATCH 09/13] Add test for running gc during an add License: MIT Signed-off-by: Jeromy --- core/coreunix/add.go | 2 +- core/coreunix/add_test.go | 130 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 1 deletion(-) diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 80a4bb6adb2..64bb6ad3062 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -155,7 +155,7 @@ func (params *Adder) PinRoot() error { return nil } - rnk, err := root.Key() + rnk, err := params.node.DAG.Add(root) if err != nil { return err } diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index 279d7ce74d5..56c921eebe0 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -1,10 +1,18 @@ package coreunix import ( + "bytes" + "io" + "io/ioutil" "testing" + "time" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + "github.com/ipfs/go-ipfs/blocks/key" + "github.com/ipfs/go-ipfs/commands/files" "github.com/ipfs/go-ipfs/core" + dag "github.com/ipfs/go-ipfs/merkledag" + "github.com/ipfs/go-ipfs/pin/gc" "github.com/ipfs/go-ipfs/repo" "github.com/ipfs/go-ipfs/repo/config" "github.com/ipfs/go-ipfs/util/testutil" @@ -29,3 +37,125 @@ func TestAddRecursive(t *testing.T) { t.Fatal("keys do not match: ", k) } } + +func TestAddGCLive(t *testing.T) { + r := &repo.Mock{ + C: config.Config{ + Identity: config.Identity{ + PeerID: "Qmfoo", // required by offline node + }, + }, + D: testutil.ThreadSafeCloserMapDatastore(), + } + node, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r}) + if err != nil { + t.Fatal(err) + } + + errs := make(chan error) + out := make(chan interface{}) + adder, err := NewAdder(context.Background(), node, out) + if err != nil { + t.Fatal(err) + } + + dataa := ioutil.NopCloser(bytes.NewBufferString("testfileA")) + rfa := files.NewReaderFile("a", "a", dataa, nil) + + // make two files with pipes so we can 'pause' the add for timing of the test + piper, pipew := io.Pipe() + hangfile := files.NewReaderFile("b", "b", piper, nil) + + datad := ioutil.NopCloser(bytes.NewBufferString("testfileD")) + rfd := files.NewReaderFile("d", "d", datad, nil) + + slf := files.NewSliceFile("files", "files", []files.File{rfa, hangfile, rfd}) + + addDone := make(chan struct{}) + go func() { + defer close(addDone) + defer close(out) + err := adder.AddFile(slf) + + if err != nil { + t.Fatal(err) + } + + }() + + addedHashes := make(map[string]struct{}) + select { + case o := <-out: + addedHashes[o.(*AddedObject).Hash] = struct{}{} + case <-addDone: + t.Fatal("add shouldnt complete yet") + } + + var gcout <-chan key.Key + gcstarted := make(chan struct{}) + go func() { + defer close(gcstarted) + gcchan, err := gc.GC(context.Background(), node.Blockstore, node.Pinning) + if err != nil { + log.Error("GC ERROR:", err) + errs <- err + return + } + + gcout = gcchan + }() + + // gc shouldnt start until we let the add finish its current file. + pipew.Write([]byte("some data for file b")) + + select { + case <-gcstarted: + t.Fatal("gc shouldnt have started yet") + case err := <-errs: + t.Fatal(err) + default: + } + + time.Sleep(time.Millisecond * 100) // make sure gc gets to requesting lock + + // finish write and unblock gc + pipew.Close() + + // receive next object from adder + select { + case o := <-out: + addedHashes[o.(*AddedObject).Hash] = struct{}{} + case err := <-errs: + t.Fatal(err) + } + + select { + case <-gcstarted: + case err := <-errs: + t.Fatal(err) + } + + for k := range gcout { + if _, ok := addedHashes[k.B58String()]; ok { + t.Fatal("gc'ed a hash we just added") + } + } + + var last key.Key + for a := range out { + // wait for it to finish + last = key.B58KeyDecode(a.(*AddedObject).Hash) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + root, err := node.DAG.Get(ctx, last) + if err != nil { + t.Fatal(err) + } + + err = dag.EnumerateChildren(ctx, node.DAG, root, key.NewKeySet()) + if err != nil { + t.Fatal(err) + } +} From c25386ea79940fc017b462ad7c65f88df010f5d1 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 6 Dec 2015 00:05:08 -0800 Subject: [PATCH 10/13] sort output in tests License: MIT Signed-off-by: Jeromy --- test/sharness/t0043-add-w.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/sharness/t0043-add-w.sh b/test/sharness/t0043-add-w.sh index f875f21b681..40e9649b74b 100755 --- a/test/sharness/t0043-add-w.sh +++ b/test/sharness/t0043-add-w.sh @@ -39,8 +39,8 @@ added QmaAHFG8cmhW3WLjofx5siSp44VV25ETN6ThzrU8iAqpkR _jo7/galecuirrj4r added QmeuSfhJNKwBESp1W9H8cfoMdBfW3AeHQDWXbNXQJYWp53 _jo7/mzo50r-1xidf5zx added QmYC3u5jGWuyFwvTxtvLYm2K3SpWZ31tg3NjpVVvh9cJaJ _jo7/wzvsihy added QmVaKAt2eVftNKFfKhiBV7Mu5HjCugffuLqWqobSSFgiA7 h3qpecj0 -added QmVPwNy8pZegpsNmsjjZvdTQn4uCeuZgtzhgWhRSQWjK9x gnz66h added QmQkib3f9XNX5sj6WEahLUPFpheTcwSRJwUCSvjcv8b9by _jo7 +added QmVPwNy8pZegpsNmsjjZvdTQn4uCeuZgtzhgWhRSQWjK9x gnz66h added QmTmc46fhKC8Liuh5soy1VotdnHcqLu3r6HpPGwDZCnqL1 ' add_w_r='QmcCksBMDuuyuyfAMMNzEAx6Z7jTrdRy9a23WpufAhG9ji' @@ -57,7 +57,7 @@ test_add_w() { random-files --seed 7547632 --files 5 --dirs 2 --depth 3 m && echo "$add_w_m" >expected && ipfs add -q -r m | tail -n1 >actual && - test_cmp expected actual + test_sort_cmp expected actual ' # test single file @@ -67,7 +67,7 @@ test_add_w() { test_expect_success "ipfs add -w (single file) is correct" ' echo "$add_w_1" >expected && - test_cmp expected actual + test_sort_cmp expected actual ' # test two files together @@ -77,7 +77,7 @@ test_add_w() { test_expect_success "ipfs add -w (multiple) is correct" ' echo "$add_w_12" >expected && - test_cmp expected actual + test_sort_cmp expected actual ' test_expect_success "ipfs add -w (multiple) succeeds" ' @@ -86,7 +86,7 @@ test_add_w() { test_expect_success "ipfs add -w (multiple) orders" ' echo "$add_w_21" >expected && - test_cmp expected actual + test_sort_cmp expected actual ' # test a directory @@ -96,7 +96,7 @@ test_add_w() { test_expect_success "ipfs add -w -r (dir) is correct" ' echo "$add_w_d1" >expected && - test_cmp expected actual + test_sort_cmp expected actual ' # test files and directory @@ -107,7 +107,7 @@ test_add_w() { test_expect_success "ipfs add -w -r is correct" ' echo "$add_w_d2" >expected && - test_cmp expected actual + test_sort_cmp expected actual ' # test -w -r m/* == -r m @@ -117,7 +117,7 @@ test_add_w() { test_expect_success "ipfs add -w -r m/* == add -r m is correct" ' echo "$add_w_m" >expected && - test_cmp expected actual + test_sort_cmp expected actual ' # test repeats together @@ -130,7 +130,7 @@ test_add_w() { test_expect_success "ipfs add -w (repeats) is correct" ' echo "$add_w_r" >expected && - test_cmp expected actual + test_sort_cmp expected actual ' } From 6af342ca23aacbad969d01f39f5a9a854d40fd03 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 6 Dec 2015 11:03:50 -0800 Subject: [PATCH 11/13] cleanup and more testing License: MIT Signed-off-by: Jeromy --- core/commands/add.go | 2 +- exchange/bitswap/workers.go | 2 +- test/sharness/t0250-files-api.sh | 15 ++++++++++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 3ec912b71f0..ef7c9748af0 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -45,7 +45,7 @@ remains to be implemented. Options: []cmds.Option{ cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive) cmds.BoolOption(quietOptionName, "q", "Write minimal output"), - cmds.BoolOption(silentOptionName, "x", "Write no output"), + cmds.BoolOption(silentOptionName, "Write no output"), cmds.BoolOption(progressOptionName, "p", "Stream progress data"), cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation"), cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk"), diff --git a/exchange/bitswap/workers.go b/exchange/bitswap/workers.go index fbf0d20db73..04d9fc2d29f 100644 --- a/exchange/bitswap/workers.go +++ b/exchange/bitswap/workers.go @@ -89,7 +89,7 @@ func (bs *Bitswap) provideWorker(px process.Process) { defer cancel() if err := bs.network.Provide(ctx, k); err != nil { - //log.Error(err) + log.Error(err) } } diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index e04f0884569..90b43081754 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -317,7 +317,13 @@ test_files_api() { ' test_expect_success "write 'no-flush' succeeds" ' - echo "testing" | ipfs files write -f -e /cats/walrus + echo "testing" | ipfs files write -f=false -e /cats/walrus + ' + + test_expect_success "root hash not bubbled up yet" ' + test -z "$ONLINE" || + (ipfs refs local > refsout && + test_expect_code 1 grep QmcwKfTMCT7AaeiD92hWjnZn9b6eh9NxnhfSzN5x2vnDpt refsout) ' test_expect_success "changes bubbled up to root on inspection" ' @@ -350,7 +356,14 @@ test_files_api() { # test offline and online test_files_api + +test_expect_success "clean up objects from previous test run" ' + ipfs repo gc +' + test_launch_ipfs_daemon + +ONLINE=1 # set online flag so tests can easily tell test_files_api test_kill_ipfs_daemon test_done From f04a79197174682db5a35dec70877e67e1906a08 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 7 Dec 2015 22:19:29 -0800 Subject: [PATCH 12/13] feedback from CR License: MIT Signed-off-by: Jeromy --- core/commands/add.go | 8 +++- core/coreunix/add.go | 89 ++++++++++++++++++------------------- test/sharness/t0080-repo.sh | 5 +++ 3 files changed, 56 insertions(+), 46 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index ef7c9748af0..152244be764 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -213,10 +213,16 @@ remains to be implemented. return } + silent, _, err := req.Option(silentOptionName).Bool() + if err != nil { + res.SetError(u.ErrCast(), cmds.ErrNormal) + return + } + var showProgressBar bool if prgFound { showProgressBar = progress - } else if !quiet { + } else if !quiet && !silent { showProgressBar = true } diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 64bb6ad3062..bd6e4f74539 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -105,97 +105,96 @@ type Adder struct { } // Perform the actual add & pin locally, outputting results to reader -func (params Adder) add(reader io.Reader) (*dag.Node, error) { - chnk, err := chunk.FromString(reader, params.Chunker) +func (adder Adder) add(reader io.Reader) (*dag.Node, error) { + chnk, err := chunk.FromString(reader, adder.Chunker) if err != nil { return nil, err } - if params.Trickle { + if adder.Trickle { return importer.BuildTrickleDagFromReader( - params.node.DAG, + adder.node.DAG, chnk, ) } return importer.BuildDagFromReader( - params.node.DAG, + adder.node.DAG, chnk, ) } -func (params *Adder) RootNode() (*dag.Node, error) { +func (adder *Adder) RootNode() (*dag.Node, error) { // for memoizing - if params.root != nil { - return params.root, nil + if adder.root != nil { + return adder.root, nil } - root, err := params.mr.GetValue().GetNode() + root, err := adder.mr.GetValue().GetNode() if err != nil { return nil, err } // if not wrapping, AND one root file, use that hash as root. - if !params.Wrap && len(root.Links) == 1 { - root, err = root.Links[0].GetNode(params.ctx, params.node.DAG) + if !adder.Wrap && len(root.Links) == 1 { + root, err = root.Links[0].GetNode(adder.ctx, adder.node.DAG) if err != nil { return nil, err } } - params.root = root + adder.root = root return root, err } -func (params *Adder) PinRoot() error { - root, err := params.RootNode() +func (adder *Adder) PinRoot() error { + root, err := adder.RootNode() if err != nil { return err } - if !params.Pin { + if !adder.Pin { return nil } - rnk, err := params.node.DAG.Add(root) + rnk, err := adder.node.DAG.Add(root) if err != nil { return err } - if params.tempRoot != "" { - err := params.node.Pinning.Unpin(params.ctx, params.tempRoot, true) + if adder.tempRoot != "" { + err := adder.node.Pinning.Unpin(adder.ctx, adder.tempRoot, true) if err != nil { return err } - params.tempRoot = rnk + adder.tempRoot = rnk } - params.node.Pinning.PinWithMode(rnk, pin.Recursive) - return params.node.Pinning.Flush() + adder.node.Pinning.PinWithMode(rnk, pin.Recursive) + return adder.node.Pinning.Flush() } -func (params *Adder) Finalize() (*dag.Node, error) { - root, err := params.mr.GetValue().GetNode() +func (adder *Adder) Finalize() (*dag.Node, error) { + // cant just call adder.RootNode() here as we need the name for printing + root, err := adder.mr.GetValue().GetNode() if err != nil { return nil, err } - params.RootNode() - var name string - if !params.Wrap { + if !adder.Wrap { name = root.Links[0].Name - child, err := root.Links[0].GetNode(params.ctx, params.node.DAG) + child, err := root.Links[0].GetNode(adder.ctx, adder.node.DAG) if err != nil { return nil, err } root = child } - err = params.outputDirs(name, root) + err = adder.outputDirs(name, root) if err != nil { return nil, err } - err = params.mr.Close() + err = adder.mr.Close() if err != nil { return nil, err } @@ -203,24 +202,24 @@ func (params *Adder) Finalize() (*dag.Node, error) { return root, nil } -func (params *Adder) outputDirs(path string, nd *dag.Node) error { +func (adder *Adder) outputDirs(path string, nd *dag.Node) error { if !bytes.Equal(nd.Data, folderData) { return nil } for _, l := range nd.Links { - child, err := l.GetNode(params.ctx, params.node.DAG) + child, err := l.GetNode(adder.ctx, adder.node.DAG) if err != nil { return err } - err = params.outputDirs(gopath.Join(path, l.Name), child) + err = adder.outputDirs(gopath.Join(path, l.Name), child) if err != nil { return err } } - return outputDagnode(params.out, path, nd) + return outputDagnode(adder.out, path, nd) } // Add builds a merkledag from the a reader, pinning all objects to the local @@ -318,7 +317,7 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.No return gopath.Join(k.String(), filename), dagnode, nil } -func (params *Adder) addNode(node *dag.Node, path string) error { +func (adder *Adder) addNode(node *dag.Node, path string) error { // patch it into the root if path == "" { key, err := node.Key() @@ -329,22 +328,22 @@ func (params *Adder) addNode(node *dag.Node, path string) error { path = key.Pretty() } - if err := mfs.PutNode(params.mr, path, node); err != nil { + if err := mfs.PutNode(adder.mr, path, node); err != nil { return err } - if !params.Silent { - return outputDagnode(params.out, path, node) + if !adder.Silent { + return outputDagnode(adder.out, path, node) } return nil } -// Add the given file while respecting the params. -func (params *Adder) AddFile(file files.File) error { - params.unlock = params.node.Blockstore.PinLock() - defer params.unlock() +// Add the given file while respecting the adder. +func (adder *Adder) AddFile(file files.File) error { + adder.unlock = adder.node.Blockstore.PinLock() + defer adder.unlock() - return params.addFile(file) + return adder.addFile(file) } func (adder *Adder) addFile(file files.File) error { @@ -394,10 +393,10 @@ func (adder *Adder) addFile(file files.File) error { return adder.addNode(dagnode, file.FileName()) } -func (params *Adder) addDir(dir files.File) error { +func (adder *Adder) addDir(dir files.File) error { log.Infof("adding directory: %s", dir.FileName()) - err := mfs.Mkdir(params.mr, dir.FileName(), true) + err := mfs.Mkdir(adder.mr, dir.FileName(), true) if err != nil { return err } @@ -411,7 +410,7 @@ func (params *Adder) addDir(dir files.File) error { break } - err = params.addFile(file) + err = adder.addFile(file) if _, ok := err.(*hiddenFileError); ok { // hidden file error, skip file continue diff --git a/test/sharness/t0080-repo.sh b/test/sharness/t0080-repo.sh index dc7c49e0ca3..1ab6238096d 100755 --- a/test/sharness/t0080-repo.sh +++ b/test/sharness/t0080-repo.sh @@ -29,6 +29,11 @@ test_expect_success "'ipfs repo gc' succeeds" ' ipfs repo gc >gc_out_actual ' +test_expect_success "'ipfs repo gc' looks good (patch root)" ' + PATCH_ROOT=QmQXirSbubiySKnqaFyfs5YzziXRB5JEVQVjU6xsd7innr && + grep "removed $PATCH_ROOT" gc_out_actual +' + test_expect_success "'ipfs repo gc' doesnt remove file" ' ipfs cat "$HASH" >out && test_cmp out afile From 9fc1a1af12acda4cb497a68167d313b786a0bc96 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 7 Dec 2015 22:34:05 -0800 Subject: [PATCH 13/13] log failure to check file size License: MIT Signed-off-by: Jeromy --- core/commands/add.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/commands/add.go b/core/commands/add.go index 152244be764..c8a17eca5b3 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -70,6 +70,7 @@ remains to be implemented. sizeFile, ok := req.Files().(files.SizeFile) if !ok { // we don't need to error, the progress bar just won't know how big the files are + log.Warning("cannnot determine size of input file") return nil } @@ -79,6 +80,7 @@ remains to be implemented. go func() { size, err := sizeFile.Size() if err != nil { + log.Warningf("error getting files size: %s", err) // see comment above return }