From 0b9e89ddfeca5ab14fb710dac1dec1f1acdbc83a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 25 Apr 2019 15:03:47 +0200 Subject: [PATCH 1/3] add: fix adding multiple files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/add.go | 98 +++++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 43 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index e6e78c1da1b..5db22940e10 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -181,24 +181,26 @@ You can now check what blocks have been created by: return err } - events := make(chan interface{}, adderOutChanSize) - - var toadd files.Node = req.Files - name := "" + toadd := []files.Node{req.Files} + name := []string{""} if !wrap { + toadd, name = nil, nil + + // process all entries in case user adds multiple files it := req.Files.Entries() - if !it.Next() { - err := it.Err() - if err == nil { - return fmt.Errorf("expected a file argument") - } + for it.Next() { + toadd = append(toadd, it.Node()) + name = append(name, it.Name()) + } + + if err := it.Err(); err != nil { return err } - toadd = it.Node() - name = it.Name() + if len(toadd) == 0 { + return fmt.Errorf("expected a file argument") + } } - _, dir := toadd.(files.Directory) opts := []options.UnixfsAddOption{ options.Unixfs.Hash(hashFunCode), @@ -215,7 +217,8 @@ You can now check what blocks have been created by: options.Unixfs.Progress(progress), options.Unixfs.Silent(silent), - options.Unixfs.Events(events), + + nil, // reserved for events, must be last } if cidVerSet { @@ -230,42 +233,51 @@ You can now check what blocks have been created by: opts = append(opts, options.Unixfs.Layout(options.TrickleLayout)) } - errCh := make(chan error, 1) - go func() { - var err error - defer func() { errCh <- err }() - defer close(events) - _, err = api.Unixfs().Add(req.Context, toadd, opts...) - }() - - for event := range events { - output, ok := event.(*coreiface.AddEvent) - if !ok { - return errors.New("unknown event type") - } + for i := range toadd { + _, dir := toadd[i].(files.Directory) + errCh := make(chan error, 1) + events := make(chan interface{}, adderOutChanSize) + opts[len(opts)-1] = options.Unixfs.Events(events) - h := "" - if output.Path != nil { - h = enc.Encode(output.Path.Cid()) - } + go func() { + var err error + defer func() { errCh <- err }() + defer close(events) + _, err = api.Unixfs().Add(req.Context, toadd[i], opts...) + }() - if !dir && name != "" { - output.Name = name - } else { - output.Name = path.Join(name, output.Name) + for event := range events { + output, ok := event.(*coreiface.AddEvent) + if !ok { + return errors.New("unknown event type") + } + + h := "" + if output.Path != nil { + h = enc.Encode(output.Path.Cid()) + } + + if !dir && name[i] != "" { + output.Name = name[i] + } else { + output.Name = path.Join(name[i], output.Name) + } + + if err := res.Emit(&AddEvent{ + Name: output.Name, + Hash: h, + Bytes: output.Bytes, + Size: output.Size, + }); err != nil { + return err + } } - if err := res.Emit(&AddEvent{ - Name: output.Name, - Hash: h, - Bytes: output.Bytes, - Size: output.Size, - }); err != nil { - return err + if err := <-errCh; err != nil { + return nil } } - - return <-errCh + return nil }, PostRun: cmds.PostRunMap{ cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { From aa0704e61a3413df62a92215e853cc752db90d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 25 Apr 2019 15:23:55 +0200 Subject: [PATCH 2/3] add: fix wrap with multiple files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/add.go | 52 ++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 5db22940e10..5416d143832 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -181,25 +181,11 @@ You can now check what blocks have been created by: return err } - toadd := []files.Node{req.Files} - name := []string{""} - if !wrap { - toadd, name = nil, nil - - // process all entries in case user adds multiple files - it := req.Files.Entries() - for it.Next() { - toadd = append(toadd, it.Node()) - name = append(name, it.Name()) - } - - if err := it.Err(); err != nil { - return err - } - - if len(toadd) == 0 { - return fmt.Errorf("expected a file argument") - } + toadd := req.Files + if wrap { + toadd = files.NewSliceDirectory([]files.DirEntry{ + files.FileEntry("", req.Files), + }) } opts := []options.UnixfsAddOption{ @@ -217,8 +203,6 @@ You can now check what blocks have been created by: options.Unixfs.Progress(progress), options.Unixfs.Silent(silent), - - nil, // reserved for events, must be last } if cidVerSet { @@ -233,17 +217,21 @@ You can now check what blocks have been created by: opts = append(opts, options.Unixfs.Layout(options.TrickleLayout)) } - for i := range toadd { - _, dir := toadd[i].(files.Directory) + opts = append(opts, nil) // events option placeholder + + var added int + addit := toadd.Entries() + for addit.Next() { + _, dir := addit.Node().(files.Directory) errCh := make(chan error, 1) events := make(chan interface{}, adderOutChanSize) opts[len(opts)-1] = options.Unixfs.Events(events) go func() { var err error - defer func() { errCh <- err }() defer close(events) - _, err = api.Unixfs().Add(req.Context, toadd[i], opts...) + _, err = api.Unixfs().Add(req.Context, addit.Node(), opts...) + errCh <- err }() for event := range events { @@ -257,10 +245,10 @@ You can now check what blocks have been created by: h = enc.Encode(output.Path.Cid()) } - if !dir && name[i] != "" { - output.Name = name[i] + if !dir && addit.Name() != "" { + output.Name = addit.Name() } else { - output.Name = path.Join(name[i], output.Name) + output.Name = path.Join(addit.Name(), output.Name) } if err := res.Emit(&AddEvent{ @@ -274,9 +262,15 @@ You can now check what blocks have been created by: } if err := <-errCh; err != nil { - return nil + return err } + added++ } + + if added == 0 { + return fmt.Errorf("expected a file argument") + } + return nil }, PostRun: cmds.PostRunMap{ From ac51d2999eed5894a8c7dcf0d1621c0421cfb578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 25 Apr 2019 15:49:13 +0200 Subject: [PATCH 3/3] add: test adding multiple files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- test/sharness/t0040-add-and-cat.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh index f1bc3c2790b..be4e2a81775 100755 --- a/test/sharness/t0040-add-and-cat.sh +++ b/test/sharness/t0040-add-and-cat.sh @@ -325,6 +325,16 @@ test_add_cat_file() { test_cmp expected actual ' + test_expect_success "ipfs add with multiple files succeeds" ' + echo "Helloo Worlds!" >mountdir/hello2.txt && + ipfs add mountdir/hello.txt mountdir/hello2.txt >actual + ' + + test_expect_success "ipfs add with multiple files output looks good" ' + echo "added QmVr26fY1tKyspEJBniVhqxQeEjhF78XerGiqWAwraVLQH hello.txt" >expected && + echo "added Qmf35k66MZNW2GijohUmXQEWKZU4cCGTCwK6idfnt152wJ hello2.txt" >> expected && + test_cmp expected actual + ' } test_add_cat_5MB() {