Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix http path retrievals #270

Merged
merged 1 commit into from
May 30, 2023
Merged

Fix http path retrievals #270

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 133 additions & 0 deletions pkg/internal/itest/http_fetch_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,26 @@ func TestHttpFetch(t *testing.T) {
validateCarBody(t, body, srcData.Root, wantCids, true)
}},
},
{
name: "http nested large sharded file, with path, dag-scope entity",
httpRemotes: 1,
generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
lsys := remotes[0].LinkSystem
return []unixfs.DirEntry{unixfs.WrapContent(t, rndReader, lsys, unixfs.GenerateFile(t, lsys, rndReader, 4<<20), wrapPath, false)}
},
paths: []string{wrapPath},
modifyQueries: []queryModifier{entityQuery},
validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) {
wantCids := append([]cid.Cid{
srcData.Root, // "/""
srcData.Children[1].Root, // "/want2"
srcData.Children[1].Children[1].Root, // "/want2/want1"
},
srcData.Children[1].Children[1].Children[1].SelfCids..., // wrapPath (full file)
)
validateCarBody(t, body, srcData.Root, wantCids, true)
}},
},
{
name: "graphsync large directory, dag-scope entity",
graphsyncRemotes: 1,
Expand All @@ -307,6 +327,18 @@ func TestHttpFetch(t *testing.T) {
validateCarBody(t, body, srcData.Root, []cid.Cid{srcData.Root}, true)
}},
},
{
name: "http large directory, dag-scope entity",
httpRemotes: 1,
generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
return []unixfs.DirEntry{unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, false)}
},
modifyQueries: []queryModifier{entityQuery},
validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) {
// expect a CAR of one block, to represent the root directory we asked for
validateCarBody(t, body, srcData.Root, []cid.Cid{srcData.Root}, true)
}},
},
{
name: "graphsync nested large directory, with path, dag-scope entity",
graphsyncRemotes: 1,
Expand Down Expand Up @@ -347,6 +379,26 @@ func TestHttpFetch(t *testing.T) {
validateCarBody(t, body, srcData.Root, wantCids, true)
}},
},
{
name: "http nested large directory, with path, dag-scope entity",
httpRemotes: 1,
generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
lsys := remotes[0].LinkSystem
return []unixfs.DirEntry{unixfs.WrapContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, false), wrapPath, false)}
},
paths: []string{wrapPath},
modifyQueries: []queryModifier{entityQuery},
validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) {
wantCids := append([]cid.Cid{
srcData.Root, // "/""
srcData.Children[1].Root, // "/want2"
srcData.Children[1].Children[1].Root, // "/want2/want1"
},
srcData.Children[1].Children[1].Children[1].SelfCids..., // wrapPath (full dir)
)
validateCarBody(t, body, srcData.Root, wantCids, true)
}},
},
{
name: "graphsync nested large directory, with path, full",
graphsyncRemotes: 1,
Expand Down Expand Up @@ -395,6 +447,30 @@ func TestHttpFetch(t *testing.T) {
unixfs.CompareDirEntries(t, srcData.Children[1].Children[1].Children[1], gotDir)
}},
},
{
name: "bitswap nested large directory, with path, full",
httpRemotes: 1,
generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
lsys := remotes[0].LinkSystem
return []unixfs.DirEntry{unixfs.WrapContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, false), wrapPath, false)}
},
paths: []string{wrapPath},
validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) {
wantCids := append([]cid.Cid{
srcData.Root, // "/""
srcData.Children[1].Root, // "/want2"
srcData.Children[1].Children[1].Root, // "/want2/want1"
},
srcData.Children[1].Children[1].Children[1].SelfCids..., // wrapPath (full dir)
)
// validate we got the dag-scope entity form
validateCarBody(t, body, srcData.Root, wantCids, false)
// validate that we got the full depth form under the path
gotDir := CarToDirEntry(t, bytes.NewReader(body), srcData.Children[1].Children[1].Children[1].Root, true)
gotDir.Path = "want0"
unixfs.CompareDirEntries(t, srcData.Children[1].Children[1].Children[1], gotDir)
}},
},
{
name: "graphsync nested large sharded directory, dag-scope entity",
graphsyncRemotes: 1,
Expand All @@ -421,6 +497,19 @@ func TestHttpFetch(t *testing.T) {
validateCarBody(t, body, srcData.Root, srcData.SelfCids, true)
}},
},
{
name: "http nested large sharded directory, dag-scope entity",
httpRemotes: 1,
generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
return []unixfs.DirEntry{unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, true)}
},
modifyQueries: []queryModifier{entityQuery},
validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) {
// sharded directory contains multiple blocks, so we expect a CAR with
// exactly those blocks
validateCarBody(t, body, srcData.Root, srcData.SelfCids, true)
}},
},
{
name: "graphsync nested large sharded directory, with path, dag-scope entity",
graphsyncRemotes: 1,
Expand Down Expand Up @@ -461,6 +550,26 @@ func TestHttpFetch(t *testing.T) {
validateCarBody(t, body, srcData.Root, wantCids, true)
}},
},
{
name: "http nested large sharded directory, with path, dag-scope entity",
httpRemotes: 1,
generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
lsys := remotes[0].LinkSystem
return []unixfs.DirEntry{unixfs.WrapContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, true), wrapPath, false)}
},
paths: []string{wrapPath},
modifyQueries: []queryModifier{entityQuery},
validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) {
wantCids := append([]cid.Cid{
srcData.Root, // "/""
srcData.Children[1].Root, // "/want2"
srcData.Children[1].Children[1].Root, // "/want2/want1"
},
srcData.Children[1].Children[1].Children[1].SelfCids..., // wrapPath (full dir)
)
validateCarBody(t, body, srcData.Root, wantCids, true)
}},
},
{
name: "graphsync nested large sharded directory, with path, full",
graphsyncRemotes: 1,
Expand Down Expand Up @@ -509,6 +618,30 @@ func TestHttpFetch(t *testing.T) {
unixfs.CompareDirEntries(t, srcData.Children[1].Children[1].Children[1], gotDir)
}},
},
{
name: "http nested large sharded directory, with path, full",
httpRemotes: 1,
generate: func(t *testing.T, rndReader io.Reader, remotes []testpeer.TestPeer) []unixfs.DirEntry {
lsys := remotes[0].LinkSystem
return []unixfs.DirEntry{unixfs.WrapContent(t, rndReader, lsys, unixfs.GenerateDirectory(t, remotes[0].LinkSystem, rndReader, 16<<20, true), wrapPath, false)}
},
paths: []string{wrapPath},
validateBodies: []bodyValidator{func(t *testing.T, srcData unixfs.DirEntry, body []byte) {
wantCids := append([]cid.Cid{
srcData.Root, // "/""
srcData.Children[1].Root, // "/want2"
srcData.Children[1].Children[1].Root, // "/want2/want1"
},
srcData.Children[1].Children[1].Children[1].SelfCids..., // wrapPath (full dir)
)
// validate we got the dag-scope entity form
validateCarBody(t, body, srcData.Root, wantCids, false)
// validate that we got the full depth form under the path
gotDir := CarToDirEntry(t, bytes.NewReader(body), srcData.Children[1].Children[1].Children[1].Root, true)
gotDir.Path = "want0"
unixfs.CompareDirEntries(t, srcData.Children[1].Children[1].Children[1], gotDir)
}},
},
{
// A very contrived example - we spread the content generated for this test across 4 peers,
// then we also make sure the root is in all of them, so the CandidateFinder will return them
Expand Down
2 changes: 1 addition & 1 deletion pkg/retriever/httpretriever.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ func makeRequest(ctx context.Context, request types.RetrievalRequest, candidate
return nil, fmt.Errorf("%w: %v", ErrBadPathForRequest, err)
}

reqURL := fmt.Sprintf("%s/ipfs/%s%s", candidateURL, request.Cid, path)
reqURL := fmt.Sprintf("%s/ipfs/%s/%s", candidateURL, request.Cid, path)
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
if err != nil {
logger.Warnf("Couldn't construct a http request %s: %v", candidate.MinerPeer.ID, err)
Expand Down