Skip to content

Commit

Permalink
Merge pull request ethersphere#541 from ethersphere/resource-rootchun…
Browse files Browse the repository at this point in the history 
…k-refactor

swarm/storage, swarm/api: Content addressed root chunks for MRU
  • Loading branch information
@zelig
zelig authored May 24, 2018
2 parents 3565149 + 6ece30c commit 3d7e57b
Show file tree
Hide file tree
Showing 6 changed files with 454 additions and 257 deletions.
91 changes: 57 additions & 34 deletions swarm/api/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ func (self *Api) Put(content, contentType string, toEncrypt bool) (k storage.Key
// Get uses iterative manifest retrieval and prefix matching
// to resolve basePath to content using dpa retrieve
// it returns a section reader, mimeType, status, the key of the actual content and an error
func (self *Api) Get(manifestKey storage.Key, path string) (reader *storage.LazyChunkReader, mimeType string, status int, contentKey storage.Key, err error) {
func (self *Api) Get(manifestKey storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, contentKey storage.Key, err error) {
log.Debug("api.get", "key", manifestKey, "path", path)
apiGetCount.Inc(1)
trie, err := loadManifest(self.dpa, manifestKey, nil)
Expand All @@ -326,40 +326,44 @@ func (self *Api) Get(manifestKey storage.Key, path string) (reader *storage.Lazy

if entry != nil {
log.Debug("trie got entry", "key", manifestKey, "path", path, "entry.Hash", entry.Hash)
// we want to be able to serve Mutable Resource Updates transparently using the bzz:// scheme
//
// we use a special manifest hack for this purpose, which is pathless and where the resource root key
// is set as the hash of the manifest (see swarm/api/manifest.go:NewResourceManifest)
//
// to avoid taking a performance hit hacking a storage.LazySectionReader to wrap the resource key,
// if the resource update is of raw type:
// we return a typed error instead. Since for all other purposes this is an invalid manifest,
// any normal interfacing code will just see an error fail accordingly.
//
// if the resource update is of multihash type:
// we validate the multihash and retrieve the manifest behind it, and resume normal operations from there
// we need to do some extra work if this is a mutable resource manifest
if entry.ContentType == ResourceContentType {

// get the resource root chunk key
log.Trace("resource type", "key", manifestKey, "hash", entry.Hash)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
rsrc, err := self.resource.LookupLatestByName(ctx, entry.Hash, true, &storage.ResourceLookupParams{})
rsrc, err := self.resource.LoadResource(storage.Key(common.FromHex(entry.Hash)))
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Debug(fmt.Sprintf("get resource content error: %v", err))
return reader, mimeType, status, nil, err

}
_, rsrcData, err := self.resource.GetContent(entry.Hash)

// use this key to retrieve the latest update
rsrc, err = self.resource.LookupLatest(ctx, rsrc.NameHash(), true, &storage.ResourceLookupParams{})
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Warn(fmt.Sprintf("get resource content error: %v", err))
log.Debug(fmt.Sprintf("get resource content error: %v", err))
return reader, mimeType, status, nil, err
}

// if it's multihash, we will transparently serve the content this multihash points to
// \TODO this resolve is rather expensive all in all, review to see if it can be achieved cheaper
if rsrc.Multihash {

// validate data as multihash
// get the data of the update
_, rsrcData, err := self.resource.GetContent(rsrc.NameHash().Hex())
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Warn(fmt.Sprintf("get resource content error: %v", err))
return reader, mimeType, status, nil, err
}

// validate that data as multihash
decodedMultihash, err := multihash.Decode(rsrcData)
if err != nil {
apiGetInvalid.Inc(1)
Expand All @@ -384,11 +388,9 @@ func (self *Api) Get(manifestKey storage.Key, path string) (reader *storage.Lazy
return reader, mimeType, status, nil, err
}

log.Trace("trie getting resource multihash entry", "key", manifestKey, "path", path)
var fullpath string
entry, fullpath = trie.getEntry(path)
log.Trace("trie got resource multihash entry", "key", manifestKey, "path", path, "entry", entry, "fullpath", fullpath)

// finally, get the manifest entry
// it will always be the entry on path ""
entry, _ = trie.getEntry(path)
if entry == nil {
status = http.StatusNotFound
apiGetNotFound.Inc(1)
Expand All @@ -398,10 +400,13 @@ func (self *Api) Get(manifestKey storage.Key, path string) (reader *storage.Lazy
}

} else {
return nil, entry.ContentType, http.StatusOK, nil, &ErrResourceReturn{entry.Hash}
// data is returned verbatim since it's not a multihash
return rsrc, "application/octet-stream", http.StatusOK, nil, nil
}
}

// regardless of resource update manifests or normal manifests we will converge at this point
// get the key the manifest entry points to and serve it if it's unambiguous
contentKey = common.Hex2Bytes(entry.Hash)
status = entry.Status
if status == http.StatusMultipleChoices {
Expand All @@ -413,6 +418,7 @@ func (self *Api) Get(manifestKey storage.Key, path string) (reader *storage.Lazy
reader, _ = self.dpa.Retrieve(contentKey)
}
} else {
// no entry found
status = http.StatusNotFound
apiGetNotFound.Inc(1)
err = fmt.Errorf("manifest entry for '%s' not found", path)
Expand Down Expand Up @@ -648,31 +654,34 @@ func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storag
}

// Look up mutable resource updates at specific periods and versions
func (self *Api) ResourceLookup(ctx context.Context, name string, period uint32, version uint32, maxLookup *storage.ResourceLookupParams) (storage.Key, []byte, error) {
func (self *Api) ResourceLookup(ctx context.Context, key storage.Key, period uint32, version uint32, maxLookup *storage.ResourceLookupParams) (string, []byte, error) {
var err error
rsrc, err := self.resource.LoadResource(key)
if err != nil {
return "", nil, err
}
if version != 0 {
if period == 0 {
return nil, nil, storage.NewResourceError(storage.ErrInvalidValue, "Period can't be 0")
return "", nil, storage.NewResourceError(storage.ErrInvalidValue, "Period can't be 0")
}
_, err = self.resource.LookupVersionByName(ctx, name, period, version, true, maxLookup)
_, err = self.resource.LookupVersion(ctx, rsrc.NameHash(), period, version, true, maxLookup)
} else if period != 0 {
_, err = self.resource.LookupHistoricalByName(ctx, name, period, true, maxLookup)
_, err = self.resource.LookupHistorical(ctx, rsrc.NameHash(), period, true, maxLookup)
} else {
_, err = self.resource.LookupLatestByName(ctx, name, true, maxLookup)
_, err = self.resource.LookupLatest(ctx, rsrc.NameHash(), true, maxLookup)
}
if err != nil {
return nil, nil, err
return "", nil, err
}
return self.resource.GetContent(name)
return self.resource.GetContent(rsrc.NameHash().Hex())
}

func (self *Api) ResourceCreate(ctx context.Context, name string, frequency uint64) (storage.Key, error) {
rsrc, err := self.resource.NewResource(ctx, name, frequency)
key, _, err := self.resource.NewResource(ctx, name, frequency)
if err != nil {
return nil, err
}
h := rsrc.NameHash()
return storage.Key(h[:]), nil
return key, nil
}

func (self *Api) ResourceUpdateMultihash(ctx context.Context, name string, data []byte) (storage.Key, uint32, uint32, error) {
Expand Down Expand Up @@ -702,3 +711,17 @@ func (self *Api) ResourceHashSize() int {
func (self *Api) ResourceIsValidated() bool {
return self.resource.IsValidated()
}

func (self *Api) ResolveResourceManifest(key storage.Key) (storage.Key, error) {
trie, err := loadManifest(self.dpa, key, nil)
if err != nil {
return nil, fmt.Errorf("cannot load resource manifest: %v", err)
}

entry, _ := trie.getEntry("")
if entry.ContentType != ResourceContentType {
return nil, fmt.Errorf("not a resource manifest: %s", key)
}

return storage.Key(common.FromHex(entry.Hash)), nil
}
Loading

0 comments on commit 3d7e57b

Please sign in to comment.