Skip to content

Commit

Permalink
Ensure layers in the local content store on FileList API
Browse files Browse the repository at this point in the history 
Signed-off-by: Kohei Tokunaga <[email protected]>
  • Loading branch information
@ktock
ktock committed Sep 11, 2023
1 parent dfcb140 commit 2c8acb0
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 13 deletions.
7 changes: 6 additions & 1 deletion cache/filelist.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,12 @@ func (sr *immutableRef) FileList(ctx context.Context, s session.Group) ([]string
}

// lazy blobs need to be pulled first
if err := sr.Extract(ctx, s); err != nil {
// if err := sr.Extract(ctx, s); err != nil {
// return nil, err
// }

// unpack the layer contents and ensure the blob exists in the content store.
if err := sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, true); err != nil {
return nil, err
}

Expand Down
2 changes: 1 addition & 1 deletion cache/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ func (cm *cacheManager) GetByBlob(ctx context.Context, desc ocispecs.Descriptor,

ref := rec.ref(true, descHandlers, nil)
if s := unlazySessionOf(opts...); s != nil {
if err := ref.unlazy(ctx, ref.descHandlers, ref.progress, s, true); err != nil {
if err := ref.unlazy(ctx, ref.descHandlers, ref.progress, s, true, false); err != nil {
return nil, err
}
}
Expand Down
25 changes: 14 additions & 11 deletions cache/refs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1002,14 +1002,14 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro
if rerr = sr.prepareRemoteSnapshotsStargzMode(ctx, s); rerr != nil {
return
}
rerr = sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true)
rerr = sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false)
}); err != nil {
return err
}
return rerr
}

return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true)
return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false)
}

func (sr *immutableRef) withRemoteSnapshotLabelsStargzMode(ctx context.Context, s session.Group, f func()) error {
Expand Down Expand Up @@ -1140,9 +1140,12 @@ func makeTmpLabelsStargzMode(labels map[string]string, s session.Group) (fields
return
}

func (sr *immutableRef) unlazy(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group, topLevel bool) error {
func (sr *immutableRef) unlazy(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group, topLevel bool, ensureContentStore bool) error {
_, err := g.Do(ctx, sr.ID()+"-unlazy", func(ctx context.Context) (_ struct{}, rerr error) {
if _, err := sr.cm.Snapshotter.Stat(ctx, sr.getSnapshotID()); err == nil {
if !ensureContentStore {
return struct{}{}, nil
}
if blob := sr.getBlob(); blob == "" {
return struct{}{}, nil
}
Expand All @@ -1153,17 +1156,17 @@ func (sr *immutableRef) unlazy(ctx context.Context, dhs DescHandlers, pg progres

switch sr.kind() {
case Merge, Diff:
return struct{}{}, sr.unlazyDiffMerge(ctx, dhs, pg, s, topLevel)
return struct{}{}, sr.unlazyDiffMerge(ctx, dhs, pg, s, topLevel, ensureContentStore)
case Layer, BaseLayer:
return struct{}{}, sr.unlazyLayer(ctx, dhs, pg, s)
return struct{}{}, sr.unlazyLayer(ctx, dhs, pg, s, ensureContentStore)
}
return struct{}{}, nil
})
return err
}

// should be called within sizeG.Do call for this ref's ID
func (sr *immutableRef) unlazyDiffMerge(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group, topLevel bool) (rerr error) {
func (sr *immutableRef) unlazyDiffMerge(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group, topLevel bool, ensureContentStore bool) (rerr error) {
eg, egctx := errgroup.WithContext(ctx)
var diffs []snapshot.Diff
sr.layerWalk(func(sr *immutableRef) {
Expand All @@ -1173,13 +1176,13 @@ func (sr *immutableRef) unlazyDiffMerge(ctx context.Context, dhs DescHandlers, p
if sr.diffParents.lower != nil {
diff.Lower = sr.diffParents.lower.getSnapshotID()
eg.Go(func() error {
return sr.diffParents.lower.unlazy(egctx, dhs, pg, s, false)
return sr.diffParents.lower.unlazy(egctx, dhs, pg, s, false, ensureContentStore)
})
}
if sr.diffParents.upper != nil {
diff.Upper = sr.diffParents.upper.getSnapshotID()
eg.Go(func() error {
return sr.diffParents.upper.unlazy(egctx, dhs, pg, s, false)
return sr.diffParents.upper.unlazy(egctx, dhs, pg, s, false, ensureContentStore)
})
}
case Layer:
Expand All @@ -1188,7 +1191,7 @@ func (sr *immutableRef) unlazyDiffMerge(ctx context.Context, dhs DescHandlers, p
case BaseLayer:
diff.Upper = sr.getSnapshotID()
eg.Go(func() error {
return sr.unlazy(egctx, dhs, pg, s, false)
return sr.unlazy(egctx, dhs, pg, s, false, ensureContentStore)
})
}
diffs = append(diffs, diff)
Expand Down Expand Up @@ -1219,7 +1222,7 @@ func (sr *immutableRef) unlazyDiffMerge(ctx context.Context, dhs DescHandlers, p
}

// should be called within sizeG.Do call for this ref's ID
func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group) (rerr error) {
func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group, ensureContentStore bool) (rerr error) {
if !sr.getBlobOnly() {
return nil
}
Expand All @@ -1246,7 +1249,7 @@ func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg pr
parentID := ""
if sr.layerParent != nil {
eg.Go(func() error {
if err := sr.layerParent.unlazy(egctx, dhs, pg, s, false); err != nil {
if err := sr.layerParent.unlazy(egctx, dhs, pg, s, false, ensureContentStore); err != nil {
return err
}
parentID = sr.layerParent.getSnapshotID()
Expand Down

0 comments on commit 2c8acb0

Please sign in to comment.