Skip to content

Commit

Permalink
Ensure layers in the local content store on FileList API
Browse files Browse the repository at this point in the history 
Signed-off-by: Kohei Tokunaga <[email protected]>
Signed-off-by: Tonis Tiigi <[email protected]>
  • Loading branch information
@ktock @tonistiigi
ktock authored and tonistiigi committed Sep 26, 2023
1 parent e371069 commit 5dccc0a
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 13 deletions.
2 changes: 1 addition & 1 deletion cache/filelist.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ func (sr *immutableRef) FileList(ctx context.Context, s session.Group) ([]string
}

// lazy blobs need to be pulled first
if err := sr.Extract(ctx, s); err != nil {
if err := sr.ensureLocalContentBlob(ctx, s); err != nil {
return nil, err
}

Expand Down
2 changes: 1 addition & 1 deletion cache/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ func (cm *cacheManager) GetByBlob(ctx context.Context, desc ocispecs.Descriptor,

ref := rec.ref(true, descHandlers, nil)
if s := unlazySessionOf(opts...); s != nil {
if err := ref.unlazy(ctx, ref.descHandlers, ref.progress, s, true); err != nil {
if err := ref.unlazy(ctx, ref.descHandlers, ref.progress, s, true, false); err != nil {
return nil, err
}
}
Expand Down
33 changes: 22 additions & 11 deletions cache/refs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -992,6 +992,14 @@ func (sr *immutableRef) Mount(ctx context.Context, readonly bool, s session.Grou
return mnt, nil
}

func (sr *immutableRef) ensureLocalContentBlob(ctx context.Context, s session.Group) error {
if (sr.kind() == Layer || sr.kind() == BaseLayer) && !sr.getBlobOnly() {
return nil
}

return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, true)
}

func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr error) {
if (sr.kind() == Layer || sr.kind() == BaseLayer) && !sr.getBlobOnly() {
return nil
Expand All @@ -1002,14 +1010,14 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro
if rerr = sr.prepareRemoteSnapshotsStargzMode(ctx, s); rerr != nil {
return
}
rerr = sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true)
rerr = sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false)
}); err != nil {
return err
}
return rerr
}

return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true)
return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false)
}

func (sr *immutableRef) withRemoteSnapshotLabelsStargzMode(ctx context.Context, s session.Group, f func()) error {
Expand Down Expand Up @@ -1149,9 +1157,12 @@ func makeTmpLabelsStargzMode(labels map[string]string, s session.Group) (fields
return
}

func (sr *immutableRef) unlazy(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group, topLevel bool) error {
func (sr *immutableRef) unlazy(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group, topLevel bool, ensureContentStore bool) error {
_, err := g.Do(ctx, sr.ID()+"-unlazy", func(ctx context.Context) (_ struct{}, rerr error) {
if _, err := sr.cm.Snapshotter.Stat(ctx, sr.getSnapshotID()); err == nil {
if !ensureContentStore {
return struct{}{}, nil
}
if blob := sr.getBlob(); blob == "" {
return struct{}{}, nil
}
Expand All @@ -1162,17 +1173,17 @@ func (sr *immutableRef) unlazy(ctx context.Context, dhs DescHandlers, pg progres

switch sr.kind() {
case Merge, Diff:
return struct{}{}, sr.unlazyDiffMerge(ctx, dhs, pg, s, topLevel)
return struct{}{}, sr.unlazyDiffMerge(ctx, dhs, pg, s, topLevel, ensureContentStore)
case Layer, BaseLayer:
return struct{}{}, sr.unlazyLayer(ctx, dhs, pg, s)
return struct{}{}, sr.unlazyLayer(ctx, dhs, pg, s, ensureContentStore)
}
return struct{}{}, nil
})
return err
}

// should be called within sizeG.Do call for this ref's ID
func (sr *immutableRef) unlazyDiffMerge(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group, topLevel bool) (rerr error) {
func (sr *immutableRef) unlazyDiffMerge(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group, topLevel bool, ensureContentStore bool) (rerr error) {
eg, egctx := errgroup.WithContext(ctx)
var diffs []snapshot.Diff
sr.layerWalk(func(sr *immutableRef) {
Expand All @@ -1182,13 +1193,13 @@ func (sr *immutableRef) unlazyDiffMerge(ctx context.Context, dhs DescHandlers, p
if sr.diffParents.lower != nil {
diff.Lower = sr.diffParents.lower.getSnapshotID()
eg.Go(func() error {
return sr.diffParents.lower.unlazy(egctx, dhs, pg, s, false)
return sr.diffParents.lower.unlazy(egctx, dhs, pg, s, false, ensureContentStore)
})
}
if sr.diffParents.upper != nil {
diff.Upper = sr.diffParents.upper.getSnapshotID()
eg.Go(func() error {
return sr.diffParents.upper.unlazy(egctx, dhs, pg, s, false)
return sr.diffParents.upper.unlazy(egctx, dhs, pg, s, false, ensureContentStore)
})
}
case Layer:
Expand All @@ -1197,7 +1208,7 @@ func (sr *immutableRef) unlazyDiffMerge(ctx context.Context, dhs DescHandlers, p
case BaseLayer:
diff.Upper = sr.getSnapshotID()
eg.Go(func() error {
return sr.unlazy(egctx, dhs, pg, s, false)
return sr.unlazy(egctx, dhs, pg, s, false, ensureContentStore)
})
}
diffs = append(diffs, diff)
Expand Down Expand Up @@ -1228,7 +1239,7 @@ func (sr *immutableRef) unlazyDiffMerge(ctx context.Context, dhs DescHandlers, p
}

// should be called within sizeG.Do call for this ref's ID
func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group) (rerr error) {
func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg progress.Controller, s session.Group, ensureContentStore bool) (rerr error) {
if !sr.getBlobOnly() {
return nil
}
Expand All @@ -1255,7 +1266,7 @@ func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg pr
parentID := ""
if sr.layerParent != nil {
eg.Go(func() error {
if err := sr.layerParent.unlazy(egctx, dhs, pg, s, false); err != nil {
if err := sr.layerParent.unlazy(egctx, dhs, pg, s, false, ensureContentStore); err != nil {
return err
}
parentID = sr.layerParent.getSnapshotID()
Expand Down

0 comments on commit 5dccc0a

Please sign in to comment.