diff --git a/pkg/commands/cache.go b/pkg/commands/cache.go
index 9f235abefc..5ce5b74cda 100644
--- a/pkg/commands/cache.go
+++ b/pkg/commands/cache.go
@@ -165,10 +165,6 @@ func (i *imageCache) get(ctx context.Context, ref name.Reference, missFunc baseF
 			return result, err
 		}
 
-		if err := i.p.AppendDescriptor(*desc); err != nil {
-			return result, err
-		}
-
 		if _, ok := result.(v1.ImageIndex); ok {
 			result = &lazyIndex{
 				ref:      ref,
@@ -202,6 +198,10 @@ func (i *imageCache) get(ctx context.Context, ref name.Reference, missFunc baseF
 				cache:    i,
 			}
 		}
+
+		if err := i.p.AppendDescriptor(*desc); err != nil {
+			return result, err
+		}
 	}
 
 	i.cache.Store(ref.String(), result)