-
Notifications
You must be signed in to change notification settings - Fork 384
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
blobinfocache: track compression types for locations
Extend the blob info cache to also cache the name of the type of compression used on a blob in a known location. (Caching the full MIME type would have required additional logic to map types in cases where we convert images during copying.) New methods for adding known locations and reading candidate locations including compression information are part of a new internal BlobInfoCache2 interface which the library's BlobInfoCache implementors implement. Pass information about compression changes to PutBlob() in its input BlobInfo, so that it can populate the blob info cache with correct compression information when it succeeds. Make sure that when TryReusingBlob successfully uses a blob from the blob info cache or can detect the compression of a blob on the filesystem, that it provides compression information in the BlobInfo that it returns, so that manifests can be updated to describe layers using the correct MIME types. When attempting to write a manifest, if a manifest can't be written because layers were compressed using an algorithm which can't be expressed using that manifest type, continue on to trying other manifest formats. Signed-off-by: Nalin Dahyabhai <[email protected]>
- Loading branch information
Showing
23 changed files
with
595 additions
and
103 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| package blobinfocache | ||
|
|
||
| import ( | ||
| "io" | ||
|
|
||
| "github.com/containers/image/v5/pkg/compression" | ||
| "github.com/containers/image/v5/types" | ||
| digest "github.com/opencontainers/go-digest" | ||
| ) | ||
|
|
||
| // FromBlobInfoCache2 returns a BlobInfoCache based on a BlobInfoCache2, returning the original | ||
| // object if it implements both BlobInfoCache and BlobInfoCache2, or a wrapper which discards | ||
| // compression information if it doesn't implement the V1 methods for storing and retrieving | ||
| // location information. | ||
| func FromBlobInfoCache2(bic BlobInfoCache2WithoutV1) BlobInfoCache2 { | ||
| if bic2, ok := bic.(BlobInfoCache2); ok { | ||
| return bic2 | ||
| } | ||
| return &wrappedBlobInfoCache2{ | ||
| BlobInfoCache2WithoutV1: bic, | ||
| } | ||
| } | ||
|
|
||
| type wrappedBlobInfoCache2 struct { | ||
| BlobInfoCache2WithoutV1 | ||
| } | ||
|
|
||
| func (bic *wrappedBlobInfoCache2) RecordKnownLocation(transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, location types.BICLocationReference) { | ||
| bic.RecordKnownLocation2(transport, scope, digest, UnknownCompression, location) | ||
| } | ||
|
|
||
| func (bic *wrappedBlobInfoCache2) CandidateLocations(transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, canSubstitute bool) []types.BICReplacementCandidate { | ||
| newCandidates := bic.CandidateLocations2(transport, scope, digest, canSubstitute) | ||
| results := make([]types.BICReplacementCandidate, 0, len(newCandidates)) | ||
| for _, c := range newCandidates { | ||
| results = append(results, types.BICReplacementCandidate{ | ||
| Digest: c.Digest, | ||
| Location: c.Location, | ||
| }) | ||
| } | ||
| return results | ||
| } | ||
|
|
||
| // FromBlobInfoCache returns a BlobInfoCache2 based on a BlobInfoCache, returning the original | ||
| // object if it implements BlobInfoCache2, or a wrapper which discards compression information | ||
| // if it only implements BlobInfoCache. | ||
| func FromBlobInfoCache(bic types.BlobInfoCache) BlobInfoCache2 { | ||
| if bic2, ok := bic.(BlobInfoCache2); ok { | ||
| return bic2 | ||
| } | ||
| return &wrappedBlobInfoCache{ | ||
| BlobInfoCache: bic, | ||
| } | ||
| } | ||
|
|
||
| type wrappedBlobInfoCache struct { | ||
| types.BlobInfoCache | ||
| } | ||
|
|
||
| func (bic *wrappedBlobInfoCache) RecordKnownLocation2(transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, compressorName string, location types.BICLocationReference) { | ||
| bic.RecordKnownLocation(transport, scope, digest, location) | ||
| } | ||
|
|
||
| func (bic *wrappedBlobInfoCache) CandidateLocations2(transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, canSubstitute bool) []BICReplacementCandidate2 { | ||
| oldCandidates := bic.CandidateLocations(transport, scope, digest, canSubstitute) | ||
| results := make([]BICReplacementCandidate2, 0, len(oldCandidates)) | ||
| for _, c := range oldCandidates { | ||
| results = append(results, BICReplacementCandidate2{ | ||
| Digest: c.Digest, | ||
| Location: c.Location, | ||
| CompressorName: UnknownCompression, | ||
| }) | ||
| } | ||
| return results | ||
| } | ||
|
|
||
| // DetectCompressionFormat wraps pkg/compression.DetectCompressionFormat and returns: | ||
| // compressionOperation: always either Compress or Decompress, PreserveOriginal on error | ||
| // compressionAlgorithm: either the specified algorithm if Compress, or nil if Decompress | ||
| // compressorName: compressionAlgorithm.Name() if Compress, Uncompressed if Decompress | ||
| // reader: as with pkg/compression.DetectCompressionFormat | ||
| // err: set on error | ||
| func DetectCompressionFormat(input io.Reader) (compressionOperation types.LayerCompression, compressionAlgorithm *compression.Algorithm, compressorName string, reader io.Reader, err error) { | ||
| algo, _, reader, err := compression.DetectCompressionFormat(input) | ||
| if err != nil { | ||
| return types.PreserveOriginal, nil, UnknownCompression, nil, err | ||
| } | ||
| if name := algo.Name(); name != "" { | ||
| return types.Compress, &algo, name, reader, nil | ||
| } else { | ||
| return types.Decompress, nil, Uncompressed, reader, nil | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| package blobinfocache | ||
|
|
||
| import ( | ||
| "archive/tar" | ||
| "bytes" | ||
| "testing" | ||
|
|
||
| "github.com/containers/image/v5/pkg/compression" | ||
| "github.com/containers/image/v5/types" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestDetectCompressionFormat(t *testing.T) { | ||
| cases := []struct { | ||
| compressor *compression.Algorithm | ||
| compressionOperation types.LayerCompression | ||
| compressorName string | ||
| }{ | ||
| {nil, types.Decompress, Uncompressed}, | ||
| {&compression.Gzip, types.Compress, compression.Gzip.Name()}, | ||
| {&compression.Zstd, types.Compress, compression.Zstd.Name()}, | ||
| {&compression.Xz, types.Compress, compression.Xz.Name()}, | ||
| } | ||
| for i, c := range cases { | ||
| var tw *tar.Writer | ||
| closeFn := func() error { return nil } | ||
| buf := bytes.Buffer{} | ||
| if c.compressor != nil { | ||
| wc, err := compression.CompressStream(&buf, *c.compressor, nil) | ||
| require.Nilf(t, err, "%d: compression.CompressStream(%q) failed", i, c.compressor.Name()) | ||
| tw = tar.NewWriter(wc) | ||
| closeFn = wc.Close | ||
| } else { | ||
| tw = tar.NewWriter(&buf) | ||
| } | ||
| err := tw.Close() | ||
| assert.Nilf(t, err, "%d: tarWriter.Close()", i) | ||
| err = closeFn() | ||
| assert.Nilf(t, err, "%d: closing compressor writer", i) | ||
| op, compressor, compressorName, _, err := DetectCompressionFormat(&buf) | ||
| assert.Nilf(t, err, "%d: DetectCompressionFormat", i) | ||
| assert.Equalf(t, c.compressionOperation, op, "%d: unexpected compression operation", i) | ||
| assert.Equalf(t, c.compressorName, compressorName, "%d: unexpected compressor name", i) | ||
| if compressor != nil { | ||
| assert.Equalf(t, c.compressorName, compressor.Name(), "%d: unexpected compressor", i) | ||
| } else { | ||
| assert.Equalf(t, c.compressorName, Uncompressed, "%d: unexpected decompressed", i) | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.