Skip to content

Commit

Permalink
Nydusify: move nydus constant to nydusify package
Browse files Browse the repository at this point in the history 
So that we can import only one `nydusify` package from buildkit/acceld.

Signed-off-by: Yan Song <[email protected]>
  • Loading branch information
@imeoer
imeoer committed Mar 25, 2022
1 parent c01190c commit e6570be
Show file tree
Hide file tree
Showing 18 changed files with 113 additions and 87 deletions.
10 changes: 5 additions & 5 deletions contrib/nydusify/pkg/backend/backend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ import (
"context"
"fmt"

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/nydusify"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
Expand Down Expand Up @@ -40,12 +40,12 @@ func blobDesc(size int64, blobID string) ocispec.Descriptor {
desc := ocispec.Descriptor{
Digest: blobDigest,
Size: size,
MediaType: utils.MediaTypeNydusBlob,
MediaType: nydusify.MediaTypeNydusBlob,
Annotations: map[string]string{
// Use `utils.LayerAnnotationUncompressed` to generate
// Use `nydusify.LayerAnnotationUncompressed` to generate
// DiffID of layer defined in OCI spec
utils.LayerAnnotationUncompressed: blobDigest.String(),
utils.LayerAnnotationNydusBlob: "true",
nydusify.LayerAnnotationUncompressed: blobDigest.String(),
nydusify.LayerAnnotationNydusBlob: "true",
},
}

Expand Down
73 changes: 37 additions & 36 deletions contrib/nydusify/pkg/cache/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"strconv"

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/backend"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/nydusify"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -78,7 +79,7 @@ func New(remote *remote.Remote, opt Opt) (*Cache, error) {
}

func (cacheRecord *Record) GetReferenceBlobs() []string {
listStr := cacheRecord.NydusBootstrapDesc.Annotations[utils.LayerAnnotationNydusReferenceBlobIDs]
listStr := cacheRecord.NydusBootstrapDesc.Annotations[nydusify.LayerAnnotationNydusReferenceBlobIDs]
if listStr == "" {
return []string{}
}
Expand Down Expand Up @@ -107,11 +108,11 @@ func (cache *Cache) recordToLayer(record *Record) (*ocispec.Descriptor, *ocispec
if record.NydusBlobDesc != nil {
if cache.opt.Backend.Type() == backend.RegistryBackend {
return nil, &ocispec.Descriptor{
MediaType: utils.MediaTypeNydusBlob,
MediaType: nydusify.MediaTypeNydusBlob,
Digest: record.NydusBlobDesc.Digest,
Size: record.NydusBlobDesc.Size,
Annotations: map[string]string{
utils.LayerAnnotationNydusBlob: "true",
nydusify.LayerAnnotationNydusBlob: "true",
},
}
}
Expand All @@ -128,14 +129,14 @@ func (cache *Cache) recordToLayer(record *Record) (*ocispec.Descriptor, *ocispec
Digest: record.NydusBootstrapDesc.Digest,
Size: record.NydusBootstrapDesc.Size,
Annotations: map[string]string{
utils.LayerAnnotationNydusBootstrap: "true",
utils.LayerAnnotationNydusSourceChainID: record.SourceChainID.String(),
nydusify.LayerAnnotationNydusBootstrap: "true",
nydusify.LayerAnnotationNydusSourceChainID: record.SourceChainID.String(),
// Use the annotation to record bootstrap layer DiffID.
utils.LayerAnnotationUncompressed: record.NydusBootstrapDiffID.String(),
nydusify.LayerAnnotationUncompressed: record.NydusBootstrapDiffID.String(),
},
}
if refenceBlobsStr, ok := record.NydusBootstrapDesc.Annotations[utils.LayerAnnotationNydusReferenceBlobIDs]; ok {
bootstrapCacheDesc.Annotations[utils.LayerAnnotationNydusReferenceBlobIDs] = refenceBlobsStr
if refenceBlobsStr, ok := record.NydusBootstrapDesc.Annotations[nydusify.LayerAnnotationNydusReferenceBlobIDs]; ok {
bootstrapCacheDesc.Annotations[nydusify.LayerAnnotationNydusReferenceBlobIDs] = refenceBlobsStr
}

var blobCacheDesc *ocispec.Descriptor
Expand All @@ -144,17 +145,17 @@ func (cache *Cache) recordToLayer(record *Record) (*ocispec.Descriptor, *ocispec
// to registry instead of storage backend.
if cache.opt.Backend.Type() == backend.RegistryBackend {
blobCacheDesc = &ocispec.Descriptor{
MediaType: utils.MediaTypeNydusBlob,
MediaType: nydusify.MediaTypeNydusBlob,
Digest: record.NydusBlobDesc.Digest,
Size: record.NydusBlobDesc.Size,
Annotations: map[string]string{
utils.LayerAnnotationNydusBlob: "true",
utils.LayerAnnotationNydusSourceChainID: record.SourceChainID.String(),
nydusify.LayerAnnotationNydusBlob: "true",
nydusify.LayerAnnotationNydusSourceChainID: record.SourceChainID.String(),
},
}
} else {
bootstrapCacheDesc.Annotations[utils.LayerAnnotationNydusBlobDigest] = record.NydusBlobDesc.Digest.String()
bootstrapCacheDesc.Annotations[utils.LayerAnnotationNydusBlobSize] = strconv.FormatInt(record.NydusBlobDesc.Size, 10)
bootstrapCacheDesc.Annotations[nydusify.LayerAnnotationNydusBlobDigest] = record.NydusBlobDesc.Digest.String()
bootstrapCacheDesc.Annotations[nydusify.LayerAnnotationNydusBlobSize] = strconv.FormatInt(record.NydusBlobDesc.Size, 10)
}
}

Expand Down Expand Up @@ -189,17 +190,17 @@ func (cache *Cache) exportRecordsToLayers() []ocispec.Descriptor {
}

func (cache *Cache) layerToRecord(layer *ocispec.Descriptor) *Record {
sourceChainIDStr, ok := layer.Annotations[utils.LayerAnnotationNydusSourceChainID]
sourceChainIDStr, ok := layer.Annotations[nydusify.LayerAnnotationNydusSourceChainID]
if !ok {
if layer.Annotations[utils.LayerAnnotationNydusBlob] == "true" {
if layer.Annotations[nydusify.LayerAnnotationNydusBlob] == "true" {
// for reference blob layers
return &Record{
NydusBlobDesc: &ocispec.Descriptor{
MediaType: layer.MediaType,
Digest: layer.Digest,
Size: layer.Size,
Annotations: map[string]string{
utils.LayerAnnotationNydusBlob: "true",
nydusify.LayerAnnotationNydusBlob: "true",
},
},
}
Expand All @@ -215,8 +216,8 @@ func (cache *Cache) layerToRecord(layer *ocispec.Descriptor) *Record {
}

// Handle bootstrap cache layer
if layer.Annotations[utils.LayerAnnotationNydusBootstrap] == "true" {
uncompressedDigestStr := layer.Annotations[utils.LayerAnnotationUncompressed]
if layer.Annotations[nydusify.LayerAnnotationNydusBootstrap] == "true" {
uncompressedDigestStr := layer.Annotations[nydusify.LayerAnnotationUncompressed]
if uncompressedDigestStr == "" {
return nil
}
Expand All @@ -229,31 +230,31 @@ func (cache *Cache) layerToRecord(layer *ocispec.Descriptor) *Record {
Digest: layer.Digest,
Size: layer.Size,
Annotations: map[string]string{
utils.LayerAnnotationNydusBootstrap: "true",
utils.LayerAnnotationUncompressed: uncompressedDigestStr,
nydusify.LayerAnnotationNydusBootstrap: "true",
nydusify.LayerAnnotationUncompressed: uncompressedDigestStr,
},
}
referenceBlobsStr := layer.Annotations[utils.LayerAnnotationNydusReferenceBlobIDs]
referenceBlobsStr := layer.Annotations[nydusify.LayerAnnotationNydusReferenceBlobIDs]
if referenceBlobsStr != "" {
bootstrapDesc.Annotations[utils.LayerAnnotationNydusReferenceBlobIDs] = referenceBlobsStr
bootstrapDesc.Annotations[nydusify.LayerAnnotationNydusReferenceBlobIDs] = referenceBlobsStr
}
var nydusBlobDesc *ocispec.Descriptor
if layer.Annotations[utils.LayerAnnotationNydusBlobDigest] != "" &&
layer.Annotations[utils.LayerAnnotationNydusBlobSize] != "" {
blobDigest := digest.Digest(layer.Annotations[utils.LayerAnnotationNydusBlobDigest])
if layer.Annotations[nydusify.LayerAnnotationNydusBlobDigest] != "" &&
layer.Annotations[nydusify.LayerAnnotationNydusBlobSize] != "" {
blobDigest := digest.Digest(layer.Annotations[nydusify.LayerAnnotationNydusBlobDigest])
if blobDigest.Validate() != nil {
return nil
}
blobSize, err := strconv.ParseInt(layer.Annotations[utils.LayerAnnotationNydusBlobSize], 10, 64)
blobSize, err := strconv.ParseInt(layer.Annotations[nydusify.LayerAnnotationNydusBlobSize], 10, 64)
if err != nil {
return nil
}
nydusBlobDesc = &ocispec.Descriptor{
MediaType: utils.MediaTypeNydusBlob,
MediaType: nydusify.MediaTypeNydusBlob,
Digest: blobDigest,
Size: blobSize,
Annotations: map[string]string{
utils.LayerAnnotationNydusBlob: "true",
nydusify.LayerAnnotationNydusBlob: "true",
},
}
}
Expand All @@ -266,13 +267,13 @@ func (cache *Cache) layerToRecord(layer *ocispec.Descriptor) *Record {
}

// Handle blob cache layer
if layer.Annotations[utils.LayerAnnotationNydusBlob] == "true" {
if layer.Annotations[nydusify.LayerAnnotationNydusBlob] == "true" {
nydusBlobDesc := &ocispec.Descriptor{
MediaType: layer.MediaType,
Digest: layer.Digest,
Size: layer.Size,
Annotations: map[string]string{
utils.LayerAnnotationNydusBlob: "true",
nydusify.LayerAnnotationNydusBlob: "true",
},
}
return &Record{
Expand Down Expand Up @@ -347,10 +348,10 @@ func (cache *Cache) Export(ctx context.Context) error {
diffIDs := []digest.Digest{}
for _, layer := range layers {
var diffID digest.Digest
if layer.MediaType == utils.MediaTypeNydusBlob {
if layer.MediaType == nydusify.MediaTypeNydusBlob {
diffID = layer.Digest
} else {
diffID = digest.Digest(layer.Annotations[utils.LayerAnnotationUncompressed])
diffID = digest.Digest(layer.Annotations[nydusify.LayerAnnotationUncompressed])
}
if diffID.Validate() == nil {
diffIDs = append(diffIDs, diffID)
Expand Down Expand Up @@ -402,7 +403,7 @@ func (cache *Cache) Export(ctx context.Context) error {
Config: *configDesc,
Layers: layers,
Annotations: map[string]string{
utils.ManifestNydusCache: cache.opt.Version,
nydusify.ManifestNydusCache: cache.opt.Version,
},
},
}
Expand Down Expand Up @@ -444,10 +445,10 @@ func (cache *Cache) Import(ctx context.Context) error {
}

// Discard the cache mismatched version
if manifest.Annotations[utils.ManifestNydusCache] != cache.opt.Version {
if manifest.Annotations[nydusify.ManifestNydusCache] != cache.opt.Version {
return fmt.Errorf(
"unmatched cache image version %s, required to be %s",
manifest.Annotations[utils.ManifestNydusCache], cache.opt.Version,
manifest.Annotations[nydusify.ManifestNydusCache], cache.opt.Version,
)
}

Expand Down Expand Up @@ -524,7 +525,7 @@ func (cache *Cache) PullBootstrap(ctx context.Context, bootstrapDesc *ocispec.De
}
defer reader.Close()

if err := utils.UnpackFile(reader, utils.BootstrapFileNameInLayer, target); err != nil {
if err := utils.UnpackFile(reader, nydusify.BootstrapFileNameInLayer, target); err != nil {
return errors.Wrap(err, "Unpack cached bootstrap layer")
}

Expand Down
20 changes: 10 additions & 10 deletions contrib/nydusify/pkg/cache/cache_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@ import (
"github.com/stretchr/testify/assert"

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/backend"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/nydusify"
)

func makeRecord(id int64, hashBlob bool) *Record {
var blobDesc *ocispec.Descriptor
idStr := strconv.FormatInt(id, 10)
if hashBlob {
blobDesc = &ocispec.Descriptor{
MediaType: utils.MediaTypeNydusBlob,
MediaType: nydusify.MediaTypeNydusBlob,
Digest: digest.FromString("blob-" + idStr),
Size: id,
}
Expand All @@ -46,27 +46,27 @@ func makeBootstrapLayer(id int64, hasBlob bool) ocispec.Descriptor {
Digest: digest.FromString("bootstrap-" + idStr),
Size: id,
Annotations: map[string]string{
utils.LayerAnnotationNydusBootstrap: "true",
utils.LayerAnnotationNydusSourceChainID: digest.FromString("chain-" + idStr).String(),
utils.LayerAnnotationUncompressed: digest.FromString("bootstrap-uncompressed-" + idStr).String(),
nydusify.LayerAnnotationNydusBootstrap: "true",
nydusify.LayerAnnotationNydusSourceChainID: digest.FromString("chain-" + idStr).String(),
nydusify.LayerAnnotationUncompressed: digest.FromString("bootstrap-uncompressed-" + idStr).String(),
},
}
if hasBlob {
desc.Annotations[utils.LayerAnnotationNydusBlobDigest] = digest.FromString("blob-" + idStr).String()
desc.Annotations[utils.LayerAnnotationNydusBlobSize] = fmt.Sprintf("%d", id)
desc.Annotations[nydusify.LayerAnnotationNydusBlobDigest] = digest.FromString("blob-" + idStr).String()
desc.Annotations[nydusify.LayerAnnotationNydusBlobSize] = fmt.Sprintf("%d", id)
}
return desc
}

func makeBlobLayer(id int64) ocispec.Descriptor {
idStr := strconv.FormatInt(id, 10)
return ocispec.Descriptor{
MediaType: utils.MediaTypeNydusBlob,
MediaType: nydusify.MediaTypeNydusBlob,
Digest: digest.FromString("blob-" + idStr),
Size: id,
Annotations: map[string]string{
utils.LayerAnnotationNydusBlob: "true",
utils.LayerAnnotationNydusSourceChainID: digest.FromString("chain-" + idStr).String(),
nydusify.LayerAnnotationNydusBlob: "true",
nydusify.LayerAnnotationNydusSourceChainID: digest.FromString("chain-" + idStr).String(),
},
}
}
Expand Down
3 changes: 2 additions & 1 deletion contrib/nydusify/pkg/checker/output.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/nydusify"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/parser"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
)
Expand Down Expand Up @@ -78,7 +79,7 @@ func (checker *Checker) Output(
}
defer bootstrapReader.Close()

if err := utils.UnpackFile(bootstrapReader, utils.BootstrapFileNameInLayer, target); err != nil {
if err := utils.UnpackFile(bootstrapReader, nydusify.BootstrapFileNameInLayer, target); err != nil {
return errors.Wrap(err, "unpack Nydus bootstrap layer")
}
}
Expand Down
7 changes: 4 additions & 3 deletions contrib/nydusify/pkg/checker/rule/manifest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/nydusify"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/parser"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
)
Expand Down Expand Up @@ -67,12 +68,12 @@ func (rule *ManifestRule) Validate() error {
layers := rule.TargetParsed.NydusImage.Manifest.Layers
for i, layer := range layers {
if i == len(layers)-1 {
if layer.Annotations[utils.LayerAnnotationNydusBootstrap] != "true" {
if layer.Annotations[nydusify.LayerAnnotationNydusBootstrap] != "true" {
return errors.New("invalid bootstrap layer in nydus image manifest")
}
} else {
if layer.MediaType != utils.MediaTypeNydusBlob ||
layer.Annotations[utils.LayerAnnotationNydusBlob] != "true" {
if layer.MediaType != nydusify.MediaTypeNydusBlob ||
layer.Annotations[nydusify.LayerAnnotationNydusBlob] != "true" {
return errors.New("invalid blob layer in nydus image manifest")
}
}
Expand Down
3 changes: 2 additions & 1 deletion contrib/nydusify/pkg/converter/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/backend"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/cache"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter/provider"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/nydusify"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
)
Expand Down Expand Up @@ -112,7 +113,7 @@ func (cg *cacheGlue) Push(ctx context.Context, layer *buildLayer) error {

// Push bootstrap layer to cache image
bootstrapReader, err := utils.PackTargz(
layer.bootstrapPath, utils.BootstrapFileNameInLayer, true,
layer.bootstrapPath, nydusify.BootstrapFileNameInLayer, true,
)
if err != nil {
return pushDone(errors.Wrapf(err, "Compress bootstrap layer"))
Expand Down
7 changes: 4 additions & 3 deletions contrib/nydusify/pkg/converter/chunk_dict.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker/tool"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter/provider"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/nydusify"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/parser"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
)
Expand Down Expand Up @@ -88,7 +89,7 @@ func getChunkDictFromRegistry(prepareDir, imageName string, insecure bool, platf
return "", err
}
defer rc.Close()
if err = utils.UnpackFile(rc, utils.BootstrapFileNameInLayer, targetFile); err != nil {
if err = utils.UnpackFile(rc, nydusify.BootstrapFileNameInLayer, targetFile); err != nil {
return "", err
}
return targetFile, nil
Expand Down Expand Up @@ -132,11 +133,11 @@ func (cvt *Converter) prepareBootstrap(prepareDir, from, info string) (string, [
for _, blobInfo := range blobsInfo {
blobDigest := digest.NewDigestFromEncoded(digest.SHA256, blobInfo.BlobID)
blobLayers = append(blobLayers, ocispec.Descriptor{
MediaType: utils.MediaTypeNydusBlob,
MediaType: nydusify.MediaTypeNydusBlob,
Size: int64(blobInfo.CompressedSize),
Digest: blobDigest,
Annotations: map[string]string{
utils.LayerAnnotationNydusBlob: "true",
nydusify.LayerAnnotationNydusBlob: "true",
},
})
}
Expand Down
Loading

0 comments on commit e6570be

Please sign in to comment.