Skip to content

Commit

Permalink
Fix #858 Add support for digests in sync
Browse files Browse the repository at this point in the history 
In addition, this changes the copy options to pass
CopyAllImages such that any digests referencing manifest
lists work as expected. While this is potentially a change
in behavior for previous sync incantations, it is most likely
desired as there is no option to override ones architecture
with the sync command anyways.

Signed-off-by: Andrew DeMaria <[email protected]>
  • Loading branch information
@muff1nman
muff1nman committed Jun 17, 2020
1 parent 0bd78a0 commit 6654bf4
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 37 deletions.
85 changes: 50 additions & 35 deletions cmd/skopeo/sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/transports"
"github.com/containers/image/v5/types"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
Expand All @@ -37,9 +38,9 @@ type syncOptions struct {

// repoDescriptor contains information of a single repository used as a sync source.
type repoDescriptor struct {
DirBasePath string // base path when source is 'dir'
TaggedImages []types.ImageReference // List of tagged image found for the repository
Context *types.SystemContext // SystemContext for the sync command
DirBasePath string // base path when source is 'dir'
ImageRefs []types.ImageReference // List of tagged image found for the repository
Context *types.SystemContext // SystemContext for the sync command
}

// tlsVerify is an implementation of the Unmarshaler interface, used to
Expand All @@ -51,7 +52,7 @@ type tlsVerifyConfig struct {
// registrySyncConfig contains information about a single registry, read from
// the source YAML file
type registrySyncConfig struct {
Images map[string][]string // Images map images name to slices with the images' tags
Images map[string][]string // Images map images name to slices with the images' references (tags, digests)
ImagesByTagRegex map[string]string `yaml:"images-by-tag-regex"` // Images map images name to regular expression with the images' tags
Credentials types.DockerAuthConfig // Username and password used to authenticate with the registry
TLSVerify tlsVerifyConfig `yaml:"tls-verify"` // TLS verification mode (enabled by default)
Expand Down Expand Up @@ -264,7 +265,7 @@ func imagesToCopyFromDir(dirPath string) ([]types.ImageReference, error) {
// in a registry configuration.
// It returns a repository descriptors slice with as many elements as the images
// found and any error encountered. Each element of the slice is a list of
// tagged image references, to be used as sync source.
// image references, to be used as sync source.
func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourceCtx types.SystemContext) ([]repoDescriptor, error) {
serverCtx := &sourceCtx
// override ctx with per-registryName options
Expand All @@ -275,7 +276,7 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
serverCtx.DockerAuthConfig = &cfg.Credentials

var repoDescList []repoDescriptor
for imageName, tags := range cfg.Images {
for imageName, refs := range cfg.Images {
repoLogger := logrus.WithFields(logrus.Fields{
"repo": imageName,
"registry": registryName,
Expand All @@ -290,24 +291,37 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
repoLogger.Info("Processing repo")

var sourceReferences []types.ImageReference
if len(tags) != 0 {
for _, tag := range tags {
tagLogger := logrus.WithFields(logrus.Fields{"tag": tag})
taggedRef, err := reference.WithTag(repoRef, tag)
if err != nil {
tagLogger.Error("Error parsing tag, skipping")
logrus.Error(err)
continue
if len(refs) != 0 {
for _, ref := range refs {
tagLogger := logrus.WithFields(logrus.Fields{"ref": ref})
var named reference.Named
// first try as digest
if d, err := digest.Parse(ref); err == nil {
named, err = reference.WithDigest(repoRef, d)
if err != nil {
tagLogger.Error("Error processing ref, skipping")
logrus.Error(err)
continue
}
} else {
tagLogger.Debugf("Ref was not a digest, trying as a tag: %s", err)
named, err = reference.WithTag(repoRef, ref)
if err != nil {
tagLogger.Error("Error parsing ref, skipping")
logrus.Error(err)
continue
}
}
imageRef, err := docker.NewReference(taggedRef)

imageRef, err := docker.NewReference(named)
if err != nil {
tagLogger.Error("Error processing tag, skipping")
tagLogger.Error("Error processing ref, skipping")
logrus.Errorf("Error getting image reference: %s", err)
continue
}
sourceReferences = append(sourceReferences, imageRef)
}
} else { // len(tags) == 0
} else { // len(refs) == 0
repoLogger.Info("Querying registry for image tags")
sourceReferences, err = imagesToCopyFromRepo(serverCtx, repoRef)
if err != nil {
Expand All @@ -318,12 +332,12 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
}

if len(sourceReferences) == 0 {
repoLogger.Warnf("No tags to sync found")
repoLogger.Warnf("No refs to sync found")
continue
}
repoDescList = append(repoDescList, repoDescriptor{
TaggedImages: sourceReferences,
Context: serverCtx})
ImageRefs: sourceReferences,
Context: serverCtx})
}

for imageName, tagRegex := range cfg.ImagesByTagRegex {
Expand Down Expand Up @@ -372,12 +386,12 @@ func imagesToCopyFromRegistry(registryName string, cfg registrySyncConfig, sourc
}

if len(sourceReferences) == 0 {
repoLogger.Warnf("No tags to sync found")
repoLogger.Warnf("No refs to sync found")
continue
}
repoDescList = append(repoDescList, repoDescriptor{
TaggedImages: sourceReferences,
Context: serverCtx})
ImageRefs: sourceReferences,
Context: serverCtx})
}

return repoDescList, nil
Expand Down Expand Up @@ -410,13 +424,13 @@ func imagesToCopy(source string, transport string, sourceCtx *types.SystemContex
if err != nil {
return nil, errors.Wrapf(err, "Cannot obtain a valid image reference for transport %q and reference %q", docker.Transport.Name(), named.String())
}
desc.TaggedImages = []types.ImageReference{srcRef}
desc.ImageRefs = []types.ImageReference{srcRef}
} else {
desc.TaggedImages, err = imagesToCopyFromRepo(sourceCtx, named)
desc.ImageRefs, err = imagesToCopyFromRepo(sourceCtx, named)
if err != nil {
return descriptors, err
}
if len(desc.TaggedImages) == 0 {
if len(desc.ImageRefs) == 0 {
return descriptors, errors.Errorf("No images to sync found in %q", source)
}
}
Expand All @@ -432,11 +446,11 @@ func imagesToCopy(source string, transport string, sourceCtx *types.SystemContex
}
desc.DirBasePath = source
var err error
desc.TaggedImages, err = imagesToCopyFromDir(source)
desc.ImageRefs, err = imagesToCopyFromDir(source)
if err != nil {
return descriptors, err
}
if len(desc.TaggedImages) == 0 {
if len(desc.ImageRefs) == 0 {
return descriptors, errors.Errorf("No images to sync found in %q", source)
}
descriptors = append(descriptors, desc)
Expand Down Expand Up @@ -526,15 +540,16 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {

imagesNumber := 0
options := copy.Options{
RemoveSignatures: opts.removeSignatures,
SignBy: opts.signByFingerprint,
ReportWriter: os.Stdout,
DestinationCtx: destinationCtx,
RemoveSignatures: opts.removeSignatures,
SignBy: opts.signByFingerprint,
ReportWriter: os.Stdout,
DestinationCtx: destinationCtx,
ImageListSelection: copy.CopyAllImages,
}

for _, srcRepo := range srcRepoList {
options.SourceCtx = srcRepo.Context
for counter, ref := range srcRepo.TaggedImages {
for counter, ref := range srcRepo.ImageRefs {
var destSuffix string
switch ref.Transport() {
case docker.Transport:
Expand All @@ -561,11 +576,11 @@ func (opts *syncOptions) run(args []string, stdout io.Writer) error {
logrus.WithFields(logrus.Fields{
"from": transports.ImageName(ref),
"to": transports.ImageName(destRef),
}).Infof("Copying image tag %d/%d", counter+1, len(srcRepo.TaggedImages))
}).Infof("Copying image ref %d/%d", counter+1, len(srcRepo.ImageRefs))

_, err = copy.Image(ctx, policyContext, destRef, ref, &options)
if err != nil {
return errors.Wrapf(err, "Error copying tag %q", transports.ImageName(ref))
return errors.Wrapf(err, "Error copying ref %q", transports.ImageName(ref))
}
imagesNumber++
}
Expand Down
3 changes: 2 additions & 1 deletion docs/skopeo-sync.1.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ registry.example.com:
redis:
- "1.0"
- "2.0"
- "sha256:0000000000000000000000000000000011111111111111111111111111111111"
images-by-tag-regex:
nginx: ^1\.13\.[12]-alpine-perl$
credentials:
Expand All @@ -162,7 +163,7 @@ skopeo sync --src yaml --dest docker sync.yml my-registry.local.lan/repo/
```
This will copy the following images:
- Repository `registry.example.com/busybox`: all images, as no tags are specified.
- Repository `registry.example.com/redis`: images tagged "1.0" and "2.0".
- Repository `registry.example.com/redis`: images tagged "1.0" and "2.0" along with image with digest "sha256:0000000000000000000000000000000011111111111111111111111111111111".
- Repository `registry.example.com/nginx`: images tagged "1.13.1-alpine-perl" and "1.13.2-alpine-perl".
- Repository `quay.io/coreos/etcd`: images tagged "latest".
Expand Down
33 changes: 32 additions & 1 deletion integration/sync_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func (s *SyncSuite) TestDocker2DirTagged(c *check.C) {
c.Assert(err, check.IsNil)

// copy docker => dir
assertSkopeoSucceeds(c, "", "copy", "docker://"+image, "dir:"+dir2)
assertSkopeoSucceeds(c, "", "copy", "--all", "docker://"+image, "dir:"+dir2)
_, err = os.Stat(path.Join(dir2, "manifest.json"))
c.Assert(err, check.IsNil)

Expand Down Expand Up @@ -289,6 +289,37 @@ docker.io:
c.Assert(nManifests, check.Equals, nTags)
}

func (s *SyncSuite) TestYamlDigest2Dir(c *check.C) {
tmpDir, err := ioutil.TempDir("", "skopeo-sync-test")
c.Assert(err, check.IsNil)
defer os.RemoveAll(tmpDir)
dir1 := path.Join(tmpDir, "dir1")

yamlConfig := `
docker.io:
images:
redis:
- sha256:61ce79d60150379787d7da677dcb89a7a047ced63406e29d6b2677b2b2163e92
`
yamlFile := path.Join(tmpDir, "registries.yaml")
ioutil.WriteFile(yamlFile, []byte(yamlConfig), 0644)
assertSkopeoSucceeds(c, "", "sync", "--scoped", "--src", "yaml", "--dest", "dir", yamlFile, dir1)

nManifests := 0
err = filepath.Walk(dir1, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && info.Name() == "manifest.json" {
nManifests++
return filepath.SkipDir
}
return nil
})
c.Assert(err, check.IsNil)
c.Assert(nManifests, check.Equals, 1)
}

func (s *SyncSuite) TestYaml2Dir(c *check.C) {
tmpDir, err := ioutil.TempDir("", "skopeo-sync-test")
c.Assert(err, check.IsNil)
Expand Down

0 comments on commit 6654bf4

Please sign in to comment.