From 629d86445961f452763b7e4a031e98a4e19daff6 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Fri, 11 Feb 2022 14:12:39 -0600 Subject: [PATCH 1/3] Temporarily pull machine images from side repo Until podman4 is in the fcos trees, we need to pull the machine images from a side repository. There is a hard coded bit that forces the side repo download right now. Simple comment or removal of the bit will revert to normal download behavior. [NO NEW TESTS NEEDED] Signed-off-by: Brent Baude --- pkg/machine/fcos.go | 57 ++++- .../stream-metadata-go/release/release.go | 112 ++++++++++ .../stream-metadata-go/release/rhcos/rhcos.go | 14 ++ .../stream-metadata-go/release/translate.go | 196 ++++++++++++++++++ vendor/modules.txt | 2 + 5 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 vendor/github.com/coreos/stream-metadata-go/release/release.go create mode 100644 vendor/github.com/coreos/stream-metadata-go/release/rhcos/rhcos.go create mode 100644 vendor/github.com/coreos/stream-metadata-go/release/translate.go diff --git a/pkg/machine/fcos.go b/pkg/machine/fcos.go index 60ab471ee8..4d3e2edf4a 100644 --- a/pkg/machine/fcos.go +++ b/pkg/machine/fcos.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/coreos/stream-metadata-go/fedoracoreos" + "github.com/coreos/stream-metadata-go/release" "github.com/coreos/stream-metadata-go/stream" "github.com/pkg/errors" @@ -28,6 +29,14 @@ var ( Format string = "qcow2.xz" ) +const ( + // Used for testing the latest podman in fcos + // special builds + podmanTesting = "podman-testing" + PodmanTestingHost = "fedorapeople.org" + PodmanTestingURL = "groups/podman/testing" +) + type FcosDownload struct { Download } @@ -111,14 +120,39 @@ func getFcosArch() string { return arch } +// getStreamURL is a wrapper for the fcos.GetStream URL +// so that we can inject a special stream and url for +// testing podman before it merges into fcos builds +func getStreamURL(streamType string) url2.URL { + // For the podmanTesting stream type, we point to + // a custom url on fedorapeople.org + if streamType == podmanTesting { + return url2.URL{ + Scheme: "https", + Host: PodmanTestingHost, + Path: fmt.Sprintf("%s/%s.json", PodmanTestingURL, "podman4"), + } + } + return fedoracoreos.GetStreamURL(streamType) +} + // This should get Exported and stay put as it will apply to all fcos downloads // getFCOS parses fedoraCoreOS's stream and returns the image download URL and the release version func getFCOSDownload(imageStream string) (*fcosDownloadInfo, error) { var ( fcosstable stream.Stream + altMeta release.Release streamType string ) + + // This is being hard set to testing. Once podman4 is in the + // fcos trees, we should remove it and re-release at least on + // macs. + imageStream = "podman-testing" + switch imageStream { + case "podman-testing": + streamType = "podman-testing" case "testing", "": streamType = fedoracoreos.StreamTesting case "next": @@ -128,7 +162,7 @@ func getFCOSDownload(imageStream string) (*fcosDownloadInfo, error) { default: return nil, errors.Errorf("invalid stream %s: valid streams are `testing` and `stable`", imageStream) } - streamurl := fedoracoreos.GetStreamURL(streamType) + streamurl := getStreamURL(streamType) resp, err := http.Get(streamurl.String()) if err != nil { return nil, err @@ -142,6 +176,27 @@ func getFCOSDownload(imageStream string) (*fcosDownloadInfo, error) { logrus.Error(err) } }() + if imageStream == podmanTesting { + if err := json.Unmarshal(body, &altMeta); err != nil { + return nil, err + } + + arches, ok := altMeta.Architectures[getFcosArch()] + if !ok { + return nil, fmt.Errorf("unable to pull VM image: no targetArch in stream") + } + qcow2, ok := arches.Media.Qemu.Artifacts["qcow2.xz"] + if !ok { + return nil, fmt.Errorf("unable to pull VM image: no qcow2.xz format in stream") + } + disk := qcow2.Disk + + return &fcosDownloadInfo{ + Location: disk.Location, + Sha256Sum: disk.Sha256, + CompressionType: "xz", + }, nil + } if err := json.Unmarshal(body, &fcosstable); err != nil { return nil, err diff --git a/vendor/github.com/coreos/stream-metadata-go/release/release.go b/vendor/github.com/coreos/stream-metadata-go/release/release.go new file mode 100644 index 0000000000..84a032703b --- /dev/null +++ b/vendor/github.com/coreos/stream-metadata-go/release/release.go @@ -0,0 +1,112 @@ +// Package release contains APIs for interacting with a +// particular "release". Avoid this unless you are sure +// you need it. It's expected that CoreOS users interact +// with streams instead. +package release + +import ( + relrhcos "github.com/coreos/stream-metadata-go/release/rhcos" +) + +// Index models the release index: +// https://github.com/coreos/fedora-coreos-tracker/tree/master/metadata/release-index +type Index struct { + Note string `json:"note"` // used to note to users not to consume the release metadata index + Releases []IndexRelease `json:"releases"` + Metadata Metadata `json:"metadata"` + Stream string `json:"stream"` +} + +// IndexRelease is a "release pointer" from a release index +type IndexRelease struct { + Commits []IndexReleaseCommit `json:"commits"` + Version string `json:"version"` + MetadataURL string `json:"metadata"` +} + +// IndexReleaseCommit describes an ostree commit plus architecture +type IndexReleaseCommit struct { + Architecture string `json:"architecture"` + Checksum string `json:"checksum"` +} + +// Release contains details from release.json +type Release struct { + Release string `json:"release"` + Stream string `json:"stream"` + Metadata Metadata `json:"metadata"` + Architectures map[string]Arch `json:"architectures"` +} + +// Metadata is common metadata that contains last-modified +type Metadata struct { + LastModified string `json:"last-modified"` +} + +// Arch release details +type Arch struct { + Commit string `json:"commit"` + Media Media `json:"media"` + RHELCoreOSExtensions *relrhcos.Extensions `json:"rhel-coreos-extensions,omitempty"` +} + +// Media contains release details for various platforms +type Media struct { + Aliyun *PlatformBase `json:"aliyun"` + Aws *PlatformAws `json:"aws"` + Azure *PlatformBase `json:"azure"` + Digitalocean *PlatformBase `json:"digitalocean"` + Exoscale *PlatformBase `json:"exoscale"` + Gcp *PlatformGcp `json:"gcp"` + Ibmcloud *PlatformBase `json:"ibmcloud"` + Metal *PlatformBase `json:"metal"` + Openstack *PlatformBase `json:"openstack"` + Qemu *PlatformBase `json:"qemu"` + Vmware *PlatformBase `json:"vmware"` + Vultr *PlatformBase `json:"vultr"` +} + +// PlatformBase with no cloud images +type PlatformBase struct { + Artifacts map[string]ImageFormat `json:"artifacts"` +} + +// PlatformAws contains AWS image information +type PlatformAws struct { + PlatformBase + Images map[string]CloudImage `json:"images"` +} + +// PlatformGcp GCP image detail +type PlatformGcp struct { + PlatformBase + Image *GcpImage `json:"image"` +} + +// ImageFormat contains all artifacts for a single OS image +type ImageFormat struct { + Disk *Artifact `json:"disk,omitempty"` + Kernel *Artifact `json:"kernel,omitempty"` + Initramfs *Artifact `json:"initramfs,omitempty"` + Rootfs *Artifact `json:"rootfs,omitempty"` +} + +// Artifact represents one image file, plus its metadata +type Artifact struct { + Location string `json:"location"` + Signature string `json:"signature"` + Sha256 string `json:"sha256"` + UncompressedSha256 string `json:"uncompressed-sha256,omitempty"` +} + +// CloudImage generic image detail +type CloudImage struct { + Image string `json:"image"` +} + +// GcpImage represents a GCP cloud image +type GcpImage struct { + Project string `json:"project,omitempty"` + Family string `json:"family,omitempty"` + Name string `json:"name,omitempty"` +} diff --git a/vendor/github.com/coreos/stream-metadata-go/release/rhcos/rhcos.go b/vendor/github.com/coreos/stream-metadata-go/release/rhcos/rhcos.go new file mode 100644 index 0000000000..aeae2c8be1 --- /dev/null +++ b/vendor/github.com/coreos/stream-metadata-go/release/rhcos/rhcos.go @@ -0,0 +1,14 @@ +package rhcos + +// Extensions is data specific to Red Hat Enterprise Linux CoreOS +type Extensions struct { + AzureDisk *AzureDisk `json:"azure-disk,omitempty"` +} + +// AzureDisk represents an Azure cloud image. +type AzureDisk struct { + // URL to an image already stored in Azure infrastructure + // that can be copied into an image gallery. Avoid creating VMs directly + // from this URL as that may lead to performance limitations. + URL string `json:"url,omitempty"` +} diff --git a/vendor/github.com/coreos/stream-metadata-go/release/translate.go b/vendor/github.com/coreos/stream-metadata-go/release/translate.go new file mode 100644 index 0000000000..518c75eb96 --- /dev/null +++ b/vendor/github.com/coreos/stream-metadata-go/release/translate.go @@ -0,0 +1,196 @@ +package release + +import ( + "github.com/coreos/stream-metadata-go/stream" + "github.com/coreos/stream-metadata-go/stream/rhcos" +) + +func mapArtifact(ra *Artifact) *stream.Artifact { + if ra == nil { + return nil + } + return &stream.Artifact{ + Location: ra.Location, + Signature: ra.Signature, + Sha256: ra.Sha256, + UncompressedSha256: ra.UncompressedSha256, + } +} + +func mapFormats(m map[string]ImageFormat) map[string]stream.ImageFormat { + r := make(map[string]stream.ImageFormat) + for k, v := range m { + r[k] = stream.ImageFormat{ + Disk: mapArtifact(v.Disk), + Kernel: mapArtifact(v.Kernel), + Initramfs: mapArtifact(v.Initramfs), + Rootfs: mapArtifact(v.Rootfs), + } + } + return r +} + +// Convert a release architecture to a stream architecture +func (releaseArch *Arch) toStreamArch(rel *Release) stream.Arch { + artifacts := make(map[string]stream.PlatformArtifacts) + cloudImages := stream.Images{} + var rhcosExt *rhcos.Extensions + relRHCOSExt := releaseArch.RHELCoreOSExtensions + if relRHCOSExt != nil { + rhcosExt = &rhcos.Extensions{} + } + if releaseArch.Media.Aws != nil { + artifacts["aws"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Aws.Artifacts), + } + awsAmis := stream.AwsImage{ + Regions: make(map[string]stream.AwsRegionImage), + } + if releaseArch.Media.Aws.Images != nil { + for region, ami := range releaseArch.Media.Aws.Images { + ri := stream.AwsRegionImage{Release: rel.Release, Image: ami.Image} + awsAmis.Regions[region] = ri + + } + cloudImages.Aws = &awsAmis + } + } + + if releaseArch.Media.Azure != nil { + artifacts["azure"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Azure.Artifacts), + } + + if relRHCOSExt != nil { + az := relRHCOSExt.AzureDisk + if az != nil { + rhcosExt.AzureDisk = &rhcos.AzureDisk{ + Release: rel.Release, + URL: az.URL, + } + } + } + // In the future this is where we'd also add FCOS Marketplace data. + // See https://github.com/coreos/stream-metadata-go/issues/13 + } + + if releaseArch.Media.Aliyun != nil { + artifacts["aliyun"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Aliyun.Artifacts), + } + } + + if releaseArch.Media.Exoscale != nil { + artifacts["exoscale"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Exoscale.Artifacts), + } + } + + if releaseArch.Media.Vultr != nil { + artifacts["vultr"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Vultr.Artifacts), + } + } + + if releaseArch.Media.Gcp != nil { + artifacts["gcp"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Gcp.Artifacts), + } + + if releaseArch.Media.Gcp.Image != nil { + cloudImages.Gcp = &stream.GcpImage{ + Name: releaseArch.Media.Gcp.Image.Name, + Family: releaseArch.Media.Gcp.Image.Family, + Project: releaseArch.Media.Gcp.Image.Project, + } + } + } + + if releaseArch.Media.Digitalocean != nil { + artifacts["digitalocean"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Digitalocean.Artifacts), + } + + /* We're producing artifacts but they're not yet available + in DigitalOcean as distribution images. + digitalOceanImage := stream.CloudImage{Image: fmt.Sprintf("fedora-coreos-%s", Stream)} + cloudImages.Digitalocean = &digitalOceanImage + */ + } + + if releaseArch.Media.Ibmcloud != nil { + artifacts["ibmcloud"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Ibmcloud.Artifacts), + } + } + + // if releaseArch.Media.Packet != nil { + // packet := StreamMediaDetails{ + // Release: rel.Release, + // Formats: releaseArch.Media.Packet.Artifacts, + // } + // artifacts.Packet = &packet + + // packetImage := StreamCloudImage{Image: fmt.Sprintf("fedora_coreos_%s", rel.Stream)} + // cloudImages.Packet = &packetImage + // } + + if releaseArch.Media.Openstack != nil { + artifacts["openstack"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Openstack.Artifacts), + } + } + + if releaseArch.Media.Qemu != nil { + artifacts["qemu"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Qemu.Artifacts), + } + } + + // if releaseArch.Media.Virtualbox != nil { + // virtualbox := StreamMediaDetails{ + // Release: rel.Release, + // Formats: releaseArch.Media.Virtualbox.Artifacts, + // } + // artifacts.Virtualbox = &virtualbox + // } + + if releaseArch.Media.Vmware != nil { + artifacts["vmware"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Vmware.Artifacts), + } + } + + if releaseArch.Media.Metal != nil { + artifacts["metal"] = stream.PlatformArtifacts{ + Release: rel.Release, + Formats: mapFormats(releaseArch.Media.Metal.Artifacts), + } + } + + return stream.Arch{ + Artifacts: artifacts, + Images: cloudImages, + RHELCoreOSExtensions: rhcosExt, + } +} + +// ToStreamArchitectures converts a release to a stream +func (rel *Release) ToStreamArchitectures() map[string]stream.Arch { + streamArch := make(map[string]stream.Arch) + for arch, releaseArch := range rel.Architectures { + streamArch[arch] = releaseArch.toStreamArch(rel) + } + return streamArch +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 9260511f5c..f6042a0417 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -291,6 +291,8 @@ github.com/coreos/go-systemd/v22/sdjournal ## explicit github.com/coreos/stream-metadata-go/fedoracoreos github.com/coreos/stream-metadata-go/fedoracoreos/internals +github.com/coreos/stream-metadata-go/release +github.com/coreos/stream-metadata-go/release/rhcos github.com/coreos/stream-metadata-go/stream github.com/coreos/stream-metadata-go/stream/rhcos # github.com/cyphar/filepath-securejoin v0.2.3 From 04badc2c9c7a3fde06d0069093840ec1d5a17eca Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Tue, 15 Feb 2022 15:21:00 -0600 Subject: [PATCH 2/3] Changes of docker descriptions It looks like some descriptions have changed on the docker registry where we had been searching for images that include 'alpine'. We are now seeing an image in the initial list that has 'alpine' in its description. Signed-off-by: Brent Baude --- test/python/docker/compat/test_images.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/python/docker/compat/test_images.py b/test/python/docker/compat/test_images.py index 485a0e4195..05d0e3e121 100644 --- a/test/python/docker/compat/test_images.py +++ b/test/python/docker/compat/test_images.py @@ -87,7 +87,8 @@ def test_list_images(self): def test_search_image(self): """Search for image""" for r in self.client.images.search("alpine"): - self.assertIn("alpine", r["Name"]) + # registry matches if string is in either one + self.assertIn("alpine", r["Name"]+" "+r["Description"].lower()) def test_search_bogus_image(self): """Search for bogus image should throw exception""" From 421b7466c2e877987248037ec3d64b1943307f2f Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Tue, 15 Feb 2022 12:31:20 -0600 Subject: [PATCH 3/3] Fix images since/after tests For the since and after imagve filter tests, instead of using the read-only cache of images, we just use the empty r/w store. We then build three images that are strictly predictable. Signed-off-by: Brent Baude --- test/e2e/images_test.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test/e2e/images_test.go b/test/e2e/images_test.go index 0a4d9b1344..6a534c9c85 100644 --- a/test/e2e/images_test.go +++ b/test/e2e/images_test.go @@ -187,23 +187,27 @@ WORKDIR /test }) It("podman images filter since image", func() { - dockerfile := `FROM quay.io/libpod/alpine:latest + dockerfile := `FROM scratch ` - podmanTest.BuildImage(dockerfile, "foobar.com/before:latest", "false") - result := podmanTest.Podman([]string{"images", "-q", "-f", "since=quay.io/libpod/alpine:latest"}) + podmanTest.BuildImage(dockerfile, "foobar.com/one:latest", "false") + podmanTest.BuildImage(dockerfile, "foobar.com/two:latest", "false") + podmanTest.BuildImage(dockerfile, "foobar.com/three:latest", "false") + result := podmanTest.PodmanNoCache([]string{"images", "-q", "-f", "since=foobar.com/one:latest"}) result.WaitWithDefaultTimeout() Expect(result).Should(Exit(0)) - Expect(result.OutputToStringArray()).To(HaveLen(9)) + Expect(result.OutputToStringArray()).To(HaveLen(2)) }) It("podman image list filter after image", func() { - dockerfile := `FROM quay.io/libpod/alpine:latest + dockerfile := `FROM scratch ` - podmanTest.BuildImage(dockerfile, "foobar.com/before:latest", "false") - result := podmanTest.Podman([]string{"image", "list", "-q", "-f", "after=quay.io/libpod/alpine:latest"}) + podmanTest.BuildImage(dockerfile, "foobar.com/one:latest", "false") + podmanTest.BuildImage(dockerfile, "foobar.com/two:latest", "false") + podmanTest.BuildImage(dockerfile, "foobar.com/three:latest", "false") + result := podmanTest.Podman([]string{"image", "list", "-q", "-f", "after=foobar.com/one:latest"}) result.WaitWithDefaultTimeout() Expect(result).Should(Exit(0)) - Expect(result.OutputToStringArray()).Should(HaveLen(9), "list filter output: %q", result.OutputToString()) + Expect(result.OutputToStringArray()).Should(HaveLen(2), "list filter output: %q", result.OutputToString()) }) It("podman images filter dangling", func() {