rhcos: implement image discovery for new pipeline

pkg/asset/installconfig/platform.go

@@ -1,6 +1,7 @@
 package installconfig
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"net/url"
@@ -12,6 +13,7 @@ import (
 	survey "gopkg.in/AlecAivazis/survey.v1"
 
 	"github.com/openshift/installer/pkg/asset"
+	"github.com/openshift/installer/pkg/rhcos"
 	"github.com/openshift/installer/pkg/types"
 )
 
@@ -52,7 +54,6 @@ var (
 
 	defaultLibvirtNetworkIfName  = "tt0"
 	defaultLibvirtNetworkIPRange = "192.168.126.0/24"
-	defaultLibvirtImageURL       = "http://aos-ostree.rhev-ci-vms.eng.rdu2.redhat.com/rhcos/images/cloud/latest/rhcos-qemu.qcow2.gz"
 )
 
 // Platform is an asset that queries the user for the platform on which to install
@@ -288,13 +289,24 @@ func (a *platform) libvirtPlatform() (*types.LibvirtPlatform, error) {
 		return nil, err
 	}
 
+	// TODO: Ideally, this would live inside of a closure which is passed to
+	//       asset.GenerateUserProvidedAsset and only called if the environment
+	//       variable isn't present. As this exists, it ruins the abstraction.
+	var qcowImage string
+	if _, ok := os.LookupEnv("OPENSHIFT_INSTALL_LIBVIRT_IMAGE"); !ok {
+		qcowImage, err = rhcos.QEMU(context.TODO(), rhcos.DefaultChannel)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to fetch QEMU image URL")
+		}
+	}
+
 	image, err := asset.GenerateUserProvidedAsset(
 		"Libvirt Image",
 		&survey.Question{
 			Prompt: &survey.Input{
 				Message: "Image",
 				Help:    "URI of the OS image.",
-				Default: defaultLibvirtImageURL,
+				Default: qcowImage,
 			},
 			Validate: survey.ComposeValidators(survey.Required, uriValidator),
 		},

pkg/rhcos/ami.go

@@ -2,86 +2,22 @@ package rhcos
 
 import (
 	"context"
-	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/pkg/errors"
 )
 
-const (
-	// DefaultChannel is the default RHCOS channel for the cluster.
-	DefaultChannel = "tested"
-)
-
-// AMI calculates a Red Hat CoreOS AMI.
-func AMI(ctx context.Context, channel, region string) (ami string, err error) {
-	if channel != DefaultChannel {
-		return "", errors.Errorf("channel %q is not yet supported", channel)
-	}
-
-	ssn := session.Must(session.NewSessionWithOptions(session.Options{
-		SharedConfigState: session.SharedConfigEnable,
-		Config: aws.Config{
-			Region: aws.String(region),
-		},
-	}))
-
-	svc := ec2.New(ssn)
-
-	result, err := svc.DescribeImagesWithContext(ctx, &ec2.DescribeImagesInput{
-		Filters: []*ec2.Filter{
-			{
-				Name:   aws.String("name"),
-				Values: aws.StringSlice([]string{"redhat-coreos-*"}),
-			},
-			{
-				Name:   aws.String("architecture"),
-				Values: aws.StringSlice([]string{"x86_64"}),
-			},
-			{
-				Name:   aws.String("virtualization-type"),
-				Values: aws.StringSlice([]string{"hvm"}),
-			},
-			{
-				Name:   aws.String("image-type"),
-				Values: aws.StringSlice([]string{"machine"}),
-			},
-			{
-				Name:   aws.String("owner-id"),
-				Values: aws.StringSlice([]string{"531415883065"}),
-			},
-			{
-				Name:   aws.String("state"),
-				Values: aws.StringSlice([]string{"available"}),
-			},
-		},
-	})
+// AMI fetches the HVM AMI ID of the latest Red Hat CoreOS release.
+func AMI(ctx context.Context, channel, region string) (string, error) {
+	meta, err := fetchLatestMetadata(ctx, channel)
 	if err != nil {
-		return "", errors.Wrap(err, "failed to describe AMIs")
+		return "", errors.Wrap(err, "failed to fetch RHCOS metadata")
 	}
 
-	var image *ec2.Image
-	var created time.Time
-	for _, nextImage := range result.Images {
-		if nextImage.ImageId == nil || nextImage.CreationDate == nil {
-			continue
+	for _, ami := range meta.AMIs {
+		if ami.Name == region {
+			return ami.HVM, nil
 		}
-		nextCreated, err := time.Parse(time.RFC3339, *nextImage.CreationDate)
-		if err != nil {
-			return "", errors.Wrap(err, "failed to parse AMIs CreationDate to time.RFC3339")
-		}
-
-		if image == nil || nextCreated.After(created) {
-			image = nextImage
-			created = nextCreated
-		}
-	}
-
-	if image == nil {
-		return "", errors.Errorf("no RHCOS AMIs found in %s", region)
 	}
 
-	return *image.ImageId, nil
+	return "", errors.Errorf("no RHCOS AMIs found in %s", region)
 }

pkg/rhcos/builds.go

@@ -0,0 +1,108 @@
+package rhcos
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// DefaultChannel is the default RHCOS channel for the cluster.
+	DefaultChannel = "maipo"
+
+	baseURL = "https://releases-rhcos.svc.ci.openshift.org/storage/releases"
+)
+
+type metadata struct {
+	AMIs []struct {
+		HVM  string `json:"hvm"`
+		Name string `json:"name"`
+	} `json:"amis"`
+	Images struct {
+		QEMU struct {
+			Path   string `json:"path"`
+			SHA256 string `json:"sha256"`
+		} `json:"qemu"`
+	} `json:"images"`
+	OSTreeVersion string `json:"ostree-version"`
+}
+
+func fetchLatestMetadata(ctx context.Context, channel string) (metadata, error) {
+	build, err := fetchLatestBuild(ctx, channel)
+	if err != nil {
+		return metadata{}, errors.Wrap(err, "failed to fetch latest build")
+	}
+
+	url := fmt.Sprintf("%s/%s/%s/meta.json", baseURL, channel, build)
+	logrus.Debugf("Fetching RHCOS metadata from %q", url)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return metadata{}, errors.Wrap(err, "failed to build request")
+	}
+
+	client := &http.Client{}
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return metadata{}, errors.Wrap(err, "failed to fetch metadata")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return metadata{}, errors.Errorf("incorrect HTTP response (%s)", resp.Status)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return metadata{}, errors.Wrap(err, "failed to read HTTP response")
+	}
+
+	var meta metadata
+	if err := json.Unmarshal(body, &meta); err != nil {
+		return meta, errors.Wrap(err, "failed to parse HTTP response")
+	}
+
+	return meta, nil
+}
+
+func fetchLatestBuild(ctx context.Context, channel string) (string, error) {
+	url := fmt.Sprintf("%s/%s/builds.json", baseURL, channel)
+	logrus.Debugf("Fetching RHCOS builds from %q", url)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to build request")
+	}
+
+	client := &http.Client{}
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return "", errors.Wrap(err, "failed to fetch builds")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", errors.Errorf("incorrect HTTP response (%s)", resp.Status)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to read HTTP response")
+	}
+
+	var builds struct {
+		Builds []string `json:"builds"`
+	}
+	if err := json.Unmarshal(body, &builds); err != nil {
+		return "", errors.Wrap(err, "failed to parse HTTP response")
+	}
+
+	if len(builds.Builds) == 0 {
+		return "", errors.Errorf("no builds found")
+	}
+
+	return builds.Builds[0], nil
+}

pkg/rhcos/qemu.go

@@ -0,0 +1,18 @@
+package rhcos
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+)
+
+// QEMU fetches the URL of the latest Red Hat CoreOS release.
+func QEMU(ctx context.Context, channel string) (string, error) {
+	meta, err := fetchLatestMetadata(ctx, channel)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to fetch RHCOS metadata")
+	}
+
+	return fmt.Sprintf("%s/%s/%s/%s", baseURL, channel, meta.OSTreeVersion, meta.Images.QEMU.Path), nil
+}