Refactor RHCOS fetch API, support fetching qemu from partial config

pkg/asset/installconfig/libvirt/libvirt.go

@@ -38,18 +38,19 @@ func Platform() (*libvirt.Platform, error) {
 		return nil, err
 	}
 
-	qcowImage, err := rhcos.QEMU(context.TODO(), rhcos.DefaultChannel)
+	rhcosBuild, err := rhcos.FetchBuild(context.TODO(), rhcos.DefaultChannel)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to fetch QEMU image URL")
 	}
+	qcowImage := rhcos.QEMU(rhcosBuild)
 
 	return &libvirt.Platform{
 		Network: libvirt.Network{
 			IfName:  defaultNetworkIfName,
 			IPRange: *defaultNetworkIPRange,
 		},
 		DefaultMachinePlatform: &libvirt.MachinePool{
-			Image: qcowImage,
+			Image:            qcowImage,
 		},
 		URI: uri,
 	}, nil

pkg/asset/machines/master.go

@@ -70,9 +70,13 @@ func (m *Master) Generate(dependencies asset.Parents) error {
 		if mpool.AMIID == "" {
 			ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second)
 			defer cancel()
-			ami, err := rhcos.AMI(ctx, rhcos.DefaultChannel, ic.Platform.AWS.Region)
+			rhcosBuild, err := rhcos.FetchBuild(ctx, rhcos.DefaultChannel)
 			if err != nil {
-				return errors.Wrap(err, "failed to determine default AMI")
+				return err
+			}
+			ami, err := rhcos.AMI(rhcosBuild, ic.Platform.AWS.Region)
+			if err != nil {
+				return err
 			}
 			mpool.AMIID = ami
 		}

pkg/asset/machines/worker.go

@@ -84,7 +84,11 @@ func (w *Worker) Generate(dependencies asset.Parents) error {
 		if mpool.AMIID == "" {
 			ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second)
 			defer cancel()
-			ami, err := rhcos.AMI(ctx, rhcos.DefaultChannel, ic.Platform.AWS.Region)
+			rhcosBuild, err := rhcos.FetchBuild(ctx, rhcos.DefaultChannel)
+			if err != nil {
+				return err
+			}
+			ami, err := rhcos.AMI(rhcosBuild, ic.Platform.AWS.Region)
 			if err != nil {
 				return errors.Wrap(err, "failed to determine default AMI")
 			}

pkg/rhcos/ami.go

@@ -1,23 +1,16 @@
 package rhcos
 
 import (
-	"context"
-
 	"github.com/pkg/errors"
 )
 
 // AMI fetches the HVM AMI ID of the latest Red Hat CoreOS release.
-func AMI(ctx context.Context, channel, region string) (string, error) {
-	meta, err := fetchLatestMetadata(ctx, channel)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to fetch RHCOS metadata")
-	}
-
-	for _, ami := range meta.AMIs {
+func AMI(build Build, region string) (string, error) {
+	for _, ami := range build.Meta.AMIs {
 		if ami.Name == region {
 			return ami.HVM, nil
 		}
 	}
 
-	return "", errors.Errorf("no RHCOS AMIs found in %s", region)
+	return "", errors.Errorf("RHCOS build %s does not have AMIs in region %s", build.Meta.OSTreeVersion, region)
 }

pkg/rhcos/builds.go

@@ -19,10 +19,19 @@ var (
 	// empty string means the first one in the build list (latest) will be used
 	buildName = ""
 
-	baseURL = "https://releases-rhcos.svc.ci.openshift.org/storage/releases"
+	defaultBaseURL = "https://releases-rhcos.svc.ci.openshift.org/storage/releases"
 )
 
-type metadata struct {
+// Represents context around a fetched build
+type Build struct {
+	BaseURL string
+	Channel string
+	Meta    Metadata
+}
+
+// Metadata is a parse of meta.json as generated by coreos-assembler which is
+// a single build.
+type Metadata struct {
 	AMIs []struct {
 		HVM  string `json:"hvm"`
 		Name string `json:"name"`
@@ -36,49 +45,53 @@ type metadata struct {
 	OSTreeVersion string `json:"ostree-version"`
 }
 
-func fetchLatestMetadata(ctx context.Context, channel string) (metadata, error) {
+func FetchBuild(ctx context.Context, channel string) (Build, error) {
 	build := buildName
 	var err error
 	if build == "" {
-		build, err = fetchLatestBuild(ctx, channel)
+		build, err = fetchLatestBuildVersion(ctx, channel)
 		if err != nil {
-			return metadata{}, errors.Wrap(err, "failed to fetch latest build")
+			return Build{}, errors.Wrap(err, "failed to fetch latest build")
 		}
 	}
 
-	url := fmt.Sprintf("%s/%s/%s/meta.json", baseURL, channel, build)
+	url := fmt.Sprintf("%s/%s/%s/meta.json", defaultBaseURL, channel, build)
 	logrus.Debugf("Fetching RHCOS metadata from %q", url)
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
-		return metadata{}, errors.Wrap(err, "failed to build request")
+		return Build{}, errors.Wrap(err, "failed to build request")
 	}
 
 	client := &http.Client{}
 	resp, err := client.Do(req.WithContext(ctx))
 	if err != nil {
-		return metadata{}, errors.Wrapf(err, "failed to fetch metadata for build %s", build)
+		return Build{}, errors.Wrapf(err, "failed to fetch metadata for build %s", build)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return metadata{}, errors.Errorf("incorrect HTTP response (%s)", resp.Status)
+		return Build{}, errors.Errorf("incorrect HTTP response (%s)", resp.Status)
 	}
 
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return metadata{}, errors.Wrap(err, "failed to read HTTP response")
+		return Build{}, errors.Wrap(err, "failed to read HTTP response")
 	}
 
-	var meta metadata
+	var meta Metadata
 	if err := json.Unmarshal(body, &meta); err != nil {
-		return meta, errors.Wrap(err, "failed to parse HTTP response")
+		return Build{}, errors.Wrap(err, "failed to parse HTTP response")
 	}
 
-	return meta, nil
+	return Build{
+		BaseURL: defaultBaseURL,
+		Channel: channel,
+		Meta:    meta,
+	}, nil
 }
 
-func fetchLatestBuild(ctx context.Context, channel string) (string, error) {
-	url := fmt.Sprintf("%s/%s/builds.json", baseURL, channel)
+func fetchLatestBuildVersion(ctx context.Context, channel string) (string, error) {
+	url := fmt.Sprintf("%s/%s/builds.json", defaultBaseURL, channel)
 	logrus.Debugf("Fetching RHCOS builds from %q", url)
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {

pkg/rhcos/qemu.go

@@ -1,18 +1,15 @@
 package rhcos
 
 import (
-	"context"
+	"os"
 	"fmt"
-
-	"github.com/pkg/errors"
 )
 
 // QEMU fetches the URL of the latest Red Hat CoreOS release.
-func QEMU(ctx context.Context, channel string) (string, error) {
-	meta, err := fetchLatestMetadata(ctx, channel)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to fetch RHCOS metadata")
+func QEMU(build Build) string {
+	qcowImage, ok := os.LookupEnv("OPENSHIFT_INSTALL_LIBVIRT_IMAGE")
+	if ok {
+		return qcowImage
 	}
-
-	return fmt.Sprintf("%s/%s/%s/%s", baseURL, channel, meta.OSTreeVersion, meta.Images.QEMU.Path), nil
+	return fmt.Sprintf("%s/%s/%s/%s", build.BaseURL, build.Channel, build.Meta.OSTreeVersion, build.Meta.Images.QEMU.Path)
 }

pkg/tfvars/libvirt/cache.go

@@ -28,7 +28,7 @@ func (libvirt *Libvirt) UseCachedImage() (err error) {
 		return nil
 	}
 
-	logrus.Infof("Fetching OS image: %s", filepath.Base(libvirt.Image))
+	logrus.Infof("Fetching qemu URL: %s", filepath.Base(libvirt.Image))
 
 	// FIXME: Use os.UserCacheDir() once we bump to Go 1.11
 	// baseCacheDir, err := os.UserCacheDir()

pkg/tfvars/libvirt/libvirt.go

@@ -11,6 +11,7 @@ import (
 type Libvirt struct {
 	URI         string `json:"libvirt_uri,omitempty"`
 	Image       string `json:"os_image,omitempty"`
+	ImageDesc   string `json:"os_image_desc,omitempty"`
 	Network     `json:",inline"`
 	MasterIPs   []string `json:"libvirt_master_ips,omitempty"`
 	BootstrapIP string   `json:"libvirt_bootstrap_ip,omitempty"`

pkg/tfvars/tfvars.go

@@ -76,9 +76,13 @@ func TFVars(cfg *types.InstallConfig, bootstrapIgn, masterIgn string) ([]byte, e
 	if cfg.Platform.AWS != nil {
 		ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
 		defer cancel()
-		ami, err := rhcos.AMI(ctx, rhcos.DefaultChannel, cfg.Platform.AWS.Region)
+		rhcosBuild, err := rhcos.FetchBuild(ctx, rhcos.DefaultChannel)
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to determine default AMI")
+			return nil, err
+		}
+		ami, err := rhcos.AMI(rhcosBuild, cfg.Platform.AWS.Region)
+		if err != nil {
+			return nil, err
 		}
 
 		config.AWS = aws.AWS{
@@ -97,13 +101,25 @@ func TFVars(cfg *types.InstallConfig, bootstrapIgn, masterIgn string) ([]byte, e
 		for i, ip := range cfg.Platform.Libvirt.MasterIPs {
 			masterIPs[i] = ip.String()
 		}
+
+		// Do an on-demand fetch if the image isn't specified (e.g. user
+		// is reusing an install config)
+		qemuImage := cfg.Platform.Libvirt.DefaultMachinePlatform.Image
+		if qemuImage == "" {
+			rhcosBuild, err := rhcos.FetchBuild(context.TODO(), rhcos.DefaultChannel)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to fetch QEMU image URL")
+			}
+			qemuImage = rhcos.QEMU(rhcosBuild)
+		}
+
 		config.Libvirt = libvirt.Libvirt{
 			URI: cfg.Platform.Libvirt.URI,
 			Network: libvirt.Network{
 				IfName:  cfg.Platform.Libvirt.Network.IfName,
 				IPRange: cfg.Platform.Libvirt.Network.IPRange.String(),
 			},
-			Image:     cfg.Platform.Libvirt.DefaultMachinePlatform.Image,
+			Image:     qemuImage,
 			MasterIPs: masterIPs,
 		}
 		if err := config.Libvirt.TFVars(config.Masters); err != nil {

pkg/types/libvirt/machinepool.go

@@ -13,6 +13,8 @@ type MachinePool struct {
 	// Image is the URL to the OS image.
 	// E.g. "http://aos-ostree.rhev-ci-vms.eng.rdu2.redhat.com/rhcos/images/cloud/latest/rhcos-qemu.qcow2.gz"
 	Image string `json:"image"`
+	// Metadata string describing the image
+	ImageDescription string `json:"imageDescription,omitempty"`
 }
 
 // Set sets the values from `required` to `a`.
@@ -30,4 +32,7 @@ func (l *MachinePool) Set(required *MachinePool) {
 	if required.Image != "" {
 		l.Image = required.Image
 	}
+	if required.ImageDescription != "" {
+		l.ImageDescription = required.ImageDescription
+	}
 }