cmd: add new image-builder binary and initial list-distros

cmd/image-builder/build.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/osbuild/images/pkg/arch"
+	"github.com/osbuild/images/pkg/osbuildmonitor"
+)
+
+// XXX: merge back into images/pkg/osbuild/osbuild-exec.go or
+// into osbuildmonitor
+func runOSBuild(manifest []byte, store, outputDirectory string, exports, extraEnv []string) error {
+	rp, wp, err := os.Pipe()
+	if err != nil {
+		return fmt.Errorf("cannot create pipe for osbuild: %w", err)
+	}
+	defer rp.Close()
+	defer wp.Close()
+
+	cmd := exec.Command(
+		"osbuild",
+		"--store", store,
+		"--output-directory", outputDirectory,
+		"--monitor=JSONSeqMonitor",
+		"--monitor-fd=3",
+		"-",
+	)
+	for _, export := range exports {
+		cmd.Args = append(cmd.Args, "--export", export)
+	}
+
+	cmd.Env = append(os.Environ(), extraEnv...)
+	cmd.Stdin = bytes.NewBuffer(manifest)
+	cmd.Stderr = os.Stderr
+	// we could use "--json" here and would get the build-result
+	// exported here
+	cmd.Stdout = nil
+	cmd.ExtraFiles = []*os.File{wp}
+
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("error starting osbuild: %v", err)
+	}
+	wp.Close()
+
+	scanner := osbuildmonitor.NewStatusScanner(rp)
+	for {
+		status, err := scanner.Status()
+		if err != nil {
+			return err
+		}
+		if status == nil {
+			break
+		}
+		// XXX: add progress bar
+		fmt.Printf("[%s] %s\n", status.Timestamp.Format("2006-01-02 15:04:05"), status.Trace)
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return fmt.Errorf("error running osbuild: %w", err)
+	}
+
+	return nil
+}
+
+func buildImage(out io.Writer, distroName, imgTypeStr, outputFilename string) error {
+	// cross arch building is not possible, we would have to download
+	// a pre-populated buildroot (tar,container) with rpm for that
+	archStr := arch.Current().String()
+	filterResult, err := getOneImage(distroName, imgTypeStr, archStr)
+	if err != nil {
+		return err
+	}
+	imgType := filterResult.ImgType
+
+	var mf bytes.Buffer
+	opts := &genManifestOptions{
+		OutputFilename: outputFilename,
+	}
+	if err := outputManifest(&mf, distroName, imgTypeStr, archStr, opts); err != nil {
+		return err
+	}
+
+	osbuildStoreDir := ".store"
+	outputDir := "."
+	buildName := fmt.Sprintf("%s-%s-%s", distroName, imgTypeStr, archStr)
+	jobOutputDir := filepath.Join(outputDir, buildName)
+	return runOSBuild(mf.Bytes(), osbuildStoreDir, jobOutputDir, imgType.Exports(), nil)
+}

cmd/image-builder/export_test.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"io"
+	"os"
+)
+
+func MockOsArgs(new []string) (restore func()) {
+	saved := os.Args
+	os.Args = append([]string{"argv0"}, new...)
+	return func() {
+		os.Args = saved
+	}
+}
+
+func MockOsStdout(new io.Writer) (restore func()) {
+	saved := osStdout
+	osStdout = new
+	return func() {
+		osStdout = saved
+	}
+}
+
+var (
+	GetOneImage = getOneImage
+	Run         = run
+)

cmd/image-builder/filters.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/osbuild/images/pkg/distrofactory"
+	"github.com/osbuild/images/pkg/imagefilter"
+)
+
+func newImageFilterDefault() (*imagefilter.ImageFilter, error) {
+	fac := distrofactory.NewDefault()
+	repos, err := newRepoRegistry()
+	if err != nil {
+		return nil, err
+	}
+	return imagefilter.New(fac, repos)
+}
+
+func getOneImage(distroName, imgTypeStr, archStr string) (*imagefilter.Result, error) {
+	imageFilter, err := newImageFilterDefault()
+	if err != nil {
+		return nil, err
+	}
+
+	// XXX: validate using "glob.QuoteMeta(distroName) == distroName",...
+	// here
+
+	filterExprs := []string{
+		fmt.Sprintf("distro:%s", distroName),
+		fmt.Sprintf("arch:%s", archStr),
+		fmt.Sprintf("type:%s", imgTypeStr),
+	}
+	filteredResults, err := imageFilter.Filter(filterExprs...)
+	if err != nil {
+		return nil, err
+	}
+	switch len(filteredResults) {
+	case 0:
+		return nil, fmt.Errorf("cannot find image for: distro:%q type:%q arch:%q", distroName, imgTypeStr, archStr)
+	case 1:
+		return &filteredResults[0], nil
+	default:
+		// XXX: imagefilter.Result should have a String() method so
+		// that this output can actually show the results
+		return nil, fmt.Errorf("internal error: found %v results for %s %s %s", len(filteredResults), distroName, imgTypeStr, archStr)
+	}
+}

cmd/image-builder/filters_test.go

@@ -0,0 +1,27 @@
+package main_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/osbuild/images/cmd/image-builder"
+)
+
+func TestGetOneImageHappy(t *testing.T) {
+	t.Setenv("IMAGE_BUILDER_EXTRA_REPOS_PATH", "../../test/data")
+
+	res, err := main.GetOneImage("centos-9", "qcow2", "x86_64")
+	require.NoError(t, err)
+	assert.Equal(t, "centos-9", res.Distro.Name())
+	assert.Equal(t, "x86_64", res.Arch.Name())
+	assert.Equal(t, "qcow2", res.ImgType.Name())
+}
+
+func TestGetOneImageSad(t *testing.T) {
+	t.Setenv("IMAGE_BUILDER_EXTRA_REPOS_PATH", "../../test/data")
+
+	_, err := main.GetOneImage("no-distro-meeh", "qcow2", "x86_64")
+	require.EqualError(t, err, `cannot find image for: distro:"no-distro-meeh" type:"qcow2" arch:"x86_64"`)
+}

cmd/image-builder/list_images.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"io"
+
+	"github.com/osbuild/images/pkg/imagefilter"
+)
+
+func listImages(out io.Writer, format string, filterExprs []string) error {
+	imageFilter, err := newImageFilterDefault()
+	if err != nil {
+		return err
+	}
+
+	filteredResult, err := imageFilter.Filter(filterExprs...)
+	if err != nil {
+		return err
+	}
+
+	fmter, err := imagefilter.NewResultsFormatter(imagefilter.OutputFormat(format))
+	if err != nil {
+		return err
+	}
+	fmter.Output(out, filteredResult)
+
+	return nil
+}

cmd/image-builder/main.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"io"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/osbuild/images/pkg/arch"
+)
+
+var osStdout io.Writer = os.Stdout
+
+func cmdListImages(cmd *cobra.Command, args []string) error {
+	filter, err := cmd.Flags().GetStringArray("filter")
+	if err != nil {
+		return err
+	}
+	format, err := cmd.Flags().GetString("format")
+	if err != nil {
+		return err
+	}
+
+	return listImages(osStdout, format, filter)
+}
+
+func cmdManifest(cmd *cobra.Command, args []string) error {
+	// support prefixes to make it easy to copy/paste from list-images
+	distroName := strings.TrimPrefix(args[0], "distro:")
+	imgType := strings.TrimPrefix(args[1], "type:")
+	var archStr string
+	if len(args) > 2 {
+		archStr = strings.TrimPrefix(args[2], "arch:")
+	} else {
+		archStr = arch.Current().String()
+	}
+
+	return outputManifest(osStdout, distroName, imgType, archStr, nil)
+}
+
+func cmdBuild(cmd *cobra.Command, args []string) error {
+	// support prefixes to make it easy to copy/paste from list-images
+	distroName := strings.TrimPrefix(args[0], "distro:")
+	imgType := strings.TrimPrefix(args[1], "type:")
+	outputFilename, err := cmd.Flags().GetString("filename")
+	if err != nil {
+		return err
+	}
+
+	return buildImage(osStdout, distroName, imgType, outputFilename)
+}
+
+func run() error {
+	// images logs a bunch of stuff to Debug/Info that we we do not
+	// want to show
+	logrus.SetLevel(logrus.WarnLevel)
+
+	rootCmd := &cobra.Command{
+		Use:   "image-builder",
+		Short: "Build operating system images from a given blueprint",
+		Long: `Build operating system images from a given blueprint
+
+Image-builder builds operating system images for a range of predefined
+operating sytsems like centos and RHEL with easy customizations support.`,
+	}
+
+	// XXX: this will list 802 images right now, we need a sensible
+	// default here, maybe without --filter just list all available
+	// distro names?
+	listImagesCmd := &cobra.Command{
+		Use:          "list-images",
+		Short:        "List buildable images, use --filter to limit further",
+		RunE:         cmdListImages,
+		SilenceUsage: true,
+	}
+	listImagesCmd.Flags().StringArray("filter", nil, "Filter distributions by a specific criteria")
+	listImagesCmd.Flags().String("format", "", "Output in a specific format (text,json)")
+	rootCmd.AddCommand(listImagesCmd)
+
+	manifestCmd := &cobra.Command{
+		Use:          "manifest <distro> <image-type> [<arch>]",
+		Short:        "Build manifest for the given distro/image-type, e.g. centos-9 qcow2",
+		RunE:         cmdManifest,
+		SilenceUsage: true,
+		// XXX: show error with available types if only one arg given
+		Args:   cobra.MinimumNArgs(2),
+		Hidden: true,
+	}
+	rootCmd.AddCommand(manifestCmd)
+
+	buildCmd := &cobra.Command{
+		Use:          "build <distro> <image-type>",
+		Short:        "Build the given distro/image-type, e.g. centos-9 qcow2",
+		RunE:         cmdBuild,
+		SilenceUsage: true,
+		// XXX: show error with available types if only one arg given
+		Args: cobra.ExactArgs(2),
+	}
+	// XXX: add this for "manifest" too in a nice way
+	buildCmd.Flags().String("filename", "", "Output as a specific filename")
+	// XXX: add --output=text,json and streaming
+	rootCmd.AddCommand(buildCmd)
+
+	return rootCmd.Execute()
+}
+
+func main() {
+	if err := run(); err != nil {
+		log.Fatalf("error: %s", err)
+	}
+}

cmd/image-builder/main_test.go

@@ -0,0 +1,32 @@
+package main_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/osbuild/images/cmd/image-builder"
+)
+
+func init() {
+	// silence logrus by default, it is quite verbose
+	logrus.SetLevel(logrus.WarnLevel)
+}
+
+func TestListImagesSmoke(t *testing.T) {
+	t.Setenv("IMAGE_BUILDER_EXTRA_REPOS_PATH", "../../test/data")
+
+	restore := main.MockOsArgs([]string{"list-images"})
+	defer restore()
+
+	var fakeStdout bytes.Buffer
+	restore = main.MockOsStdout(&fakeStdout)
+	defer restore()
+
+	err := main.Run()
+	assert.NoError(t, err)
+	// output is sorted
+	assert.Regexp(t, `(?ms)rhel-8.9.*rhel-8.10`, fakeStdout.String())
+}