MGMT-3026 - invoke fio from installcmd

- Invoke fio_perf_check agent command as part of installcmd flow.
- Handled error in inventory (handleReplyError) by looking for a specific
  fio exit code (FioPerfCheckCmdExitCode).

internal/bminventory/inventory.go

@@ -2433,6 +2433,10 @@ func (b *bareMetalInventory) PostStepReply(ctx context.Context, params installer
 func (b *bareMetalInventory) handleReplyError(params installer.PostStepReplyParams, ctx context.Context, log logrus.FieldLogger, h *models.Host) error {
 
 	if params.Reply.StepType == models.StepTypeInstall {
+		if params.Reply.ExitCode == host.FioPerfCheckCmdExitCode {
+			log.Warnf("FIO performance check: %s", params.Reply.Error)
+			return b.hostApi.HandlePrepareInstallationFailure(ctx, h, params.Reply.Error)
+		}
 		// Handle case of installation error due to an already running assisted-installer.
 		if params.Reply.ExitCode == ContainerAlreadyRunningExitCode && strings.Contains(params.Reply.Error, "the container name \"assisted-installer\" is already in use") {
 			log.Warnf("Install command failed due to an already running installation: %s", params.Reply.Error)

internal/host/fioperfcheckcmd.go

@@ -0,0 +1,80 @@
+package host
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/openshift/assisted-service/models"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	FioPerfCheckCmdExitCode int64 = 222
+	FioDurationThreshold    int64 = 20
+)
+
+type fioPerfCheckCmd struct {
+	baseCmd
+	fioPerfCheckImage string
+	path              string
+	durationThreshold int64
+}
+
+func NewFioPerfCheckCmd(log logrus.FieldLogger, fioPerfCheckImage string, path string, durationThreshold int64) *fioPerfCheckCmd {
+	return &fioPerfCheckCmd{
+		baseCmd:           baseCmd{log: log},
+		fioPerfCheckImage: fioPerfCheckImage,
+		path:              path,
+		durationThreshold: durationThreshold,
+	}
+}
+
+func (c *fioPerfCheckCmd) GetSteps(ctx context.Context, host *models.Host) ([]*models.Step, error) {
+	args, err := c.GetArgs()
+	if err != nil {
+		return nil, err
+	}
+
+	step := &models.Step{
+		StepType: models.StepTypeFioPerfCheck,
+		Command:  "podman",
+		Args:     args,
+	}
+	return []*models.Step{step}, nil
+}
+
+func (c *fioPerfCheckCmd) GetArgs() ([]string, error) {
+	exitCode := FioPerfCheckCmdExitCode
+	request := models.FioPerfCheckRequest{
+		Path:              &c.path,
+		DurationThreshold: &c.durationThreshold,
+		ExitCode:          &exitCode,
+	}
+	requestBytes, err := json.Marshal(request)
+	if err != nil {
+		c.log.WithError(err).Errorf("failed to marshal FioPerfCheckRequest")
+		return nil, err
+	}
+
+	return []string{
+		"run", "--privileged", "--net=host", "--rm", "--quiet",
+		"-v", "/dev:/dev:rw",
+		"-v", "/var/log:/var/log",
+		"-v", "/run/systemd/journal/socket:/run/systemd/journal/socket",
+		c.fioPerfCheckImage,
+		"fio_perf_check",
+		strconv.Quote(string(requestBytes)),
+	}, nil
+}
+
+func (c *fioPerfCheckCmd) GetCommandString() string {
+	args, err := c.GetArgs()
+	if err != nil {
+		return ""
+	}
+
+	return fmt.Sprintf("podman %s && ", strings.Join(args, " "))
+}

internal/host/host.go

@@ -91,6 +91,7 @@ type API interface {
 	RegisterHost(ctx context.Context, h *models.Host, db *gorm.DB) error
 	RegisterInstalledOCPHost(ctx context.Context, h *models.Host, db *gorm.DB) error
 	HandleInstallationFailure(ctx context.Context, h *models.Host) error
+	HandlePrepareInstallationFailure(ctx context.Context, h *models.Host, reason string) error
 	UpdateInstallProgress(ctx context.Context, h *models.Host, progress *models.HostProgress) error
 	RefreshStatus(ctx context.Context, h *models.Host, db *gorm.DB) error
 	SetBootstrap(ctx context.Context, h *models.Host, isbootstrap bool, db *gorm.DB) error
@@ -240,6 +241,20 @@ func (m *Manager) populateDisksEligibility(inventoryString string) (string, erro
 	return string(result), nil
 }
 
+func (m *Manager) HandlePrepareInstallationFailure(ctx context.Context, h *models.Host, reason string) error {
+
+	lastStatusUpdateTime := h.StatusUpdatedAt
+	err := m.sm.Run(TransitionTypeHostInstallationFailed, newStateHost(h), &TransitionArgsHostInstallationFailed{
+		ctx:    ctx,
+		reason: reason,
+	})
+	if err == nil {
+		m.reportInstallationMetrics(ctx, h, &models.HostProgressInfo{CurrentStage: "installation command failed",
+			StageStartedAt: lastStatusUpdateTime}, models.HostStageFailed)
+	}
+	return err
+}
+
 func (m *Manager) UpdateInventory(ctx context.Context, h *models.Host, inventory string) error {
 	hostStatus := swag.StringValue(h.Status)
 	allowedStatuses := append(hostStatusesBeforeInstallation[:], models.HostStatusInstallingInProgress)

internal/host/installcmd.go

@@ -147,14 +147,14 @@ func (i *installCmd) GetSteps(ctx context.Context, host *models.Host) ([]*models
 	if err = t.Execute(buf, data); err != nil {
 		return nil, err
 	}
-	step.Args = []string{"-c", buf.String()}
 
 	unbootableCmd, err := i.getDiskUnbootableCmd(ctx, *host)
 	if err != nil {
 		return nil, err
 	}
 
-	step.Args = []string{"-c", unbootableCmd + buf.String()}
+	fioPerfCheckCmd := NewFioPerfCheckCmd(i.log, i.instructionConfig.FioPerfCheckImage, bootdevice, FioDurationThreshold)
+	step.Args = []string{"-c", unbootableCmd + fioPerfCheckCmd.GetCommandString() + buf.String()}
 
 	if _, err := UpdateHost(i.log, i.db, host.ClusterID, *host.ID, *host.Status,
 		"installer_version", i.instructionConfig.InstallerImage, "installation_disk_path", bootdevice); err != nil {

internal/host/installcmd_test.go

@@ -30,6 +30,7 @@ var DefaultInstructionConfig = InstructionConfig{
 	InstallerImage:      "quay.io/ocpmetal/assisted-installer:latest",
 	ControllerImage:     "quay.io/ocpmetal/assisted-installer-controller:latest",
 	InventoryImage:      "quay.io/ocpmetal/assisted-installer-agent:latest",
+	FioPerfCheckImage:   "quay.io/ocpmetal/assisted-installer-agent:latest",
 	InstallationTimeout: 120,
 	ReleaseImageMirror:  "local.registry:5000/ocp@sha256:eab93b4591699a5a4ff50ad3517892653f04fb840127895bb3609b3cc68f98f3",
 }
@@ -365,6 +366,13 @@ func validateInstallCommand(reply *models.Step, role models.HostRole, clusterId
 		"--boot-device /dev/sdb --host-id %s --openshift-version %s --mco-image mcoImage " +
 		"--controller-image %s --url %s --insecure=false --agent-image %s --installation-timeout %s"
 
+	fioPerfCheckCmd := "podman run --privileged --net=host --rm --quiet -v /dev:/dev:rw -v /var/log:/var/log " +
+		"-v /run/systemd/journal/socket:/run/systemd/journal/socket " +
+		"quay.io/ocpmetal/assisted-installer-agent:latest fio_perf_check " +
+		"\"{\\\"duration_threshold\\\":20,\\\"exit_code\\\":222,\\\"path\\\":\\\"/dev/sdb\\\"}\" && "
+
+	installCommand = fioPerfCheckCmd + installCommand
+
 	if proxy != "" {
 		installCommand += fmt.Sprintf(" %s", proxy)
 	} else if bootableDisks != nil {

internal/host/instructionmanager.go

@@ -57,6 +57,7 @@ type InstructionConfig struct {
 	DhcpLeaseAllocatorImage      string `envconfig:"DHCP_LEASE_ALLOCATOR_IMAGE" default:"quay.io/ocpmetal/assisted-installer-agent:latest"`
 	APIVIPConnectivityCheckImage string `envconfig:"API_VIP_CONNECTIVITY_CHECK_IMAGE" default:"quay.io/ocpmetal/assisted-installer-agent:latest"`
 	NtpSynchronizerImage         string `envconfig:"NTP_SYNCHRONIZER_IMAGE" default:"quay.io/ocpmetal/assisted-installer-agent:latest"`
+	FioPerfCheckImage            string `envconfig:"FIO_PERF_CHECK_IMAGE" default:"quay.io/ocpmetal/assisted-installer-agent:latest"`
 	SkipCertVerification         bool   `envconfig:"SKIP_CERT_VERIFICATION" default:"false"`
 	SupportL2                    bool   `envconfig:"SUPPORT_L2" default:"true"`
 	InstallationTimeout          uint   `envconfig:"INSTALLATION_TIMEOUT" default:"0"`