Skip to content

Commit

Permalink
Fix fork/exec error running under kind with podman.
Browse files Browse the repository at this point in the history 
/output is getting mounted with noexec in this environment and thus we
quickly fail all provisions as we can't run the openshift-install
binary.

Copy the binaries to /bin as soon as we see they've been written to
/output where we know we can exec them. Uses cp vs any Go copy
just for simplicity and to avoid memory problems.
  • Loading branch information
@dgoodwin
dgoodwin committed May 12, 2020
1 parent a9c2252 commit ebe2c72
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 105 deletions.
77 changes: 30 additions & 47 deletions pkg/installmanager/installmanager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ const (
defaultPullSecretMountPath = "/pullsecret/" + corev1.DockerConfigJsonKey
defaultManifestsMountPath = "/manifests"
defaultHomeDir = "/home/hive" // Used if no HOME env var set.
defaultBinaryDir = "/bin" // location to copy binaries too as /output is sometimes mounted with noexec.
)

var (
Expand Down Expand Up @@ -110,6 +111,7 @@ type InstallManager struct {
waitForProvisioningStage func(*hivev1.ClusterProvision, *InstallManager) error
isGatherLogsEnabled func() bool
waitForInstallCompleteExecutions int
binaryDir string
}

// NewInstallManagerCommand is the entrypoint to create the 'install-manager' subcommand
Expand All @@ -134,6 +136,7 @@ func NewInstallManagerCommand() *cobra.Command {
im.InstallConfigMountPath = defaultInstallConfigMountPath
im.PullSecretMountPath = defaultPullSecretMountPath
im.ManifestsMountPath = defaultManifestsMountPath
im.binaryDir = defaultBinaryDir

if err := im.Validate(); err != nil {
log.WithError(err).Error("invalid command options")
Expand Down Expand Up @@ -277,7 +280,10 @@ func (m *InstallManager) Run() error {

m.ClusterName = cd.Spec.ClusterName

m.waitForInstallerBinaries()
if err := m.waitForInstallerBinaries(); err != nil {
m.log.WithError(err).Error("error waiting for binaries")
return err
}

go m.tailFullInstallLog(scrubInstallLog)

Expand Down Expand Up @@ -466,12 +472,32 @@ func (m *InstallManager) waitForFiles(files []string) {
m.log.Infof("all files found, ready to proceed")
}

func (m *InstallManager) waitForInstallerBinaries() {
func (m *InstallManager) waitForInstallerBinaries() error {
fileList := []string{
filepath.Join(m.WorkDir, "openshift-install"),
filepath.Join(m.WorkDir, "oc"),
}
m.waitForFiles(fileList)

// copy each binary to /bin to avoid situations where the workdir may be mounted with noexec.
// (i.e. kind + podman)
for _, src := range fileList {
dest := filepath.Join(m.binaryDir, filepath.Base(src))
if err := m.copyFile(src, dest); err != nil {
return err
}
m.log.Infof("copied %s to %s", src, dest)
}
return nil
}

func (m *InstallManager) copyFile(src, dst string) error {
cmd := exec.Command("cp", "-p", src, dst)
if err := cmd.Run(); err != nil {
log.WithError(err).Errorf("error copying file %s to %s", src, dst)
return err
}
return nil
}

// cleanupFailedInstall allows recovering from an installation error and allows retries
Expand Down Expand Up @@ -652,7 +678,7 @@ func (m *InstallManager) provisionCluster() error {

func (m *InstallManager) runOpenShiftInstallCommand(args ...string) error {
m.log.WithField("args", args).Info("running openshift-install binary")
cmd := exec.Command("./openshift-install", args...)
cmd := exec.Command(filepath.Join(m.binaryDir, "openshift-install"), args...)
cmd.Dir = m.WorkDir

// save the commands' stdout/stderr to a file
Expand Down Expand Up @@ -819,7 +845,7 @@ func (m *InstallManager) gatherLogs(cd *hivev1.ClusterDeployment, sshPrivKeyPath
func (m *InstallManager) gatherClusterLogs(cd *hivev1.ClusterDeployment) error {
m.log.Info("attempting to gather logs with oc adm must-gather")
destDir := filepath.Join(m.LogsDir, fmt.Sprintf("%s-must-gather", time.Now().Format("20060102150405")))
cmd := exec.Command(filepath.Join(m.WorkDir, "oc"), "adm", "must-gather", "--dest-dir", destDir)
cmd := exec.Command(filepath.Join(m.binaryDir, "oc"), "adm", "must-gather", "--dest-dir", destDir)
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, fmt.Sprintf("KUBECONFIG=%s", filepath.Join(m.WorkDir, "auth", "kubeconfig")))
stdout, err := cmd.Output()
Expand Down Expand Up @@ -994,49 +1020,6 @@ func (m *InstallManager) gatherBootstrapNodeLogs(cd *hivev1.ClusterDeployment, n
return nil
}

func (m *InstallManager) runGatherScript(bootstrapIP, scriptTemplate, workDir string) (string, error) {

tmpFile, err := ioutil.TempFile(workDir, "gatherlog")
if err != nil {
m.log.WithError(err).Error("failed to create temp log gathering file")
return "", err
}
defer os.Remove(tmpFile.Name())

destTarball := filepath.Join(m.LogsDir, fmt.Sprintf("%s-log-bundle.tar.gz", time.Now().Format("20060102150405")))
script := fmt.Sprintf(scriptTemplate, bootstrapIP, bootstrapIP, destTarball)
m.log.Debugf("generated script: %s", script)

if _, err := tmpFile.Write([]byte(script)); err != nil {
m.log.WithError(err).Error("failed to write to log gathering file")
return "", err
}
if err := tmpFile.Chmod(0555); err != nil {
m.log.WithError(err).Error("failed to set script as executable")
return "", err
}
if err := tmpFile.Close(); err != nil {
m.log.WithError(err).Error("failed to close script")
return "", err
}

m.log.Info("Gathering logs from bootstrap node")
gatherCmd := exec.Command(tmpFile.Name())
if err := gatherCmd.Run(); err != nil {
m.log.WithError(err).Error("failed while running gather script")
return "", err
}

_, err = os.Stat(destTarball)
if err != nil {
m.log.WithError(err).Error("error while stat-ing log tarball")
return "", err
}
m.log.Infof("cluster logs gathered: %s", destTarball)

return destTarball, nil
}

func (m *InstallManager) isBootstrapComplete() bool {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
Expand Down
63 changes: 5 additions & 58 deletions pkg/installmanager/installmanager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,12 +149,13 @@ func TestInstallManager(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
tempDir, err := ioutil.TempDir("", "installmanagertest")
if !assert.NoError(t, err) {
t.Fail()
}
require.NoError(t, err)
defer os.RemoveAll(tempDir)
defer os.Remove(installerConsoleLogFilePath)

binaryTempDir, err := ioutil.TempDir(tempDir, "bin")
require.NoError(t, err)

pullSecret := testSecret(corev1.SecretTypeDockerConfigJson, pullSecretSecretName, corev1.DockerConfigJsonKey, "{}")
existing := test.existing
existing = append(existing, pullSecret)
Expand All @@ -181,6 +182,7 @@ func TestInstallManager(t *testing.T) {
DynamicClient: fakeClient,
InstallConfigMountPath: mountedInstallConfigFile,
PullSecretMountPath: mountedPullSecretFile,
binaryDir: binaryTempDir,
}
im.Complete([]string{})

Expand Down Expand Up @@ -461,61 +463,6 @@ REDACTED LINE OF OUTPUT`,

}

func TestGatherLogs(t *testing.T) {
fakeBootstrapIP := "1.2.3.4"

tests := []struct {
name string
scriptTemplate string
expectedLogData string
expectedError bool
}{
{
name: "cannot execute script",
scriptTemplate: "not a bash script %s %s %s",
expectedError: true,
},
{
name: "successfully run script file",
scriptTemplate: `#!/bin/bash
echo "fake log output %s %s" > %s`,
expectedLogData: fmt.Sprintf("fake log output %s %s\n", fakeBootstrapIP, fakeBootstrapIP),
},
{
name: "error running script",
scriptTemplate: `#!/bin/bash
exit 2`,
expectedError: true,
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
im := InstallManager{
LogLevel: "debug",
isGatherLogsEnabled: func() bool {
return true
},
}
assert.NoError(t, im.Complete([]string{}))
result, err := im.runGatherScript(fakeBootstrapIP, test.scriptTemplate, "/tmp")
if test.expectedError {
assert.Error(t, err, "expected error for test case %s", test.name)
} else {
t.Logf("result file: %s", result)
data, err := ioutil.ReadFile(result)
assert.NoError(t, err, "error reading returned log file data")
assert.Equal(t, test.expectedLogData, string(data))

// cleanup saved/copied logfile
if err := os.RemoveAll(result); err != nil {
t.Logf("couldn't delete saved log file: %v", err)
}
}
})
}
}

func TestInstallManagerSSH(t *testing.T) {
apis.AddToScheme(scheme.Scheme)

Expand Down

0 comments on commit ebe2c72

Please sign in to comment.