Skip to content

Commit

Permalink
refactor: Leverage the ssh client in favor of Docker adapter
Browse files Browse the repository at this point in the history
Use an implementation based on ssh'ing directly to nodes rather than through their node container.
Move the scripts directory on the node during provisioning and remove it after finishing

Signed-off-by: aerosouund <[email protected]>
  • Loading branch information
aerosouund committed Jul 4, 2024
1 parent f23ddf4 commit 101007b
Showing 1 changed file with 39 additions and 4 deletions.
43 changes: 39 additions & 4 deletions cluster-provision/gocli/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,13 @@ func run(cmd *cobra.Command, args []string) (retErr error) {
return err
}

dm, err := cli.ContainerInspect(context.Background(), dnsmasq.ID)
if err != nil {
return err
}

sshPort, err := utils.GetPublicPort(utils.PortSSH, dm.NetworkSettings.Ports)

// Pull the registry image
err = docker.ImagePull(cli, ctx, utils.DockerRegistryImage, types.ImagePullOptions{})
if err != nil {
Expand Down Expand Up @@ -443,12 +450,18 @@ func run(cmd *cobra.Command, args []string) (retErr error) {

nodeName := nodeNameFromIndex(x + 1)
nodeNum := fmt.Sprintf("%02d", x+1)
sshClient = docker.NewDockerAdapter(cli, nodeContainer(prefix, nodeName))

sshClient, err = sshutils.NewSSHClient(sshPort, x+1, false)
if err != nil {
return err
}
if reverse {
nodeName = nodeNameFromIndex((int(nodes) - x))
nodeNum = fmt.Sprintf("%02d", (int(nodes) - x))
sshClient = docker.NewDockerAdapter(cli, nodeContainer(prefix, nodeName))
sshClient, err = sshutils.NewSSHClient(sshPort, (int(nodes) - x), false)

if err != nil {
return err
}
}

// assign a GPU to one node
Expand Down Expand Up @@ -591,6 +604,20 @@ func run(cmd *cobra.Command, args []string) (retErr error) {
return err
}

// the scripts required for running the provider exist in the node container, in order to execute them directly on the node they must be copied to the node first
devNull, _ := os.Open(os.DevNull)
success, err = docker.Exec(cli, nodeContainer(prefix, nodeName), []string{"/bin/bash", "-c", "scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i vagrant.key " + fmt.Sprintf("-r /scripts [email protected]%d:/home/vagrant/scripts", x+1)}, devNull)
if err != nil {
return err
}

// move the scripts to the same location they were in in the nodecontainer to enable command abstraction
for _, cmd := range []string{"sudo mkdir /scripts", "sudo cp -r /home/vagrant/scripts/* /scripts"} {
if err = sshClient.SSH(cmd); err != nil {
return err
}
}

n := nodesconfig.NewNodeLinuxConfig(x+1, prefix, dockerProxy, etcdDataMountSize, gpuAddress,
fipsEnabled, runEtcdOnMemory, singleStack, enableAudit, realtimeSchedulingEnabled, psaEnabled)

Expand All @@ -604,12 +631,20 @@ func run(cmd *cobra.Command, args []string) (retErr error) {
}(node.ID)
}

sshClient = docker.NewDockerAdapter(cli, nodeContainer(prefix, nodeNameFromIndex(1)))
sshClient, _ := sshutils.NewSSHClient(sshPort, 1, false)
n := nodesconfig.NewNodeK8sConfig(cephEnabled, prometheusEnabled, prometheusAlertmanagerEnabled, grafanaEnabled, istioEnabled, nfsCsiEnabled)
if err = provisionK8sOptions(sshClient, n); err != nil {
return err
}

// clean up scripts directory
for i := 0; i < int(nodes); i++ {
sshClient, _ := sshutils.NewSSHClient(sshPort, i+1, false)
if err = sshClient.SSH("rm -rf /home/vagrant/scripts"); err != nil {
return fmt.Errorf("Cleaning up scripts dir failed: %s", err)
}
}

// If background flag was specified, we don't want to clean up if we reach that state
if !background {
wg.Wait()
Expand Down

0 comments on commit 101007b

Please sign in to comment.