Fix container stop and destroy procedure

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Fixed
+- Fix pot-stop and pot-destroy command invocations (#49)
+
 ## [0.9.1] - 2023-09-29
 ### Added
 - Add optional keyword "attributes" to set pot attributes like `devfs_ruleset` on the task (#42)

driver/driver.go

@@ -26,14 +26,14 @@ const (
 
 	// pluginVersion allows the client to identify and use newer versions of
 	// an installed plugin
-	pluginVersion = "v0.2.1"
+	pluginVersion = "v0.10.0"
 
 	// fingerprintPeriod is the interval at which the driver will send fingerprint responses
 	fingerprintPeriod = 30 * time.Second
 
 	// taskHandleVersion is the version of task handle which this driver sets
 	// and understands how to decode driver state
-	taskHandleVersion = 1
+	taskHandleVersion = 2
 
 	// potBIN is the singularity binary path.
 	potBIN = "/usr/local/bin/pot"
@@ -350,7 +350,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error {
 	if err := handle.SetDriverState(&driverState); err != nil {
 		d.logger.Error("failed to start task, error setting driver state", "error", err)
 		//Destroy container if err on setting driver state
-		se.destroyContainer(handle.Config)
+		se.destroyContainer(handle.Config, &h.calledDestroy)
 		return fmt.Errorf("failed to set driver state: %v", err)
 	}
 
@@ -411,7 +411,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		if exists == 0 {
 			if err := se.createContainer(cfg); err != nil {
 				//Destroy container if err on creation
-				err := se.destroyContainer(cfg)
+				err := se.destroyContainer(cfg, nil)
 				if err != nil {
 					d.logger.Error("Error destroying container with err: ", err)
 				}
@@ -420,7 +420,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 			d.logger.Trace("StartTask", "Created container, se:", se)
 
 			if err := se.startContainer(cfg); err != nil {
-				err := se.destroyContainer(cfg)
+				err := se.destroyContainer(cfg, nil)
 				if err != nil {
 					d.logger.Error("Error destroying container with err: ", err)
 				}
@@ -430,7 +430,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 		} else {
 			d.logger.Trace("StartTask", "Container existed, se", se)
 			if err := se.startContainer(cfg); err != nil {
-				err := se.destroyContainer(cfg)
+				err := se.destroyContainer(cfg, nil)
 				if err != nil {
 					d.logger.Error("Error destroying container with err: ", err)
 				}
@@ -473,7 +473,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 	if err := handle.SetDriverState(&driverState); err != nil {
 		d.logger.Error("failed to start task, error setting driver state", "error", err)
 		//Destroy container if err on setting driver state
-		se.destroyContainer(cfg)
+		se.destroyContainer(cfg, &h.calledDestroy)
 		return nil, nil, fmt.Errorf("failed to set driver state: %v", err)
 	}
 
@@ -521,7 +521,7 @@ OuterLoop:
 		handle.exitResult.ExitCode = last_run.ExitCode
 	}
 
-	err = se.destroyContainer(handle.taskConfig)
+	err = se.destroyContainer(handle.taskConfig, &handle.calledDestroy)
 	if err != nil {
 		d.logger.Error("Error destroying container with err: ", err)
 	}
@@ -547,7 +547,7 @@ func (d *Driver) potWait(taskID string, se syexec) {
 		}
 	}
 
-	err = se.destroyContainer(handle.taskConfig)
+	err = se.destroyContainer(handle.taskConfig, &handle.calledDestroy)
 	if err != nil {
 		d.logger.Error("Error destroying container with err: ", err)
 	}
@@ -603,19 +603,11 @@ func (d *Driver) StopTask(taskID string, timeout time.Duration, signal string) e
 		d.logger.Error("unable to decode driver in STOPTASK:", err)
 	}
 
-	se := prepareStop(handle.taskConfig, driverConfig)
+	se := prepareDestroy(handle.taskConfig, driverConfig)
 
 	se.logger = d.logger
 
-	if err := se.stopContainer(handle.taskConfig); err != nil {
-		se.logger.Error("unable to run stopContainer: %v", err)
-	}
-
-	se = prepareDestroy(handle.taskConfig, driverConfig)
-
-	se.logger = d.logger
-
-	if err := se.destroyContainer(handle.taskConfig); err != nil {
+	if err := se.destroyContainer(handle.taskConfig, &handle.calledDestroy); err != nil {
 		return fmt.Errorf("unable to run destroyContainer: %v", err)
 	}
 

driver/handle.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strconv"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/hashicorp/go-hclog"
@@ -16,6 +17,8 @@ type taskHandle struct {
 	pid    int
 	logger hclog.Logger
 
+	calledDestroy atomic.Bool
+
 	// stateLock syncs access to all fields below
 	stateLock sync.RWMutex
 

driver/pot.go

@@ -11,6 +11,7 @@ import (
 	"os/exec"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -36,7 +37,6 @@ type syexec struct {
 	argvEnv           string
 	argvExtraHosts    string
 	argvStart         []string
-	argvStop          []string
 	argvStats         []string
 	argvLastRunStats  []string
 	argvDestroy       []string
@@ -130,41 +130,13 @@ func (s *syexec) startContainer(commandCfg *drivers.TaskConfig) error {
 	return nil
 }
 
-func (s *syexec) stopContainer(commandCfg *drivers.TaskConfig) error {
-	s.logger.Debug("launching StopContainer command", strings.Join(s.argvStop, " "))
-
-	cmd := exec.Command(potBIN, s.argvStop...)
-
-	// set the task dir as the working directory for the command
-	cmd.Dir = commandCfg.TaskDir().Dir
-	cmd.Path = potBIN
-	cmd.Args = append([]string{cmd.Path}, s.argvStop...)
-
-	// Start the process
-	if err := cmd.Run(); err != nil {
-		// try to get the exit code
-		if exitError, ok := err.(*exec.ExitError); ok {
-			ws := exitError.Sys().(syscall.WaitStatus)
-			s.exitCode = ws.ExitStatus()
-		} else {
-			s.logger.Error("Could not get exit code for stopping container ", "pot", s.argvStop)
-			s.exitCode = defaultFailedCode
-		}
-	} else {
-		// success, exitCode should be 0 if go is ok
-		ws := cmd.ProcessState.Sys().(syscall.WaitStatus)
-		s.exitCode = ws.ExitStatus()
+func (s *syexec) destroyContainer(commandCfg *drivers.TaskConfig, calledDestroy *atomic.Bool) error {
+	if calledDestroy != nil && !calledDestroy.CompareAndSwap(false, true) {
+		return nil
 	}
 
-	s.cmd = cmd
-
-	s.state = &psState{Pid: s.cmd.Process.Pid, ExitCode: s.exitCode, Time: time.Now()}
-	return nil
-}
-
-func (s *syexec) destroyContainer(commandCfg *drivers.TaskConfig) error {
 	s.argvDestroy = append(s.argvDestroy, "-F")
-	s.logger.Debug("launching DestroyContainer command", strings.Join(s.argvDestroy, " "))
+	s.logger.Debug("launching DestroyContainer command (stops as well)", strings.Join(s.argvDestroy, " "))
 
 	cmd := exec.Command(potBIN, s.argvDestroy...)
 

driver/prepare.go

@@ -163,10 +163,6 @@ func prepareContainer(cfg *drivers.TaskConfig, taskCfg TaskConfig) (syexec, erro
 	argvStart = append(argvStart, "start", potName)
 	se.argvStart = argvStart
 
-	argvStop := make([]string, 0, 50)
-	argvStop = append(argvStop, "stop", potName)
-	se.argvStop = argvStop
-
 	argvStats := make([]string, 0, 50)
 	argvStats = append(argvStats, "get-rss", "-p", potName, "-J")
 	se.argvStats = argvStats
@@ -229,27 +225,6 @@ func (s *syexec) Close() {
 	}
 }
 
-func prepareStop(cfg *drivers.TaskConfig, taskCfg TaskConfig) syexec {
-	argv := make([]string, 0, 50)
-	var se syexec
-	se.taskConfig = taskCfg
-	se.cfg = cfg
-	se.env = cfg.EnvList()
-
-	// action can be run/exec
-
-	argv = append(argv, "stop")
-
-	parts := strings.Split(cfg.ID, "/")
-	completeName := parts[1] + "_" + parts[2] + "_" + parts[0]
-
-	argv = append(argv, completeName)
-
-	se.argvStop = append(argv, taskCfg.Args...)
-
-	return se
-}
-
 func prepareDestroy(cfg *drivers.TaskConfig, taskCfg TaskConfig) syexec {
 	argv := make([]string, 0, 50)
 	var se syexec
@@ -264,9 +239,7 @@ func prepareDestroy(cfg *drivers.TaskConfig, taskCfg TaskConfig) syexec {
 	parts := strings.Split(cfg.ID, "/")
 	completeName := parts[1] + "_" + parts[2] + "_" + parts[0]
 
-	argv = append(argv, "-p", completeName)
-
-	se.argvDestroy = append(argv, taskCfg.Args...)
+	se.argvDestroy = append(argv, "-p", completeName)
 
 	return se