Make collector shutdown timeout configurable

`collector_shutdown_timeout: "10s"`

cfgfile/schema.go

@@ -28,6 +28,8 @@ type SidecarConfig struct {
 	CollectorValidationTimeoutString string        `config:"collector_validation_timeout"`
 	CollectorValidationTimeout       time.Duration // set from CollectorValidationTimeoutString
 	CollectorConfigurationDirectory  string        `config:"collector_configuration_directory"`
+	CollectorShutdownTimeoutString   string        `config:"collector_shutdown_timeout"`
+	CollectorShutdownTimeout         time.Duration // set from CollectorShutdownTimeoutString
 	LogRotateMaxFileSizeString       string        `config:"log_rotate_max_file_size"`
 	LogRotateMaxFileSize             int64         // set from LogRotateMaxFileSizeString
 	LogRotateKeepFiles               int           `config:"log_rotate_keep_files"`
@@ -54,6 +56,7 @@ log_path: "/var/log/graylog-sidecar"
 log_rotate_max_file_size: "10MiB"
 log_rotate_keep_files: 10
 collector_validation_timeout: "1m"
+collector_shutdown_timeout: "10s"
 collector_configuration_directory: "/var/lib/graylog-sidecar/generated"
 collector_binaries_accesslist:
   - "/usr/bin/filebeat"

context/context.go

@@ -112,6 +112,10 @@ func (ctx *Ctx) LoadConfig(path *string) error {
 	if err != nil {
 		log.Fatal("Cannot parse validation timeout duration: ", err)
 	}
+	ctx.UserConfig.CollectorShutdownTimeout, err = time.ParseDuration(ctx.UserConfig.CollectorShutdownTimeoutString)
+	if err != nil {
+		log.Fatal("Cannot parse shutdown timeout duration: ", err)
+	}
 
 	// collector_configuration_directory
 	if ctx.UserConfig.CollectorConfigurationDirectory == "" {

daemon/exec_helper.go

@@ -28,7 +28,7 @@ func Setpgid(cmd *exec.Cmd) {
 	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 }
 
-func KillProcess(r *ExecRunner, timeout time.Duration) {
+func KillProcess(r *ExecRunner) {
 	pid := r.cmd.Process.Pid
 
 	if pid == -1 {
@@ -48,7 +48,7 @@ func KillProcess(r *ExecRunner, timeout time.Duration) {
 		log.Infof("[%s] Failed to SIGTERM process group %s", r.Name(), err)
 	}
 
-	limit := timeout.Milliseconds()
+	limit := r.context.UserConfig.CollectorShutdownTimeout.Milliseconds()
 	tick := 100 * time.Millisecond
 	for t := tick.Milliseconds(); r.Running() && t < limit; t += tick.Milliseconds() {
 		log.Debugf("[%s] Waiting for process group to finish (%vms / %vms)", r.Name(), t, limit)

daemon/exec_helper_windows.go

@@ -52,7 +52,7 @@ import (
 func Setpgid(cmd *exec.Cmd) {
 	// nop on windows
 }
-func KillProcess(r *ExecRunner, _ time.Duration) {
+func KillProcess(r *ExecRunner) {
 	err := r.cmd.Process.Kill()
 	if err != nil {
 		log.Debugf("[%s] Failed to kill process %s", r.Name(), err)

daemon/exec_runner.go

@@ -237,7 +237,7 @@ func (r *ExecRunner) stop() error {
 
 	log.Infof("[%s] Stopping", r.name)
 
-	KillProcess(r, 5*time.Second)
+	KillProcess(r)
 
 	if !r.Running() {
 		r.backend.SetStatus(backends.StatusStopped, "Stopped", "")

sidecar-example.yml

@@ -59,6 +59,10 @@ server_api_token: ""
 # How long to wait for the config validation command.
 #collector_validation_timeout: "1m"
 
+# How long to wait for the collector to gracefully shutdown.
+# After this timeout the sidecar tries to terminate the collector with SIGKILL
+#collector_shutdown_timeout: "10s"
+
 # Directory where the sidecar generates configurations for collectors.
 #collector_configuration_directory: "/var/lib/graylog-sidecar/generated"