template: lock task handle before trying script check (#23917)

In #23663 we fixed the template hook so that `change_mode="script"` didn't lose
track of the task handle during restores. But this revealed a second bug which
is that access to the handle is not locked while in use, which can allow it to
be removed concurrently.

Fixes: #23875

client/allocrunner/taskrunner/template/template.go

@@ -583,6 +583,8 @@ func (tm *TaskTemplateManager) handleScriptError(script *structs.ChangeScript, m
 func (tm *TaskTemplateManager) processScript(script *structs.ChangeScript, wg *sync.WaitGroup) {
 	defer wg.Done()
 
+	tm.handleLock.Lock()
+	defer tm.handleLock.Unlock()
 	if tm.handle == nil {
 		failureMsg := fmt.Sprintf(
 			"Template failed to run script %v with arguments %v because task driver handle is not available",

client/allocrunner/taskrunner/template/template_test.go

@@ -1146,6 +1146,7 @@ func TestTaskTemplateManager_ScriptExecution(t *testing.T) {
 	key2 := "bar"
 	content1_1 := "cat"
 	content1_2 := "dog"
+	content1_3 := "goldfish"
 	t1 := &structs.Template{
 		EmbeddedTmpl: `
 FOO={{key "bam"}}
@@ -1219,6 +1220,29 @@ OUTER:
 			t.Fatal(t, "should have received an event")
 		}
 	}
+
+	// remove the reference to the task handle and update the template contents
+	// again
+	harness.manager.SetDriverHandle(nil)
+	harness.consul.SetKV(t, key1, []byte(content1_3))
+	timeout = time.After(time.Duration(5*testutil.TestMultiplier()) * time.Second)
+
+OUTER2:
+	for {
+		select {
+		case <-harness.mockHooks.RestartCh:
+			t.Fatal(t, "restart not expected")
+		case ev := <-harness.mockHooks.EmitEventCh:
+			if strings.Contains(
+				ev.DisplayMessage, "task driver handle is not available") {
+				break OUTER2
+			}
+		case <-harness.mockHooks.SignalCh:
+			t.Fatal(t, "signal not expected")
+		case <-timeout:
+			t.Fatal(t, "should have received an event that task driver handle is unavailable")
+		}
+	}
 }
 
 // TestTaskTemplateManager_ScriptExecutionFailTask tests whether we fail the