Fix nil job on allocation

The way the copying was happening on the alloc_runner was by temporarily
setting the alloc.Job to nil, copying and then restoring it. This
created an issue in which when the alloc was shared (which it is in
server/client mode and between alloc_runner/task_runner) there were race
conditions that could create a panic.

Fixes #2605

client/alloc_runner.go

@@ -425,18 +425,8 @@ func copyTaskStates(states map[string]*structs.TaskState) map[string]*structs.Ta
 func (r *AllocRunner) Alloc() *structs.Allocation {
 	r.allocLock.Lock()
 
-	// Clear the job before copying
-	job := r.alloc.Job
-
-	// Since we are clearing the job, anything that access the alloc.Job field
-	// must acquire the lock or access it via this method.
-	r.alloc.Job = nil
-
-	alloc := r.alloc.Copy()
-
-	// Restore
-	r.alloc.Job = job
-	alloc.Job = job
+	// Don't do a deep copy of the job
+	alloc := r.alloc.CopySkipJob()
 
 	// The status has explicitly been set.
 	if r.allocClientStatus != "" || r.allocClientDescription != "" {
@@ -774,7 +764,7 @@ func (r *AllocRunner) handleDestroy() {
 
 			return
 		case <-r.updateCh:
-			r.logger.Printf("[ERR] client: dropping update to terminal alloc '%s'", r.alloc.ID)
+			r.logger.Printf("[DEBUG] client: dropping update to terminal alloc '%s'", r.alloc.ID)
 		}
 	}
 }

nomad/structs/structs.go

@@ -3680,13 +3680,25 @@ type Allocation struct {
 }
 
 func (a *Allocation) Copy() *Allocation {
+	return a.copyImpl(true)
+}
+
+// Copy provides a copy of the allocation but doesn't deep copy the job
+func (a *Allocation) CopySkipJob() *Allocation {
+	return a.copyImpl(false)
+}
+
+func (a *Allocation) copyImpl(job bool) *Allocation {
 	if a == nil {
 		return nil
 	}
 	na := new(Allocation)
 	*na = *a
 
-	na.Job = na.Job.Copy()
+	if job {
+		na.Job = na.Job.Copy()
+	}
+
 	na.Resources = na.Resources.Copy()
 	na.SharedResources = na.SharedResources.Copy()