UI: Server-side reschedule tracking

ui/app/components/allocation-row.js

@@ -59,12 +59,11 @@ export default Component.extend({
     const allocation = this.get('allocation');
 
     if (allocation) {
-      this.get('fetchStats').perform(allocation);
+      run.scheduleOnce('afterRender', this, qualifyAllocation);
     } else {
       this.get('fetchStats').cancelAll();
       this.set('stats', null);
     }
-    run.scheduleOnce('afterRender', this, qualifyJob);
   },
 
   fetchStats: task(function*(allocation) {
@@ -84,6 +83,14 @@ export default Component.extend({
   }).drop(),
 });
 
+function qualifyAllocation() {
+  const allocation = this.get('allocation');
+  return allocation.reload().then(() => {
+    this.get('fetchStats').perform(allocation);
+    run.scheduleOnce('afterRender', this, qualifyJob);
+  });
+}
+
 function qualifyJob() {
   const allocation = this.get('allocation');
   if (allocation.get('originalJobId')) {

ui/app/components/reschedule-event-row.js

@@ -0,0 +1,20 @@
+import Component from '@ember/component';
+import { computed } from '@ember/object';
+import { inject as service } from '@ember/service';
+
+export default Component.extend({
+  store: service(),
+  tagName: '',
+
+  // When given a string, the component will fetch the allocation
+  allocationId: null,
+
+  // An allocation can also be provided directly
+  allocation: computed('allocationId', function() {
+    return this.get('store').findRecord('allocation', this.get('allocationId'));
+  }),
+
+  time: null,
+  linkToAllocation: true,
+  label: '',
+});

ui/app/models/allocation.js

@@ -37,6 +37,13 @@ export default Model.extend({
     return STATUS_ORDER[this.get('clientStatus')] || 100;
   }),
 
+  // When allocations are server-side rescheduled, a paper trail
+  // is left linking all reschedule attempts.
+  previousAllocation: belongsTo('allocation', { inverse: 'nextAllocation' }),
+  nextAllocation: belongsTo('allocation', { inverse: 'previousAllocation' }),
+
+  followUpEvaluation: belongsTo('evaluation'),
+
   statusClass: computed('clientStatus', function() {
     const classMap = {
       pending: 'is-pending',
@@ -67,4 +74,22 @@ export default Model.extend({
   },
 
   states: fragmentArray('task-state'),
+  rescheduleEvents: fragmentArray('reschedule-event'),
+
+  hasRescheduleEvents: computed('rescheduleEvents.length', 'nextAllocation', function() {
+    return this.get('rescheduleEvents.length') > 0 || this.get('nextAllocation');
+  }),
+
+  hasStoppedRescheduling: computed(
+    'nextAllocation',
+    'clientStatus',
+    'followUpEvaluation',
+    function() {
+      return (
+        !this.get('nextAllocation.content') &&
+        !this.get('followUpEvaluation.content') &&
+        this.get('clientStatus') === 'failed'
+      );
+    }
+  ),
 });

ui/app/models/evaluation.js

@@ -23,4 +23,6 @@ export default Model.extend({
   job: belongsTo('job'),
 
   modifyIndex: attr('number'),
+
+  waitUntil: attr('date'),
 });

ui/app/models/node.js

@@ -36,5 +36,5 @@ export default Model.extend({
     return this.get('httpAddr') == null;
   }),
 
-  allocations: hasMany('allocations'),
+  allocations: hasMany('allocations', { inverse: 'node' }),
 });

ui/app/models/reschedule-event.js

@@ -0,0 +1,16 @@
+import Fragment from 'ember-data-model-fragments/fragment';
+import attr from 'ember-data/attr';
+import { fragmentOwner } from 'ember-data-model-fragments/attributes';
+import shortUUIDProperty from '../utils/properties/short-uuid';
+
+export default Fragment.extend({
+  allocation: fragmentOwner(),
+
+  previousAllocationId: attr('string'),
+  previousNodeId: attr('string'),
+  time: attr('date'),
+  delay: attr('string'),
+
+  previousAllocationShortId: shortUUIDProperty('previousAllocationId'),
+  previousNodeShortId: shortUUIDProperty('previousNodeShortId'),
+});

ui/app/serializers/allocation.js

@@ -37,6 +37,13 @@ export default ApplicationSerializer.extend({
     hash.ModifyTimeNanos = hash.ModifyTime % 1000000;
     hash.ModifyTime = Math.floor(hash.ModifyTime / 1000000);
 
+    hash.RescheduleEvents = (hash.RescheduleTracker || {}).Events;
+
+    // API returns empty strings instead of null
+    hash.PreviousAllocationID = hash.PreviousAllocation ? hash.PreviousAllocation : null;
+    hash.NextAllocationID = hash.NextAllocation ? hash.NextAllocation : null;
+    hash.FollowUpEvaluationID = hash.FollowupEvalID ? hash.FollowupEvalID : null;
+
     return this._super(typeHash, hash);
   },
 });

ui/app/serializers/reschedule-event.js

@@ -0,0 +1,17 @@
+import ApplicationSerializer from './application';
+
+export default ApplicationSerializer.extend({
+  normalize(typeHash, hash) {
+    // Time is in the form of nanoseconds since epoch, but JS dates
+    // only understand time to the millisecond precision. So store
+    // the time (precise to ms) as a date, and store the remaining ns
+    // as a number to deal with when it comes up.
+    hash.TimeNanos = hash.RescheduleTime % 1000000;
+    hash.Time = Math.floor(hash.RescheduleTime / 1000000);
+
+    hash.PreviousAllocationId = hash.PrevAllocID ? hash.PrevAllocID : null;
+    hash.PreviousNodeId = hash.PrevNodeID ? hash.PrevNodeID : null;
+
+    return this._super(typeHash, hash);
+  },
+});

ui/app/styles/components/boxed-section.scss

@@ -28,6 +28,15 @@
       background: $white;
     }
 
+    &.is-hollow {
+      border-bottom: none;
+      background: transparent;
+
+      & + .boxed-section-body {
+        border-top: none;
+      }
+    }
+
     & + .boxed-section-body {
       padding: 1.5em;
       border-top-left-radius: 0;

ui/app/styles/core/icon.scss

@@ -28,6 +28,10 @@ $icon-dimensions-large: 2rem;
     width: $icon-dimensions-large;
   }
 
+  &.is-faded {
+    fill: $grey-light;
+  }
+
   @each $name, $pair in $colors {
     $color: nth($pair, 1);
 

ui/app/styles/core/table.scss

@@ -78,19 +78,23 @@
       width: 25%;
     }
 
+    &.is-truncatable {
+      overflow: hidden;
+      text-overflow: ellipsis;
+      white-space: nowrap;
+    }
+
+    &.is-narrow {
+      padding: 1.25em 0 1.25em 0.5em;
+    }
+
     // Only use px modifiers when text needs to be truncated.
     // In this and only this scenario are percentages not effective.
     &.is-200px {
       width: 200px;
       max-width: 200px;
     }
 
-    &.is-truncatable {
-      overflow: hidden;
-      text-overflow: ellipsis;
-      white-space: nowrap;
-    }
-
     @for $i from 1 through 11 {
       &.is-#{$i} {
         width: 100% / 12 * $i;

ui/app/templates/allocations/allocation/index.hbs

@@ -91,5 +91,16 @@
         {{/list-table}}
       </div>
     </div>
+
+    {{#if model.hasRescheduleEvents}}
+      <div class="boxed-section" data-test-reschedule-events>
+        <div class="boxed-section-head is-hollow">
+          Reschedule Events
+        </div>
+        <div class="boxed-section-body">
+          {{reschedule-event-timeline allocation=model}}
+        </div>
+      </div>
+    {{/if}}
   </section>
 {{/gutter-menu}}

ui/app/templates/clients/client.hbs

@@ -43,6 +43,7 @@
             sortDescending=sortDescending
             class="with-foot" as |t|}}
             {{#t.head}}
+              <th class="is-narrow"></th>
               {{#t.sort-by prop="shortId"}}ID{{/t.sort-by}}
               {{#t.sort-by prop="modifyIndex" title="Modify Index"}}Modified{{/t.sort-by}}
               {{#t.sort-by prop="name"}}Name{{/t.sort-by}}

ui/app/templates/components/allocation-row.hbs

@@ -1,3 +1,10 @@
+<td data-test-indicators class="is-narrow">
+  {{#if allocation.nextAllocation}}
+    <span class="tooltip text-center" aria-label="Allocation was rescheduled">
+      {{x-icon "history" class="is-faded"}}
+    </span>
+  {{/if}}
+</td>
 <td data-test-short-id>
   {{#link-to "allocations.allocation" allocation class="is-primary"}}
     {{allocation.shortId}}

ui/app/templates/components/job-deployment/deployment-allocations.hbs

@@ -7,6 +7,7 @@
       source=deployment.allocations
       class="allocations" as |t|}}
       {{#t.head}}
+        <th class="is-narrow"></th>
         <th>ID</th>
         <th>Modified</th>
         <th>Name</th>

ui/app/templates/components/reschedule-event-row.hbs

@@ -0,0 +1,46 @@
+<li class="timeline-note">
+  {{#if label}}
+    <strong data-test-reschedule-label>{{label}}</strong>
+  {{/if}}
+  {{moment-format time "MMMM D, YYYY HH:mm:ss"}}
+</li>
+<li class="timeline-object" data-test-allocation={{allocation.id}}>
+  <div class="boxed-section">
+    {{#unless linkToAllocation}}
+      <div class="boxed-section-head" data-test-row-heading>
+        This Allocation
+      </div>
+    {{/unless}}
+    <div class="boxed-section-body inline-definitions">
+      <div class="columns">
+        <div class="column is-centered is-minimum">
+          <span data-test-allocation-status class="tag {{allocation.statusClass}}">
+            {{allocation.clientStatus}}
+          </span>
+        </div>
+        <div class="column">
+          <div class="boxed-section-row">
+            <span class="pair">
+              <span class="term">Allocation</span>
+              {{#if linkToAllocation}}
+                {{#link-to "allocations.allocation" allocation.id}}
+                  <code data-test-allocation-link>{{allocation.shortId}}</code>
+                {{/link-to}}
+              {{else}}
+                <code data-test-allocation-link>{{allocation.shortId}}</code>
+              {{/if}}
+            </span>
+            <span class="pair">
+              <span class="term">Client</span>
+              <span>
+                {{#link-to "clients.client" data-test-node-link allocation.node.id}}
+                  <code>{{allocation.node.id}}</code>
+                {{/link-to}}
+              </span>
+            </span>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</li>

ui/app/templates/components/reschedule-event-timeline.hbs

@@ -0,0 +1,29 @@
+<ol class="timeline">
+  {{#if allocation.nextAllocation}}
+    {{reschedule-event-row
+      label="Next Allocation"
+      allocation=allocation.nextAllocation
+      time=allocation.nextAllocation.modifyTime}}
+  {{/if}}
+  {{#if allocation.hasStoppedRescheduling}}
+    <li class="timeline-note" data-test-stop-warning>
+      {{x-icon "warning" class="is-warning is-text"}} Nomad has stopped attempting to reschedule this allocation.
+    </li>
+  {{/if}}
+  {{#if (and allocation.followUpEvaluation.waitUntil (not allocation.nextAllocation))}}
+    <li class="timeline-note" data-test-attempt-notice>
+      {{x-icon "clock" class="is-info is-text"}} Nomad will attempt to reschedule
+      {{moment-from-now allocation.followUpEvaluation.waitUntil interval=1000}}
+    </li>
+  {{/if}}
+  {{reschedule-event-row
+    allocation=allocation
+    linkToAllocation=false
+    time=allocation.modifyTime}}
+
+  {{#each (reverse allocation.rescheduleEvents) as |event|}}
+    {{reschedule-event-row
+      allocationId=event.previousAllocationId
+      time=event.time}}
+  {{/each}}
+</ol>

ui/app/templates/jobs/job/task-group.hbs

@@ -73,6 +73,7 @@
             sortDescending=sortDescending
             class="with-foot" as |t|}}
             {{#t.head}}
+              <th class="is-narrow"></th>
               {{#t.sort-by prop="shortId"}}ID{{/t.sort-by}}
               {{#t.sort-by prop="modifyIndex" title="Modify Index"}}Modified{{/t.sort-by}}
               {{#t.sort-by prop="name"}}Name{{/t.sort-by}}
@@ -83,7 +84,7 @@
               <th>Memory</th>
             {{/t.head}}
             {{#t.body as |row|}}
-              {{allocation-row data-test-allocation allocation=row.model context="job" onClick=(action "gotoAllocation" row.model)}}
+              {{allocation-row data-test-allocation=row.model.id allocation=row.model context="job" onClick=(action "gotoAllocation" row.model)}}
             {{/t.body}}
           {{/list-table}}
           <div class="table-foot">