From 9bcf8c8d3da6a59b37e51a9adc0d5fb11ebd5b80 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Mon, 24 Aug 2020 09:21:53 -0700 Subject: [PATCH 1/3] Parameterize generateResources to enable property overrides --- ui/mirage/common.js | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/ui/mirage/common.js b/ui/mirage/common.js index 45508c5c977..acfd6069d1d 100644 --- a/ui/mirage/common.js +++ b/ui/mirage/common.js @@ -27,10 +27,10 @@ export const STORAGE_PROVIDERS = ['ebs', 'zfs', 'nfs', 'cow', 'moo']; export function generateResources(options = {}) { return { - CPU: faker.helpers.randomize(CPU_RESERVATIONS), - MemoryMB: faker.helpers.randomize(MEMORY_RESERVATIONS), - DiskMB: faker.helpers.randomize(DISK_RESERVATIONS), - IOPS: faker.helpers.randomize(IOPS_RESERVATIONS), + CPU: options.CPU || faker.helpers.randomize(CPU_RESERVATIONS), + MemoryMB: options.MemoryMB || faker.helpers.randomize(MEMORY_RESERVATIONS), + DiskMB: options.DiskMB || faker.helpers.randomize(DISK_RESERVATIONS), + IOPS: options.IOPS || faker.helpers.randomize(IOPS_RESERVATIONS), Networks: generateNetworks(options.networks), Ports: generatePorts(options.networks), }; @@ -73,15 +73,17 @@ export function generateNetworks(options = {}) { } export function generatePorts(options = {}) { - return Array(faker.random.number({ - min: options.minPorts != null ? options.minPorts : 0, - max: options.maxPorts != null ? options.maxPorts : 2 - })) + return Array( + faker.random.number({ + min: options.minPorts != null ? options.minPorts : 0, + max: options.maxPorts != null ? options.maxPorts : 2, + }) + ) .fill(null) .map(() => ({ Label: faker.hacker.noun(), Value: faker.random.number({ min: 5000, max: 60000 }), To: faker.random.number({ min: 5000, max: 60000 }), HostIP: faker.random.boolean() ? faker.internet.ip() : faker.internet.ipv6(), - })) -} \ No newline at end of file + })); +} From b46955f9e13e8784d2c2c379ff2d59d34ca0fcf1 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Mon, 24 Aug 2020 09:24:32 -0700 Subject: [PATCH 2/3] Allow for custom resource requirements for jobs that are recognized by allocations The job factory will now accept an array of resourceSpecs that is a shorthand notation for memory, cpu, disk, and iops requirements. These specs get passed down to task groups. The task group factory will split the resource requirements near evenly (there is variance threshold) across all expected tasks. Allocations then construct task-resource objects based on the resources from the matching task. --- ui/mirage/factories/allocation.js | 36 +++++++++------- ui/mirage/factories/job.js | 33 ++++++++++++-- ui/mirage/factories/task-group.js | 71 ++++++++++++++++++++++++++++++- 3 files changed, 120 insertions(+), 20 deletions(-) diff --git a/ui/mirage/factories/allocation.js b/ui/mirage/factories/allocation.js index c91ba9c95ae..c571271d039 100644 --- a/ui/mirage/factories/allocation.js +++ b/ui/mirage/factories/allocation.js @@ -40,16 +40,18 @@ export default Factory.extend({ withTaskWithPorts: trait({ afterCreate(allocation, server) { const taskGroup = server.db.taskGroups.findBy({ name: allocation.taskGroup }); - const resources = taskGroup.taskIds.map(id => - server.create( + const resources = taskGroup.taskIds.map(id => { + const task = server.db.tasks.find(id); + return server.create( 'task-resource', { allocation, - name: server.db.tasks.find(id).name, + name: task.name, + resources: task.Resources, }, 'withReservedPorts' - ) - ); + ); + }); allocation.update({ taskResourceIds: resources.mapBy('id') }); }, @@ -58,16 +60,18 @@ export default Factory.extend({ withoutTaskWithPorts: trait({ afterCreate(allocation, server) { const taskGroup = server.db.taskGroups.findBy({ name: allocation.taskGroup }); - const resources = taskGroup.taskIds.map(id => - server.create( + const resources = taskGroup.taskIds.map(id => { + const task = server.db.tasks.find(id); + return server.create( 'task-resource', { allocation, - name: server.db.tasks.find(id).name, + name: task.name, + resources: task.Resources, }, 'withoutReservedPorts' - ) - ); + ); + }); allocation.update({ taskResourceIds: resources.mapBy('id') }); }, @@ -191,12 +195,14 @@ export default Factory.extend({ }) ); - const resources = taskGroup.taskIds.map(id => - server.create('task-resource', { + const resources = taskGroup.taskIds.map(id => { + const task = server.db.tasks.find(id); + return server.create('task-resource', { allocation, - name: server.db.tasks.find(id).name, - }) - ); + name: task.name, + resources: task.Resources, + }); + }); allocation.update({ taskStateIds: allocation.clientStatus === 'pending' ? [] : states.mapBy('id'), diff --git a/ui/mirage/factories/job.js b/ui/mirage/factories/job.js index 04fb128df18..c4958bb5b69 100644 --- a/ui/mirage/factories/job.js +++ b/ui/mirage/factories/job.js @@ -20,7 +20,19 @@ export default Factory.extend({ version: 1, - groupsCount: () => faker.random.number({ min: 1, max: 2 }), + // When provided, the resourceSpec will inform how many task groups to create + // and how much of each resource that task group reserves. + // + // One task group, 256 MiB memory and 500 Mhz cpu + // resourceSpec: ['M: 256, C: 500'] + // + // Two task groups + // resourceSpec: ['M: 256, C: 500', 'M: 1024, C: 1200'] + resourceSpec: null, + + groupsCount() { + return this.resourceSpec ? this.resourceSpec.length : faker.random.number({ min: 1, max: 2 }); + }, region: () => 'global', type: () => faker.helpers.randomize(JOB_TYPES), @@ -135,9 +147,22 @@ export default Factory.extend({ groupProps.count = job.groupTaskCount; } - const groups = job.noHostVolumes - ? server.createList('task-group', job.groupsCount, 'noHostVolumes', groupProps) - : server.createList('task-group', job.groupsCount, groupProps); + let groups; + if (job.noHostVolumes) { + groups = provide(job.groupsCount, (_, idx) => + server.create('task-group', 'noHostVolumes', { + ...groupProps, + resourceSpec: job.resourceSpec && job.resourceSpec.length && job.resourceSpec[idx], + }) + ); + } else { + groups = provide(job.groupsCount, (_, idx) => + server.create('task-group', { + ...groupProps, + resourceSpec: job.resourceSpec && job.resourceSpec.length && job.resourceSpec[idx], + }) + ); + } job.update({ taskGroupIds: groups.mapBy('id'), diff --git a/ui/mirage/factories/task-group.js b/ui/mirage/factories/task-group.js index 7d6822c22e6..8e3cbc9ac8b 100644 --- a/ui/mirage/factories/task-group.js +++ b/ui/mirage/factories/task-group.js @@ -1,6 +1,7 @@ import { Factory, trait } from 'ember-cli-mirage'; import faker from 'nomad-ui/mirage/faker'; import { provide } from '../utils'; +import { generateResources } from '../common'; const DISK_RESERVATIONS = [200, 500, 1000, 2000, 5000, 10000, 100000]; @@ -36,6 +37,9 @@ export default Factory.extend({ // When true, only creates allocations shallow: false, + // When set, passed into tasks to set resource values + resourceSpec: null, + afterCreate(group, server) { let taskIds = []; let volumes = Object.keys(group.volumes); @@ -66,12 +70,20 @@ export default Factory.extend({ } if (!group.shallow) { - const tasks = provide(group.count, () => { + const resources = + group.resourceSpec && divide(group.count, parseResourceSpec(group.resourceSpec)); + const tasks = provide(group.count, (_, idx) => { const mounts = faker.helpers .shuffle(volumes) .slice(0, faker.random.number({ min: 1, max: 3 })); + + const maybeResources = {}; + if (resources) { + maybeResources.Resources = generateResources(resources[idx]); + } return server.create('task', { taskGroup: group, + ...maybeResources, volumeMounts: mounts.map(mount => ({ Volume: mount, Destination: `/${faker.internet.userName()}/${faker.internet.domainWord()}/${faker.internet.color()}`, @@ -136,3 +148,60 @@ function makeHostVolumes() { return hash; }, {}); } + +function parseResourceSpec(spec) { + const mapping = { + M: 'MemoryMB', + C: 'CPU', + D: 'DiskMB', + I: 'IOPS', + }; + + const terms = spec.split(',').map(t => { + const [k, v] = t + .trim() + .split(':') + .map(kv => kv.trim()); + return [k, +v]; + }); + + return terms.reduce((hash, term) => { + hash[mapping[term[0]]] = term[1]; + return hash; + }, {}); +} + +// Split a single resources object into N resource objects where +// the sum of each property of the new resources objects equals +// the original resources properties +// ex: divide(2, { Mem: 400, Cpu: 250 }) -> [{ Mem: 80, Cpu: 50 }, { Mem: 320, Cpu: 200 }] +function divide(count, resources) { + const wheel = roulette(1, count); + + const ret = provide(count, (_, idx) => { + return Object.keys(resources).reduce((hash, key) => { + hash[key] = Math.round(resources[key] * wheel[idx]); + return hash; + }, {}); + }); + + return ret; +} + +// Roulette splits a number into N divisions +// Variance is a value between 0 and 1 that determines how much each division in +// size. At 0 each division is even, at 1, it's entirely random but the sum of all +// divisions is guaranteed to equal the total value. +function roulette(total, divisions, variance = 0.8) { + let roulette = new Array(divisions).fill(total / divisions); + roulette.forEach((v, i) => { + if (i === roulette.length - 1) return; + roulette.splice(i, 2, ...rngDistribute(roulette[i], roulette[i + 1], variance)); + }); + return roulette; +} + +function rngDistribute(a, b, variance = 0.8) { + const move = a * faker.random.number({ min: 0, max: variance, precision: 0.01 }); + return [a - move, b + move]; +} From d55c3ca809e8879fcc2e8b014487211246f7ba83 Mon Sep 17 00:00:00 2001 From: Michael Lange Date: Mon, 24 Aug 2020 09:27:43 -0700 Subject: [PATCH 3/3] Derive allocation Resources property from TaskResources --- ui/mirage/serializers/allocation.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ui/mirage/serializers/allocation.js b/ui/mirage/serializers/allocation.js index 9f5801a6f25..eeaaf28f6f4 100644 --- a/ui/mirage/serializers/allocation.js +++ b/ui/mirage/serializers/allocation.js @@ -18,5 +18,14 @@ export default ApplicationSerializer.extend({ function serializeAllocation(allocation) { allocation.TaskStates = allocation.TaskStates.reduce(arrToObj('Name'), {}); + allocation.Resources = allocation.TaskResources.mapBy('Resources').reduce( + (hash, resources) => { + ['CPU', 'DiskMB', 'IOPS', 'MemoryMB'].forEach(key => (hash[key] += resources[key])); + hash.Networks = resources.Networks; + hash.Ports = resources.Ports; + return hash; + }, + { CPU: 0, DiskMB: 0, IOPS: 0, MemoryMB: 0 } + ); allocation.TaskResources = allocation.TaskResources.reduce(arrToObj('Name', 'Resources'), {}); }