Skip to content

Commit

Permalink
BatchAI. Specification for new 2018-05-01 API version (#2896)
Browse files Browse the repository at this point in the history
* BatchAI. SDK 2018-05-01

Changes:
- Added first class support for Horovod framework
- Added first class support for custom mpi commands
- Introduced workspaces
- Introduces experiments as a grouping mechanism for related jobs
- Moved jobs from the top level into workpace/experiment
- Moved clusters and file servers from the top level into workspaces
- Removing filter and select parameters as they are not suppported yet
- Removing FileServer type as only NFS is supporter
- Removed type of output directories - server doesn't support it
- Removed 'createNew' attribute of output directory because there is no
use-case scenario for having it equal to false.

* Addressed CR feedback

* BatchAI. Get rid of booleans and added readonly attributes to where applicable
  • Loading branch information
AlexanderYukhanov authored and dsgouda committed Apr 27, 2018
1 parent ee40ba8 commit 5f40c8e
Show file tree
Hide file tree
Showing 35 changed files with 5,542 additions and 3 deletions.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
{
"parameters": {
"resourceGroupName": "demo_resource_group",
"workspaceName": "demo_workspace",
"clusterName": "demo_cluster",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"api-version": "2018-05-01"
},
"responses": {
"200": {
"body": {
"value": [
{
"nodeId": "tvm-3601533753_1-20170719t162906z",
"ipAddress": "13.84.190.124",
"port": 50000
},
{
"nodeId": "tvm-3601533753_2-20170719t162906z",
"ipAddress": "13.84.190.124",
"port": 50001
}
]
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"parameters": {
"resourceGroupName": "demo_resource_group",
"workspaceName": "demo_workspace",
"clusterName": "demo_cluster",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"api-version": "2018-05-01"
},
"responses": {
"202": {
"headers": {
"Location": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.BatchAI/locations/eastus/operationresults/7e69f371-db62-4183-bba0-5ee577d43bd7?api-version=2018-05-01",
"Retry-After": 15
}
},
"200": {
},
"204": {
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"parameters": {
"resourceGroupName": "demo_resource_group",
"workspaceName": "demo_workspace",
"experimentName": "demo_experiment",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"api-version": "2018-05-01"
},
"responses": {
"202": {
"headers": {
"Location": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.BatchAI/locations/eastus/operationresults/7e69f371-db62-4183-bba0-5ee577d43bd7?api-version=2018-05-01",
"Retry-After": 15
}
},
"200": {
},
"204": {
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"parameters": {
"resourceGroupName": "demo_resource_group",
"workspaceName": "demo_workspace",
"fileServerName": "demo_nfs",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"api-version": "2018-05-01"
},
"responses": {
"202": {
"headers": {
"Location": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.BatchAI/locations/eastus/operationresults/7e69f371-db62-4183-bba0-5ee577d43bd7?api-version=2018-05-01",
"Retry-After": 15
}
},
"200": {
},
"204": {
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{
"parameters": {
"resourceGroupName": "demo_resource_group",
"workspaceName": "demo_workspace",
"experimentName": "demo_experiment",
"jobName": "demo_job",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"api-version": "2018-05-01"
},
"responses": {
"202": {
"headers": {
"Location": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.BatchAI/locations/eastus/operationresults/7e69f371-db62-4183-bba0-5ee577d43bd7?api-version=2018-05-01",
"Retry-After": 15
}
},
"200": {
},
"204": {
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"parameters": {
"resourceGroupName": "demo_resource_group",
"workspaceName": "demo_workspace",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"api-version": "2018-05-01"
},
"responses": {
"202": {
"headers": {
"Location": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/providers/Microsoft.BatchAI/locations/eastus/operationresults/7e69f371-db62-4183-bba0-5ee577d43bd7?api-version=2018-05-01",
"Retry-After": 15
}
},
"200": {
},
"204": {
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
{
"parameters": {
"resourceGroupName": "demo_resource_group",
"workspaceName": "demo_workspace",
"clusterName": "demo_cluster",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"api-version": "2018-05-01"
},
"responses": {
"200": {
"body": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/demo_resource_group/providers/Microsoft.BatchAI/workspaces/demo_workspace/clusters/demo_cluster",
"name": "demo_cluster",
"type": "Microsoft.BatchAI/Clusters",
"location": "eastus",
"properties": {
"provisioningState": "succeeded",
"allocationState": "resizing",
"creationTime": "2017-09-27T21:50:02.377Z",
"allocationStateTransitionTime": "2017-09-27T21:50:04.521Z",
"provisioningStateTransitionTime": "2017-09-27T21:50:03.828Z",
"vmSize": "STANDARD_NC6",
"currentNodeCount": 0,
"nodeStateCounts": {
"runningNodeCount": 0,
"idleNodeCount": 0,
"unusableNodeCount": 0,
"preparingNodeCount": 0,
"leavingNodeCount": 0
},
"vmPriority": "dedicated",
"scaleSettings": {
"manual": {
"targetNodeCount": 1,
"nodeDeallocationOption": "requeue"
}
},
"virtualMachineConfiguration": {
"imageReference": {
"publisher": "Canonical",
"offer": "UbuntuServer",
"sku": "16.04-LTS",
"version": "latest"
}
},
"userAccountSettings": {
"adminUserName": "admin_user_name"
},
"nodeSetup": {
"mountVolumes": {
"azureFileShares": [
{
"accountName": "storage_account_name",
"azureFileUrl": "https://storage_account_name.file.core.windows.net/azure_file_share_name",
"credentials": {},
"relativeMountPath": "azfiles",
"fileMode": "0777",
"directoryMode": "0777"
}
],
"fileServers": [
{
"fileServer": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/demo_resource_group/providers/Microsoft.BatchAI/workspaces/demo_workspace/fileservers/fileservercedd134b"
},
"relativeMountPath": "nfs",
"mountOptions": "rw"
}
]
}
},
"subnet": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/demo_resource_group/providers/Microsoft.Network/virtualNetworks/7feb1976-8c31-4f1f-bea2-86cb1839a7bavnet/subnets/Subnet-1"
}
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"parameters": {
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"resourceGroupName": "demo_resource_group",
"workspaceName": "demo_workspace",
"experimentName": "demo_experiment",
"api-version": "2018-05-01"
},
"responses": {
"200": {
"body": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/demo_resource_group/providers/Microsoft.BatchAI/workspace/demo_workspace/experiments/demo_experiment",
"name": "demo_experiment",
"type": "Microsoft.BatchAI/Experiments",
"properties": {
"creationTime": "2017-09-27T22:27:52.611Z",
"provisioningState": "succeeded",
"provisioningStateTransitionTime": "2017-09-27T22:28:08.327Z"
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"parameters": {
"resourceGroupName": "demo_resource_group",
"workspaceName": "demo_workspace",
"fileServerName": "demo_nfs",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"api-version": "2018-05-01"
},
"responses": {
"200": {
"body": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/demo_resource_group/providers/Microsoft.BatchAI/workspaces/demo_workspace/fileservers/fileservercedd134b",
"name": "fileservercedd134b",
"type": "Microsoft.BatchAI/FileServers",
"location": "eastus",
"properties": {
"provisioningState": "succeeded",
"creationTime": "2017-09-27T21:45:28.159Z",
"provisioningStateTransitionTime": "2017-09-27T21:49:25.416Z",
"vmSize": "STANDARD_NC6",
"sshConfiguration": {
"userAccountSettings": {
"adminUserName": "admin_user_name"
}
},
"dataDisks": {
"diskSizeInGB": 10,
"cachingType": "none",
"diskCount": 2,
"storageAccountType": "Standard_LRS"
},
"subnet": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/demo_resource_group/providers/Microsoft.Network/virtualNetworks/7feb1976-8c31-4f1f-bea2-86cb1839a7bavnet/subnets/Subnet-1"
},
"mountSettings": {
"mountPoint": "/mnt/data",
"fileServerPublicIP": "13.90.141.66",
"fileServerInternalIP": "10.0.0.4"
}
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
{
"parameters": {
"resourceGroupName": "demo_resource_group",
"workspaceName": "demo_workspace",
"experimentName": "demo_experiment",
"jobName": "demo_job",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"api-version": "2018-05-01"
},
"responses": {
"200": {
"body": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/demo_resource_group/providers/Microsoft.BatchAI/workspaces/demo_workspace/experiments/demo_experiment/jobs/demo_job",
"name": "job",
"type": "Microsoft.BatchAI/Jobs",
"properties": {
"priority": "normal",
"cluster": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/demo_resource_group/providers/Microsoft.BatchAI/clusters/demo_cluster"
},
"nodeCount": 1,
"containerSettings": {
"imageSourceRegistry": {
"image": "ubuntu"
}
},
"toolType": "custom",
"customToolkitSettings": {
"commandLine": "echo hi | tee $AZ_BATCHAI_OUTPUT_OUTPUTS/hi.txt"
},
"stdOutErrPathPrefix": "$AZ_BATCHAI_MOUNT_ROOT/azfiles",
"jobOutputDirectoryPathSegment": "00000000-0000-0000-0000-000000000000/demo_resource_group/workspaces/demo_workspace/00000000-0000-0000-0000-000000000000/experiments/demo_experiment/00000000-0000-0000-0000-000000000000/jobs/demo_job/00000000-0000-0000-0000-000000000000",
"inputDirectories": [
{
"id": "INPUT",
"path": "$AZ_BATCHAI_MOUNT_ROOT/azfiles/input"
}
],
"outputDirectories": [
{
"id": "OUTPUTS",
"pathPrefix": "$AZ_BATCHAI_MOUNT_ROOT/azfiles/",
"pathSuffix": "files"
}
],
"constraints": {
"maxWallClockTime": "P7D"
},
"creationTime": "2017-09-27T23:36:52.611Z",
"provisioningState": "succeeded",
"provisioningStateTransitionTime": "2017-09-27T23:36:53.701Z",
"executionState": "running",
"executionStateTransitionTime": "2017-09-27T23:36:56.395Z",
"executionInfo": {
"startTime": "2017-09-27T23:36:54.115Z"
}
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"parameters": {
"resourceGroupName": "demo_resource_group",
"workspaceName": "demo_workspace",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"api-version": "2018-05-01"
},
"responses": {
"200": {
"body": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/demo_resource_group/providers/Microsoft.BatchAI/workspaces/demo_workspace",
"name": "demo_workspace",
"type": "Microsoft.BatchAI/Workspace",
"location": "eastus",
"properties": {
"creationTime": "2017-09-27T21:50:03.828Z",
"provisioningState": "succeeded",
"provisioningStateTransitionTime": "2017-09-27T21:50:03.828Z"
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"parameters": {
"resourceGroupName": "demo_resource_group",
"workspaceName": "demo_workspace",
"experimentName": "demo_experiment",
"jobName": "demo_job",
"subscriptionId": "00000000-0000-0000-0000-000000000000",
"api-version": "2018-05-01"
},
"responses": {
"200": {
"body": {
"value": [
{
"nodeId": "tvm-3601533753_1-20170719t162906z",
"ipAddress": "13.84.190.124",
"port": 50000
}
]
}
}
}
}
Loading

0 comments on commit 5f40c8e

Please sign in to comment.