Skip to content

Commit

Permalink
[HDInsight] Support kafka rest proxy feature (Azure#7844)
Browse files Browse the repository at this point in the history
  • Loading branch information
aim-for-better authored and AnuTalluri committed Dec 11, 2019
1 parent e48c9c2 commit c6e58aa
Show file tree
Hide file tree
Showing 20 changed files with 1,485 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@
},
"Create HDInsight cluster with Autoscale configuration": {
"$ref": "./examples/CreateHDInsightClusterWithAutoscaleConfig.json"
},
"Create Kafka cluster with Kafka Rest Proxy": {
"$ref": "./examples/CreateKafkaClusterWithKafkaRestProxy.json"
}
},
"parameters": [
Expand Down Expand Up @@ -553,6 +556,28 @@
}
}
},
"ClientGroupInfo": {
"description": "The information of AAD security group.",
"properties": {
"groupName": {
"type": "string",
"description": "The AAD security group name."
},
"groupId": {
"type": "string",
"description": "The AAD security group id."
}
}
},
"KafkaRestProperties": {
"description": "The kafka rest proxy configuration which contains AAD security group information.",
"properties": {
"clientGroupInfo": {
"$ref": "#/definitions/ClientGroupInfo",
"description": "The information of AAD security group."
}
}
},
"SecurityProfile": {
"description": "The security profile which contains Ssh public key for the HDInsight cluster.",
"properties": {
Expand Down Expand Up @@ -931,6 +956,10 @@
"$ref": "#/definitions/ClusterDefinition",
"description": "The cluster definition."
},
"kafkaRestProperties": {
"$ref": "#/definitions/KafkaRestProperties",
"description": "The cluster kafka rest proxy configuration."
},
"securityProfile": {
"$ref": "#/definitions/SecurityProfile",
"description": "The security profile."
Expand Down Expand Up @@ -1066,6 +1095,10 @@
"$ref": "#/definitions/ClusterDefinition",
"description": "The cluster definition."
},
"kafkaRestProperties": {
"$ref": "#/definitions/KafkaRestProperties",
"description": "The cluster kafka rest proxy configuration."
},
"securityProfile": {
"$ref": "#/definitions/SecurityProfile",
"description": "The security profile."
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,85 @@
"200": {
"headers": {
"location": "https://management.azure.com/subscriptions/subid/providers/Microsoft.HDInsight/pathToOperationResult"
},
"body": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/zzy-test-rg/providers/Microsoft.HDInsight/clusters/cluster1",
"name": "cluster1",
"type": "Microsoft.HDInsight/clusters",
"location": "East US",
"etag": "fdf2a6e8-ce83-42cc-8c2d-0ceb11a370ff",
"properties": {
"clusterVersion": "4.0.1000.1",
"osType": "Linux",
"clusterDefinition": {
"blueprint": "https://blueprints.azurehdinsight.net/hadoop-4.0.1000.1.1910270459.json",
"kind": "HADOOP",
"componentVersion": {
"Hadoop": "3.1"
}
},
"computeProfile": {
"roles": [
{
"name": "headnode",
"targetInstanceCount": 2,
"hardwareProfile": {
"vmSize": "standard_d12_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
}
},
{
"name": "workernode",
"targetInstanceCount": 4,
"hardwareProfile": {
"vmSize": "standard_d4_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
}
},
{
"name": "zookeepernode",
"targetInstanceCount": 3,
"hardwareProfile": {
"vmSize": "standard_a2_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
}
}
]
},
"provisioningState": "Succeeded",
"clusterState": "Running",
"createdDate": "2019-11-18T12:25:43.48",
"quotaInfo": {
"coresUsed": 40
},
"connectivityEndpoints": [
{
"name": "SSH",
"protocol": "TCP",
"location": "cluster1-ssh.azurehdinsight.net",
"port": 22
},
{
"name": "HTTPS",
"protocol": "TCP",
"location": "cluster1.azurehdinsight.net",
"port": 443
}
],
"tier": "Standard"
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,223 @@
{
"parameters": {
"clusterName": "cluster1",
"resourceGroupName": "rg1",
"api-version": "2018-06-01-preview",
"subscriptionId": "subid",
"parameters": {
"properties": {
"clusterVersion": "4.0",
"osType": "Linux",
"tier": "Standard",
"clusterDefinition": {
"kind": "kafka",
"componentVersion": {
"Kafka": "2.1"
},
"configurations": {
"gateway": {
"restAuthCredential.isEnabled": true,
"restAuthCredential.username": "admin",
"restAuthCredential.password": "**********"
}
}
},
"kafkaRestProperties": {
"clientGroupInfo": {
"groupName": "Kafka security group name",
"groupId": "00000000-0000-0000-0000-111111111111"
}
},
"computeProfile": {
"roles": [
{
"name": "headnode",
"targetInstanceCount": 2,
"hardwareProfile": {
"vmSize": "Large"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser",
"password": "**********"
}
}
},
{
"name": "workernode",
"targetInstanceCount": 3,
"hardwareProfile": {
"vmSize": "Large"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser",
"password": "**********"
}
},
"dataDisksGroups": [
{
"disksPerNode": 8
}
]
},
{
"name": "zookeepernode",
"targetInstanceCount": 3,
"hardwareProfile": {
"vmSize": "Small"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser",
"password": "**********"
}
}
},
{
"name": "kafkamanagementnode",
"targetInstanceCount": 2,
"hardwareProfile": {
"vmSize": "Standard_D4_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "kafkauser",
"password": "**********"
}
}
}
]
},
"storageProfile": {
"storageaccounts": [
{
"name": "mystorage.blob.core.windows.net",
"isDefault": true,
"container": "containername",
"key": "storagekey"
}
]
}
}
}
},
"responses": {
"200": {
"headers": {
"location": "https://management.azure.com/subscriptions/subid/providers/Microsoft.HDInsight/pathToOperationResult"
},
"body": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg1/providers/Microsoft.HDInsight/clusters/cluster1",
"name": "cluster1",
"type": "Microsoft.HDInsight/clusters",
"location": "South Central US",
"etag": "e1266b83-9bda-4797-a906-1bf82c8eb09a",
"tags": null,
"properties": {
"clusterVersion": "4.0.1000.1",
"osType": "Linux",
"clusterDefinition": {
"blueprint": "https://blueprints.azurehdinsight.net/kafka-4.0.1000.1.1911212244.json",
"kind": "KAFKA",
"componentVersion": {
"Kafka": "2.1"
}
},
"computeProfile": {
"roles": [
{
"name": "headnode",
"targetInstanceCount": 2,
"hardwareProfile": {
"vmSize": "standard_d3_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
}
},
{
"name": "workernode",
"targetInstanceCount": 4,
"hardwareProfile": {
"vmSize": "standard_d3_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
},
"dataDisksGroups": [
{
"disksPerNode": 2,
"storageAccountType": "Standard_LRS",
"diskSizeGB": 1023
}
]
},
{
"name": "kafkamanagementnode",
"targetInstanceCount": 2,
"hardwareProfile": {
"vmSize": "standard_d4_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
}
},
{
"name": "zookeepernode",
"targetInstanceCount": 3,
"hardwareProfile": {
"vmSize": "standard_a4_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
}
}
]
},
"provisioningState": "Succeeded",
"clusterState": "Running",
"createdDate": "2019-11-25T03:43:23.663",
"quotaInfo": {
"coresUsed": 52
},
"connectivityEndpoints": [
{
"name": "SSH",
"protocol": "TCP",
"location": "cluster1-ssh.azurehdinsight.net",
"port": 22
},
{
"name": "HTTPS",
"protocol": "TCP",
"location": "cluster1.azurehdinsight.net",
"port": 443
},
{
"name": "KafkaRestProxyPublicEndpoint",
"protocol": "TCP",
"location": "cluster1-kafkarest.azurehdinsight.net",
"port": 443
}
],
"tier": "Standard",
"kafkaRestProperties": {
"clientGroupInfo": {
"groupName": "security group name",
"groupId": "00000000-0000-0000-0000-000000000000"
}
}
}
}
}
}
}
Loading

0 comments on commit c6e58aa

Please sign in to comment.