diff --git a/.gitignore b/.gitignore index c7727735..cd1b46ca 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,6 @@ ingestors/python/.idea ingestors/java/spring/.gradle/ ingestors/java/spring/.idea/ ingestors/java/spring/gradle.properties -ingestors/java/spring/target/ \ No newline at end of file +ingestors/java/spring/target/ + +.meta/* \ No newline at end of file diff --git a/README.md b/README.md index 9c2be11c..1823c6d9 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,15 @@
+
+ Deploy to Azure → +
+ + + +
+
+ **Deploy with Docker** Run the following in your cloud environment: diff --git a/deploy/azure/deployment.bicep b/deploy/azure/deployment.bicep new file mode 100644 index 00000000..b13eacef --- /dev/null +++ b/deploy/azure/deployment.bicep @@ -0,0 +1,238 @@ +@description('Name for the Metlo Instance') +param vmName string = 'Metlo-Manager' + +var adminUsername = 'azureuser' + +var authenticationType = 'sshPublicKey' + +@description('SSH Key or password for the Virtual Machine. SSH key is recommended.') +@secure() +param adminPasswordOrKey string + +var dnsLabelPrefix = toLower('${vmName}-${uniqueString(resourceGroup().id)}') + +@description('Location for all resources. Leave as default to take value from resource group') +param location string = resourceGroup().location + +var vmSize = 'Standard_B2s' + +var virtualNetworkName = 'metloVNet' + +var subnetName = 'metloSubnet' + +var networkSecurityGroupName = 'metloSecGroupNet' + +var customDataName = 'metloPostDeploymentScript' +// var customData = '[base64("sudo metlo init-env && sudo metlo update && sudo metlo start")]' + +var publicIPAddressName = '${vmName}PublicIP' +var networkInterfaceName = '${vmName}NetInt' +var osDiskType = 'Standard_LRS' +var subnetAddressPrefix = '10.1.0.0/24' +var addressPrefix = '10.1.0.0/16' +var linuxConfiguration = { + disablePasswordAuthentication: true + ssh: { + publicKeys: [ + { + path: '/home/${adminUsername}/.ssh/authorized_keys' + keyData: adminPasswordOrKey + } + ] + } +} + +resource nic 'Microsoft.Network/networkInterfaces@2021-05-01' = { + name: networkInterfaceName + location: location + properties: { + ipConfigurations: [ + { + name: 'ipconfig1' + properties: { + subnet: { + id: subnet.id + } + privateIPAllocationMethod: 'Dynamic' + publicIPAddress: { + id: publicIP.id + } + } + } + ] + networkSecurityGroup: { + id: nsg.id + } + } +} + +resource nsg 'Microsoft.Network/networkSecurityGroups@2021-05-01' = { + name: networkSecurityGroupName + location: location + properties: { + securityRules: [ + { + name: 'MetloSSH' + properties: { + priority: 1000 + protocol: 'Tcp' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: '*' + sourcePortRange: '*' + destinationAddressPrefix: '*' + destinationPortRange: '22' + } + } + { + name: 'Metlo-Collector' + properties: { + priority: 1001 + protocol: 'Tcp' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: '*' + sourcePortRange: '*' + destinationAddressPrefix: '*' + destinationPortRange: '8081' + } + } + { + name: 'Metlo-Frontend-10-8' + properties: { + priority: 1002 + protocol: 'Tcp' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: '10.0.0.0/8' + sourcePortRange: '*' + destinationAddressPrefix: '*' + destinationPortRange: '8000' + } + } + { + name: 'Metlo-Frontend-172.16-12' + properties: { + priority: 1003 + protocol: 'Tcp' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: '172.16.0.0/12' + sourcePortRange: '*' + destinationAddressPrefix: '*' + destinationPortRange: '8000' + } + } + { + name: 'Metlo-Frontend-192.168-16' + properties: { + priority: 1004 + protocol: 'Tcp' + access: 'Allow' + direction: 'Inbound' + sourceAddressPrefix: '192.168.0.0/16' + sourcePortRange: '*' + destinationAddressPrefix: '*' + destinationPortRange: '8000' + } + } + ] + } +} + +resource vnet 'Microsoft.Network/virtualNetworks@2021-05-01' = { + name: virtualNetworkName + location: location + properties: { + addressSpace: { + addressPrefixes: [ + addressPrefix + ] + } + } +} + +resource subnet 'Microsoft.Network/virtualNetworks/subnets@2021-05-01' = { + parent: vnet + name: subnetName + properties: { + addressPrefix: subnetAddressPrefix + privateEndpointNetworkPolicies: 'Enabled' + privateLinkServiceNetworkPolicies: 'Enabled' + } +} + +resource publicIP 'Microsoft.Network/publicIPAddresses@2021-05-01' = { + name: publicIPAddressName + location: location + sku: { + name: 'Basic' + } + properties: { + publicIPAllocationMethod: 'Dynamic' + publicIPAddressVersion: 'IPv4' + dnsSettings: { + domainNameLabel: dnsLabelPrefix + } + idleTimeoutInMinutes: 4 + } +} + +resource vm 'Microsoft.Compute/virtualMachines@2021-11-01' = { + name: vmName + location: location + properties: { + userData: '' + hardwareProfile: { + vmSize: vmSize + } + storageProfile: { + osDisk: { + createOption: 'FromImage' + managedDisk: { + storageAccountType: osDiskType + } + } + imageReference: { + publisher: 'Canonical' + offer: 'UbuntuServer' + sku: '18.04-LTS' + version: 'latest' + } + } + networkProfile: { + networkInterfaces: [ + { + id: nic.id + } + ] + } + osProfile: { + computerName: vmName + adminUsername: adminUsername + adminPassword: adminPasswordOrKey + linuxConfiguration: ((authenticationType == 'password') ? null : linuxConfiguration) + } + } +} + +resource script 'Microsoft.Compute/virtualMachines/extensions@2022-08-01' = { + name: format('{0}/{1}', vmName, customDataName) + location: location + dependsOn: [ vm ] + properties: { + publisher: 'Microsoft.OSTCExtensions' + type: 'CustomScriptForLinux' + typeHandlerVersion: '1.2' + settings: { + fileUris: [ + 'https://raw.githubusercontent.com/metlo-labs/metlo/master/deploy/deploy_script.sh' + ] + commandToExecute: 'sudo /bin/bash deploy_script.sh' + } + } +} + +output adminUsername string = adminUsername +output hostname string = publicIP.properties.dnsSettings.fqdn +output sshCommand string = 'ssh ${adminUsername}@${publicIP.properties.dnsSettings.fqdn}' diff --git a/deploy/azure/deployment.json b/deploy/azure/deployment.json new file mode 100644 index 00000000..888bfe9f --- /dev/null +++ b/deploy/azure/deployment.json @@ -0,0 +1,285 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.12.40.16777", + "templateHash": "15490251908371106898" + } + }, + "parameters": { + "vmName": { + "type": "string", + "defaultValue": "Metlo-Manager", + "metadata": { + "description": "Name for the Metlo Instance" + } + }, + "adminPasswordOrKey": { + "type": "secureString", + "metadata": { + "description": "SSH Key or password for the Virtual Machine. SSH key is recommended." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location for all resources. Leave as default to take value from resource group" + } + } + }, + "variables": { + "adminUsername": "azureuser", + "authenticationType": "sshPublicKey", + "dnsLabelPrefix": "[toLower(format('{0}-{1}', parameters('vmName'), uniqueString(resourceGroup().id)))]", + "vmSize": "Standard_B2s", + "virtualNetworkName": "metloVNet", + "subnetName": "metloSubnet", + "networkSecurityGroupName": "metloSecGroupNet", + "customDataName": "metloPostDeploymentScript", + "publicIPAddressName": "[format('{0}PublicIP', parameters('vmName'))]", + "networkInterfaceName": "[format('{0}NetInt', parameters('vmName'))]", + "osDiskType": "Standard_LRS", + "subnetAddressPrefix": "10.1.0.0/24", + "addressPrefix": "10.1.0.0/16", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "[format('/home/{0}/.ssh/authorized_keys', variables('adminUsername'))]", + "keyData": "[parameters('adminPasswordOrKey')]" + } + ] + } + } + }, + "resources": [ + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2021-05-01", + "name": "[variables('networkInterfaceName')]", + "location": "[parameters('location')]", + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('virtualNetworkName'), variables('subnetName'))]" + }, + "privateIPAllocationMethod": "Dynamic", + "publicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName'))]" + } + } + } + ], + "networkSecurityGroup": { + "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('networkSecurityGroupName'))]" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', variables('networkSecurityGroupName'))]", + "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName'))]", + "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('virtualNetworkName'), variables('subnetName'))]" + ] + }, + { + "type": "Microsoft.Network/networkSecurityGroups", + "apiVersion": "2021-05-01", + "name": "[variables('networkSecurityGroupName')]", + "location": "[parameters('location')]", + "properties": { + "securityRules": [ + { + "name": "MetloSSH", + "properties": { + "priority": 1000, + "protocol": "Tcp", + "access": "Allow", + "direction": "Inbound", + "sourceAddressPrefix": "*", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "22" + } + }, + { + "name": "Metlo-Collector", + "properties": { + "priority": 1001, + "protocol": "Tcp", + "access": "Allow", + "direction": "Inbound", + "sourceAddressPrefix": "*", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "8081" + } + }, + { + "name": "Metlo-Frontend-10-8", + "properties": { + "priority": 1002, + "protocol": "Tcp", + "access": "Allow", + "direction": "Inbound", + "sourceAddressPrefix": "10.0.0.0/8", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "8000" + } + }, + { + "name": "Metlo-Frontend-172.16-12", + "properties": { + "priority": 1003, + "protocol": "Tcp", + "access": "Allow", + "direction": "Inbound", + "sourceAddressPrefix": "172.16.0.0/12", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "8000" + } + }, + { + "name": "Metlo-Frontend-192.168-16", + "properties": { + "priority": 1004, + "protocol": "Tcp", + "access": "Allow", + "direction": "Inbound", + "sourceAddressPrefix": "192.168.0.0/16", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "8000" + } + } + ] + } + }, + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2021-05-01", + "name": "[variables('virtualNetworkName')]", + "location": "[parameters('location')]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "[variables('addressPrefix')]" + ] + } + } + }, + { + "type": "Microsoft.Network/virtualNetworks/subnets", + "apiVersion": "2021-05-01", + "name": "[format('{0}/{1}', variables('virtualNetworkName'), variables('subnetName'))]", + "properties": { + "addressPrefix": "[variables('subnetAddressPrefix')]", + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]" + ] + }, + { + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2021-05-01", + "name": "[variables('publicIPAddressName')]", + "location": "[parameters('location')]", + "sku": { + "name": "Basic" + }, + "properties": { + "publicIPAllocationMethod": "Dynamic", + "publicIPAddressVersion": "IPv4", + "dnsSettings": { + "domainNameLabel": "[variables('dnsLabelPrefix')]" + }, + "idleTimeoutInMinutes": 4 + } + }, + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2021-11-01", + "name": "[parameters('vmName')]", + "location": "[parameters('location')]", + "properties": { + "userData": "", + "hardwareProfile": { + "vmSize": "[variables('vmSize')]" + }, + "storageProfile": { + "osDisk": { + "createOption": "FromImage", + "managedDisk": { + "storageAccountType": "[variables('osDiskType')]" + } + }, + "imageReference": { + "publisher": "Canonical", + "offer": "UbuntuServer", + "sku": "18.04-LTS", + "version": "latest" + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]" + } + ] + }, + "osProfile": { + "computerName": "[parameters('vmName')]", + "adminUsername": "[variables('adminUsername')]", + "adminPassword": "[parameters('adminPasswordOrKey')]", + "linuxConfiguration": "[if(equals(variables('authenticationType'), 'password'), null(), variables('linuxConfiguration'))]" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]" + ] + }, + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "apiVersion": "2022-08-01", + "name": "[format('{0}/{1}', parameters('vmName'), variables('customDataName'))]", + "location": "[parameters('location')]", + "properties": { + "publisher": "Microsoft.OSTCExtensions", + "type": "CustomScriptForLinux", + "typeHandlerVersion": "1.2", + "settings": { + "fileUris": [ + "https://raw.githubusercontent.com/metlo-labs/metlo/master/deploy/deploy_script.sh" + ], + "commandToExecute": "sudo /bin/bash deploy_script.sh" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', parameters('vmName'))]" + ] + } + ], + "outputs": { + "adminUsername": { + "type": "string", + "value": "[variables('adminUsername')]" + }, + "hostname": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName')), '2021-05-01').dnsSettings.fqdn]" + }, + "sshCommand": { + "type": "string", + "value": "[format('ssh {0}@{1}', variables('adminUsername'), reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName')), '2021-05-01').dnsSettings.fqdn)]" + } + } + } \ No newline at end of file diff --git a/deploy/deploy_script.sh b/deploy/deploy_script.sh index 44e9a74d..dddc460a 100644 --- a/deploy/deploy_script.sh +++ b/deploy/deploy_script.sh @@ -24,5 +24,6 @@ sudo chmod +x /opt/metlo/manage-deployment.py sudo ln -s /opt/metlo/manage-deployment.py /usr/bin/metlo-deploy export METLO_DIR=/opt/metlo -sudo metlo-deploy init -sudo metlo-deploy update +echo "METLO_DIR=/opt/metlo" >> /home/azureuser/.bashrc +sudo -E metlo-deploy init +sudo -E metlo-deploy update diff --git a/manage-deployment.py b/manage-deployment.py index 912f995f..556658f5 100755 --- a/manage-deployment.py +++ b/manage-deployment.py @@ -118,7 +118,7 @@ def update(): def main(): parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(dest='command', required=True) + subparsers = parser.add_subparsers(dest='command') init_cmd = subparsers.add_parser('init') init_env_cmd = subparsers.add_parser('init-env')