diff --git a/backend/src/api/connections/index.ts b/backend/src/api/connections/index.ts index 48481685..a55b237a 100644 --- a/backend/src/api/connections/index.ts +++ b/backend/src/api/connections/index.ts @@ -8,13 +8,18 @@ import { import ApiResponseHandler from "api-response-handler" import { decrypt } from "utils/encryption" import { delete_connection as delete_connection_request } from "suricata_setup/" +import { ConnectionType } from "@common/enums" const list_connections = async (req: Request, res: Response) => { try { const connections = (await list_connections_service()).map(v => { - delete v.aws.keypair - delete v.aws.access_id - delete v.aws.secret_access_key + if (v.connectionType === ConnectionType.AWS) { + delete v.aws.keypair + delete v.aws.access_id + delete v.aws.secret_access_key + } else if (v.connectionType === ConnectionType.GCP) { + delete v.gcp.key_file + } return v }) diff --git a/backend/src/api/setup/index.ts b/backend/src/api/setup/index.ts index 2df92f32..e50a03bc 100644 --- a/backend/src/api/setup/index.ts +++ b/backend/src/api/setup/index.ts @@ -1,21 +1,24 @@ import { Request, Response } from "express" import ApiResponseHandler from "api-response-handler" -import { AWS_CONNECTION, STEP_RESPONSE } from "@common/types" +import { AWS_CONNECTION, SSH_INFO, STEP_RESPONSE } from "@common/types" import { ConnectionType } from "@common/enums" import { setup } from "suricata_setup" import "express-session" import { EC2_CONN } from "suricata_setup/aws-services/create-ec2-instance" import { VirtualizationType } from "@aws-sdk/client-ec2" -import { save_connection } from "services/connections" import { deleteKeyFromRedis, getFromRedis } from "suricata_setup/utils" +import { + list_images, + list_machines, +} from "suricata_setup/gcp-services/gcp_setup" declare module "express-session" { interface SessionData { connection_config: Record< string, // id { - step?: STEP_RESPONSE["step_number"] - status?: STEP_RESPONSE["status"] + step?: STEP_RESPONSE["step_number"] + status?: STEP_RESPONSE["status"] id?: string type?: ConnectionType data?: STEP_RESPONSE["data"] @@ -53,17 +56,6 @@ export const setup_connection = async ( ...resp, } - if (resp.status === "COMPLETE") { - const { - params: { name }, - } = req.body - await save_connection({ - conn_meta: req.session.connection_config[id].data as AWS_CONNECTION, - id: id, - name: name, - }) - } - delete resp.data await ApiResponseHandler.success(res, resp) @@ -77,8 +69,8 @@ export const aws_os_choices = async ( res: Response, ): Promise => { const { id } = req.body - const { access_id, secret_access_key, region } = - req.session.connection_config[id].data + const { access_id, secret_access_key, region } = req.session + .connection_config[id].data as STEP_RESPONSE["data"] let conn = new EC2_CONN(access_id, secret_access_key, region) let choices = await conn.get_latest_image() await ApiResponseHandler.success(res, [ @@ -86,14 +78,32 @@ export const aws_os_choices = async ( ]) } +export const gcp_os_choices = async ( + req: Request, + res: Response, +): Promise => { + try { + const { id } = req.body + const { key_file, zone, project } = req.session.connection_config[id] + .data as STEP_RESPONSE["data"] + + let choices = await list_images({ key_file, project, zone }) + let resp = choices.map(v => [v.description, v.selfLink]) + await ApiResponseHandler.success(res, resp) + } catch (err) { + await ApiResponseHandler.error(res, err) + } +} + export const aws_instance_choices = async ( req: Request, res: Response, ): Promise => { try { const { id, specs } = req.body - const { access_id, secret_access_key, virtualization_type, region } = - req.session.connection_config[id].data + const { access_id, secret_access_key, virtualization_type, region } = req + .session.connection_config[id] + .data as STEP_RESPONSE["data"] let conn = new EC2_CONN(access_id, secret_access_key, region) let choices = await conn.get_valid_types( virtualization_type as VirtualizationType, @@ -107,6 +117,32 @@ export const aws_instance_choices = async ( ApiResponseHandler.error(res, err) } } +export const gcp_instance_choices = async ( + req: Request, + res: Response, +): Promise => { + try { + const { id, specs } = req.body + const { key_file, zone, project } = req.session.connection_config[id] + .data as STEP_RESPONSE["data"] + + let choices = await list_machines({ + key_file, + zone, + project, + minCpu: specs.minCpu, + maxCpu: specs.maxCpu, + minMem: specs.minMem, + maxMem: specs.maxMem, + }) + await ApiResponseHandler.success( + res, + choices.map(v => [v.name, v.selfLink]), + ) + } catch (err) { + ApiResponseHandler.error(res, err) + } +} export const get_setup_state = async (req: Request, res: Response) => { const { uuid } = req.params @@ -115,6 +151,10 @@ export const get_setup_state = async (req: Request, res: Response) => { if (["OK", "FAIL"].includes(resp.success)) { await deleteKeyFromRedis(uuid) } + req.session.connection_config[resp.data.id] = { + ...req.session.connection_config[resp.data.id], + ...resp, + } delete resp.data await ApiResponseHandler.success(res, resp) } catch (err) { diff --git a/backend/src/index.ts b/backend/src/index.ts index a84548e3..45559efc 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -29,6 +29,8 @@ import { MulterSource } from "multer-source" import { aws_instance_choices, aws_os_choices, + gcp_instance_choices, + gcp_os_choices, get_setup_state, setup_connection, } from "./api/setup" @@ -99,6 +101,8 @@ app.post("/api/v1/setup_connection", setup_connection) app.get("/api/v1/setup_connection/fetch/:uuid", get_setup_state) app.post("/api/v1/setup_connection/aws/os", aws_os_choices) app.post("/api/v1/setup_connection/aws/instances", aws_instance_choices) +app.post("/api/v1/setup_connection/gcp/os", gcp_os_choices) +app.post("/api/v1/setup_connection/gcp/instances", gcp_instance_choices) app.get("/api/v1/list_connections", list_connections) app.get("/api/v1/list_connections/:uuid", get_connection_for_uuid) app.get( diff --git a/backend/src/models/connections.ts b/backend/src/models/connections.ts index 90d7b548..5e8bfe4d 100644 --- a/backend/src/models/connections.ts +++ b/backend/src/models/connections.ts @@ -8,7 +8,13 @@ import { BeforeInsert, } from "typeorm" import { ConnectionType } from "@common/enums" -import { AWS_CONNECTION, ENCRYPTED_AWS_CONNECTION__META } from "@common/types" +import { + AWS_CONNECTION, + ENCRYPTED_AWS_CONNECTION__META, + ENCRYPTED_GCP_CONNECTION__META, + GCP_CONNECTION, + SSH_INFO, +} from "@common/types" import { encrypt, generate_iv } from "utils/encryption" @Entity() @@ -29,11 +35,17 @@ export class Connections extends BaseEntity { name: string @Column({ nullable: true, type: "jsonb" }) - aws?: AWS_CONNECTION + aws?: AWS_CONNECTION & SSH_INFO @Column({ nullable: true, type: "jsonb" }) aws_meta?: ENCRYPTED_AWS_CONNECTION__META + @Column({ nullable: true, type: "jsonb" }) + gcp?: GCP_CONNECTION + + @Column({ nullable: true, type: "jsonb" }) + gcp_meta?: ENCRYPTED_GCP_CONNECTION__META + @BeforeInsert() beforeInsert() { let key = process.env.ENCRYPTION_KEY @@ -75,6 +87,21 @@ export class Connections extends BaseEntity { secret_access_key_iv: sa_key_iv.toString("base64"), } } + } else if (this.connectionType == ConnectionType.GCP && this.gcp) { + // Encrypt GCP Key File + let key_file_iv = generate_iv() + let { encrypted: encrypted_k, tag: tag_k } = encrypt( + this.gcp.key_file, + encryptionKey, + key_file_iv, + ) + this.gcp.key_file = encrypted_k + if (!this.gcp_meta) { + this.gcp_meta = { + key_file_tag: tag_k.toString("base64"), + key_file_iv: key_file_iv.toString("base64"), + } + } } } } diff --git a/backend/src/services/connections/index.ts b/backend/src/services/connections/index.ts index db46f38c..cee38b9d 100644 --- a/backend/src/services/connections/index.ts +++ b/backend/src/services/connections/index.ts @@ -1,15 +1,15 @@ import { ConnectionType } from "@common/enums" -import { AWS_CONNECTION } from "@common/types" +import { AWS_CONNECTION, GCP_CONNECTION, SSH_INFO } from "@common/types" import { AppDataSource } from "data-source" import Error500InternalServer from "errors/error-500-internal-server" import { Connections } from "models" -const save_connection = async ({ +const save_connection_aws = async ({ conn_meta, name, id, }: { - conn_meta: AWS_CONNECTION + conn_meta: AWS_CONNECTION & SSH_INFO name: string id: string }) => { @@ -52,7 +52,7 @@ const save_connection = async ({ remote_machine_url, keypair_name, keypair_id, - } as AWS_CONNECTION + } as AWS_CONNECTION & SSH_INFO conn.connectionType = ConnectionType.AWS conn.uuid = id conn.name = name @@ -64,6 +64,74 @@ const save_connection = async ({ throw new Error500InternalServer(err) } } +const save_connection_gcp = async ({ + conn_meta, + name, + id, +}: { + conn_meta: GCP_CONNECTION + name: string + id: string +}) => { + const { + key_file, + project, + zone, + network_url, + ip_range, + source_subnetwork_url, + firewall_rule_url, + destination_subnetwork_url, + router_url, + machine_type, + source_image, + image_template_url, + instance_url, + managed_group_url, + health_check_url, + backend_service_url, + forwarding_rule_url, + source_instance_url, + packet_mirror_url, + source_instance_name, + source_private_ip, + } = conn_meta + const conn = new Connections() + + conn.gcp = { + key_file, + project, + zone, + network_url, + ip_range, + source_subnetwork_url, + firewall_rule_url, + destination_subnetwork_url, + router_url, + machine_type, + source_image, + image_template_url, + instance_url, + managed_group_url, + health_check_url, + backend_service_url, + forwarding_rule_url, + source_instance_url, + packet_mirror_url, + source_instance_name, + source_private_ip, + } as GCP_CONNECTION + conn.connectionType = ConnectionType.GCP + conn.uuid = id + conn.name = name + try { + const connectionRepository = AppDataSource.getRepository(Connections) + await connectionRepository.save(conn) + } catch (err) { + console.error(`Error in saving connection: ${err}`) + throw new Error500InternalServer(err) + } +} const list_connections = async () => { try { @@ -77,6 +145,7 @@ const list_connections = async () => { "conn.updatedAt", "conn.connectionType", "conn.aws", + "conn.gcp", ]) .getMany() return resp @@ -149,7 +218,8 @@ const delete_connection_for_uuid = async ({ uuid }) => { } } export { - save_connection, + save_connection_aws, + save_connection_gcp, list_connections, get_connection_for_uuid, update_connection_for_uuid, diff --git a/backend/src/suricata_setup/aws-services/aws_setup.ts b/backend/src/suricata_setup/aws-services/aws_setup.ts index e628b306..b5603aa9 100644 --- a/backend/src/suricata_setup/aws-services/aws_setup.ts +++ b/backend/src/suricata_setup/aws-services/aws_setup.ts @@ -22,13 +22,15 @@ import { } from "./utils" import { STEP_RESPONSE } from "@common/types" +import { ConnectionType } from "@common/enums" +type RESPONSE = STEP_RESPONSE export async function aws_key_setup({ access_id, secret_access_key, region, ...rest -}: STEP_RESPONSE["data"]): Promise { +}: RESPONSE["data"]): Promise { try { let client = new STSClient({ credentials: { @@ -77,7 +79,7 @@ export async function aws_source_identification({ source_instance_id, region: _region, ...rest -}: STEP_RESPONSE["data"]): Promise { +}: RESPONSE["data"]): Promise { try { let client = new EC2Client({ credentials: { @@ -144,7 +146,7 @@ export async function aws_os_selection({ ami, region, ...rest -}: STEP_RESPONSE["data"]): Promise { +}: RESPONSE["data"]): Promise { try { let conn = new EC2_CONN(access_id, secret_access_key, region) let resp = await conn.image_from_ami(ami) @@ -163,6 +165,7 @@ export async function aws_os_selection({ region, ami, virtualization_type: resp[0].VirtualizationType, + username: "ubuntu", ...rest, }, } @@ -197,7 +200,7 @@ export async function aws_instance_selection({ selected_instance_type, ami, ...rest -}: STEP_RESPONSE["data"]): Promise { +}: RESPONSE["data"]): Promise { try { return { success: "OK", @@ -250,7 +253,7 @@ export async function aws_instance_creation({ selected_instance_type, id, ...rest -}: STEP_RESPONSE["data"]): Promise { +}: RESPONSE["data"]): Promise { try { let conn = new EC2_CONN(access_id, secret_access_key, region) let resp = await conn.create_new_instance(ami, selected_instance_type, id) @@ -309,7 +312,7 @@ export async function get_public_ip({ region, destination_eni_id, ...rest -}: STEP_RESPONSE["data"]): Promise { +}: RESPONSE["data"]): Promise { try { let client = new EC2Client({ credentials: { @@ -369,7 +372,7 @@ export async function aws_mirror_target_creation({ destination_eni_id, id, ...rest -}: STEP_RESPONSE["data"]): Promise { +}: RESPONSE["data"]): Promise { try { let client = new EC2Client({ credentials: { @@ -428,7 +431,7 @@ export async function aws_mirror_filter_creation({ mirror_rules, id, ...rest -}: STEP_RESPONSE["data"]): Promise { +}: RESPONSE["data"]): Promise { let client = new EC2Client({ credentials: { secretAccessKey: secret_access_key, @@ -523,7 +526,7 @@ export async function aws_mirror_session_creation({ mirror_target_id, id, ...rest -}: STEP_RESPONSE["data"]): Promise { +}: RESPONSE["data"]): Promise { try { let client = new EC2Client({ credentials: { diff --git a/backend/src/suricata_setup/aws-services/delete.ts b/backend/src/suricata_setup/aws-services/delete.ts index 2f6aa6ca..2baf577e 100644 --- a/backend/src/suricata_setup/aws-services/delete.ts +++ b/backend/src/suricata_setup/aws-services/delete.ts @@ -1,5 +1,6 @@ import { EC2Client } from "@aws-sdk/client-ec2" import { STEP_RESPONSE } from "@common/types" +import { ConnectionType } from "@common/enums" import { randomUUID } from "crypto" import { EC2_CONN } from "./create-ec2-instance" import { @@ -8,7 +9,9 @@ import { delete_mirror_target, } from "./mirroring" -export async function delete_aws_data(aws: STEP_RESPONSE["data"]) { +export async function delete_aws_data( + aws: STEP_RESPONSE["data"], +) { let client = new EC2Client({ credentials: { accessKeyId: aws.access_id, diff --git a/backend/src/suricata_setup/gcp-services/gcp_apis.ts b/backend/src/suricata_setup/gcp-services/gcp_apis.ts new file mode 100644 index 00000000..d2d48463 --- /dev/null +++ b/backend/src/suricata_setup/gcp-services/gcp_apis.ts @@ -0,0 +1,540 @@ +import { + InstancesClient, + SubnetworksClient, + AddressesClient, + GlobalAddressesClient, + FirewallsClient, + NetworksClient, + RoutersClient, + ImagesClient, + InstanceTemplatesClient, + InstanceGroupManagersClient, + HealthChecksClient, + RegionBackendServicesClient, + ForwardingRulesClient, + ZonesClient, + PacketMirroringsClient, + RegionOperationsClient, + GlobalOperationsClient, + ZoneOperationsClient, + MachineTypesClient, +} from "@google-cloud/compute" + +const PREFIX_LENGTH = 24 +const METLO_DATA_COLLECTOR_TAG = "metlo-capture" +const COOL_DOWN_PERIOD = 180 + +export class GCP_CONN { + private zone: string + private region: string + private project: string + private keyfile: Object + + constructor(key_file: Object | string, zone: string, project: string) { + this.keyfile = key_file instanceof Object ? key_file : JSON.parse(key_file) + this.zone = zone + this.region = zone.substring(0, zone.length - 2) + this.project = project + } + + public async test_connection() { + // We run `initialize` in get_conn, which tests the connection internally. + let conn = new InstancesClient({ credentials: this.keyfile }) + conn.initialize() + } + + public async list_zones() { + let conn = new ZonesClient({ credentials: this.keyfile }) + return conn.list() + } + + public async get_networks({ name }) { + let conn = new NetworksClient({ credentials: this.keyfile }) + return conn.get({ project: this.project, network: name }) + } + + public async get_regional_operation_status(operation: string) { + let conn = new RegionOperationsClient({ credentials: this.keyfile }) + return conn.get({ operation, region: this.region, project: this.project }) + } + + public async get_zonal_operation_status(operation: string) { + let conn = new ZoneOperationsClient({ credentials: this.keyfile }) + return conn.get({ operation, zone: this.zone, project: this.project }) + } + + public async get_global_operation_status(operation: string) { + let conn = new GlobalOperationsClient({ credentials: this.keyfile }) + return conn.get({ operation, project: this.project }) + } + + public async get_instance(name: string) { + let conn = new InstancesClient({ credentials: this.keyfile }) + return conn.get({ + instance: name, + project: this.project, + zone: this.zone, + }) + } + + public async get_instance_by_name({ instanceName }) { + let conn = new InstancesClient({ credentials: this.keyfile }) + const resp = conn.list({ + project: this.project, + // zone: this.zone, + filter: `name eq ${instanceName}`, + }) + return resp + } + + public async get_subnet_information({ subnetName }) { + let conn = new SubnetworksClient({ credentials: this.keyfile }) + const resp = conn.get({ + project: this.project, + subnetwork: subnetName, + region: "us-west1", + }) + return resp + } + + public async create_new_subnet({ name, ipCidr, network }) { + let conn = new SubnetworksClient({ credentials: this.keyfile }) + return conn.insert({ + project: this.project, + region: this.region, + subnetworkResource: { + network, + ipCidrRange: ipCidr, + name, + }, + }) + } + + public async get_address_information({ addressName }) { + const resp = new GlobalAddressesClient({ credentials: this.keyfile }).get({ + project: this.project, + address: addressName, + }) + return resp + } + + public async create_new_internal_address({ + addressName, + network, + prefixLength, + }) { + const conn = new GlobalAddressesClient({ credentials: this.keyfile }) + let resp = conn.insert({ + project: this.project, + addressResource: { + region: this.region, + name: addressName, + prefixLength: prefixLength, + addressType: "INTERNAL", + purpose: "VPC_PEERING", + network, + }, + }) + return resp + } + + public async create_new_external_address({ addressName, network }) { + const conn = new AddressesClient({ credentials: this.keyfile }) + let resp = conn.insert({ + project: this.project, + region: this.region, + addressResource: { + region: this.region, + name: addressName, + addressType: "EXTERNAL", + network, + }, + }) + return resp + } + + public async get_external_address({ addressName }) { + let conn = new AddressesClient({ credentials: this.keyfile }) + return conn.get({ + project: this.project, + address: addressName, + region: this.region, + }) + } + + public async attach_external_ip({ instanceURL, address }) { + let conn = new InstancesClient({ credentials: this.keyfile }) + let instance = await conn.get({ + zone: this.zone, + project: this.project, + instance: instanceURL, + }) + return conn.addAccessConfig({ + zone: this.zone, + project: this.project, + instance: instanceURL, + networkInterface: instance[0].networkInterfaces[0].name, + accessConfigResource: { + natIP: address, + name: `${instance[0].name}-EX_ADDR`, + }, + }) + } + + public async detach_external_ip({ instanceURL }) { + let conn = new InstancesClient({ credentials: this.keyfile }) + let instance = await conn.get({ + zone: this.zone, + project: this.project, + instance: instanceURL, + }) + + return conn.deleteAccessConfig({ + zone: this.zone, + project: this.project, + instance: instanceURL, + networkInterface: instance[0].networkInterfaces[0].name, + accessConfig: `${instance[0].name}-EX_ADDR`, + }) + } + + public async describe_new_address({ addressName }) { + const conn = new GlobalAddressesClient({ credentials: this.keyfile }) + let resp = conn.get({ + project: this.project, + address: addressName, + }) + return resp + } + + public async delete_new_address({ addressName }) { + const conn = new GlobalAddressesClient({ credentials: this.keyfile }) + let resp = conn.delete({ + project: this.project, + address: addressName, + }) + return resp + } + + public async create_firewall_rule({ firewallName, networkName, ipRange }) { + const conn = new FirewallsClient({ credentials: this.keyfile }) + return conn.insert({ + project: this.project, + firewallResource: { + direction: "INGRESS", + network: networkName, + targetTags: [METLO_DATA_COLLECTOR_TAG], + sourceRanges: ["0.0.0.0/0"], + name: firewallName, + priority: 65534, + logConfig: { + enable: false, + }, + allowed: [ + { + IPProtocol: "all", + }, + ], + }, + }) + } + + public async list_routers() { + let conn = new RoutersClient({ credentials: this.keyfile }) + return conn.list({ region: this.region, project: this.project }) + } + + public async list_nats({ router }) { + let conn = new RoutersClient({ credentials: this.keyfile }) + return conn.getNatMappingInfo({ + router, + region: this.region, + project: this.project, + }) + } + + public async create_router({ routerName, networkURL, subnetURL }) { + let conn = new RoutersClient({ credentials: this.keyfile }) + return conn.insert({ + project: this.project, + region: this.region, + routerResource: { + region: this.region, + name: routerName, + network: networkURL, + nats: [ + { + natIpAllocateOption: "AUTO_ONLY", + name: `${routerName}-nats`, + subnetworks: [ + { + sourceIpRangesToNat: ["PRIMARY_IP_RANGE"], + name: subnetURL, + }, + ], + enableEndpointIndependentMapping: false, + sourceSubnetworkIpRangesToNat: "LIST_OF_SUBNETWORKS", + }, + ], + }, + }) + } + + public async create_health_check({ healthCheckName }) { + let conn = new HealthChecksClient({ credentials: this.keyfile }) + return conn.insert({ + project: this.project, + healthCheckResource: { + unhealthyThreshold: 2, + healthyThreshold: 2, + checkIntervalSec: 30, + timeoutSec: 10, + name: healthCheckName, + description: "METLO health check for backend", + type: "TCP", + tcpHealthCheck: { + proxyHeader: null, + port: 80, + }, + logConfig: { + enable: false, + }, + }, + }) + } + + public async create_image_template({ + machineType, + sourceImage, + network, + subnet, + imageTemplateName, + }) { + let conn = new InstanceTemplatesClient({ credentials: this.keyfile }) + return conn.insert({ + project: this.project, + instanceTemplateResource: { + name: imageTemplateName, + properties: { + metadata: { + items: [ + { + key: "startup-script", + value: `#! /bin/bash + apt-get update + apt-get install apache2 -y + vm_hostname="$(curl -H "Metadata-Flavor:Google" \ + http://169.254.169.254/computeMetadata/v1/instance/name)" + echo "Page served from: $vm_hostname" | \ + tee /var/www/html/index.html + systemctl restart apache2`, + }, + ], + }, + machineType, + disks: [ + { + autoDelete: true, + boot: true, + deviceName: `${imageTemplateName}-disk`, + diskEncryptionKey: {}, + initializeParams: { + diskSizeGb: "10", + diskType: "pd-balanced", + labels: {}, + sourceImage: sourceImage, + }, + mode: "READ_WRITE", + type: "PERSISTENT", + }, + ], + networkInterfaces: [ + { + network, + subnetwork: subnet, + }, + ], + tags: { + items: [METLO_DATA_COLLECTOR_TAG], + }, + }, + }, + }) + } + + public async create_instance_manager({ templateURL, instanceName }) { + let conn = new InstanceGroupManagersClient({ credentials: this.keyfile }) + return conn.insert({ + project: this.project, + zone: this.zone, + instanceGroupManagerResource: { + zone: this.zone, + name: instanceName, + instanceTemplate: templateURL, + targetSize: 1, + }, + }) + } + + public async list_instance_for_group({ managedGroupName }) { + let conn = new InstanceGroupManagersClient({ credentials: this.keyfile }) + return conn.listManagedInstances({ + instanceGroupManager: managedGroupName, + zone: this.zone, + project: this.project, + }) + } + + public async create_instance({ instanceGroupURL, instance_name }) { + let conn = new InstanceGroupManagersClient({ credentials: this.keyfile }) + return conn.createInstances({ + project: this.project, + zone: this.zone, + instanceGroupManager: instanceGroupURL, + instanceGroupManagersCreateInstancesRequestResource: { + instances: [{ name: instance_name }], + }, + }) + } + + public async add_key({ username, publicKey, instance }) { + let conn = new InstancesClient({ credentials: this.keyfile }) + let project_metadata = await conn.get({ + project: this.project, + instance: instance, + zone: this.zone, + }) + let project_fingerprint = project_metadata[0].metadata.fingerprint + + let project_meta_without_ssh = project_metadata[0].metadata.items.filter( + v => v.key !== "ssh-keys", + ) + + let ssh_keys = project_metadata[0].metadata.items.find( + v => v.key === "ssh-keys", + ) + return conn.setMetadata({ + instance, + zone: this.zone, + project: this.project, + metadataResource: { + fingerprint: project_fingerprint, + items: [ + ...project_meta_without_ssh, + { + key: "ssh-keys", + value: `${ssh_keys.value}\n${username}:${publicKey}`, + }, + ], + }, + }) + } + + public async create_backend_service({ + networkURL, + managedGroupURL, + healthCheckURL, + name, + }) { + let conn = new RegionBackendServicesClient({ credentials: this.keyfile }) + return conn.insert({ + project: this.project, + region: this.region, + backendServiceResource: { + name: name, + region: this.region, + network: networkURL, + healthChecks: [healthCheckURL], + loadBalancingScheme: "INTERNAL", + backends: [{ group: managedGroupURL }], + protocol: "TCP", + }, + }) + } + + // public async add_backend({ + // backendServiceName, + // managedGroupURL, + // healthCheckURL, + // }) { + // let conn = new RegionBackendServicesClient({ credentials: this.keyfile }) + // return await conn.update({ + // project: this.project, + // region: this.region, + // backendService: backendServiceName, + // backendServiceResource: { + // region: this.region, + // backends: [{ group: managedGroupURL }], + // healthChecks: healthCheckURL ? [healthCheckURL] : [], + // loadBalancingScheme: "INTERNAL", + // }, + // }) + // } + + public async create_forwarding_rule({ + name, + networkURL, + subnetURL, + backendServiceURL, + }) { + let conn = new ForwardingRulesClient({ credentials: this.keyfile }) + return conn.insert({ + region: this.region, + project: this.project, + forwardingRuleResource: { + name, + loadBalancingScheme: "INTERNAL", + backendService: backendServiceURL, + isMirroringCollector: true, + network: networkURL, + region: this.region, + subnetwork: subnetURL, + IPProtocol: "TCP", + allPorts: true, + }, + }) + } + + public async start_packet_mirroring({ + name, + networkURL, + mirroredInstanceURL, + loadBalancerURL, + }) { + let conn = new PacketMirroringsClient({ credentials: this.keyfile }) + return conn.insert({ + project: this.project, + region: this.region, + packetMirroringResource: { + name, + collectorIlb: { url: loadBalancerURL }, + enable: "TRUE", + network: { + url: networkURL, + }, + mirroredResources: { + instances: [ + { + url: mirroredInstanceURL, + }, + ], + }, + region: this.region, + }, + }) + } + + public async list_machine_images({ project, filters }) { + let conn = new ImagesClient({ credentials: this.keyfile }) + return conn.list({ project: project, filter: filters }) + } + + public async list_machine_types({ filters }) { + let conn = new MachineTypesClient({ credentials: this.keyfile }) + return conn.list({ + project: this.project, + zone: this.zone, + filter: filters, + }) + } +} diff --git a/backend/src/suricata_setup/gcp-services/gcp_setup.ts b/backend/src/suricata_setup/gcp-services/gcp_setup.ts new file mode 100644 index 00000000..2530ec7a --- /dev/null +++ b/backend/src/suricata_setup/gcp-services/gcp_setup.ts @@ -0,0 +1,999 @@ +import { MachineSpecifications, STEP_RESPONSE } from "@common/types" +import { ConnectionType, GCP_STEPS } from "@common/enums" +import { GCP_CONN } from "./gcp_apis" +import AsyncRetry from "async-retry" +import { promisify } from "util" +import { exec } from "child_process" +import { writeFileSync, unlinkSync } from "fs" +import { + put_data_file, + remove_file, + format, +} from "suricata_setup/ssh-services/ssh-setup" +import path from "path" + +const promiseExec = promisify(exec) + +type RESPONSE = STEP_RESPONSE + +export async function gcp_key_setup({ + key_file, + zone, + project, + network_name, + id, + ...rest +}: RESPONSE["data"]): Promise> { + const fileName = `authFile-${id}` + try { + let conn = new GCP_CONN(key_file, zone, project) + await conn.test_connection() + let network_resp = await conn.get_networks({ name: network_name }) + + const file = writeFileSync(fileName, key_file, { flag: "w" }) + await promiseExec( + `gcloud auth activate-service-account --key-file=${fileName}`, + ) + + return { + success: "OK", + status: "IN-PROGRESS", + step_number: 1, + next_step: 2, + last_completed: 1, + message: "Verified GCP Credentials", + error: null, + data: { + id, + key_file, + zone, + project, + network_url: network_resp[0].selfLink, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: 1, + next_step: 2, + last_completed: 0, + message: + "Couldn't verify GCP Credentials. Please verify that access id and secret access key are correct.", + error: { + err: JSON.stringify(err), + }, + data: { + id, + ...rest, + }, + } + } finally { + unlinkSync(fileName) + } +} + +export async function gcp_source_identification({ + key_file, + zone, + project, + source_instance_name, + ...rest +}: RESPONSE["data"]): Promise> { + try { + let conn = new GCP_CONN(key_file, zone, project) + let resp = await conn.get_instance(source_instance_name) + return { + success: "OK", + status: "IN-PROGRESS", + step_number: 2, + next_step: 3, + last_completed: 2, + message: "Verified GCP source instance", + error: null, + data: { + key_file, + zone, + project, + source_instance_name, + source_private_ip: resp[0].networkInterfaces[0].networkIP, + source_subnetwork_url: resp[0].networkInterfaces[0].subnetwork, + source_instance_url: resp[0].selfLink, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: 1, + next_step: 2, + last_completed: 0, + message: + "Couldn't verify mirroring source presence in region/zone. Please verify name and zone .", + error: { + err: JSON.stringify(err), + }, + data: { key_file, zone, project, ...rest }, + } + } +} + +export async function get_destination_subnet({ + key_file, + project, + zone, + network_url, + id, + ...rest +}: RESPONSE["data"]): Promise> { + try { + let addressName = `metlo-address-temporary-${id}` + let conn = new GCP_CONN(key_file, zone, project) + + let address_resp = await conn.create_new_internal_address({ + addressName: addressName, + network: network_url, + prefixLength: 24, + }) + + await wait_for_global_operation(address_resp[0].latestResponse.name, conn) + + let connectionReadyResp = await AsyncRetry( + async (f, at) => { + let resp = await conn.get_address_information({ + addressName: addressName, + }) + if (resp[0].status === "RESERVED") { + return resp + } else { + throw Error("Couldn't reserve address") + } + }, + { retries: 5 }, + ) + + const ip_range = `${connectionReadyResp[0].address}/${connectionReadyResp[0].prefixLength}` + + let delete_resp = await conn.delete_new_address({ + addressName: addressName, + }) + await wait_for_global_operation(delete_resp[0].latestResponse.name, conn) + + let destination_subnetwork = await conn.create_new_subnet({ + network: network_url, + ipCidr: ip_range, + name: `metlo-subnet-${id}`, + }) + await wait_for_regional_operation( + destination_subnetwork[0].latestResponse.name, + conn, + ) + return { + success: "OK", + status: "IN-PROGRESS", + step_number: 3, + next_step: 4, + last_completed: 3, + message: "Creating GCP Address", + error: null, + data: { + key_file, + zone, + project, + ip_range: ip_range, + network_url, + destination_subnetwork_url: + //@ts-ignore + destination_subnetwork[0].latestResponse.targetLink, + id, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: 3, + next_step: 3, + last_completed: 2, + message: "Couldn't create subnet", + error: { + err: JSON.stringify(err), + }, + data: { key_file, project, zone, network_url, id, ...rest }, + } + } +} + +export async function create_firewall_rule({ + key_file, + project, + zone, + network_url, + ip_range, + id, + ...rest +}: RESPONSE["data"]): Promise> { + try { + let conn = new GCP_CONN(key_file, zone, project) + const firewallName = `metlo-firewall-${id}` + let resp = await conn.create_firewall_rule({ + firewallName, + networkName: network_url, + ipRange: ip_range, + }) + + return { + success: "OK", + status: "IN-PROGRESS", + step_number: 4, + next_step: 5, + last_completed: 4, + message: "Creating GCP Address", + error: null, + data: { + key_file, + zone, + project, + network_url, + ip_range, + //@ts-ignore + firewall_url: resp[0].latestResponse.targetLink, + id, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: 4, + next_step: 4, + last_completed: 3, + message: "Couldn't create firewall rule.", + error: { + err: JSON.stringify(err), + }, + data: { key_file, zone, project, network_url, ip_range, id, ...rest }, + } + } +} + +export async function create_cloud_router({ + key_file, + project, + zone, + network_url, + destination_subnetwork_url, + ip_range, + id, + ...rest +}: RESPONSE["data"]): Promise { + try { + let conn = new GCP_CONN(key_file, zone, project) + let resp = await conn.list_routers() + let useful_routers = resp[0].filter(v => { + const usesfulNats = v.nats.filter(nat => + [ + "ALL_SUBNETWORKS_ALL_IP_RANGES", + "ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES", + ].includes(nat.sourceSubnetworkIpRangesToNat), + ) + return v.network === network_url && usesfulNats.length > 0 + }) + var router_url + if (useful_routers.length > 0) { + const useful_nats = useful_routers[0].nats.find(nat => + [ + "ALL_SUBNETWORKS_ALL_IP_RANGES", + "ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES", + ].includes(nat.sourceSubnetworkIpRangesToNat), + ) + if (useful_nats) { + router_url = useful_routers[0].selfLink + } + } + if (!router_url) { + let new_router = await conn.create_router({ + routerName: `metlo-router-${id}`, + networkURL: network_url, + subnetURL: destination_subnetwork_url, + }) + // @ts-ignore + router_url = new_router[0].latestResponse.targetLink + } + ;("https://www.googleapis.com/compute/v1/projects/metlo-crypto/regions/us-west1/subnetworks/metlo-subnet-3e63bb14-bdb4-4c6d-bc41-31512ca50ad2") + return { + success: "OK", + status: "IN-PROGRESS", + step_number: 5, + next_step: 6, + last_completed: 5, + message: "Creating GCP Address", + error: null, + data: { + key_file, + project, + zone, + network_url, + destination_subnetwork_url, + router_url, + ip_range, + id, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: 5, + next_step: 5, + last_completed: 4, + message: "Couldn't create firewall rule.", + error: { + err: JSON.stringify(err), + }, + data: { + key_file, + project, + zone, + network_url, + destination_subnetwork_url, + ip_range, + id, + ...rest, + }, + } + } +} + +export async function create_mig({ + key_file, + project, + zone, + network_url, + destination_subnetwork_url, + machine_type, + source_image, + id, + ...rest +}: RESPONSE["data"]): Promise { + try { + let conn = new GCP_CONN(key_file, zone, project) + const imageTemplateName = `metlo-image-${id}` + let image_resp = await conn.create_image_template({ + machineType: machine_type, + sourceImage: source_image, + network: network_url, + subnet: destination_subnetwork_url, + imageTemplateName: imageTemplateName, + }) + let img_resp = await wait_for_global_operation( + image_resp[0].latestResponse.name, + conn, + ) + + // const imgUrl = + // "https://www.googleapis.com/compute/v1/projects/metlo-crypto/global/instanceTemplates/metlo-image-98c897b9-9660-4176-a986-9c60fde2bc11" + + const instanceGroupName = `metlo-mig-${id}` + let instance_manager = await conn.create_instance_manager({ + templateURL: img_resp[0].targetLink, + instanceName: instanceGroupName, + }) + let resp = await wait_for_zonal_operation( + instance_manager[0].latestResponse.name, + conn, + ) + + const instance_name = `metlo-scaler-${id}` + + let instance = await conn.list_instance_for_group({ + managedGroupName: instanceGroupName, + }) + + return { + success: "OK", + status: "IN-PROGRESS", + step_number: 5, + next_step: 6, + last_completed: 5, + message: "Creating GCP Address", + error: null, + data: { + key_file, + project, + zone, + network_url, + destination_subnetwork_url, + machine_type, + source_image, + id, + // @ts-ignore + image_template_url: image_resp[0].latestResponse.targetLink, + // @ts-ignore + instance_url: instance[0][0].instance, + managed_group_url: `https://www.googleapis.com/compute/v1/projects/${project}/zones/${zone}/instanceGroups/${instanceGroupName}`, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: 5, + next_step: 5, + last_completed: 4, + message: "Couldn't create firewall rule.", + error: { + err: JSON.stringify(err), + }, + data: { + key_file, + project, + zone, + network_url, + destination_subnetwork_url, + machine_type, + source_image, + id, + ...rest, + }, + } + } +} + +export async function create_health_check({ + key_file, + project, + zone, + id, + ...rest +}: RESPONSE["data"]): Promise { + try { + let conn = new GCP_CONN(key_file, zone, project) + const health_check_name = `metlo-health-check-${id}` + let resp = await conn.create_health_check({ + healthCheckName: health_check_name, + }) + await wait_for_global_operation(resp[0].latestResponse.name, conn) + return { + success: "OK", + status: "IN-PROGRESS", + step_number: 8, + next_step: 9, + last_completed: 8, + message: "Creating GCP Keypair", + error: null, + data: { + key_file, + project, + zone, + id, + //@ts-ignore + health_check_url: resp[0].latestResponse.targetLink, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: 8, + next_step: 8, + last_completed: 7, + message: "Couldn't add ssh-keys", + error: { + err: JSON.stringify(err), + }, + data: { + key_file, + project, + zone, + id, + ...rest, + }, + } + } +} + +export async function create_backend_service({ + key_file, + project, + zone, + network_url, + instance_url, + managed_group_url, + health_check_url, + id, + ...rest +}: RESPONSE["data"]): Promise { + try { + let conn = new GCP_CONN(key_file, zone, project) + const backend_name = `metlo-backend-${id}` + let resp = await conn.create_backend_service({ + networkURL: network_url, + managedGroupURL: managed_group_url, + healthCheckURL: health_check_url, + name: backend_name, + }) + await wait_for_regional_operation(resp[0].latestResponse.name, conn) + return { + success: "OK", + status: "IN-PROGRESS", + step_number: 9, + next_step: 10, + last_completed: 9, + message: "Creating GCP Keypair", + error: null, + data: { + key_file, + project, + zone, + network_url, + instance_url, + managed_group_url, + health_check_url, + //@ts-ignore + backend_service_url: resp[0].latestResponse.targetLink, + id, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: 9, + next_step: 9, + last_completed: 8, + message: "Couldn't add ssh-keys", + error: { + err: JSON.stringify(err), + }, + data: { + key_file, + project, + zone, + network_url, + instance_url, + managed_group_url, + health_check_url, + id, + ...rest, + }, + } + } +} + +export async function create_load_balancer({ + key_file, + project, + zone, + network_url, + destination_subnetwork_url, + backend_service_url, + id, + ...rest +}: RESPONSE["data"]): Promise { + try { + let conn = new GCP_CONN(key_file, zone, project) + const rule_name = `metlo-forwarding-rule-${id}` + let resp = await conn.create_forwarding_rule({ + networkURL: network_url, + name: rule_name, + subnetURL: destination_subnetwork_url, + backendServiceURL: backend_service_url, + }) + await wait_for_regional_operation(resp[0].latestResponse.name, conn) + return { + success: "OK", + status: "IN-PROGRESS", + step_number: 10, + next_step: 11, + last_completed: 10, + message: "Creating GCP Keypair", + error: null, + data: { + key_file, + project, + zone, + network_url, + destination_subnetwork_url, + //@ts-ignore + forwarding_rule_url: resp[0].latestResponse.targetLink, + backend_service_url, + id, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: 10, + next_step: 10, + last_completed: 9, + message: "Couldn't add ssh-keys", + error: { + err: JSON.stringify(err), + }, + data: { + key_file, + project, + zone, + network_url, + destination_subnetwork_url, + backend_service_url, + id, + ...rest, + }, + } + } +} + +export async function packet_mirroring({ + key_file, + project, + zone, + network_url, + forwarding_rule_url, + source_instance_url, + id, + ...rest +}: RESPONSE["data"]): Promise { + try { + let conn = new GCP_CONN(key_file, zone, project) + const packet_mirror_name = `metlo-packet-mirroring-${id}` + let resp = await conn.start_packet_mirroring({ + networkURL: network_url, + name: packet_mirror_name, + mirroredInstanceURL: source_instance_url, + loadBalancerURL: forwarding_rule_url, + }) + await wait_for_regional_operation(resp[0].latestResponse.name, conn) + return { + success: "OK", + status: "IN-PROGRESS", + step_number: 11, + next_step: 12, + last_completed: 11, + message: "Creating GCP Keypair", + error: null, + data: { + key_file, + project, + zone, + network_url, + forwarding_rule_url, + source_instance_url, + //@ts-ignore + packet_mirror_url: resp[0].latestResponse.targetLink, + id, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: 11, + next_step: 11, + last_completed: 10, + message: "Couldn't add ssh-keys", + error: { + err: JSON.stringify(err), + }, + data: { + key_file, + project, + zone, + network_url, + forwarding_rule_url, + source_instance_url, + id, + ...rest, + }, + } + } +} + +export async function test_ssh({ + key_file, + instance_url, + project, + ...rest +}: RESPONSE["data"]): Promise { + try { + const cmd = `gcloud compute ssh ${instance_url.split("/").at(-1)} \ + --command="lsb_release -i" \ + --account=${JSON.parse(key_file).client_email} \ + --tunnel-through-iap \ + --project=${project.trim()}` + + let { stderr, stdout } = await promiseExec(cmd) + if (!stdout.trim().includes(`Distributor ID:`)) { + throw new Error("Couldn't test ssh connection") + } + return { + success: "OK", + status: "IN-PROGRESS", + step_number: GCP_STEPS.TEST_SSH, + next_step: GCP_STEPS.TEST_SSH + 1, + last_completed: GCP_STEPS.TEST_SSH, + message: "Testing SSH connection to remote machine.", + error: null, + data: { + key_file, + instance_url, + project, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: GCP_STEPS.TEST_SSH, + next_step: GCP_STEPS.TEST_SSH, + last_completed: GCP_STEPS.TEST_SSH - 1, + message: `Couldn't connect to ssh. Please check if key was constructed`, + error: { + err: JSON.stringify(err), + }, + data: { + key_file, + instance_url, + project, + ...rest, + }, + } + } +} + +export async function push_files({ + key_file, + source_private_ip, + project, + id, + instance_url, + ...rest +}: RESPONSE["data"]): Promise { + const endpoint = "api/v1/log-request/single" + const instance_name = instance_url.split("/").at(-1) + try { + let filepath_ingestor_out = path.normalize( + `${__dirname}/../generics/scripts/metlo-ingestor-${id}.service`, + ) + let filepath_ingestor_in = path.normalize( + `${__dirname}/../generics/scripts/metlo-ingestor-template.service`, + ) + let filepath_rules_out = path.normalize( + `${__dirname}/../generics/scripts/local-${id}.rules`, + ) + let filepath_rules_in = path.normalize( + `${__dirname}/../generics/scripts/local.rules`, + ) + + put_data_file( + format(filepath_ingestor_in, [`${process.env.BACKEND_URL}/${endpoint}`]), + filepath_ingestor_out, + ) + put_data_file( + format(filepath_rules_in, [source_private_ip]), + filepath_rules_out, + ) + + const fileMap = [ + path.normalize(`${__dirname}/../generics/scripts/install.sh`) + + ` ${instance_name}:~/install.sh`, + path.normalize(`${__dirname}/../generics/scripts/install-deps.sh`) + + ` ${instance_name}:~/install-deps.sh`, + path.normalize(`${__dirname}/../generics/scripts/suricata.yaml`) + + ` ${instance_name}:~/suricata.yaml`, + filepath_rules_out + ` ${instance_name}:"~/local.rules"`, + filepath_ingestor_out + ` ${instance_name}:~/metlo-ingestor.service`, + ] + + let pushes = fileMap.map(file => { + return promiseExec( + `gcloud compute scp ${file} \ + --account=${JSON.parse(key_file).client_email}\ + --tunnel-through-iap \ + --project=${project}`, + ) + }) + let resp = await Promise.all(pushes) + + remove_file(filepath_ingestor_out) + remove_file(filepath_rules_out) + + return { + success: "OK", + status: "IN-PROGRESS", + step_number: GCP_STEPS.PUSH_FILES, + next_step: GCP_STEPS.PUSH_FILES + 1, + last_completed: GCP_STEPS.PUSH_FILES, + message: "Pushed configuration files to remote machine", + error: null, + data: { + key_file, + source_private_ip, + project, + id, + instance_url, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: GCP_STEPS.PUSH_FILES, + next_step: GCP_STEPS.PUSH_FILES, + last_completed: GCP_STEPS.PUSH_FILES - 1, + message: `Couldn't push configuration files to remote machine`, + error: { + err: JSON.stringify(err), + }, + data: { + key_file, + source_private_ip, + project, + id, + instance_url, + ...rest, + }, + } + } +} + +export async function execute_commands({ + instance_url, + key_file, + project, + ...rest +}: RESPONSE["data"]): Promise { + const instance_name = instance_url.split("/").at(-1) + const acct_email = JSON.parse(key_file).client_email + + try { + await promiseExec( + `gcloud compute ssh ${instance_name} \ + --account=${acct_email} \ + --tunnel-through-iap \ + --project=${project} \ + --command="cd ~ && chmod +x install-deps.sh && ./install-deps.sh"`, + ) + await promiseExec( + `gcloud compute ssh ${instance_name} \ + --account=${JSON.parse(key_file).client_email} \ + --tunnel-through-iap \ + --project=${project} \ + --command="source ~/.nvm/nvm.sh && cd ~ && chmod +x install.sh && ./install.sh"`, + ) + + return { + success: "OK", + status: "COMPLETE", + step_number: GCP_STEPS.EXEC_COMMAND, + next_step: GCP_STEPS.EXEC_COMMAND + 1, + last_completed: GCP_STEPS.EXEC_COMMAND, + message: "Executed configuration files on remote machine succesfully", + error: null, + data: { + instance_url, + key_file, + project, + ...rest, + }, + } + } catch (err) { + console.log(err) + return { + success: "FAIL", + status: "IN-PROGRESS", + step_number: GCP_STEPS.EXEC_COMMAND, + next_step: GCP_STEPS.EXEC_COMMAND, + last_completed: GCP_STEPS.EXEC_COMMAND - 1, + message: `Couldn't exec commands to install things`, + error: { + err: JSON.stringify(err), + }, + data: { + instance_url, + key_file, + project, + ...rest, + }, + } + } +} + +async function wait_for_global_operation(operation_id, conn: GCP_CONN) { + return await AsyncRetry( + async (f, at) => { + let resp = await conn.get_global_operation_status(operation_id) + if (resp[0].status === "DONE") { + return resp + } else { + throw Error("Couldn't fetch global operation") + } + }, + { retries: 5 }, + ) +} + +async function wait_for_regional_operation(operation_id, conn: GCP_CONN) { + return await AsyncRetry( + async (f, at) => { + let resp = await conn.get_regional_operation_status(operation_id) + + if (resp[0].status === "DONE") { + return resp + } else { + throw Error("Couldn't fetch regional operation") + } + }, + { retries: 5 }, + ) +} + +async function wait_for_zonal_operation(operation_id, conn: GCP_CONN) { + return await AsyncRetry( + async (f, at) => { + let resp = await conn.get_zonal_operation_status(operation_id) + if (resp[0].status === "DONE") { + return resp + } else { + throw Error("Couldn't fetch regional operation") + } + }, + { retries: 5 }, + ) +} + +export async function list_images({ + key_file, + project, + zone, +}: Partial) { + let conn = new GCP_CONN(key_file, zone, project) + let resps = await conn.list_machine_images({ + project: "ubuntu-os-cloud", + filters: "id = 5824089035104633064", + }) + return resps[0] +} + +export async function list_machines({ + key_file, + project, + zone, + minCpu, + maxCpu, + minMem, + maxMem, +}: Partial & MachineSpecifications) { + let conn = new GCP_CONN(key_file, zone, project) + return ( + await conn.list_machine_types({ + filters: `memoryMb >=${minMem} AND memoryMb <= ${maxMem} AND guestCpus <= ${maxCpu} AND guestCpus >= ${minCpu}`, + }) + )[0] +} diff --git a/backend/src/suricata_setup/gcp-services/index.ts b/backend/src/suricata_setup/gcp-services/index.ts deleted file mode 100644 index 88730711..00000000 --- a/backend/src/suricata_setup/gcp-services/index.ts +++ /dev/null @@ -1,275 +0,0 @@ -import compute, { - InstancesClient, - SubnetworksClient, - AddressesClient, - GlobalAddressesClient, - FirewallsClient, - NetworksClient, - RoutersClient, - ImagesClient, - InstanceTemplatesClient, - InstanceGroupManagersClient, - HealthChecksClient, - AutoscalersClient, -} from "@google-cloud/compute" - -const PREFIX_LENGTH = 24 -const METLO_DATA_COLLECTOR_TAG = "metlo-capture" -const COOL_DOWN_PERIOD = 180 - -export class GCP_CONN { - private zone: string - private region: string - private project: string - private keyfile: Object - - constructor(key_file: Object, zone: string, project: string) { - this.keyfile = key_file - this.zone = zone - this.region = zone.substring(0, zone.length - 2) - this.project = project - } - - public async test_connection() { - try { - // We run `initialize` in get_conn, which tests the connection internally. - let conn = new InstancesClient({ credentials: this.keyfile }) - conn.initialize() - return true - } catch (err) { - return false - } - } - - public async list_instances() { - let conn = new InstancesClient({ credentials: this.keyfile }) - return conn.list({ project: this.project, zone: this.zone }) - } - - public async get_instance_by_name({ instanceName }) { - let conn = new InstancesClient({ credentials: this.keyfile }) - const resp = conn.list({ - project: this.project, - zone: this.zone, - filter: `name eq ${instanceName}`, - }) - return resp - } - - public async get_subnet_information({ subnetName }) { - let conn = new SubnetworksClient({ credentials: this.keyfile }) - const resp = conn.get({ - project: this.project, - subnetwork: "default", - region: "us-west1", - }) - return resp - } - - public async get_address_information({ addressName }) { - const resp = new AddressesClient({ credentials: this.keyfile }).get({ - project: this.project, - address: addressName, - region: "us-west1", - }) - return resp - } - - public async create_new_address({ addressName, network }) { - const conn = new GlobalAddressesClient({ credentials: this.keyfile }) - let resp = conn.insert({ - project: this.project, - addressResource: { - name: addressName, - prefixLength: PREFIX_LENGTH, - network, - }, - }) - return resp - } - - public async delete_new_address({ addressName }) { - const conn = new GlobalAddressesClient({ credentials: this.keyfile }) - let resp = conn.delete({ - project: this.project, - address: addressName, - }) - return resp - } - - public async create_firewall({ firewallName, networkName, ipRange }) { - const conn = new FirewallsClient({ credentials: this.keyfile }) - return conn.insert({ - project: this.project, - firewallResource: { - network: networkName, - targetTags: [METLO_DATA_COLLECTOR_TAG], - sourceRanges: [ipRange], - name: firewallName, - priority: 65534, - allowed: [ - { - IPProtocol: "all", - }, - ], - }, - }) - } - - public async list_routers() { - let conn = new RoutersClient({ credentials: this.keyfile }) - return conn.list({ region: this.region, project: this.project }) - } - - public async list_nats({ router }) { - let conn = new RoutersClient({ credentials: this.keyfile }) - return conn.getNatMappingInfo({ - router, - region: this.region, - project: this.project, - }) - } - - public async create_router({ routerName, networkURL, subnetURL }) { - let conn = new RoutersClient({ credentials: this.keyfile }) - return conn.insert({ - project: this.project, - region: this.region, - routerResource: { - region: this.region, - name: routerName, - network: networkURL, - nats: [ - { - name: `${routerName}-nats`, - sourceSubnetworkIpRangesToNat: "LIST_OF_SUBNETWORKS", - subnetworks: [{ name: subnetURL }], - natIpAllocateOption: "AUTO_ONLY", - }, - ], - }, - }) - } - - public async create_health_check({ healthCheckName }) { - let conn = new HealthChecksClient({ credentials: this.keyfile }) - return conn.insert({ - project: this.project, - healthCheckResource: { - unhealthyThreshold: 2, - healthyThreshold: 2, - checkIntervalSec: 30, - timeoutSec: 10, - name: healthCheckName, - description: "METLO health check for backend", - type: "TCP", - tcpHealthCheck: { - proxyHeader: null, - port: 22, - }, - logConfig: { - enable: false, - }, - }, - }) - } - - public async create_image_template({ - machineType, - sourceImage, - network, - subnet, - imageTemplateName, - }) { - let conn = new InstanceTemplatesClient({ credentials: this.keyfile }) - return conn.insert({ - project: this.project, - instanceTemplateResource: { - name: imageTemplateName, - properties: { - metadata: { - items: [ - { - key: "startup-script", - value: `#! /bin/bash - apt-get update - apt-get install apache2 -y - vm_hostname="$(curl -H "Metadata-Flavor:Google" \ - http://169.254.169.254/computeMetadata/v1/instance/name)" - echo "Page served from: $vm_hostname" | \ - tee /var/www/html/index.html - systemctl restart apache2`, - }, - ], - }, - machineType, - disks: [ - { - boot: true, - initializeParams: { sourceImage: sourceImage, diskSizeGb: 10 }, - }, - ], - networkInterfaces: [ - { - network, - subnetwork: subnet, - }, - ], - tags: { - items: ["http-server", "https-server", METLO_DATA_COLLECTOR_TAG], - }, - }, - }, - }) - } - - public async create_managed_instance({ - templateUrl, - healthCheckURL, - instanceName, - }) { - let conn = new InstanceGroupManagersClient({ credentials: this.keyfile }) - return conn.insert( - { - project: this.project, - zone: this.zone, - instanceGroupManagerResource: { - region: this.region, - zone: this.zone, - name: instanceName, - instanceTemplate: templateUrl, - targetSize: 1, - autoHealingPolicies: [ - { - healthCheck: healthCheckURL, - initialDelaySec: COOL_DOWN_PERIOD, - }, - ], - }, - }, - {}, - ) - } - - public async create_auto_scaling({ - managedGroupURL, - scalingGroupName, - maxReplicas, - minReplicas, - }) { - let conn = new AutoscalersClient({ credentials: this.keyfile }) - return conn.insert({ - project: this.project, - zone: this.zone, - autoscalerResource: { - target: managedGroupURL, - name: scalingGroupName, - autoscalingPolicy: { - maxNumReplicas: maxReplicas, - minNumReplicas: minReplicas, - coolDownPeriodSec: COOL_DOWN_PERIOD, - }, - }, - }) - } -} diff --git a/backend/src/suricata_setup/generics/scripts/install-deps.sh b/backend/src/suricata_setup/generics/scripts/install-deps.sh index bcfa3a18..df34f065 100644 --- a/backend/src/suricata_setup/generics/scripts/install-deps.sh +++ b/backend/src/suricata_setup/generics/scripts/install-deps.sh @@ -10,8 +10,12 @@ sudo chmod 777 /etc/suricata-logs sudo mkdir /var/lib/suricata sudo mkdir /var/lib/suricata/rules + +echo "Get network interface" sudo mv ~/local.rules /var/lib/suricata/rules/local.rules -f +INTERFACE=$(ip link | egrep "ens[0-9]*" -o) +sed -i "s/%interface/$INTERFACE/" ~/local.rules sudo mv ~/suricata.yaml /etc/suricata/suricata.yaml -f sudo mkdir /usr/local/nvm @@ -20,4 +24,5 @@ sudo mkdir /etc/metlo-ingestor echo "CLONING INGESTOR" cd /etc sudo chmod 777 /etc/metlo-ingestor +sudo rm -rf /etc/metlo-ingestor/* git clone https://github.com/metlo-labs/metlo.git metlo-ingestor \ No newline at end of file diff --git a/backend/src/suricata_setup/generics/scripts/install.sh b/backend/src/suricata_setup/generics/scripts/install.sh index 566a8956..bb8009b4 100644 --- a/backend/src/suricata_setup/generics/scripts/install.sh +++ b/backend/src/suricata_setup/generics/scripts/install.sh @@ -8,6 +8,11 @@ cd /etc/metlo-ingestor/ingestors/suricata yarn install yarn build +cd ~ +# Use ~ as separator since HOME can have escapable characters which will conflict with forward-slash +# Replace home directory to properly set nvm directory +sed -i "s~%home~$HOME~" ~/metlo-ingestor.service + echo "ADDING SERVICE" sudo mv ~/metlo-ingestor.service /lib/systemd/system/metlo-ingestor.service -f diff --git a/backend/src/suricata_setup/generics/scripts/metlo-ingestor-template.service b/backend/src/suricata_setup/generics/scripts/metlo-ingestor-template.service index ea063819..1cd28c7d 100644 --- a/backend/src/suricata_setup/generics/scripts/metlo-ingestor-template.service +++ b/backend/src/suricata_setup/generics/scripts/metlo-ingestor-template.service @@ -5,7 +5,7 @@ After=network-online.target [Service] Restart=on-failure WorkingDirectory=/etc/metlo-ingestor/ingestors/suricata/dist -ExecStart=/home/ubuntu/.nvm/versions/node/v17.9.1/bin/node /etc/metlo-ingestor/ingestors/suricata/dist/index.js -s /etc/suricata-logs/eve.sock -u %s +ExecStart=%home/.nvm/versions/node/v17.9.1/bin/node /etc/metlo-ingestor/ingestors/suricata/dist/index.js -s /etc/suricata-logs/eve.sock -u %s [Install] WantedBy=multi-user.target diff --git a/backend/src/suricata_setup/generics/scripts/suricata.yaml b/backend/src/suricata_setup/generics/scripts/suricata.yaml index 4fe5b5ad..3cf3b61b 100644 --- a/backend/src/suricata_setup/generics/scripts/suricata.yaml +++ b/backend/src/suricata_setup/generics/scripts/suricata.yaml @@ -607,7 +607,7 @@ logging: # Linux high speed capture support af-packet: - - interface: ens5 + - interface: %interface # Number of receive threads. "auto" uses the number of cores #threads: auto # Default clusterid. AF_PACKET will load balance packets based on flow. diff --git a/backend/src/suricata_setup/index.ts b/backend/src/suricata_setup/index.ts index 938fac0b..3cb1fe65 100644 --- a/backend/src/suricata_setup/index.ts +++ b/backend/src/suricata_setup/index.ts @@ -1,4 +1,4 @@ -import { ConnectionType, STEPS } from "@common/enums" +import { ConnectionType, AWS_STEPS, GCP_STEPS } from "@common/enums" import { STEP_RESPONSE } from "@common/types" import { aws_instance_creation, @@ -15,29 +15,59 @@ import { delete_aws_data } from "./aws-services/delete" import { test_ssh, push_files, execute_commands } from "./ssh-services" import { v4 as uuidv4 } from "uuid" import { addToRedis, addToRedisFromPromise } from "./utils" -import { save_connection } from "services/connections" +import { save_connection_aws, save_connection_gcp } from "services/connections" +import { + get_destination_subnet, + gcp_key_setup, + gcp_source_identification, + create_firewall_rule, + create_cloud_router, + create_mig, + create_health_check, + create_backend_service, + create_load_balancer, + packet_mirroring, + test_ssh as gcp_test_ssh, + push_files as gcp_push_files, + execute_commands as gcp_execute_commands, +} from "./gcp-services/gcp_setup" -function dummy_response(uuid, step, data) { - const resp = { - success: "FETCHING", - status: "IN-PROGRESS", - retry_id: uuid, - next_step: step, - step_number: step, - last_completed: step - 1, - message: `Fetching data for step ${STEPS[step]}`, - data: data, - } as STEP_RESPONSE - return resp +function dummy_response(uuid, step, data, type: ConnectionType) { + if (type == ConnectionType.AWS) { + const resp = { + success: "FETCHING", + status: "IN-PROGRESS", + retry_id: uuid, + next_step: step, + step_number: step, + last_completed: step - 1, + message: `Fetching data for step ${AWS_STEPS[step]}`, + data: data, + } as STEP_RESPONSE + return resp + } else if (type == ConnectionType.GCP) { + const resp = { + success: "FETCHING", + status: "IN-PROGRESS", + retry_id: uuid, + next_step: step, + step_number: step, + last_completed: step - 1, + message: `Fetching data for step ${GCP_STEPS[step]}`, + data: data, + } as STEP_RESPONSE + return resp + } } export async function setup( step: number = 0, type: ConnectionType, metadata_for_step: STEP_RESPONSE["data"], -): Promise { +): Promise> { var uuid, resp if (type == ConnectionType.AWS) { + type connType = STEP_RESPONSE switch (step) { case 1: return await aws_key_setup(metadata_for_step as any) @@ -59,40 +89,43 @@ export async function setup( return await aws_mirror_session_creation(metadata_for_step as any) case 10: uuid = uuidv4() - resp = dummy_response(uuid, 10, metadata_for_step) + resp = dummy_response(uuid, 10, metadata_for_step, ConnectionType.AWS) await addToRedis(uuid, resp) addToRedisFromPromise( uuid, test_ssh({ ...metadata_for_step, + step: 10, }), ) return resp case 11: uuid = uuidv4() - resp = dummy_response(uuid, 11, metadata_for_step) + resp = dummy_response(uuid, 11, metadata_for_step, ConnectionType.AWS) await addToRedis(uuid, resp) addToRedisFromPromise( uuid, push_files({ ...metadata_for_step, + step: 11, }), ) return resp case 12: uuid = uuidv4() - resp = dummy_response(uuid, 12, metadata_for_step) + resp = dummy_response(uuid, 12, metadata_for_step, ConnectionType.AWS) await addToRedis(uuid, resp) addToRedisFromPromise( uuid, execute_commands({ ...metadata_for_step, + step: 12, } as any).then(resp => { if (resp.status === "COMPLETE") { - save_connection({ + save_connection_aws({ id: resp.data.id, name: resp.data.name, - conn_meta: { ...resp.data } as Required, + conn_meta: { ...resp.data } as Required, }) } return resp @@ -104,17 +137,108 @@ export async function setup( break } } else if (type == ConnectionType.GCP) { - return { - success: "FAIL", - status: "COMPLETE", - step_number: 1, - next_step: 2, - last_completed: 1, - message: "Not configured yet for GCP", - error: { - err: "Not configured yet for GCP", - }, - data: metadata_for_step, + type connType = STEP_RESPONSE + let metadata = metadata_for_step as connType["data"] + switch (step) { + case GCP_STEPS.GCP_KEY_SETUP: + return await gcp_key_setup(metadata_for_step as any) + case GCP_STEPS.SOURCE_INSTANCE_ID: + return await gcp_source_identification(metadata_for_step as any) + case GCP_STEPS.CREATE_DESTINATION_SUBNET: + uuid = uuidv4() + resp = await dummy_response( + uuid, + 3, + metadata_for_step, + ConnectionType.GCP, + ) + await addToRedis(uuid, resp) + addToRedisFromPromise(uuid, get_destination_subnet(metadata)) + return resp + case GCP_STEPS.CREATE_FIREWALL: + return await create_firewall_rule(metadata_for_step as any) + case GCP_STEPS.CREATE_CLOUD_ROUTER: + return await create_cloud_router(metadata_for_step as any) + case GCP_STEPS.CREATE_MIG: + uuid = uuidv4() + resp = dummy_response(uuid, 6, metadata_for_step, ConnectionType.GCP) + await addToRedis(uuid, resp) + addToRedisFromPromise(uuid, create_mig(metadata_for_step)) + return resp + case GCP_STEPS.CREATE_HEALTH_CHECK: + uuid = uuidv4() + resp = dummy_response(uuid, 8, metadata_for_step, ConnectionType.GCP) + await addToRedis(uuid, resp) + addToRedisFromPromise(uuid, create_health_check(metadata_for_step)) + return resp + case GCP_STEPS.CREATE_BACKEND_SERVICE: + uuid = uuidv4() + resp = dummy_response(uuid, 9, metadata_for_step, ConnectionType.GCP) + await addToRedis(uuid, resp) + + addToRedisFromPromise(uuid, create_backend_service(metadata_for_step)) + return resp + case GCP_STEPS.CREATE_ILB: + uuid = uuidv4() + resp = dummy_response(uuid, 10, metadata_for_step, ConnectionType.GCP) + await addToRedis(uuid, resp) + + addToRedisFromPromise(uuid, create_load_balancer(metadata_for_step)) + return resp + case GCP_STEPS.START_PACKET_MIRRORING: + uuid = uuidv4() + resp = dummy_response(uuid, 11, metadata_for_step, ConnectionType.GCP) + await addToRedis(uuid, resp) + + addToRedisFromPromise(uuid, packet_mirroring(metadata_for_step)) + return resp + case GCP_STEPS.TEST_SSH: + uuid = uuidv4() + resp = dummy_response( + uuid, + GCP_STEPS.TEST_SSH, + metadata_for_step, + ConnectionType.GCP, + ) + await addToRedis(uuid, resp) + addToRedisFromPromise(uuid, gcp_test_ssh(metadata_for_step)) + return resp + case GCP_STEPS.PUSH_FILES: + uuid = uuidv4() + resp = dummy_response( + uuid, + GCP_STEPS.PUSH_FILES, + metadata_for_step, + ConnectionType.GCP, + ) + await addToRedis(uuid, resp) + addToRedisFromPromise(uuid, gcp_push_files(metadata_for_step)) + return resp + case GCP_STEPS.EXEC_COMMAND: + uuid = uuidv4() + resp = dummy_response( + uuid, + GCP_STEPS.EXEC_COMMAND, + metadata_for_step, + ConnectionType.GCP, + ) + await addToRedis(uuid, resp) + addToRedisFromPromise( + uuid, + gcp_execute_commands(metadata_for_step).then(resp => { + if (resp.status === "COMPLETE") { + save_connection_gcp({ + id: resp.data.id, + name: resp.data.name, + conn_meta: { ...resp.data } as Required, + }) + } + return resp + }), + ) + return resp + default: + throw Error(`Don't have step ${step} registered`) } } } @@ -124,7 +248,9 @@ export async function delete_connection( connection_data: STEP_RESPONSE["data"], ): Promise { if (type === ConnectionType.AWS) { - return await delete_aws_data(connection_data) + return await delete_aws_data( + connection_data as STEP_RESPONSE["data"], + ) } else if (type === ConnectionType.GCP) { throw new Error("GCP connections are not defined yet") } else { diff --git a/backend/src/suricata_setup/ssh-services/index.ts b/backend/src/suricata_setup/ssh-services/index.ts index 856ecace..c79e668a 100644 --- a/backend/src/suricata_setup/ssh-services/index.ts +++ b/backend/src/suricata_setup/ssh-services/index.ts @@ -1,28 +1,34 @@ +import { ConnectionType } from "@common/enums" import { STEP_RESPONSE } from "@common/types" import { randomUUID } from "crypto" import { SSH_CONN, put_data_file, format, remove_file } from "./ssh-setup" +type RESPONSE = STEP_RESPONSE + export async function test_ssh({ keypair, remote_machine_url, + username, + step, ...rest -}: STEP_RESPONSE["data"]): Promise { +}: RESPONSE["data"] & { step: number }): Promise { var conn try { - conn = new SSH_CONN(keypair, remote_machine_url, "ubuntu") + conn = new SSH_CONN(keypair, remote_machine_url, username) await conn.test_connection() conn.disconnect() return { success: "OK", status: "IN-PROGRESS", - step_number: 10, - next_step: 11, - last_completed: 10, + step_number: step, + next_step: step + 1, + last_completed: step, message: "Testing SSH connection to remote machine.", error: null, data: { keypair, remote_machine_url, + username, ...rest, }, } @@ -31,9 +37,9 @@ export async function test_ssh({ return { success: "FAIL", status: "IN-PROGRESS", - step_number: 10, - next_step: 11, - last_completed: 9, + step_number: step, + next_step: step, + last_completed: step - 1, message: `Couldn't connect to ssh. Please check if key was constructed`, error: { err: err, @@ -41,6 +47,7 @@ export async function test_ssh({ data: { keypair, remote_machine_url, + username, ...rest, }, } @@ -51,10 +58,12 @@ export async function push_files({ keypair, remote_machine_url, source_private_ip, + username, + step, ...rest -}: STEP_RESPONSE["data"]): Promise { +}: RESPONSE["data"] & { step: number }): Promise { const endpoint = "api/v1/log-request/single" - let conn = new SSH_CONN(keypair, remote_machine_url, "ubuntu") + let conn = new SSH_CONN(keypair, remote_machine_url, username) try { let filepath_ingestor = `${__dirname}/../generics/scripts/metlo-ingestor-${randomUUID()}.service` let filepath_rules = `${__dirname}/../generics/scripts/local-${randomUUID()}.rules` @@ -93,14 +102,16 @@ export async function push_files({ return { success: "OK", status: "IN-PROGRESS", - step_number: 11, - next_step: 12, - last_completed: 11, + step_number: step, + next_step: step + 1, + last_completed: step, message: "Pushed configuration files to remote machine", error: null, data: { keypair, remote_machine_url, + username, + source_private_ip, ...rest, }, } @@ -109,9 +120,9 @@ export async function push_files({ return { success: "FAIL", status: "IN-PROGRESS", - step_number: 11, - next_step: 12, - last_completed: 10, + step_number: step, + next_step: step, + last_completed: step - 1, message: `Couldn't push configuration files to remote machine`, error: { err: err, @@ -119,6 +130,8 @@ export async function push_files({ data: { keypair, remote_machine_url, + username, + source_private_ip, ...rest, }, } @@ -128,29 +141,32 @@ export async function push_files({ export async function execute_commands({ keypair, remote_machine_url, + username, + step, ...rest -}: STEP_RESPONSE["data"]): Promise { - let conn = new SSH_CONN(keypair, remote_machine_url, "ubuntu") +}: RESPONSE["data"] & { step: number }): Promise { + let conn = new SSH_CONN(keypair, remote_machine_url, username) try { await conn.run_command( "cd ~ && chmod +x install-deps.sh && ./install-deps.sh ", ) await conn.run_command( - "source $HOME/.nvm/nvm.sh && cd ~ && chmod +x install.sh && ./install.sh ", + "source ~/.nvm/nvm.sh && cd ~ && chmod +x install.sh && ./install.sh ", ) conn.disconnect() return { success: "OK", status: "COMPLETE", - step_number: 12, - next_step: null, - last_completed: 12, + step_number: step, + next_step: step + 1, + last_completed: step, message: "Executed configuration files on remote machine succesfully", error: null, data: { keypair, remote_machine_url, + username, ...rest, }, } @@ -159,9 +175,9 @@ export async function execute_commands({ return { success: "FAIL", status: "IN-PROGRESS", - step_number: 12, - next_step: null, - last_completed: 11, + step_number: step, + next_step: step, + last_completed: step - 1, message: `Couldn't exec commands to install things`, error: { err: err, @@ -169,6 +185,7 @@ export async function execute_commands({ data: { keypair, remote_machine_url, + username, ...rest, }, } diff --git a/common/src/enums.ts b/common/src/enums.ts index 2be911b0..6566db3a 100644 --- a/common/src/enums.ts +++ b/common/src/enums.ts @@ -53,7 +53,7 @@ export enum SpecExtension { YAML = "yaml", } -export enum STEPS { +export enum AWS_STEPS { // SETUP MIRROR INSTANCE AWS_KEY_SETUP = 1, SOURCE_INSTANCE_ID = 2, @@ -69,6 +69,22 @@ export enum STEPS { EXEC_COMMAND = 12, } +export enum GCP_STEPS{ + GCP_KEY_SETUP=1, // Get key, region, network + SOURCE_INSTANCE_ID=2, // Get GCP Source Instance ID/Name. Also get subnet info from that + CREATE_DESTINATION_SUBNET=3, // Reserve an Address, use that to create subnet from that and delete address + CREATE_FIREWALL=4, // Create firewall targeting certain things, say tags, also create rules + CREATE_CLOUD_ROUTER=5, // Includes creation of NAT + CREATE_MIG=6, // Create Image template and MIG in one step + CREATE_HEALTH_CHECK=7, + CREATE_BACKEND_SERVICE=8, // Also add health check to backend service + CREATE_ILB=9, + START_PACKET_MIRRORING=10, + TEST_SSH = 11, + PUSH_FILES = 12, + EXEC_COMMAND = 13, +} + export enum protocols { TCP = 6, UDP = 17, diff --git a/common/src/maps.ts b/common/src/maps.ts index 8333ebd6..f2142bbc 100644 --- a/common/src/maps.ts +++ b/common/src/maps.ts @@ -1,33 +1,50 @@ -import { DataSection, STEPS } from "./enums" +import { DataSection, AWS_STEPS, GCP_STEPS } from "./enums" -export const NEXT_STEP: Record = { - [STEPS.AWS_KEY_SETUP]: STEPS.SOURCE_INSTANCE_ID, - [STEPS.SOURCE_INSTANCE_ID]: STEPS.SELECT_OS, - [STEPS.SELECT_OS]: STEPS.SELECT_INSTANCE_TYPE, - [STEPS.SELECT_INSTANCE_TYPE]: STEPS.CREATE_INSTANCE, - [STEPS.CREATE_INSTANCE]: STEPS.INSTANCE_IP, - [STEPS.INSTANCE_IP]: STEPS.CREATE_MIRROR_TARGET, - [STEPS.CREATE_MIRROR_TARGET]: STEPS.CREATE_MIRROR_FILTER, - [STEPS.CREATE_MIRROR_FILTER]: STEPS.CREATE_MIRROR_SESSION, - [STEPS.CREATE_MIRROR_SESSION]: STEPS.TEST_SSH, - [STEPS.TEST_SSH]: STEPS.PUSH_FILES, - [STEPS.PUSH_FILES]: STEPS.EXEC_COMMAND, - [STEPS.EXEC_COMMAND]: null, +export const AWS_NEXT_STEP: Record = { + [AWS_STEPS.AWS_KEY_SETUP]: AWS_STEPS.SOURCE_INSTANCE_ID, + [AWS_STEPS.SOURCE_INSTANCE_ID]: AWS_STEPS.SELECT_OS, + [AWS_STEPS.SELECT_OS]: AWS_STEPS.SELECT_INSTANCE_TYPE, + [AWS_STEPS.SELECT_INSTANCE_TYPE]: AWS_STEPS.CREATE_INSTANCE, + [AWS_STEPS.CREATE_INSTANCE]: AWS_STEPS.INSTANCE_IP, + [AWS_STEPS.INSTANCE_IP]: AWS_STEPS.CREATE_MIRROR_TARGET, + [AWS_STEPS.CREATE_MIRROR_TARGET]: AWS_STEPS.CREATE_MIRROR_FILTER, + [AWS_STEPS.CREATE_MIRROR_FILTER]: AWS_STEPS.CREATE_MIRROR_SESSION, + [AWS_STEPS.CREATE_MIRROR_SESSION]: AWS_STEPS.TEST_SSH, + [AWS_STEPS.TEST_SSH]: AWS_STEPS.PUSH_FILES, + [AWS_STEPS.PUSH_FILES]: AWS_STEPS.EXEC_COMMAND, + [AWS_STEPS.EXEC_COMMAND]: null, } -export const STEP_TO_TITLE_MAP: Record = { - [STEPS.AWS_KEY_SETUP]: "AWS Credentials Setup", - [STEPS.SOURCE_INSTANCE_ID]: "EC2 Instance for mirroring source", - [STEPS.SELECT_OS]: "OS Selection", - [STEPS.SELECT_INSTANCE_TYPE]: "EC2 Instance type selection", - [STEPS.CREATE_INSTANCE]: "EC2 Instance Instantiation", - [STEPS.INSTANCE_IP]: "Obtain Mirror Instance IP", - [STEPS.CREATE_MIRROR_TARGET]: "Traffic Mirror Target Creation", - [STEPS.CREATE_MIRROR_FILTER]: "Traffic Mirror Filter Creation", - [STEPS.CREATE_MIRROR_SESSION]: "Traffic Mirror Session Creation", - [STEPS.TEST_SSH]: "SSH Connection Test", - [STEPS.PUSH_FILES]: "Push installation files to remote instance", - [STEPS.EXEC_COMMAND]: "Install metlo", +export const AWS_STEP_TO_TITLE_MAP: Record = { + [AWS_STEPS.AWS_KEY_SETUP]: "AWS Credentials Setup", + [AWS_STEPS.SOURCE_INSTANCE_ID]: "EC2 Instance for mirroring source", + [AWS_STEPS.SELECT_OS]: "OS Selection", + [AWS_STEPS.SELECT_INSTANCE_TYPE]: "EC2 Instance type selection", + [AWS_STEPS.CREATE_INSTANCE]: "EC2 Instance Instantiation", + [AWS_STEPS.INSTANCE_IP]: "Obtain Mirror Instance IP", + [AWS_STEPS.CREATE_MIRROR_TARGET]: "Traffic Mirror Target Creation", + [AWS_STEPS.CREATE_MIRROR_FILTER]: "Traffic Mirror Filter Creation", + [AWS_STEPS.CREATE_MIRROR_SESSION]: "Traffic Mirror Session Creation", + [AWS_STEPS.TEST_SSH]: "SSH Connection Test", + [AWS_STEPS.PUSH_FILES]: "Push installation files to remote instance", + [AWS_STEPS.EXEC_COMMAND]: "Install metlo", +} + +export const GCP_STEP_TO_TITLE_MAP: Record = { + [GCP_STEPS.GCP_KEY_SETUP]: "GCP Credentials Setup", + [GCP_STEPS.SOURCE_INSTANCE_ID]: "GCP Mirrored Instance Sselection", + [GCP_STEPS.CREATE_DESTINATION_SUBNET]: "Destination Subnet Creation", + [GCP_STEPS.CREATE_FIREWALL]: "GCP Firewall creation", + [GCP_STEPS.CREATE_CLOUD_ROUTER]: "GCP Cloud Router Creation for Collector", + [GCP_STEPS.CREATE_MIG]: "GCP Collector Instance Creation", + // [GCP_STEPS.PUSH_KEY]: "SSH Key Creation", + [GCP_STEPS.CREATE_HEALTH_CHECK]: "Collector Health Check Creation", + [GCP_STEPS.CREATE_BACKEND_SERVICE]: "GCP Packet Routing Service Creation", + [GCP_STEPS.CREATE_ILB]: "Internal Load Balancer Creation", + [GCP_STEPS.START_PACKET_MIRRORING]: "Start Data Mirroring", + [GCP_STEPS.TEST_SSH]: "SSH Connection Test", + [GCP_STEPS.PUSH_FILES]: "Push installation files to remote instance", + [GCP_STEPS.EXEC_COMMAND]: "Install metlo", } export const DATA_SECTION_TO_LABEL_MAP: Record = { diff --git a/common/src/types.ts b/common/src/types.ts index 1975fb6e..aa562af5 100644 --- a/common/src/types.ts +++ b/common/src/types.ts @@ -11,8 +11,9 @@ import { RiskScore, SpecExtension, Status, - STEPS, + AWS_STEPS, UpdateAlertType, + GCP_STEPS, } from "./enums" import "axios" @@ -220,18 +221,18 @@ export interface Usage { count: number } -export interface STEP_RESPONSE { +export interface STEP_RESPONSE { success: "OK" | "FAIL" | "FETCHING"; status: "STARTED" | "COMPLETE" | "IN-PROGRESS"; retry_id?: string; - next_step: STEPS; - step_number: STEPS; - last_completed: STEPS; + next_step: AWS_STEPS|GCP_STEPS; + step_number: AWS_STEPS|GCP_STEPS; + last_completed: AWS_STEPS|GCP_STEPS; message: string; error?: { err: string; }; - data: CONNECTIONS_BASE & Partial; + data: CONNECTIONS_BASE & (T extends ConnectionType.AWS ? Partial<(AWS_CONNECTION & AWS_CONNECTION_MISC & SSH_INFO)> : T extends ConnectionType.GCP ? Partial:never); returns?: { os_types?: [{ name: string; ami: string }] instance_types?: string[] @@ -259,6 +260,12 @@ export interface CONNECTIONS_BASE { name: string; } +export interface SSH_INFO{ + keypair:string, + username:string, + remote_machine_url:string, +} + export interface AWS_CONNECTION { secret_access_key: string; access_id: string; @@ -271,14 +278,12 @@ export interface AWS_CONNECTION { mirror_filter_id: string; mirror_session_id: string; mirror_rules: Array; - keypair: string; destination_eni_id: string; source_eni_id: string; - source_private_ip: string; backend_url: string; - remote_machine_url: string; keypair_id: string; keypair_name: string; + source_private_ip:string, } export interface AWS_CONNECTION_MISC { @@ -296,11 +301,45 @@ export interface ENCRYPTED_AWS_CONNECTION__META { access_id_iv: string } +export interface GCP_CONNECTION { + key_file: string; + project: string; + zone: string; + network_url: string; + ip_range:string; + source_subnetwork_url: string; + firewall_rule_url:string; + destination_subnetwork_url: string; + router_url:string; + machine_type:string; + source_image:string; + image_template_url:string; + instance_url:string; + managed_group_url:string; + health_check_url:string; + backend_service_url:string; + forwarding_rule_url:string; + source_instance_url:string; + packet_mirror_url:string; + source_instance_name: string; + source_private_ip:string, +} + +export interface GCP_CONNECTION_MISC{ + network_name:string; +} + +export interface ENCRYPTED_GCP_CONNECTION__META { + key_file_tag: string; + key_file_iv: string; +} + export interface ConnectionInfo { - uuid: string - connectionType: ConnectionType - createdAt: Date - updatedAt: Date - name: string - aws?: Omit + uuid: string; + connectionType: ConnectionType; + createdAt: Date; + updatedAt: Date; + name: string; + aws?: Omit; + gcp?: Omit; } diff --git a/frontend/package.json b/frontend/package.json index 345e191c..6e9b43ac 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -27,6 +27,7 @@ "json-source-map": "^0.6.1", "luxon": "^3.0.1", "next": "latest", + "parse-json": "^6.0.2", "prism-react-renderer": "^1.3.5", "react": "^18.2.0", "react-chartjs-2": "^4.3.1", diff --git a/frontend/src/components/ConnectionConfiguration/AWS/configureAws.tsx b/frontend/src/components/ConnectionConfiguration/AWS/configureAws.tsx index 0c986ca2..f508d7c4 100644 --- a/frontend/src/components/ConnectionConfiguration/AWS/configureAws.tsx +++ b/frontend/src/components/ConnectionConfiguration/AWS/configureAws.tsx @@ -7,30 +7,29 @@ import { Spinner, Flex, } from "@chakra-ui/react" -import { ConnectionType, STEPS } from "@common/enums" +import { ConnectionType, AWS_STEPS } from "@common/enums" import { useState } from "react" import KeySetup from "./key_setup" import { v4 as uuidv4 } from "uuid" import SourceInstanceID from "./source_instance_id" import { STEP_RESPONSE } from "@common/types" -import { STEP_TO_TITLE_MAP } from "@common/maps" +import { AWS_STEP_TO_TITLE_MAP } from "@common/maps" import axios, { AxiosResponse, AxiosError, AxiosRequestConfig } from "axios" -import { getAPIURL } from "~/constants" import OsSelection from "./os_selection" import InstanceSelection from "./instance_selection" -import GenericStepAWS from "./genericStepAws" +import GenericStep from "../common/genericStep" import SetupRulesFilter from "./mirrorFilters" import { useToast } from "@chakra-ui/react" import { api_call_retry } from "utils" interface configureAWSParams { - selected: STEPS - updateSelected: (x: STEPS) => void + selected: AWS_STEPS + updateSelected: (x: AWS_STEPS) => void } const incrementStep = ( id: string, params: Record, - step: STEPS, + step: AWS_STEPS, onStepSuccess: ( data: AxiosResponse, any>, ) => void, @@ -64,7 +63,7 @@ const incrementStep = ( const getRetryId = async ( id: string, params: Record, - step: STEPS, + step: AWS_STEPS, onError: (err) => void, ) => { try { @@ -78,8 +77,6 @@ const getRetryId = async ( } } -const MAX_RETRY = 3 - const ConfigureAWS: React.FC = ({ selected, updateSelected, @@ -88,9 +85,9 @@ const ConfigureAWS: React.FC = ({ const [id] = useState(uuidv4()) const [name, setName] = useState(`Metlo-Connection-${id}`) const toast = useToast() - const create_toast_with_message = (msg: string, step: STEPS) => { + const create_toast_with_message = (msg: string, step: AWS_STEPS) => { toast({ - title: `Encountered an error on step ${STEPS[step]}`, + title: `Encountered an error on step ${AWS_STEPS[step]}`, description: msg, status: "error", duration: 6000, @@ -100,7 +97,7 @@ const ConfigureAWS: React.FC = ({ const step_increment_function = ( params: Record, - step: STEPS, + step: AWS_STEPS, ) => { let retries = 0 incrementStep( @@ -125,7 +122,7 @@ const ConfigureAWS: React.FC = ({ params, onComplete, }: { - step: STEPS + step: AWS_STEPS params: Record onComplete?: () => void }) => { @@ -162,124 +159,124 @@ const ConfigureAWS: React.FC = ({ }) } - let internals = (selectedIndex: STEPS): React.ReactElement => { + let internals = (selectedIndex: AWS_STEPS): React.ReactElement => { switch (selectedIndex) { - case STEPS.AWS_KEY_SETUP: + case AWS_STEPS.AWS_KEY_SETUP: return ( { - step_increment_function(params, STEPS.AWS_KEY_SETUP) + step_increment_function(params, AWS_STEPS.AWS_KEY_SETUP) }} name={name} setName={setName} /> ) - case STEPS.SOURCE_INSTANCE_ID: + case AWS_STEPS.SOURCE_INSTANCE_ID: return ( { - step_increment_function(params, STEPS.SOURCE_INSTANCE_ID) + step_increment_function(params, AWS_STEPS.SOURCE_INSTANCE_ID) }} /> ) - case STEPS.SELECT_OS: + case AWS_STEPS.SELECT_OS: return ( { - step_increment_function(params, STEPS.SELECT_OS) + step_increment_function(params, AWS_STEPS.SELECT_OS) }} id={id} /> ) - case STEPS.SELECT_INSTANCE_TYPE: + case AWS_STEPS.SELECT_INSTANCE_TYPE: return ( { - step_increment_function(params, STEPS.SELECT_INSTANCE_TYPE) + step_increment_function(params, AWS_STEPS.SELECT_INSTANCE_TYPE) }} isCurrent={selectedIndex == selected} setLoadingState={setUpdating} /> ) - case STEPS.CREATE_INSTANCE: + case AWS_STEPS.CREATE_INSTANCE: return ( - { - step_increment_function(params, STEPS.CREATE_INSTANCE) + step_increment_function(params, AWS_STEPS.CREATE_INSTANCE) }} isCurrent={selectedIndex == selected} - > + > ) - case STEPS.INSTANCE_IP: + case AWS_STEPS.INSTANCE_IP: return ( - { - await step_increment_function(params, STEPS.INSTANCE_IP) + await step_increment_function(params, AWS_STEPS.INSTANCE_IP) }} isCurrent={selectedIndex == selected} /> ) - case STEPS.CREATE_MIRROR_TARGET: + case AWS_STEPS.CREATE_MIRROR_TARGET: return ( - { - step_increment_function(params, STEPS.CREATE_MIRROR_TARGET) + step_increment_function(params, AWS_STEPS.CREATE_MIRROR_TARGET) }} isCurrent={selectedIndex == selected} - > + > ) - case STEPS.CREATE_MIRROR_FILTER: + case AWS_STEPS.CREATE_MIRROR_FILTER: return ( { - step_increment_function(params, STEPS.CREATE_MIRROR_FILTER) + step_increment_function(params, AWS_STEPS.CREATE_MIRROR_FILTER) }} isCurrent={selectedIndex == selected} /> ) - case STEPS.CREATE_MIRROR_SESSION: + case AWS_STEPS.CREATE_MIRROR_SESSION: return ( - { - step_increment_function(params, STEPS.CREATE_MIRROR_SESSION) + step_increment_function(params, AWS_STEPS.CREATE_MIRROR_SESSION) }} isCurrent={selectedIndex == selected} - > + > ) - case STEPS.TEST_SSH: + case AWS_STEPS.TEST_SSH: return ( - { - await retrier({ step: STEPS.TEST_SSH, params }) + await retrier({ step: AWS_STEPS.TEST_SSH, params }) }} isCurrent={selectedIndex == selected} /> ) - case STEPS.PUSH_FILES: + case AWS_STEPS.PUSH_FILES: return ( - { - retrier({ step: STEPS.PUSH_FILES, params }) + retrier({ step: AWS_STEPS.PUSH_FILES, params }) }} isCurrent={selectedIndex == selected} /> ) - case STEPS.EXEC_COMMAND: + case AWS_STEPS.EXEC_COMMAND: return ( - { retrier({ - step: STEPS.EXEC_COMMAND, + step: AWS_STEPS.EXEC_COMMAND, params, onComplete: () => { toast({ @@ -309,7 +306,7 @@ const ConfigureAWS: React.FC = ({

- Step {i + 1}: {STEP_TO_TITLE_MAP[i + 1]} + Step {i + 1}: {AWS_STEP_TO_TITLE_MAP[i + 1]}

diff --git a/frontend/src/components/ConnectionConfiguration/GCP/configureGcp.tsx b/frontend/src/components/ConnectionConfiguration/GCP/configureGcp.tsx new file mode 100644 index 00000000..88dfa39c --- /dev/null +++ b/frontend/src/components/ConnectionConfiguration/GCP/configureGcp.tsx @@ -0,0 +1,361 @@ +import { + Box, + Accordion, + AccordionItem, + AccordionButton, + AccordionPanel, + Spinner, + Flex, +} from "@chakra-ui/react" +import { ConnectionType, GCP_STEPS } from "@common/enums" +import { useState } from "react" +import { v4 as uuidv4 } from "uuid" +import { STEP_RESPONSE } from "@common/types" +import { GCP_STEP_TO_TITLE_MAP } from "@common/maps" +import axios, { AxiosResponse, AxiosError, AxiosRequestConfig } from "axios" +import { useToast } from "@chakra-ui/react" +import { api_call_retry } from "utils" +import GenericStep from "../common/genericStep" +import KeySetup from "./key_setup" +import SourceInstanceID from "./source_instance_id" +import SourceMigConfig from "./destination_instance_config" +interface configureAWSParams { + selected: GCP_STEPS + updateSelected: (x: GCP_STEPS) => void +} + +type connData = Omit, "data"> + +const incrementStep = ( + id: string, + params: Record, + step: GCP_STEPS, + onStepSuccess: (data: AxiosResponse) => void, + onStepError: (data: AxiosResponse) => void, + onError: (data: AxiosError) => void, + setUpdating: (x: boolean) => void, +) => { + axios + .post(`/api/v1/setup_connection`, { + id: id, + params: params, + type: ConnectionType.GCP, + step: step, + }) + .then(value => { + if (value.data.success === "OK") { + onStepSuccess(value) + } else if (value.data.success === "FAIL") { + onStepError(value) + } + }) + .catch((err: AxiosError) => { + onError(err) + }) + .finally(() => { + setUpdating(false) + }) + setUpdating(true) +} + +const getRetryId = async ( + id: string, + params: Record, + step: GCP_STEPS, + onError: (err) => void, +) => { + try { + let resp = await axios.post(`/api/v1/setup_connection`, { + id: id, + params: params, + type: ConnectionType.GCP, + step: step, + }) + return resp.data.retry_id + } catch (err) { + onError(err) + } +} + +const ConfigureGCP: React.FC = ({ + selected, + updateSelected, +}) => { + const [isUpdating, setUpdating] = useState(false) + const [id] = useState(uuidv4()) + const [name, setName] = useState(`Metlo-Connection-${id}`) + const toast = useToast() + const create_toast_with_message = (msg: string, step: GCP_STEPS) => { + toast({ + title: `Encountered an error on step ${GCP_STEPS[step]}`, + description: msg, + status: "error", + duration: 6000, + isClosable: true, + }) + } + + const step_increment_function = ( + params: Record, + step: GCP_STEPS, + onSuccess = () => {}, + ) => { + let retries = 0 + incrementStep( + id, + { ...params, name: name }, + step, + () => { + updateSelected(step + 1) + onSuccess() + }, + err => { + create_toast_with_message(err.data.message, step) + console.log(err.data.error) + }, + error => { + create_toast_with_message(error.message as string, step) + console.log(error) + }, + setUpdating, + ) + } + + const retrier = async ({ + step, + params, + onComplete, + }: { + step: GCP_STEPS + params: Record + onComplete?: () => void + }) => { + setUpdating(true) + let _params = { ...params, name: name } + let retry_id = await getRetryId(id, _params, step, () => {}) + + if (retry_id) { + console.log("Attempting to fetch") + api_call_retry({ + url: `/api/v1/setup_connection/fetch/${retry_id}`, + requestParams: { + params: { id, step, ..._params }, + } as AxiosRequestConfig, + onAPIError: (err: AxiosError) => { + create_toast_with_message(err.message, step) + }, + onError: (err: Error) => { + create_toast_with_message(err.message, step) + }, + onSuccess: (resp: AxiosResponse) => { + if (resp.data.success === "OK") { + updateSelected(step + 1) + if (resp.data.status === "COMPLETE") { + onComplete() + } + } else { + create_toast_with_message(resp.data.message, step) + console.log(resp.data.error) + } + }, + onFinally: () => { + setUpdating(false) + }, + shouldRetry: (resp: AxiosResponse) => { + return resp.data.success === "FETCHING" + }, + }) + } else { + console.log("Couldn't attempt to fetch") + } + } + + let internals = (renderIndex: GCP_STEPS): React.ReactElement => { + switch (renderIndex) { + case GCP_STEPS.GCP_KEY_SETUP: + return ( + { + step_increment_function(params, GCP_STEPS.GCP_KEY_SETUP) + }} + name={name} + setName={setName} + /> + ) + case GCP_STEPS.SOURCE_INSTANCE_ID: + return ( + { + step_increment_function(params, GCP_STEPS.SOURCE_INSTANCE_ID) + }} + /> + ) + case GCP_STEPS.CREATE_DESTINATION_SUBNET: + return ( + { + await retrier({ + step: GCP_STEPS.CREATE_DESTINATION_SUBNET, + params, + }) + }} + isCurrent={GCP_STEPS.CREATE_DESTINATION_SUBNET == selected} + /> + ) + case GCP_STEPS.CREATE_FIREWALL: + return ( + { + await step_increment_function(params, GCP_STEPS.CREATE_FIREWALL) + }} + isCurrent={GCP_STEPS.CREATE_FIREWALL == selected} + /> + ) + case GCP_STEPS.CREATE_CLOUD_ROUTER: + return ( + { + await step_increment_function( + params, + GCP_STEPS.CREATE_CLOUD_ROUTER, + ) + }} + isCurrent={GCP_STEPS.CREATE_CLOUD_ROUTER == selected} + /> + ) + case GCP_STEPS.CREATE_MIG: + return ( + { + await retrier({ step: GCP_STEPS.CREATE_MIG, params }) + }} + isSelected={GCP_STEPS.CREATE_MIG === selected} + id={id} + /> + ) + case GCP_STEPS.CREATE_HEALTH_CHECK: + return ( + { + await retrier({ params, step: GCP_STEPS.CREATE_HEALTH_CHECK }) + }} + isCurrent={GCP_STEPS.CREATE_HEALTH_CHECK == selected} + /> + ) + case GCP_STEPS.CREATE_BACKEND_SERVICE: + return ( + { + await retrier({ params, step: GCP_STEPS.CREATE_BACKEND_SERVICE }) + }} + isCurrent={GCP_STEPS.CREATE_BACKEND_SERVICE == selected} + /> + ) + case GCP_STEPS.CREATE_ILB: + return ( + { + await retrier({ params, step: GCP_STEPS.CREATE_ILB }) + }} + isCurrent={GCP_STEPS.CREATE_ILB == selected} + /> + ) + case GCP_STEPS.START_PACKET_MIRRORING: + return ( + { + await retrier({ params, step: GCP_STEPS.START_PACKET_MIRRORING }) + }} + isCurrent={GCP_STEPS.START_PACKET_MIRRORING == selected} + /> + ) + case GCP_STEPS.TEST_SSH: + return ( + { + await retrier({ step: GCP_STEPS.TEST_SSH, params }) + }} + isCurrent={GCP_STEPS.TEST_SSH == selected} + /> + ) + case GCP_STEPS.PUSH_FILES: + return ( + { + retrier({ step: GCP_STEPS.PUSH_FILES, params }) + }} + isCurrent={GCP_STEPS.PUSH_FILES == selected} + /> + ) + case GCP_STEPS.EXEC_COMMAND: + return ( + { + retrier({ + step: GCP_STEPS.EXEC_COMMAND, + params, + }) + }} + isCurrent={GCP_STEPS.EXEC_COMMAND == selected} + /> + ) + } + } + + return ( + <> + + {Array.from(Array(Object.values(GCP_STEPS).length / 2)).map((_, i) => { + return ( + +

+ + + Step {i + 1}: {GCP_STEP_TO_TITLE_MAP[i + 1]} + + +

+ + {isUpdating && ( + + + + )} + + {internals(i + 1)} + + + + ) + })} + + + ) +} + +export default ConfigureGCP diff --git a/frontend/src/components/ConnectionConfiguration/GCP/destination_instance_config.tsx b/frontend/src/components/ConnectionConfiguration/GCP/destination_instance_config.tsx new file mode 100644 index 00000000..f17f8e95 --- /dev/null +++ b/frontend/src/components/ConnectionConfiguration/GCP/destination_instance_config.tsx @@ -0,0 +1,108 @@ +import { + Box, + Button, + Flex, + Grid, + GridItem, + Input, + Select, + useToast, +} from "@chakra-ui/react" +import axios from "axios" +import { useEffect, useState } from "react" + +interface KeySetupInterface { + complete: (params: Record) => void + isSelected: boolean + id: string +} + +type osChoiceResp = [description: string, link: string] + +const SourceMigConfig: React.FC = ({ + complete, + isSelected, + id, +}) => { + const [imageTemplateURL, setImageTemplateURL] = useState("") + const [instanceName, setInstanceName] = useState("") + const [osChoices, setOSChoices] = useState>([]) + const toast = useToast() + + useEffect(() => { + if (isSelected) { + axios + .post>("/api/v1/setup_connection/gcp/os", { + id: id, + }) + .then(os_choice => { + setOSChoices(os_choice.data) + setImageTemplateURL(os_choice.data[0][1]) + }) + .catch(err => { + toast({ + title: "Encountered an error fetching OS Choices", + description: "Check the console for more details", + }) + console.warn(err) + }) + } + }, [isSelected]) + + return ( + + + Compute Engine Image + + + + + + + + Compute Engine Instance + + + + setInstanceName(e.target.value)} + value={instanceName} + /> + + + + + + + + + ) +} +export default SourceMigConfig diff --git a/frontend/src/components/ConnectionConfiguration/GCP/key_setup.tsx b/frontend/src/components/ConnectionConfiguration/GCP/key_setup.tsx new file mode 100644 index 00000000..99271183 --- /dev/null +++ b/frontend/src/components/ConnectionConfiguration/GCP/key_setup.tsx @@ -0,0 +1,105 @@ +import { + Box, + Button, + Flex, + HStack, + Input, + Textarea, + VStack, +} from "@chakra-ui/react" +import { useState } from "react" + +interface KeySetupInterface { + complete: (params: Record) => void + name: string + setName: (name: string) => void +} + +const KeySetup: React.FC = ({ complete, name, setName }) => { + const [key, setKey] = useState(``) + const [project, setProject] = useState("") + const [zone, setZone] = useState("") + const [network, setNetwork] = useState("") + + return ( + + + + Connection Name + + + + setName(e.target.value)} value={name} /> + + + + + + Project + + + + setProject(e.target.value)} value={project} /> + + + + + + Network Name + + + + setNetwork(e.target.value)} value={network} /> + + + + + + Zone + + + + setZone(e.target.value)} value={zone} /> + + + + + + Key File + + + +