From b831d7243ce38a5bab91d1098e601af34b05f072 Mon Sep 17 00:00:00 2001 From: Ninad Sinha Date: Wed, 16 Nov 2022 17:50:53 +0530 Subject: [PATCH 1/6] Add list, remove and update to gcp cli mirroring --- cli/src/gcp/delete.ts | 191 ++++++++++++++++++++++++++++++++++++++++ cli/src/gcp/gcp_apis.ts | 73 +++++++++++++++ cli/src/gcp/list.ts | 111 +++++++++++++++++++++++ cli/src/gcp/setup.ts | 150 +++++++++++++++++++++++-------- cli/src/index.ts | 4 + 5 files changed, 491 insertions(+), 38 deletions(-) create mode 100644 cli/src/gcp/delete.ts create mode 100644 cli/src/gcp/list.ts diff --git a/cli/src/gcp/delete.ts b/cli/src/gcp/delete.ts new file mode 100644 index 00000000..47b582a3 --- /dev/null +++ b/cli/src/gcp/delete.ts @@ -0,0 +1,191 @@ +import AsyncRetry from "async-retry" +import { v4 as uuidv4, validate } from "uuid" +import fs from "fs" +import { prompt } from "enquirer" +import { GCP_REGIONS_SUPPORTED, wait_for_global_operation, wait_for_regional_operation, wait_for_zonal_operation } from "./gcpUtils" +import { GCP_CONN } from "./gcp_apis" +import chalk from "chalk" +import ora from "ora"; +import { google } from "@google-cloud/compute/build/protos/protos" + +const spinner = ora() + +const verifyAccountDetails = async () => { + const gcp_regions = GCP_REGIONS_SUPPORTED.map(e => ({ + name: e, + })) + const resp = await prompt([ + { + type: "input", + name: "_projectName", + message: "GCP Project Name", + }, { + type: "input", + initial: "default", + name: "_networkName", + message: "GCP Network to mirror", + }, { + type: "autocomplete", + name: "_zoneName", + message: "Select your GCP zone", + initial: 1, + choices: gcp_regions, + }, { + type: "input", + name: "_keyPath", + message: "Path to GCP key file", + validate: (path: string) => { + if (fs.existsSync(path)) { + return true + } else { + // @ts-ignore + let text = chalk.redBright(`GCP Key file not found at ${path}`) + return text + } + } + } + ]) + + spinner.text = "Validating account details" + spinner.start() + // @ts-ignore Destructuring is improperly done + const { _projectName: project, _networkName: network, _zoneName: zone, _keyPath: keyFilePath } = resp; + + const key = (fs.readFileSync(keyFilePath)).toString("utf-8"); + + let conn = new GCP_CONN(key, zone, project) + await conn.test_connection() + await conn.get_zone({ zone }) + spinner.succeed("Validated account details") + spinner.stop() + spinner.clear() + return { project, network, zone, key } +} + + +const deletePacketMirroringResources = async ( + conn: GCP_CONN, + mirroring: google.cloud.compute.v1.IPacketMirroring[] +) => { + if (mirroring.length == 0) { + throw new Error("No existing packet mirroring instances found") + } + const sourceTypeResp = await prompt([ + { + type: "autocomplete", + name: "_packetMirrorName", + message: "Select Packet Mirroring instance", + initial: 0, + choices: mirroring.filter((inst) => inst.name.startsWith("metlo")).map((inst) => inst.name) + }, { + type: "select", + name: "_sourceType", + message: "Select your mirror source type", + initial: 0, + choices: ["INSTANCE", "SUBNET", "TAG"], + }, + ]) + let sourceType = sourceTypeResp["_sourceType"] + let packetMirrorName = sourceTypeResp["_packetMirrorName"] + + if (sourceType === "INSTANCE") { + const instanceNameResp = await prompt([ + { + type: "autocomplete", + name: "_name", + message: "Enter the mirror source instance name to remove", + choices: mirroring.flatMap( + (mirror) => mirror.mirroredResources.instances.map( + inst => { + const splits = inst.url.split("/"); + return splits[splits.length - 1] + } + ) + ) + } + ]) + spinner.start("Verifying mirror source details") + const instanceName = instanceNameResp['_name'].trim() + + const resources = mirroring.find((mirror) => mirror.name === packetMirrorName).mirroredResources + resources.instances = resources.instances.filter((inst) => !inst.url.includes(instanceName)) + resources.instances = resources.instances.length > 0 ? resources.instances : null + + console.log(resources.instances) + + let resp = await conn.remove_packet_mirroring_resources({ packetMirrorName, newMirroredResources: resources }) + } else if (sourceType === "SUBNET") { + const subnetNameResp = await prompt([ + { + type: "autocomplete", + name: "_name", + message: "Enter the mirror source subnet name to remove", + choices: mirroring.flatMap( + (mirror) => mirror.mirroredResources.subnetworks.map( + inst => { + const splits = inst.url.split("/"); + return splits[splits.length - 1] + } + ) + ) + } + ]) + spinner.start("Verifying mirror source details") + const subnetName = subnetNameResp['_name'].trim() + + const resources = mirroring.find((mirror) => mirror.name === packetMirrorName).mirroredResources + resources.subnetworks = resources.subnetworks.filter((inst) => !inst.url.includes(subnetName)) + resources.subnetworks = resources.subnetworks.length > 0 ? resources.subnetworks : null + + console.log(resources.subnetworks) + + let resp = await conn.remove_packet_mirroring_resources({ packetMirrorName, newMirroredResources: resources }) + } else if (sourceType === "TAG") { + const tagNameResp = await prompt([ + { + type: "autocomplete", + name: "_name", + message: "Enter the mirror source tag name to remove", + choices: mirroring.flatMap( + (mirror) => mirror.mirroredResources.tags + ) + } + ]) + spinner.start("Verifying mirror source details") + let tagName = tagNameResp["_name"].trim() + + const resources = mirroring.find((mirror) => mirror.name === packetMirrorName).mirroredResources + resources.tags = resources.tags.filter((inst) => !inst.includes(tagName)) + if (resources.tags.length == 0) { + delete resources.tags + } + + let resp = await conn.remove_packet_mirroring_resources({ packetMirrorName, newMirroredResources: resources }) + } + spinner.succeed("Deleted resource from packet mirroring Succesfully") + spinner.stop() + spinner.clear() + return {} +} + +export const gcpTrafficMirrorDelete = async () => { + const id = uuidv4() + const data = {} + try { + const { project, zone, network, key } = await verifyAccountDetails() + console.log("Validated account details succesfully") + const networkUrl = `https://www.googleapis.com/compute/v1/projects/${project}/global/networks/${network}` + const conn = new GCP_CONN(key, zone, project); + data["key"] = key + data["zone"] = zone + data["project"] = project + + const [packetMirrors] = await conn.list_packet_mirroring() + + await deletePacketMirroringResources(conn, packetMirrors) + } catch (e) { + spinner.fail() + console.log(e) + console.log(data) + } +} diff --git a/cli/src/gcp/gcp_apis.ts b/cli/src/gcp/gcp_apis.ts index 550bc19e..450eb5a1 100644 --- a/cli/src/gcp/gcp_apis.ts +++ b/cli/src/gcp/gcp_apis.ts @@ -19,6 +19,8 @@ import { ZoneOperationsClient, MachineTypesClient, } from "@google-cloud/compute" +import { google } from "@google-cloud/compute/build/protos/protos" +import { wait_for_regional_operation } from "./gcpUtils" const PREFIX_LENGTH = 24 const METLO_DATA_COLLECTOR_TAG = "metlo-capture" @@ -523,4 +525,75 @@ export class GCP_CONN { packetMirroring: packetMirroringURL, }) } + + public async list_packet_mirroring() { + let conn = new PacketMirroringsClient({ credentials: this.keyfile }) + return conn.list({ + project: this.project, + region: this.region + }) + } + + public async get_packet_mirroring({ packetMirrorName }) { + let conn = new PacketMirroringsClient({ credentials: this.keyfile }) + return conn.get({ + project: this.project, + region: this.region, + packetMirroring: packetMirrorName + }) + } + + public async update_packet_mirroring({ packetMirrorName, updateInstance, updateTag, updateSubnet }: updatePacketMirroringInterface) { + let conn = new PacketMirroringsClient({ credentials: this.keyfile }) + let [mirrorInfo, ,] = await this.get_packet_mirroring({ packetMirrorName }) + let updatedMirrorInfo: google.cloud.compute.v1.IPacketMirroring = { + ...mirrorInfo, mirroredResources: { + tags: updateTag ? [updateTag, ...mirrorInfo.mirroredResources.tags] : mirrorInfo.mirroredResources.tags, + instances: updateInstance ? [updateInstance, ...mirrorInfo.mirroredResources.instances] : mirrorInfo.mirroredResources.instances, + subnetworks: updateSubnet ? [updateSubnet, ...mirrorInfo.mirroredResources.subnetworks] : mirrorInfo.mirroredResources.subnetworks, + } + } + return conn.patch({ + project: this.project, + region: this.region, + packetMirroring: packetMirrorName, + packetMirroringResource: updatedMirrorInfo + }) + } + + public async remove_packet_mirroring_resources({ packetMirrorName, newMirroredResources }: deletePacketMirroringResourcesInterface) { + let conn = new PacketMirroringsClient({ credentials: this.keyfile }) + let [mirrorInfo, ,] = await this.get_packet_mirroring({ packetMirrorName }) + let updatedMirrorInfo: google.cloud.compute.v1.IPacketMirroring = { + ...mirrorInfo, mirroredResources: newMirroredResources + } + + const [delOpRegional, ,] = await conn.delete({ project: this.project, region: this.region, packetMirroring: packetMirrorName }) + await wait_for_regional_operation(delOpRegional.latestResponse.name, this) + + const [createOpRegional, ,] = await this.start_packet_mirroring({ + name: mirrorInfo.name, + networkURL: mirrorInfo.network.url, + mirroredInstanceURLs: newMirroredResources.instances.map((inst) => inst.url), + mirroredSubnetURLS: newMirroredResources.subnetworks.map((inst) => inst.url), + mirroredTagURLs: newMirroredResources.tags, + loadBalancerURL: mirrorInfo.collectorIlb.url + }) + await wait_for_regional_operation(createOpRegional.latestResponse.name, this) + + return + } + } + +interface updatePacketMirroringInterface { + packetMirrorName: string; + updateTag?: string; + updateInstance?: google.cloud.compute.v1.IPacketMirroringMirroredResourceInfoInstanceInfo; + updateSubnet?: google.cloud.compute.v1.IPacketMirroringMirroredResourceInfoSubnetInfo; +} + +interface deletePacketMirroringResourcesInterface { + packetMirrorName: string + newMirroredResources: google.cloud.compute.v1.IPacketMirroringMirroredResourceInfo +} \ No newline at end of file diff --git a/cli/src/gcp/list.ts b/cli/src/gcp/list.ts new file mode 100644 index 00000000..cfcbc6d2 --- /dev/null +++ b/cli/src/gcp/list.ts @@ -0,0 +1,111 @@ +import { v4 as uuidv4 } from "uuid" +import fs from "fs" +import { prompt } from "enquirer" +import { GCP_REGIONS_SUPPORTED } from "./gcpUtils" +import { GCP_CONN } from "./gcp_apis" +import chalk from "chalk" +import ora from "ora"; +import { Table } from "console-table-printer" + +const spinner = ora() + +const verifyAccountDetails = async () => { + const gcp_regions = GCP_REGIONS_SUPPORTED.map(e => ({ + name: e, + })) + const resp = await prompt([ + { + type: "input", + name: "_projectName", + message: "GCP Project Name", + }, { + type: "input", + initial: "default", + name: "_networkName", + message: "GCP Network to mirror", + }, { + type: "autocomplete", + name: "_zoneName", + message: "Select your GCP zone", + initial: 1, + choices: gcp_regions, + }, { + type: "input", + name: "_keyPath", + message: "Path to GCP key file", + validate: (path: string) => { + if (fs.existsSync(path)) { + return true + } else { + // @ts-ignore + let text = chalk.redBright(`GCP Key file not found at ${path}`) + return text + } + } + } + ]) + + spinner.text = "Validating account details" + spinner.start() + // @ts-ignore Destructuring is improperly done + const { _projectName: project, _networkName: network, _zoneName: zone, _keyPath: keyFilePath } = resp; + + const key = (fs.readFileSync(keyFilePath)).toString("utf-8"); + + let conn = new GCP_CONN(key, zone, project) + await conn.test_connection() + await conn.get_zone({ zone }) + spinner.succeed("Validated account details") + spinner.stop() + spinner.clear() + return { project, network, zone, key } +} + +const listPacketMirroring = async (conn: GCP_CONN, zone: string) => { + const p = new Table({ + columns: [ + { name: "packetMirror", alignment: "left", title: "Packet Mirror" }, + { name: "type", alignment: "left", title: "Mirror Type" }, + { name: "source", alignment: "left", title: "Mirror Source" }, + ], + }) + const [resp] = await conn.list_packet_mirroring() + resp.forEach((mirror) => { + mirror.mirroredResources.instances.forEach(inst => { + const instanceName = inst.url.split("/") + const instanceUrl = `https://console.cloud.google.com/compute/instancesDetail/zones/${zone}/instances/${instanceName[instanceName.length - 1]}` + p.addRow({ packetMirror: mirror.name, type: "Instance", source: instanceUrl }) + }) + mirror.mirroredResources.subnetworks.forEach(inst => { + const instanceName = inst.url.split("/") + const instanceUrl = `https://console.cloud.google.com/networking/subnetworks/details/${zone}/${instanceName[instanceName.length - 1]}` + p.addRow({ packetMirror: mirror.name, type: "Subnet", source: instanceUrl }) + }) + mirror.mirroredResources.tags.forEach(inst => p.addRow({ packetMirror: mirror.name, type: "Tag", source: inst })) + }) + + console.log(chalk.bold("\n Metlo Mirroring Sessions")) + p.printTable() +} + +const imageURL = "https://www.googleapis.com/compute/v1/projects/metlo-security/global/images/metlo-ingestor-v2" + +export const gcpTrafficMirrorList = async () => { + const id = uuidv4() + const data = {} + try { + const { project, zone, network, key } = await verifyAccountDetails() + console.log("Validated account details succesfully") + const networkUrl = `https://www.googleapis.com/compute/v1/projects/${project}/global/networks/${network}` + const conn = new GCP_CONN(key, zone, project); + data["key"] = key + data["zone"] = zone + data["project"] = project + + const { } = await listPacketMirroring(conn, zone) + } catch (e) { + spinner.fail() + console.log(e) + console.log(data) + } +} diff --git a/cli/src/gcp/setup.ts b/cli/src/gcp/setup.ts index 05232b36..b6689f89 100644 --- a/cli/src/gcp/setup.ts +++ b/cli/src/gcp/setup.ts @@ -1,11 +1,12 @@ import AsyncRetry from "async-retry" -import { v4 as uuidv4 } from "uuid" +import { v4 as uuidv4, validate } from "uuid" import fs from "fs" import { prompt } from "enquirer" import { GCP_REGIONS_SUPPORTED, wait_for_global_operation, wait_for_regional_operation, wait_for_zonal_operation } from "./gcpUtils" import { GCP_CONN } from "./gcp_apis" import chalk from "chalk" import ora from "ora"; +import { google } from "@google-cloud/compute/build/protos/protos" const spinner = ora() @@ -96,10 +97,11 @@ const sourceSelection = async (conn: GCP_CONN) => { message: "Enter the mirror source subnet name", } ]) + spinner.start("Verifying mirror source details") let resp = await conn.get_subnet_information({ subnetName: subnetNameResp['_name'].trim(), }) - spinner.start("Verifying mirror source details") + source_private_ip = resp[0].ipCidrRange source_subnetwork_url = resp[0].selfLink source_instance_url = resp[0].selfLink @@ -116,7 +118,7 @@ const sourceSelection = async (conn: GCP_CONN) => { let tagName = tagNameResp["_name"].trim() if (!resp[0].find(v => v.tags.items.includes(tagName))) { throw new Error( - `No instances with tag ${tagName} found in specifiec zone`, + `No instances with tag ${tagName} found in specific zone`, ) } sourceTag: tagNameResp["_name"] @@ -265,7 +267,6 @@ const create_mig = async ( network_url: string, destination_subnetwork_url: string, source_image: string, - source_ip: string, id: string, ) => { @@ -328,13 +329,9 @@ const create_mig = async ( imageTemplateName: imageTemplateName, startupScript: `#!/bin/bash echo "METLO_ADDR=${machineInfoResp['_url']}" >> /opt/metlo/credentials - echo "METLO_KEY=${machineInfoResp['_apiKey']}" >> /opt/metlo/credentials - echo "alert http ${source_ip} any -> any any (msg:\\"TEST\\"; flow:established,to_client; http.response_body; pcre:/./; sid:1; rev:1; threshold: type limit, track by_rule, seconds 1, count 30;)" >> /opt/metlo/local.rules - sudo mv /opt/metlo/local.rules /var/lib/suricata/rules/local.rules + echo "METLO_KEY=${machineInfoResp['_apiKey']}" >> /opt/metlo/credentials sudo systemctl enable metlo-ingestor.service - sudo systemctl start metlo-ingestor.service - sudo systemctl enable suricata.service - sudo systemctl start suricata.service` + sudo systemctl start metlo-ingestor.service` }) let img_resp = await wait_for_global_operation( image_resp[0].latestResponse.name, @@ -490,7 +487,77 @@ const packetMirroring = async ( } -const imageURL = "https://www.googleapis.com/compute/v1/projects/metlo-security/global/images/metlo-ingestor-v2" +const updatePacketMirroring = async ( + conn: GCP_CONN, + mirroring: google.cloud.compute.v1.IPacketMirroring[] +) => { + const sourceTypeResp = await prompt([ + { + type: "autocomplete", + name: "_packetMirrorName", + message: "Select Packet Mirroring instance", + initial: 0, + choices: mirroring.map((inst) => inst.name) + }, { + type: "select", + name: "_sourceType", + message: "Select your mirror source type", + initial: 0, + choices: ["INSTANCE", "SUBNET", "TAG"], + }, + ]) + let sourceType = sourceTypeResp["_sourceType"] + let packetMirrorName = sourceTypeResp["_packetMirrorName"] + + if (sourceType === "INSTANCE") { + const instanceNameResp = await prompt([ + { + type: "input", + name: "_name", + message: "Enter the mirror source instance name", + } + ]) + spinner.start("Verifying mirror source details") + let [resp] = await conn.get_instance(instanceNameResp['_name'].trim()) + await conn.update_packet_mirroring({ packetMirrorName, updateInstance: { url: resp.selfLink } }) + } else if (sourceType === "SUBNET") { + const subnetNameResp = await prompt([ + { + type: "input", + name: "_name", + message: "Enter the mirror source subnet name", + } + ]) + spinner.start("Verifying mirror source details") + let [resp] = await conn.get_subnet_information({ + subnetName: subnetNameResp['_name'].trim(), + }) + await conn.update_packet_mirroring({ packetMirrorName, updateSubnet: { url: resp.selfLink } }) + } else if (sourceType === "TAG") { + const tagNameResp = await prompt([ + { + type: "input", + name: "_name", + message: "Enter the mirror source tag name", + } + ]) + spinner.start("Verifying mirror source details") + let [resp] = await conn.list_instances() + let tagName = tagNameResp["_name"].trim() + if (!resp.find(v => v.tags.items.includes(tagName))) { + throw new Error( + `No instances with tag ${tagName} found in specific zone`, + ) + } + await conn.update_packet_mirroring({ packetMirrorName, updateTag: tagName }) + } + spinner.succeed("Updated packet mirroring") + spinner.stop() + spinner.clear() + return {} +} + +const imageURL = "https://www.googleapis.com/compute/v1/projects/metlo-security/global/images/metlo-ingestor-v3" export const gcpTrafficMirrorSetup = async () => { const id = uuidv4() @@ -504,33 +571,40 @@ export const gcpTrafficMirrorSetup = async () => { data["zone"] = zone data["project"] = project - const { sourceType, sourceInstanceURL, sourcePrivateIP, sourceSubnetworkURL, sourceTag } = await sourceSelection(conn) - data["sourceType"] = sourceType - data["sourceInstanceURL"] = sourceInstanceURL - data["sourcePrivateIP"] = sourcePrivateIP - data["sourceSubnetworkURL"] = sourceSubnetworkURL - data["sourceTag"] = sourceTag - const { ipRange, destinationSubnetworkUrl } = await getDestinationSubnet(conn, networkUrl, id) - data["ipRange"] = ipRange - data["destinationSubnetworkUrl"] = destinationSubnetworkUrl - const { firewallRuleUrl } = await createFirewallRule(conn, networkUrl, ipRange, id) - data["firewallRuleUrl"] = firewallRuleUrl - const { routerURL } = await createCloudRouter(conn, networkUrl, destinationSubnetworkUrl, id) - data["routerURL"] = routerURL - const { imageTemplateUrl, instanceGroupName, instanceUrl } = await create_mig(conn, networkUrl, destinationSubnetworkUrl, imageURL, sourcePrivateIP, id) - data['mageTemplateUrl'] = imageTemplateUrl - data['instanceGroupName'] = instanceGroupName - data['instanceUrl'] = instanceUrl - const managedGroupUrl = `https://www.googleapis.com/compute/v1/projects/${project}/zones/${zone}/instanceGroups/${instanceGroupName}` - data['managedGroupUrl'] = managedGroupUrl - const { healthCheckUrl } = await createHealthCheck(conn, id) - data['healthCheckUrl'] = healthCheckUrl - const { backendServiceUrl } = await createBackendService(conn, networkUrl, managedGroupUrl, healthCheckUrl, id) - data['backendServiceUrl'] = backendServiceUrl - const { forwardingRuleUrl } = await createLoadBalancer(conn, networkUrl, destinationSubnetworkUrl, backendServiceUrl, id) - data['forwardingRuleUrl'] = forwardingRuleUrl - const { packetMirrorUrl } = await packetMirroring(conn, networkUrl, forwardingRuleUrl, sourceInstanceURL, sourceTag, sourceType, id) - data["packetMirrorUrl"] = packetMirrorUrl + const [packetMirrors] = await conn.list_packet_mirroring() + + if (packetMirrors.length > 0) { + console.log(chalk.blue("Updating the existing Packet Mirroring instance instead of creating new.")) + await updatePacketMirroring(conn, packetMirrors) + } else { + const { sourceType, sourceInstanceURL, sourcePrivateIP, sourceSubnetworkURL, sourceTag } = await sourceSelection(conn) + data["sourceType"] = sourceType + data["sourceInstanceURL"] = sourceInstanceURL + data["sourcePrivateIP"] = sourcePrivateIP + data["sourceSubnetworkURL"] = sourceSubnetworkURL + data["sourceTag"] = sourceTag + const { ipRange, destinationSubnetworkUrl } = await getDestinationSubnet(conn, networkUrl, id) + data["ipRange"] = ipRange + data["destinationSubnetworkUrl"] = destinationSubnetworkUrl + const { firewallRuleUrl } = await createFirewallRule(conn, networkUrl, ipRange, id) + data["firewallRuleUrl"] = firewallRuleUrl + const { routerURL } = await createCloudRouter(conn, networkUrl, destinationSubnetworkUrl, id) + data["routerURL"] = routerURL + const { imageTemplateUrl, instanceGroupName, instanceUrl } = await create_mig(conn, networkUrl, destinationSubnetworkUrl, imageURL, id) + data['mageTemplateUrl'] = imageTemplateUrl + data['instanceGroupName'] = instanceGroupName + data['instanceUrl'] = instanceUrl + const managedGroupUrl = `https://www.googleapis.com/compute/v1/projects/${project}/zones/${zone}/instanceGroups/${instanceGroupName}` + data['managedGroupUrl'] = managedGroupUrl + const { healthCheckUrl } = await createHealthCheck(conn, id) + data['healthCheckUrl'] = healthCheckUrl + const { backendServiceUrl } = await createBackendService(conn, networkUrl, managedGroupUrl, healthCheckUrl, id) + data['backendServiceUrl'] = backendServiceUrl + const { forwardingRuleUrl } = await createLoadBalancer(conn, networkUrl, destinationSubnetworkUrl, backendServiceUrl, id) + data['forwardingRuleUrl'] = forwardingRuleUrl + const { packetMirrorUrl } = await packetMirroring(conn, networkUrl, forwardingRuleUrl, sourceInstanceURL, sourceTag, sourceType, id) + data["packetMirrorUrl"] = packetMirrorUrl + } } catch (e) { spinner.fail() console.log(e) diff --git a/cli/src/index.ts b/cli/src/index.ts index 0451a60d..d4caca1e 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -7,6 +7,8 @@ import { awsTrafficMirrorRemove } from "./aws/remove" import { gcpTrafficMirrorSetup } from "./gcp/setup" import init from "./init" import testAPI from "./testAPI" +import { gcpTrafficMirrorList } from "./gcp/list" +import { gcpTrafficMirrorDelete } from "./gcp/delete" program.name("metlo").description("Metlo's command line tool.").version("0.0.0") @@ -37,5 +39,7 @@ const trafficMirrorGcp = trafficMirror .command("gcp") .description("Set up traffic mirroring for GCP") trafficMirrorGcp.command("new").action(gcpTrafficMirrorSetup) +trafficMirrorGcp.command("list").action(gcpTrafficMirrorList) +trafficMirrorGcp.command("remove").action(gcpTrafficMirrorDelete) program.parseAsync() From e66f156a2abfcea30f98d671a26f110292da8ae7 Mon Sep 17 00:00:00 2001 From: Ninad Sinha Date: Wed, 16 Nov 2022 17:52:15 +0530 Subject: [PATCH 2/6] bump version --- cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/package.json b/cli/package.json index 393d5749..810ef268 100644 --- a/cli/package.json +++ b/cli/package.json @@ -1,6 +1,6 @@ { "name": "@metlo/cli", - "version": "0.0.7", + "version": "0.0.8", "license": "MIT", "main": "dist/index.js", "types": "dist/index.d.ts", From 4d403681a33d6ce401d4abbd60419a9fca3b447b Mon Sep 17 00:00:00 2001 From: Ninad Sinha Date: Wed, 16 Nov 2022 22:25:45 +0530 Subject: [PATCH 3/6] Cleanup and fixes --- cli/src/gcp/gcp_apis.ts | 3 ++- cli/src/gcp/setup.ts | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/src/gcp/gcp_apis.ts b/cli/src/gcp/gcp_apis.ts index 450eb5a1..668551b0 100644 --- a/cli/src/gcp/gcp_apis.ts +++ b/cli/src/gcp/gcp_apis.ts @@ -211,7 +211,8 @@ export class GCP_CONN { }, allowed: [ { - IPProtocol: "all", + IPProtocol: "UDP", + ports: ["4789"] }, ], }, diff --git a/cli/src/gcp/setup.ts b/cli/src/gcp/setup.ts index b6689f89..18f742cf 100644 --- a/cli/src/gcp/setup.ts +++ b/cli/src/gcp/setup.ts @@ -121,7 +121,7 @@ const sourceSelection = async (conn: GCP_CONN) => { `No instances with tag ${tagName} found in specific zone`, ) } - sourceTag: tagNameResp["_name"] + sourceTag = tagNameResp["_name"] source_private_ip = "0.0.0.0/0" // Allow any since filtering is done on tags by gcp source_subnetwork_url = "" source_instance_url = "" @@ -473,7 +473,7 @@ const packetMirroring = async ( let resp = await conn.start_packet_mirroring({ networkURL: network_url, name: packet_mirror_name, - mirroredTagURLs: mirror_source_value, + mirroredTagURLs: [mirror_source_value], loadBalancerURL: forwarding_rule_url, }) packet_mirror_url = ( @@ -557,14 +557,13 @@ const updatePacketMirroring = async ( return {} } -const imageURL = "https://www.googleapis.com/compute/v1/projects/metlo-security/global/images/metlo-ingestor-v3" +const imageURL = "https://www.googleapis.com/compute/v1/projects/metlo-security/global/images/metlo-ingestor-v5" export const gcpTrafficMirrorSetup = async () => { const id = uuidv4() const data = {} try { const { project, zone, network, key } = await verifyAccountDetails() - console.log("Validated account details succesfully") const networkUrl = `https://www.googleapis.com/compute/v1/projects/${project}/global/networks/${network}` const conn = new GCP_CONN(key, zone, project); data["key"] = key From 185eb03024dbc91d40961d3fa5d93fdb60eae883 Mon Sep 17 00:00:00 2001 From: Ninad Sinha Date: Wed, 16 Nov 2022 23:59:13 +0530 Subject: [PATCH 4/6] Cleanup delete code - List only item types that are deletable - Don't print debug data on crash --- cli/src/gcp/delete.ts | 67 +++++++++++++++++++++++------------------ cli/src/gcp/gcp_apis.ts | 38 +++++++++++++++-------- cli/src/gcp/list.ts | 3 +- cli/src/gcp/setup.ts | 3 +- 4 files changed, 65 insertions(+), 46 deletions(-) diff --git a/cli/src/gcp/delete.ts b/cli/src/gcp/delete.ts index 47b582a3..17a242c7 100644 --- a/cli/src/gcp/delete.ts +++ b/cli/src/gcp/delete.ts @@ -70,6 +70,38 @@ const deletePacketMirroringResources = async ( if (mirroring.length == 0) { throw new Error("No existing packet mirroring instances found") } + + const instanceChoices = mirroring.flatMap( + (mirror) => mirror.mirroredResources.instances.map( + inst => { + const splits = inst.url.split("/"); + return splits[splits.length - 1] + } + ) + ) + const subnetChoices = mirroring.flatMap( + (mirror) => mirror.mirroredResources.subnetworks.map( + inst => { + const splits = inst.url.split("/"); + return splits[splits.length - 1] + } + ) + ) + const tagChoices = mirroring.flatMap( + (mirror) => mirror.mirroredResources.tags + ) + + const availableChoices = [] + if (instanceChoices.length > 0) { + availableChoices.push("INSTANCE") + } + if (subnetChoices.length > 0) { + availableChoices.push("SUBNET") + } + if (tagChoices.length > 0) { + availableChoices.push("TAG") + } + const sourceTypeResp = await prompt([ { type: "autocomplete", @@ -82,7 +114,7 @@ const deletePacketMirroringResources = async ( name: "_sourceType", message: "Select your mirror source type", initial: 0, - choices: ["INSTANCE", "SUBNET", "TAG"], + choices: availableChoices, }, ]) let sourceType = sourceTypeResp["_sourceType"] @@ -94,14 +126,8 @@ const deletePacketMirroringResources = async ( type: "autocomplete", name: "_name", message: "Enter the mirror source instance name to remove", - choices: mirroring.flatMap( - (mirror) => mirror.mirroredResources.instances.map( - inst => { - const splits = inst.url.split("/"); - return splits[splits.length - 1] - } - ) - ) + choices: instanceChoices + } ]) spinner.start("Verifying mirror source details") @@ -111,8 +137,6 @@ const deletePacketMirroringResources = async ( resources.instances = resources.instances.filter((inst) => !inst.url.includes(instanceName)) resources.instances = resources.instances.length > 0 ? resources.instances : null - console.log(resources.instances) - let resp = await conn.remove_packet_mirroring_resources({ packetMirrorName, newMirroredResources: resources }) } else if (sourceType === "SUBNET") { const subnetNameResp = await prompt([ @@ -120,14 +144,7 @@ const deletePacketMirroringResources = async ( type: "autocomplete", name: "_name", message: "Enter the mirror source subnet name to remove", - choices: mirroring.flatMap( - (mirror) => mirror.mirroredResources.subnetworks.map( - inst => { - const splits = inst.url.split("/"); - return splits[splits.length - 1] - } - ) - ) + choices: subnetChoices } ]) spinner.start("Verifying mirror source details") @@ -137,8 +154,6 @@ const deletePacketMirroringResources = async ( resources.subnetworks = resources.subnetworks.filter((inst) => !inst.url.includes(subnetName)) resources.subnetworks = resources.subnetworks.length > 0 ? resources.subnetworks : null - console.log(resources.subnetworks) - let resp = await conn.remove_packet_mirroring_resources({ packetMirrorName, newMirroredResources: resources }) } else if (sourceType === "TAG") { const tagNameResp = await prompt([ @@ -146,9 +161,7 @@ const deletePacketMirroringResources = async ( type: "autocomplete", name: "_name", message: "Enter the mirror source tag name to remove", - choices: mirroring.flatMap( - (mirror) => mirror.mirroredResources.tags - ) + choices: tagChoices } ]) spinner.start("Verifying mirror source details") @@ -156,9 +169,6 @@ const deletePacketMirroringResources = async ( const resources = mirroring.find((mirror) => mirror.name === packetMirrorName).mirroredResources resources.tags = resources.tags.filter((inst) => !inst.includes(tagName)) - if (resources.tags.length == 0) { - delete resources.tags - } let resp = await conn.remove_packet_mirroring_resources({ packetMirrorName, newMirroredResources: resources }) } @@ -175,8 +185,7 @@ export const gcpTrafficMirrorDelete = async () => { const { project, zone, network, key } = await verifyAccountDetails() console.log("Validated account details succesfully") const networkUrl = `https://www.googleapis.com/compute/v1/projects/${project}/global/networks/${network}` - const conn = new GCP_CONN(key, zone, project); - data["key"] = key + const conn = new GCP_CONN(key, zone, project); data["zone"] = zone data["project"] = project diff --git a/cli/src/gcp/gcp_apis.ts b/cli/src/gcp/gcp_apis.ts index 668551b0..d13e5296 100644 --- a/cli/src/gcp/gcp_apis.ts +++ b/cli/src/gcp/gcp_apis.ts @@ -565,22 +565,34 @@ export class GCP_CONN { public async remove_packet_mirroring_resources({ packetMirrorName, newMirroredResources }: deletePacketMirroringResourcesInterface) { let conn = new PacketMirroringsClient({ credentials: this.keyfile }) let [mirrorInfo, ,] = await this.get_packet_mirroring({ packetMirrorName }) - let updatedMirrorInfo: google.cloud.compute.v1.IPacketMirroring = { - ...mirrorInfo, mirroredResources: newMirroredResources - } const [delOpRegional, ,] = await conn.delete({ project: this.project, region: this.region, packetMirroring: packetMirrorName }) await wait_for_regional_operation(delOpRegional.latestResponse.name, this) - - const [createOpRegional, ,] = await this.start_packet_mirroring({ - name: mirrorInfo.name, - networkURL: mirrorInfo.network.url, - mirroredInstanceURLs: newMirroredResources.instances.map((inst) => inst.url), - mirroredSubnetURLS: newMirroredResources.subnetworks.map((inst) => inst.url), - mirroredTagURLs: newMirroredResources.tags, - loadBalancerURL: mirrorInfo.collectorIlb.url - }) - await wait_for_regional_operation(createOpRegional.latestResponse.name, this) + if ( + newMirroredResources.instances.length == 0 && + newMirroredResources.subnetworks.length == 0 && + newMirroredResources.tags.length == 0 + ) { + const [createOpRegional, ,] = await this.start_packet_mirroring({ + name: mirrorInfo.name, + networkURL: mirrorInfo.network.url, + mirroredInstanceURLs: [], + mirroredSubnetURLS: [], + mirroredTagURLs: ["metlo-provided-placeholder-network-tag"], + loadBalancerURL: mirrorInfo.collectorIlb.url + }) + await wait_for_regional_operation(createOpRegional.latestResponse.name, this) + } else { + const [createOpRegional, ,] = await this.start_packet_mirroring({ + name: mirrorInfo.name, + networkURL: mirrorInfo.network.url, + mirroredInstanceURLs: newMirroredResources.instances.map((inst) => inst.url), + mirroredSubnetURLS: newMirroredResources.subnetworks.map((inst) => inst.url), + mirroredTagURLs: newMirroredResources.tags, + loadBalancerURL: mirrorInfo.collectorIlb.url + }) + await wait_for_regional_operation(createOpRegional.latestResponse.name, this) + } return } diff --git a/cli/src/gcp/list.ts b/cli/src/gcp/list.ts index cfcbc6d2..39a47471 100644 --- a/cli/src/gcp/list.ts +++ b/cli/src/gcp/list.ts @@ -97,8 +97,7 @@ export const gcpTrafficMirrorList = async () => { const { project, zone, network, key } = await verifyAccountDetails() console.log("Validated account details succesfully") const networkUrl = `https://www.googleapis.com/compute/v1/projects/${project}/global/networks/${network}` - const conn = new GCP_CONN(key, zone, project); - data["key"] = key + const conn = new GCP_CONN(key, zone, project); data["zone"] = zone data["project"] = project diff --git a/cli/src/gcp/setup.ts b/cli/src/gcp/setup.ts index 18f742cf..59864453 100644 --- a/cli/src/gcp/setup.ts +++ b/cli/src/gcp/setup.ts @@ -565,8 +565,7 @@ export const gcpTrafficMirrorSetup = async () => { try { const { project, zone, network, key } = await verifyAccountDetails() const networkUrl = `https://www.googleapis.com/compute/v1/projects/${project}/global/networks/${network}` - const conn = new GCP_CONN(key, zone, project); - data["key"] = key + const conn = new GCP_CONN(key, zone, project); data["zone"] = zone data["project"] = project From 8ce4843545c35192c7428f774543d774867562e5 Mon Sep 17 00:00:00 2001 From: Ninad Sinha Date: Thu, 17 Nov 2022 00:03:44 +0530 Subject: [PATCH 5/6] Mention debug data --- cli/src/gcp/delete.ts | 1 + cli/src/gcp/list.ts | 1 + cli/src/gcp/setup.ts | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/src/gcp/delete.ts b/cli/src/gcp/delete.ts index 17a242c7..cb34b738 100644 --- a/cli/src/gcp/delete.ts +++ b/cli/src/gcp/delete.ts @@ -194,6 +194,7 @@ export const gcpTrafficMirrorDelete = async () => { await deletePacketMirroringResources(conn, packetMirrors) } catch (e) { spinner.fail() + console.log(chalk.bgRedBright("Metlo packet mirroring item removal failed. This might help in debugging it.")) console.log(e) console.log(data) } diff --git a/cli/src/gcp/list.ts b/cli/src/gcp/list.ts index 39a47471..53792c38 100644 --- a/cli/src/gcp/list.ts +++ b/cli/src/gcp/list.ts @@ -104,6 +104,7 @@ export const gcpTrafficMirrorList = async () => { const { } = await listPacketMirroring(conn, zone) } catch (e) { spinner.fail() + console.log(chalk.bgRedBright("Metlo packet mirroring list failed. This might help in debugging it.")) console.log(e) console.log(data) } diff --git a/cli/src/gcp/setup.ts b/cli/src/gcp/setup.ts index 59864453..d2fdf3fc 100644 --- a/cli/src/gcp/setup.ts +++ b/cli/src/gcp/setup.ts @@ -589,7 +589,7 @@ export const gcpTrafficMirrorSetup = async () => { const { routerURL } = await createCloudRouter(conn, networkUrl, destinationSubnetworkUrl, id) data["routerURL"] = routerURL const { imageTemplateUrl, instanceGroupName, instanceUrl } = await create_mig(conn, networkUrl, destinationSubnetworkUrl, imageURL, id) - data['mageTemplateUrl'] = imageTemplateUrl + data['imageTemplateUrl'] = imageTemplateUrl data['instanceGroupName'] = instanceGroupName data['instanceUrl'] = instanceUrl const managedGroupUrl = `https://www.googleapis.com/compute/v1/projects/${project}/zones/${zone}/instanceGroups/${instanceGroupName}` @@ -605,6 +605,7 @@ export const gcpTrafficMirrorSetup = async () => { } } catch (e) { spinner.fail() + console.log(chalk.bgRedBright("Metlo packet mirroring creation failed. This might help in debugging it.")) console.log(e) console.log(data) } From aa9e4c1b8f470897f4e1e67ab754ba0124ee786c Mon Sep 17 00:00:00 2001 From: Ninad Sinha Date: Thu, 17 Nov 2022 00:10:19 +0530 Subject: [PATCH 6/6] Remove extra console logs --- cli/src/gcp/delete.ts | 1 - cli/src/gcp/list.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/cli/src/gcp/delete.ts b/cli/src/gcp/delete.ts index cb34b738..d3b8b8be 100644 --- a/cli/src/gcp/delete.ts +++ b/cli/src/gcp/delete.ts @@ -183,7 +183,6 @@ export const gcpTrafficMirrorDelete = async () => { const data = {} try { const { project, zone, network, key } = await verifyAccountDetails() - console.log("Validated account details succesfully") const networkUrl = `https://www.googleapis.com/compute/v1/projects/${project}/global/networks/${network}` const conn = new GCP_CONN(key, zone, project); data["zone"] = zone diff --git a/cli/src/gcp/list.ts b/cli/src/gcp/list.ts index 53792c38..ec669ca1 100644 --- a/cli/src/gcp/list.ts +++ b/cli/src/gcp/list.ts @@ -95,7 +95,6 @@ export const gcpTrafficMirrorList = async () => { const data = {} try { const { project, zone, network, key } = await verifyAccountDetails() - console.log("Validated account details succesfully") const networkUrl = `https://www.googleapis.com/compute/v1/projects/${project}/global/networks/${network}` const conn = new GCP_CONN(key, zone, project); data["zone"] = zone