diff --git a/perf/README.md b/perf/README.md index 57d3ed68f..8ae94832d 100644 --- a/perf/README.md +++ b/perf/README.md @@ -15,11 +15,11 @@ Benchmark results can be visualized with https://observablehq.com/@libp2p-worksp 3. Wait for action run to finish and to push a commit to your branch. 4. Visualize results on https://observablehq.com/@libp2p-workspace/performance-dashboard. -## Running with Terraform on AWS manually +## Running manually ### Prerequisites -- Terraform 1.5.5 or later +- Terraform 1.5.4 or later - Node.js 18 or later - [an AWS IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users.html) @@ -34,7 +34,6 @@ Benchmark results can be visualized with https://observablehq.com/@libp2p-worksp 6. `SERVER_IP=$(terraform output -raw server_ip)` **Notes** -- You may need to reset the infrastructure if you encounter any errors, you can do that by running `terraform destroy` and then `terraform apply`. - While running terraform you may encounter the following error: ```bash Error: collecting instance settings: reading EC2 Launch Template versions: couldn't find resource @@ -43,7 +42,7 @@ Benchmark results can be visualized with https://observablehq.com/@libp2p-worksp │ on ../../modules/short_lived/main.tf line 15, in resource "aws_instance" "perf": │ 15: resource "aws_instance" "perf" { ``` -- If you set *TF_VAR* [`long_lived_enabled`](./terraform/configs/local/terraform.tf#L42) env variable to default to **true** terraform should spin up the long-lived resources that are required for the short-lived resources to be created. +- This implies that you haven't deployed the long-lived infrastructure on your AWS account. To do so along with each short-lived deployment, you can set *TF_VAR* [`long_lived_enabled`](./terraform/configs/local/terraform.tf#L42) env variable to default to `true`. Terraform should then spin up the long-lived resources that are required for the short-lived resources to be created. - It's best to destroy the infrastructure after you're done with your testing, you can do that by running `terraform destroy`. @@ -80,7 +79,7 @@ Given you have provisioned your infrastructure, you can now build and run the li - `--download-bytes` number of bytes to download per stream. - Output - Logging MUST go to `stderr`. - - Measurement output is printed to **stdout** as JSON in the form of: + - Measurement output is printed to `stdout` as JSON in the form of: ```json {"latency": 0.246442851} ``` diff --git a/perf/runner/src/index.ts b/perf/runner/src/index.ts index 3e78103e4..8f78c103f 100644 --- a/perf/runner/src/index.ts +++ b/perf/runner/src/index.ts @@ -4,9 +4,9 @@ import yargs from 'yargs'; import fs from 'fs'; import { BenchmarkResults, Benchmark, Result, IperfResults, PingResults, ResultValue } from './benchmark-result-type'; -async function main(clientPublicIP: string, serverPublicIP: string, iterations: number) { - const pings = runPing(clientPublicIP, serverPublicIP); - const iperf = runIPerf(clientPublicIP, serverPublicIP); +async function main(clientPublicIP: string, serverPublicIP: string, testing: boolean) { + const pings = runPing(clientPublicIP, serverPublicIP, testing); + const iperf = runIPerf(clientPublicIP, serverPublicIP, testing); copyAndBuildPerfImplementations(serverPublicIP); copyAndBuildPerfImplementations(clientPublicIP); @@ -19,7 +19,7 @@ async function main(clientPublicIP: string, serverPublicIP: string, iterations: uploadBytes: 100 << 20, downloadBytes: 0, unit: "bit/s", - iterations, + iterations: testing ? 1 : 10, }), runBenchmarkAcrossVersions({ name: "Single Connection throughput – Download 100 MiB", @@ -28,7 +28,7 @@ async function main(clientPublicIP: string, serverPublicIP: string, iterations: uploadBytes: 0, downloadBytes: 100 << 20, unit: "bit/s", - iterations, + iterations: testing ? 1 : 10, }), runBenchmarkAcrossVersions({ name: "Connection establishment + 1 byte round trip latencies", @@ -37,7 +37,7 @@ async function main(clientPublicIP: string, serverPublicIP: string, iterations: uploadBytes: 1, downloadBytes: 1, unit: "s", - iterations: iterations * 10, + iterations: testing ? 1 : 100, }), ]; @@ -53,10 +53,11 @@ async function main(clientPublicIP: string, serverPublicIP: string, iterations: console.error("== done"); } -function runPing(clientPublicIP: string, serverPublicIP: string): PingResults { - console.error(`= run 100 pings from client to server`); +function runPing(clientPublicIP: string, serverPublicIP: string, testing: boolean): PingResults { + const pingCount = testing ? 1 : 100; + console.error(`= run ${pingCount} pings from client to server`); - const cmd = `ssh -o StrictHostKeyChecking=no ec2-user@${clientPublicIP} 'ping -c 100 ${serverPublicIP}'`; + const cmd = `ssh -o StrictHostKeyChecking=no ec2-user@${clientPublicIP} 'ping -c ${pingCount} ${serverPublicIP}'`; const stdout = execCommand(cmd).toString(); // Extract the time from each ping @@ -71,9 +72,9 @@ function runPing(clientPublicIP: string, serverPublicIP: string): PingResults { return { unit: "s", results: times } } -function runIPerf(clientPublicIP: string, serverPublicIP: string): IperfResults { - const iterations = 60; - console.error(`= run ${iterations} iPerf TCP from client to server`); +function runIPerf(clientPublicIP: string, serverPublicIP: string, testing: boolean): IperfResults { + const iPerfIterations = testing ? 1 : 60; + console.error(`= run ${iPerfIterations} iPerf TCP from client to server`); const killCMD = `ssh -o StrictHostKeyChecking=no ec2-user@${serverPublicIP} 'kill $(cat pidfile); rm pidfile; rm server.log || true'`; const killSTDOUT = execCommand(killCMD); @@ -83,7 +84,7 @@ function runIPerf(clientPublicIP: string, serverPublicIP: string): IperfResults const serverSTDOUT = execCommand(serverCMD); console.error(serverSTDOUT); - const cmd = `ssh -o StrictHostKeyChecking=no ec2-user@${clientPublicIP} 'iperf3 -c ${serverPublicIP} -b 25g -t ${iterations}'`; + const cmd = `ssh -o StrictHostKeyChecking=no ec2-user@${clientPublicIP} 'iperf3 -c ${serverPublicIP} -b 25g -t ${iPerfIterations}'`; const stdout = execSync(cmd).toString(); // Extract the bitrate from each relevant line @@ -232,14 +233,14 @@ const argv = yargs demandOption: true, description: 'Server public IP address', }, - 'iterations': { - type: 'number', - default: 10, - description: 'Number of iterations to run', + 'testing': { + type: 'boolean', + default: false, + description: 'Run in testing mode', demandOption: false, } }) .command('help', 'Print usage information', yargs.help) .parseSync(); -main(argv['client-public-ip'] as string, argv['server-public-ip'] as string, argv['iterations'] as number); +main(argv['client-public-ip'] as string, argv['server-public-ip'] as string, argv['testing'] as boolean);