diff --git a/perf/README.md b/perf/README.md index f063facb7..57d3ed68f 100644 --- a/perf/README.md +++ b/perf/README.md @@ -15,7 +15,14 @@ Benchmark results can be visualized with https://observablehq.com/@libp2p-worksp 3. Wait for action run to finish and to push a commit to your branch. 4. Visualize results on https://observablehq.com/@libp2p-workspace/performance-dashboard. -## Running via local machine +## Running with Terraform on AWS manually + +### Prerequisites + +- Terraform 1.5.5 or later +- Node.js 18 or later +- [an AWS IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users.html) + ### Provision infrastructure @@ -26,13 +33,28 @@ Benchmark results can be visualized with https://observablehq.com/@libp2p-worksp 5. `CLIENT_IP=$(terraform output -raw client_ip)` 6. `SERVER_IP=$(terraform output -raw server_ip)` -### Build and run implementations +**Notes** +- You may need to reset the infrastructure if you encounter any errors, you can do that by running `terraform destroy` and then `terraform apply`. +- While running terraform you may encounter the following error: + ```bash + Error: collecting instance settings: reading EC2 Launch Template versions: couldn't find resource + │ + │ with module.short_lived_server[0].aws_instance.perf, + │ on ../../modules/short_lived/main.tf line 15, in resource "aws_instance" "perf": + │ 15: resource "aws_instance" "perf" { + ``` +- If you set *TF_VAR* [`long_lived_enabled`](./terraform/configs/local/terraform.tf#L42) env variable to default to **true** terraform should spin up the long-lived resources that are required for the short-lived resources to be created. + +- It's best to destroy the infrastructure after you're done with your testing, you can do that by running `terraform destroy`. + +### Build and run libp2p implementations -_WARNING_: Running the perf tests might take a while. +Given you have provisioned your infrastructure, you can now build and run the libp2p implementations on the AWS instances. 1. `cd runner` 2. `npm ci` -3. `npm run start -- --client-public-ip $CLIENT_IP --server-public-ip $SERVER_IP` +3. `npm run start -- --client-public-ip $CLIENT_IP --server-public-ip $SERVER_IP` + * Note: The default number of iterations that perf will run is 10; desired iterations can be set with the `--iterations ` option. ### Deprovision infrastructure @@ -41,27 +63,28 @@ _WARNING_: Running the perf tests might take a while. ## Adding a new implementation or a new version -1. Add implementation to `impl/`. - - For a new implementation, create a folder `impl//` +1. Add the implementation to new subdirectory in [`impl/*`](./impl/). + - For a new implementation, create a folder `impl//` e.g. `go-libp2p` - For a new version of an existing implementation, create a folder `impl//`. - In that folder include a `Makefile` that builds an executable and stores it next to the `Makefile` under the name `perf`. - Requirements for the executable: - - Running as a libp2p-perf server - - Command line flags - - `--run-server` + - Running as a libp2p-perf server: + - The perf server must not exit as it will be closed by the test runner. + - The executable must accept the command flag `--run-server` which indicates it's running as server. - Running as a libp2p-perf client + - Given that perf is a client driven set of benchmarks, the performance will be measured by the client. - Input via command line - - `--server-ip-address` - - `--transport` (see `runner/versions.ts` for possible variants) + - `--server-address` + - `--transport` (see [`runner/versions.ts`](./runner/src/versions.ts#L7-L43) for possible variants) - `--upload-bytes` number of bytes to upload per stream. - `--download-bytes` number of bytes to download per stream. - Output - - Logging MUST go to stderr. - - Measurement output is printed to stdout as JSON in the form of: + - Logging MUST go to `stderr`. + - Measurement output is printed to **stdout** as JSON in the form of: ```json {"latency": 0.246442851} ``` Note that the measurement includes the time to (1) establish the connection, (2) upload the bytes and (3) download the bytes. -2. For a new implementation, in `impl/Makefile` include your implementation in the `all` target. -3. For a new version, reference version in `runner/src/versions.ts`. +2. For a new implementation, in [`impl/Makefile` include your implementation in the `all` target.](./impl/Makefile#L7) +3. For a new version, reference version in [`runner/src/versions.ts`](./runner/src/versions.ts#L7-L43). diff --git a/perf/runner/src/benchmark-result-type.ts b/perf/runner/src/benchmark-result-type.ts index 34d1088ef..7645ecb16 100644 --- a/perf/runner/src/benchmark-result-type.ts +++ b/perf/runner/src/benchmark-result-type.ts @@ -36,9 +36,7 @@ export type Result = { }; export type ResultValue = { - connectionEstablishedSeconds: number, - uploadSeconds: number, - downloadSeconds: number, + latency: number }; export type Comparison = { diff --git a/perf/runner/src/index.ts b/perf/runner/src/index.ts index 1f6020331..3e78103e4 100644 --- a/perf/runner/src/index.ts +++ b/perf/runner/src/index.ts @@ -4,7 +4,7 @@ import yargs from 'yargs'; import fs from 'fs'; import { BenchmarkResults, Benchmark, Result, IperfResults, PingResults, ResultValue } from './benchmark-result-type'; -async function main(clientPublicIP: string, serverPublicIP: string) { +async function main(clientPublicIP: string, serverPublicIP: string, iterations: number) { const pings = runPing(clientPublicIP, serverPublicIP); const iperf = runIPerf(clientPublicIP, serverPublicIP); @@ -19,7 +19,7 @@ async function main(clientPublicIP: string, serverPublicIP: string) { uploadBytes: 100 << 20, downloadBytes: 0, unit: "bit/s", - iterations: 10, + iterations, }), runBenchmarkAcrossVersions({ name: "Single Connection throughput – Download 100 MiB", @@ -28,7 +28,7 @@ async function main(clientPublicIP: string, serverPublicIP: string) { uploadBytes: 0, downloadBytes: 100 << 20, unit: "bit/s", - iterations: 10, + iterations, }), runBenchmarkAcrossVersions({ name: "Connection establishment + 1 byte round trip latencies", @@ -37,7 +37,7 @@ async function main(clientPublicIP: string, serverPublicIP: string) { uploadBytes: 1, downloadBytes: 1, unit: "s", - iterations: 100, + iterations: iterations * 10, }), ]; @@ -232,8 +232,14 @@ const argv = yargs demandOption: true, description: 'Server public IP address', }, + 'iterations': { + type: 'number', + default: 10, + description: 'Number of iterations to run', + demandOption: false, + } }) .command('help', 'Print usage information', yargs.help) .parseSync(); -main(argv['client-public-ip'] as string, argv['server-public-ip'] as string); +main(argv['client-public-ip'] as string, argv['server-public-ip'] as string, argv['iterations'] as number);