From 0bafd2346d1dd7ab5a339c44789fb3a7be4dc747 Mon Sep 17 00:00:00 2001 From: Sumu Date: Thu, 13 Jun 2024 17:48:48 -0400 Subject: [PATCH 1/9] WIP Signed-off-by: Sumu --- projenrc/benchmark-test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/projenrc/benchmark-test.ts b/projenrc/benchmark-test.ts index fdd9a877..ec1ba879 100644 --- a/projenrc/benchmark-test.ts +++ b/projenrc/benchmark-test.ts @@ -8,6 +8,7 @@ import * as tar from 'tar'; import * as ts from 'typescript'; import * as yargs from 'yargs'; import { ACTIONS_SETUP_NODE, YARN_INSTALL } from './common'; +import { SUPPORT_POLICY } from './support'; export class BenchmarkTest { public constructor( @@ -21,6 +22,8 @@ export class BenchmarkTest { receiveArgs: true, }); + const jsiiVersion = SUPPORT_POLICY.current; + const iterations = 20; const indices = Array.from({ length: iterations }, (_, idx) => idx); @@ -149,6 +152,10 @@ export class BenchmarkTest { 'aws cloudwatch put-metric-data --metric-name TSC-average --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.duration-tsc }}', 'aws cloudwatch put-metric-data --metric-name JSII-average --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.duration-jsii }}', 'aws cloudwatch put-metric-data --metric-name JSII-slowdown --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.jsii-slowdown }}', + 'aws cloudwatch put-metric-data --metric-name JSII-average --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.duration-jsii }} ' + + `--dimensions JsiiVersion=${jsiiVersion}`, + 'aws cloudwatch put-metric-data --metric-name JSII-slowdown --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.jsii-slowdown }} ' + + `--dimensions JsiiVersion=${jsiiVersion}`, ].join('\n'), }, ], From 9a383f3b96472188d0c1251aea74a29abb9567ff Mon Sep 17 00:00:00 2001 From: Sumu Date: Thu, 13 Jun 2024 17:49:26 -0400 Subject: [PATCH 2/9] run npx projen Signed-off-by: Sumu --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 96f1e300..03d02b94 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -432,3 +432,5 @@ jobs: aws cloudwatch put-metric-data --metric-name TSC-average --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.duration-tsc }} aws cloudwatch put-metric-data --metric-name JSII-average --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.duration-jsii }} aws cloudwatch put-metric-data --metric-name JSII-slowdown --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.jsii-slowdown }} + aws cloudwatch put-metric-data --metric-name JSII-average --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.duration-jsii }} --dimensions JsiiVersion=5.4 + aws cloudwatch put-metric-data --metric-name JSII-slowdown --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.jsii-slowdown }} --dimensions JsiiVersion=5.4 From 8200b46c0ec2195813f9cc3a7325f3208bca5aa0 Mon Sep 17 00:00:00 2001 From: Sumu Date: Fri, 14 Jun 2024 14:06:19 -0400 Subject: [PATCH 3/9] add tsc version, use one aws cli call Signed-off-by: Sumu --- .github/workflows/build.yml | 58 ++++++++++++++++++++++++++++++---- projenrc/benchmark-test.ts | 62 +++++++++++++++++++++++++++++++------ 2 files changed, 105 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 03d02b94..931de1d9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -426,11 +426,57 @@ jobs: role-to-assume: arn:aws:iam::590183883712:role/Ops-jsiiTeamOIDC-Role1ABCC5F0-jL37v7e7I15P role-session-name: github-diff-action@cdk-ops output-credentials: true + - name: Get TSC version + id: get_tsc_version + if: github.event.repository.fork == false && github.ref == 'refs/heads/main' + uses: aws-actions/configure-aws-credentials@v4 + run: echo "TSC_VERSION=$(tsc --version | awk '{print $2}')" >> $GITHUB_ENV - name: Publish Metrics if: github.event.repository.fork == false && github.ref == 'refs/heads/main' - run: |- - aws cloudwatch put-metric-data --metric-name TSC-average --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.duration-tsc }} - aws cloudwatch put-metric-data --metric-name JSII-average --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.duration-jsii }} - aws cloudwatch put-metric-data --metric-name JSII-slowdown --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.jsii-slowdown }} - aws cloudwatch put-metric-data --metric-name JSII-average --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.duration-jsii }} --dimensions JsiiVersion=5.4 - aws cloudwatch put-metric-data --metric-name JSII-slowdown --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.jsii-slowdown }} --dimensions JsiiVersion=5.4 + run: |2- + + |- + aws cloudwatch put-metric-data --namespace JsiiPerformance --metric-data '[ + { + "MetricName": "TSC-average", + "Value": ${{steps.output_summary.outputs.duration-tsc}} + }, + { + "MetricName": "JSII-average", + "Value": ${{steps.output_summary.outputs.duration-jsii}} + }, + { + "MetricName": "JSII-slowdown", + "Value": ${{steps.output_summary.outputs.jsii-slowdown}} + }, + { + "MetricName": "TSC-average", + "Value": ${{steps.output_summary.outputs.duration-tsc}}, + "Dimensions": [ + { + "Name": "TscVersion", + "Value": "${{ env.TSC_VERSION }}" + } + ] + }, + { + "MetricName": "JSII-average", + "Value": ${{steps.output_summary.outputs.duration-jsii}}, + "Dimensions": [ + { + "Name": "JsiiVersion", + "Value": 5.4 + } + ] + }, + { + "MetricName": "JSII-slowdown", + "Value": ${{steps.output_summary.outputs.jsii-slowdown}}, + "Dimensions": [ + { + "Name": "JsiiVersion", + "Value": 5.4 + } + ] + } + ]' diff --git a/projenrc/benchmark-test.ts b/projenrc/benchmark-test.ts index ec1ba879..d8f193da 100644 --- a/projenrc/benchmark-test.ts +++ b/projenrc/benchmark-test.ts @@ -145,18 +145,62 @@ export class BenchmarkTest { 'output-credentials': true, }, }, + { + name: 'Get TSC version', + id: 'get_tsc_version', + if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, + uses: 'aws-actions/configure-aws-credentials@v4', + run: `echo "TSC_VERSION=$(tsc --version | awk '{print $2}')" >> $GITHUB_ENV`, + }, { name: 'Publish Metrics', if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, - run: [ - 'aws cloudwatch put-metric-data --metric-name TSC-average --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.duration-tsc }}', - 'aws cloudwatch put-metric-data --metric-name JSII-average --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.duration-jsii }}', - 'aws cloudwatch put-metric-data --metric-name JSII-slowdown --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.jsii-slowdown }}', - 'aws cloudwatch put-metric-data --metric-name JSII-average --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.duration-jsii }} ' + - `--dimensions JsiiVersion=${jsiiVersion}`, - 'aws cloudwatch put-metric-data --metric-name JSII-slowdown --namespace JsiiPerformance --value ${{ steps.output_summary.outputs.jsii-slowdown }} ' + - `--dimensions JsiiVersion=${jsiiVersion}`, - ].join('\n'), + run: ` + |- + aws cloudwatch put-metric-data --namespace JsiiPerformance --metric-data '[ + { + "MetricName": "TSC-average", + "Value": \${{steps.output_summary.outputs.duration-tsc}} + }, + { + "MetricName": "JSII-average", + "Value": \${{steps.output_summary.outputs.duration-jsii}} + }, + { + "MetricName": "JSII-slowdown", + "Value": \${{steps.output_summary.outputs.jsii-slowdown}} + }, + { + "MetricName": "TSC-average", + "Value": \${{steps.output_summary.outputs.duration-tsc}}, + "Dimensions": [ + { + "Name": "TscVersion", + "Value": "\${{ env.TSC_VERSION }}" + } + ] + }, + { + "MetricName": "JSII-average", + "Value": \${{steps.output_summary.outputs.duration-jsii}}, + "Dimensions": [ + { + "Name": "JsiiVersion", + "Value": ${jsiiVersion} + } + ] + }, + { + "MetricName": "JSII-slowdown", + "Value": \${{steps.output_summary.outputs.jsii-slowdown}}, + "Dimensions": [ + { + "Name": "JsiiVersion", + "Value": ${jsiiVersion} + } + ] + } + ]'`, }, ], }, From d9f716b8fbfcfddfe17505c2ff103a6d510aace6 Mon Sep 17 00:00:00 2001 From: Sumu Date: Fri, 14 Jun 2024 14:08:52 -0400 Subject: [PATCH 4/9] typo Signed-off-by: Sumu --- .github/workflows/build.yml | 1 - projenrc/benchmark-test.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 931de1d9..3431e919 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -429,7 +429,6 @@ jobs: - name: Get TSC version id: get_tsc_version if: github.event.repository.fork == false && github.ref == 'refs/heads/main' - uses: aws-actions/configure-aws-credentials@v4 run: echo "TSC_VERSION=$(tsc --version | awk '{print $2}')" >> $GITHUB_ENV - name: Publish Metrics if: github.event.repository.fork == false && github.ref == 'refs/heads/main' diff --git a/projenrc/benchmark-test.ts b/projenrc/benchmark-test.ts index d8f193da..218f7fce 100644 --- a/projenrc/benchmark-test.ts +++ b/projenrc/benchmark-test.ts @@ -149,7 +149,6 @@ export class BenchmarkTest { name: 'Get TSC version', id: 'get_tsc_version', if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, - uses: 'aws-actions/configure-aws-credentials@v4', run: `echo "TSC_VERSION=$(tsc --version | awk '{print $2}')" >> $GITHUB_ENV`, }, { From 27d4be0175a3ad369e634e334b5cc998e94e6f21 Mon Sep 17 00:00:00 2001 From: Sumu Date: Fri, 14 Jun 2024 14:27:42 -0400 Subject: [PATCH 5/9] uncomment if to test workflow on branch Signed-off-by: Sumu --- .github/workflows/build.yml | 3 --- projenrc/benchmark-test.ts | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3431e919..e7f3b74c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -418,7 +418,6 @@ jobs: fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary.join("\n"), "utf-8"); EOF - name: Authenticate Via OIDC Role - if: github.event.repository.fork == false && github.ref == 'refs/heads/main' uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -428,10 +427,8 @@ jobs: output-credentials: true - name: Get TSC version id: get_tsc_version - if: github.event.repository.fork == false && github.ref == 'refs/heads/main' run: echo "TSC_VERSION=$(tsc --version | awk '{print $2}')" >> $GITHUB_ENV - name: Publish Metrics - if: github.event.repository.fork == false && github.ref == 'refs/heads/main' run: |2- |- diff --git a/projenrc/benchmark-test.ts b/projenrc/benchmark-test.ts index 218f7fce..d9af53e9 100644 --- a/projenrc/benchmark-test.ts +++ b/projenrc/benchmark-test.ts @@ -135,7 +135,7 @@ export class BenchmarkTest { }, { name: 'Authenticate Via OIDC Role', - if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, + //if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, uses: 'aws-actions/configure-aws-credentials@v4', with: { 'aws-region': 'us-east-1', @@ -148,12 +148,12 @@ export class BenchmarkTest { { name: 'Get TSC version', id: 'get_tsc_version', - if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, + //if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, run: `echo "TSC_VERSION=$(tsc --version | awk '{print $2}')" >> $GITHUB_ENV`, }, { name: 'Publish Metrics', - if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, + //if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, run: ` |- aws cloudwatch put-metric-data --namespace JsiiPerformance --metric-data '[ From 8501121c20027d6440196123cabedf5512b2d6dc Mon Sep 17 00:00:00 2001 From: Sumu Date: Fri, 14 Jun 2024 14:41:48 -0400 Subject: [PATCH 6/9] fix run script Signed-off-by: Sumu --- .github/workflows/build.yml | 5 ++--- projenrc/benchmark-test.ts | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7f3b74c..92d23c59 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -429,9 +429,8 @@ jobs: id: get_tsc_version run: echo "TSC_VERSION=$(tsc --version | awk '{print $2}')" >> $GITHUB_ENV - name: Publish Metrics - run: |2- - - |- + run: |- + |- aws cloudwatch put-metric-data --namespace JsiiPerformance --metric-data '[ { "MetricName": "TSC-average", diff --git a/projenrc/benchmark-test.ts b/projenrc/benchmark-test.ts index d9af53e9..fedcf51a 100644 --- a/projenrc/benchmark-test.ts +++ b/projenrc/benchmark-test.ts @@ -154,8 +154,7 @@ export class BenchmarkTest { { name: 'Publish Metrics', //if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, - run: ` - |- + run: `|- aws cloudwatch put-metric-data --namespace JsiiPerformance --metric-data '[ { "MetricName": "TSC-average", From 2b432d42f6e870d274ad58889a0966c9f554fc75 Mon Sep 17 00:00:00 2001 From: Sumu Date: Fri, 14 Jun 2024 14:53:09 -0400 Subject: [PATCH 7/9] formatting Signed-off-by: Sumu --- .github/workflows/build.yml | 3 +-- projenrc/benchmark-test.ts | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 92d23c59..239230a1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -430,8 +430,7 @@ jobs: run: echo "TSC_VERSION=$(tsc --version | awk '{print $2}')" >> $GITHUB_ENV - name: Publish Metrics run: |- - |- - aws cloudwatch put-metric-data --namespace JsiiPerformance --metric-data '[ + aws cloudwatch put-metric-data --namespace JsiiPerformance --metric-data '[ { "MetricName": "TSC-average", "Value": ${{steps.output_summary.outputs.duration-tsc}} diff --git a/projenrc/benchmark-test.ts b/projenrc/benchmark-test.ts index fedcf51a..ac857fab 100644 --- a/projenrc/benchmark-test.ts +++ b/projenrc/benchmark-test.ts @@ -154,8 +154,7 @@ export class BenchmarkTest { { name: 'Publish Metrics', //if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, - run: `|- - aws cloudwatch put-metric-data --namespace JsiiPerformance --metric-data '[ + run: `aws cloudwatch put-metric-data --namespace JsiiPerformance --metric-data '[ { "MetricName": "TSC-average", "Value": \${{steps.output_summary.outputs.duration-tsc}} From 496ac45916dfef2d930162f7f83ff3ce364bddd2 Mon Sep 17 00:00:00 2001 From: Sumu Date: Fri, 14 Jun 2024 15:13:16 -0400 Subject: [PATCH 8/9] add quotes to jsii version dimension Signed-off-by: Sumu --- .github/workflows/build.yml | 4 ++-- projenrc/benchmark-test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 239230a1..ee75d9f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -459,7 +459,7 @@ jobs: "Dimensions": [ { "Name": "JsiiVersion", - "Value": 5.4 + "Value": "5.4" } ] }, @@ -469,7 +469,7 @@ jobs: "Dimensions": [ { "Name": "JsiiVersion", - "Value": 5.4 + "Value": "5.4" } ] } diff --git a/projenrc/benchmark-test.ts b/projenrc/benchmark-test.ts index ac857fab..fb8bfd9f 100644 --- a/projenrc/benchmark-test.ts +++ b/projenrc/benchmark-test.ts @@ -183,7 +183,7 @@ export class BenchmarkTest { "Dimensions": [ { "Name": "JsiiVersion", - "Value": ${jsiiVersion} + "Value": "${jsiiVersion}" } ] }, @@ -193,7 +193,7 @@ export class BenchmarkTest { "Dimensions": [ { "Name": "JsiiVersion", - "Value": ${jsiiVersion} + "Value": "${jsiiVersion}" } ] } From 683504c7c4eb048a19540d1c2cee82a1737808fb Mon Sep 17 00:00:00 2001 From: Sumu Date: Fri, 14 Jun 2024 15:28:28 -0400 Subject: [PATCH 9/9] add conditionals back, testing succeeded Signed-off-by: Sumu --- .github/workflows/build.yml | 3 +++ projenrc/benchmark-test.ts | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ee75d9f4..56036cda 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -418,6 +418,7 @@ jobs: fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary.join("\n"), "utf-8"); EOF - name: Authenticate Via OIDC Role + if: github.event.repository.fork == false && github.ref == 'refs/heads/main' uses: aws-actions/configure-aws-credentials@v4 with: aws-region: us-east-1 @@ -427,8 +428,10 @@ jobs: output-credentials: true - name: Get TSC version id: get_tsc_version + if: github.event.repository.fork == false && github.ref == 'refs/heads/main' run: echo "TSC_VERSION=$(tsc --version | awk '{print $2}')" >> $GITHUB_ENV - name: Publish Metrics + if: github.event.repository.fork == false && github.ref == 'refs/heads/main' run: |- aws cloudwatch put-metric-data --namespace JsiiPerformance --metric-data '[ { diff --git a/projenrc/benchmark-test.ts b/projenrc/benchmark-test.ts index fb8bfd9f..a8b61e7b 100644 --- a/projenrc/benchmark-test.ts +++ b/projenrc/benchmark-test.ts @@ -135,7 +135,7 @@ export class BenchmarkTest { }, { name: 'Authenticate Via OIDC Role', - //if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, + if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, uses: 'aws-actions/configure-aws-credentials@v4', with: { 'aws-region': 'us-east-1', @@ -148,12 +148,12 @@ export class BenchmarkTest { { name: 'Get TSC version', id: 'get_tsc_version', - //if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, + if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, run: `echo "TSC_VERSION=$(tsc --version | awk '{print $2}')" >> $GITHUB_ENV`, }, { name: 'Publish Metrics', - //if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, + if: `github.event.repository.fork == false && github.ref == 'refs/heads/main'`, run: `aws cloudwatch put-metric-data --namespace JsiiPerformance --metric-data '[ { "MetricName": "TSC-average",