From 19abef787b948be4c90a7a6db1e1a97168936938 Mon Sep 17 00:00:00 2001 From: George Fu Date: Tue, 29 Oct 2024 12:40:55 -0400 Subject: [PATCH] test: throw final error in turbo script (#6594) * test: throw final error in turbo script * test: add region * test(middleware-user-agent): add client region in integ test * test(middleware-websocket): set mock credentials * chore(scripts): no retry in turbo * test(aws-util-test): delete AWS env vars, set mock credentials in integ tests * test(aws-middleware-test): add mock creds for integration tests * test(aws-util-test): add httpAuthScheme mocking for integ tests * test: override selected httpAuthScheme.identity in integration tests * test: override signer credential provider in integration * test: use static credentials in integration * test: remove unused --------- Co-authored-by: Kamat, Trivikram <16024985+trivikr@users.noreply.github.com> --- .../src/middleware-eventstream.integ.spec.ts | 9 +++- ...iddleware-flexible-checksums.integ.spec.ts | 2 +- .../middleware-s3-express.integ.spec.ts | 2 +- .../src/middleware-user-agent.integ.spec.ts | 1 + .../src/middleware-websocket.integ.spec.ts | 4 ++ .../middleware-apply-body-checksum.spec.ts | 11 ++-- .../src/middleware-content-length.spec.ts | 43 ++++++++++++--- .../src/middleware-endpoint.spec.ts | 11 +++- .../src/middleware-retry.spec.ts | 7 ++- .../src/middleware-serde.spec.ts | 35 +++++++++--- .../src/util-stream.spec.ts | 15 ++++-- .../src/clients/Weather.integ.spec.ts | 4 +- .../src/requests/test-http-handler.ts | 53 +++++++++---------- scripts/turbo/index.js | 5 +- 14 files changed, 141 insertions(+), 61 deletions(-) diff --git a/packages/middleware-eventstream/src/middleware-eventstream.integ.spec.ts b/packages/middleware-eventstream/src/middleware-eventstream.integ.spec.ts index 781f563e53004..b8cf25b96cbcf 100644 --- a/packages/middleware-eventstream/src/middleware-eventstream.integ.spec.ts +++ b/packages/middleware-eventstream/src/middleware-eventstream.integ.spec.ts @@ -40,7 +40,14 @@ describe("middleware-eventstream", () => { describe(RekognitionStreaming.name, () => { it("should set streaming headers", async () => { - const client = new RekognitionStreaming({ region: "us-west-2", logger }); + const client = new RekognitionStreaming({ + region: "us-west-2", + logger, + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); requireRequestsFrom(client).toMatch({ headers: { diff --git a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts index 7dae9a2145fb3..5d304c5a467fe 100644 --- a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts +++ b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts @@ -138,7 +138,7 @@ describe("middleware-flexible-checksums", () => { requireRequestsFrom(client).toMatch({ headers: { - "user-agent": new RegExp(`(.*?) m\/${id},E$`), + "user-agent": new RegExp(`(.*?) m\/(.*?)${id}(.*?)$`), }, }); diff --git a/packages/middleware-sdk-s3/src/s3-express/middleware-s3-express.integ.spec.ts b/packages/middleware-sdk-s3/src/s3-express/middleware-s3-express.integ.spec.ts index 7b3727f83959a..6273d3938f530 100644 --- a/packages/middleware-sdk-s3/src/s3-express/middleware-s3-express.integ.spec.ts +++ b/packages/middleware-sdk-s3/src/s3-express/middleware-s3-express.integ.spec.ts @@ -94,7 +94,7 @@ describe("middleware-s3-express", () => { requireRequestsFrom(client).toMatch({ headers: { - "user-agent": /(.*?) m\/J,E$/, + "user-agent": /(.*?) m\/(.*?)J(.*?)$/, }, }); diff --git a/packages/middleware-user-agent/src/middleware-user-agent.integ.spec.ts b/packages/middleware-user-agent/src/middleware-user-agent.integ.spec.ts index 0289ad462e1d8..657c7b81ca18d 100644 --- a/packages/middleware-user-agent/src/middleware-user-agent.integ.spec.ts +++ b/packages/middleware-user-agent/src/middleware-user-agent.integ.spec.ts @@ -30,6 +30,7 @@ describe("middleware-user-agent", () => { describe("features", () => { it("should detect DDB mapper, account id, and account id mode", async () => { const client = new DynamoDB({ + region: "us-west-2", accountIdEndpointMode: async () => "required" as const, }); diff --git a/packages/middleware-websocket/src/middleware-websocket.integ.spec.ts b/packages/middleware-websocket/src/middleware-websocket.integ.spec.ts index 926bac4fb6695..d8d9119dc4b7e 100644 --- a/packages/middleware-websocket/src/middleware-websocket.integ.spec.ts +++ b/packages/middleware-websocket/src/middleware-websocket.integ.spec.ts @@ -16,6 +16,10 @@ describe("middleware-websocket", () => { const client = new RekognitionStreaming({ region: "us-west-2", logger, + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, }); requireRequestsFrom(client).toMatch({ protocol: "wss:", diff --git a/private/aws-middleware-test/src/middleware-apply-body-checksum.spec.ts b/private/aws-middleware-test/src/middleware-apply-body-checksum.spec.ts index 6b8ee965e9c3c..d9fd64247a079 100644 --- a/private/aws-middleware-test/src/middleware-apply-body-checksum.spec.ts +++ b/private/aws-middleware-test/src/middleware-apply-body-checksum.spec.ts @@ -1,13 +1,18 @@ -import { test as it, describe, expect } from "vitest"; - import { S3Control } from "@aws-sdk/client-s3-control"; +import { describe, expect, test as it } from "vitest"; import { requireRequestsFrom } from "../../aws-util-test/src"; describe("middleware-apply-body-checksum", () => { describe(S3Control.name, () => { it("should add body-checksum", async () => { - const client = new S3Control({ region: "us-west-2" }); + const client = new S3Control({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); requireRequestsFrom(client).toMatch({ headers: { "content-md5": /^.{22}(==)?$/i, diff --git a/private/aws-middleware-test/src/middleware-content-length.spec.ts b/private/aws-middleware-test/src/middleware-content-length.spec.ts index 2c9d73db112b4..95b19fc468c62 100644 --- a/private/aws-middleware-test/src/middleware-content-length.spec.ts +++ b/private/aws-middleware-test/src/middleware-content-length.spec.ts @@ -1,15 +1,20 @@ -import { test as it, describe, expect } from "vitest"; - import { AccessAnalyzer } from "@aws-sdk/client-accessanalyzer"; import { S3 } from "@aws-sdk/client-s3"; import { XRay } from "@aws-sdk/client-xray"; +import { describe, expect, test as it } from "vitest"; import { requireRequestsFrom } from "../../../private/aws-util-test/src"; describe("middleware-content-length", () => { describe(AccessAnalyzer.name, () => { it("should not add content-length if no body", async () => { - const client = new AccessAnalyzer({ region: "us-west-2" }); + const client = new AccessAnalyzer({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); requireRequestsFrom(client).toMatch({ headers: { "content-length": /undefined/, @@ -24,7 +29,13 @@ describe("middleware-content-length", () => { }); it("should add content-length if body present", async () => { - const client = new AccessAnalyzer({ region: "us-west-2" }); + const client = new AccessAnalyzer({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); requireRequestsFrom(client).toMatch({ headers: { "content-length": /106/, @@ -42,7 +53,13 @@ describe("middleware-content-length", () => { describe(S3.name, () => { it("should not add content-length if no body", async () => { - const client = new S3({ region: "us-west-2" }); + const client = new S3({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); requireRequestsFrom(client).toMatch({ headers: { "content-length": /undefined/, @@ -58,7 +75,13 @@ describe("middleware-content-length", () => { }); it("should add content-length if body present", async () => { - const client = new S3({ region: "us-west-2" }); + const client = new S3({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); requireRequestsFrom(client).toMatch({ headers: { "content-length": /4/, @@ -77,7 +100,13 @@ describe("middleware-content-length", () => { describe(XRay.name, () => { it("should add content-length if body present", async () => { - const client = new XRay({ region: "us-west-2" }); + const client = new XRay({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); requireRequestsFrom(client).toMatch({ headers: { "content-length": /24/, diff --git a/private/aws-middleware-test/src/middleware-endpoint.spec.ts b/private/aws-middleware-test/src/middleware-endpoint.spec.ts index c3b504b8da6ed..2f2a8eb5aa8b9 100644 --- a/private/aws-middleware-test/src/middleware-endpoint.spec.ts +++ b/private/aws-middleware-test/src/middleware-endpoint.spec.ts @@ -1,7 +1,6 @@ -import { test as it, describe, expect } from "vitest"; - import { S3 } from "@aws-sdk/client-s3"; import { S3Control } from "@aws-sdk/client-s3-control"; +import { describe, expect, test as it } from "vitest"; import { requireRequestsFrom } from "../../aws-util-test/src"; @@ -13,6 +12,10 @@ describe("middleware-endpoint", () => { it("should resolve endpoints", async () => { const client = new S3({ region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, useFipsEndpoint: true, useDualstackEndpoint: true, useArnRegion: true, @@ -33,6 +36,10 @@ describe("middleware-endpoint", () => { it("should resolve endpoints", async () => { const client = new S3Control({ region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, useFipsEndpoint: true, useDualstackEndpoint: true, useArnRegion: true, diff --git a/private/aws-middleware-test/src/middleware-retry.spec.ts b/private/aws-middleware-test/src/middleware-retry.spec.ts index 83e13baf61bf5..e36ab8b540f99 100644 --- a/private/aws-middleware-test/src/middleware-retry.spec.ts +++ b/private/aws-middleware-test/src/middleware-retry.spec.ts @@ -1,6 +1,5 @@ -import { test as it, describe, expect } from "vitest"; - import { Lambda } from "@aws-sdk/client-lambda"; +import { describe, expect, test as it } from "vitest"; import { requireRequestsFrom } from "../../aws-util-test/src"; @@ -9,6 +8,10 @@ describe("middleware-retry", () => { it("should set retry headers", async () => { const client = new Lambda({ region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, }); requireRequestsFrom(client).toMatch({ diff --git a/private/aws-middleware-test/src/middleware-serde.spec.ts b/private/aws-middleware-test/src/middleware-serde.spec.ts index 89c5e26fcbc18..0d2f4ea3cfe21 100644 --- a/private/aws-middleware-test/src/middleware-serde.spec.ts +++ b/private/aws-middleware-test/src/middleware-serde.spec.ts @@ -1,16 +1,21 @@ -import { test as it, describe } from "vitest"; - import { EC2 } from "@aws-sdk/client-ec2"; import { S3 } from "@aws-sdk/client-s3"; import { SageMaker } from "@aws-sdk/client-sagemaker"; import { SageMakerRuntime } from "@aws-sdk/client-sagemaker-runtime"; +import { describe, test as it } from "vitest"; import { requireRequestsFrom } from "../../aws-util-test/src"; describe("middleware-serde", () => { describe(S3.name, () => { it("should serialize xml", async () => { - const client = new S3({ region: "us-west-2" }); + const client = new S3({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); requireRequestsFrom(client).toMatch({ method: "PUT", hostname: "s3.us-west-2.amazonaws.com", @@ -55,7 +60,13 @@ describe("middleware-serde", () => { describe(EC2.name, () => { it("should serialize query", async () => { - const client = new EC2({ region: "us-west-2" }); + const client = new EC2({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); requireRequestsFrom(client).toMatch({ method: "POST", hostname: "ec2.us-west-2.amazonaws.com", @@ -79,7 +90,13 @@ describe("middleware-serde", () => { describe(SageMaker.name, () => { it("should serialize json", async () => { - const client = new SageMaker({ region: "us-west-2" }); + const client = new SageMaker({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); requireRequestsFrom(client).toMatch({ method: "POST", hostname: "api.sagemaker.us-west-2.amazonaws.com", @@ -102,7 +119,13 @@ describe("middleware-serde", () => { describe(SageMakerRuntime.name, () => { it("should serialize json", async () => { - const client = new SageMakerRuntime({ region: "us-west-2" }); + const client = new SageMakerRuntime({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); requireRequestsFrom(client).toMatch({ method: "POST", hostname: "runtime.sagemaker.us-west-2.amazonaws.com", diff --git a/private/aws-middleware-test/src/util-stream.spec.ts b/private/aws-middleware-test/src/util-stream.spec.ts index a069a53c7d875..ca897549397ab 100644 --- a/private/aws-middleware-test/src/util-stream.spec.ts +++ b/private/aws-middleware-test/src/util-stream.spec.ts @@ -1,18 +1,23 @@ -import { test as it, describe, expect } from "vitest"; - import { Lambda } from "@aws-sdk/client-lambda"; import { HttpHandler, HttpResponse } from "@smithy/protocol-http"; import { HttpRequest as IHttpRequest } from "@smithy/types"; import { Uint8ArrayBlobAdapter } from "@smithy/util-stream"; import { fromUtf8 } from "@smithy/util-utf8"; import { Readable } from "stream"; +import { describe, expect, test as it } from "vitest"; import { requireRequestsFrom } from "../../aws-util-test/src"; describe("util-stream", () => { describe(Lambda.name, () => { it("should be uniform between string and Uint8Array payloads", async () => { - const client = new Lambda({ region: "us-west-2" }); + const client = new Lambda({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); requireRequestsFrom(client).toMatch({ method: "POST", hostname: "lambda.us-west-2.amazonaws.com", @@ -54,6 +59,10 @@ describe("util-stream", () => { const lambda = new Lambda({ region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, }); requireRequestsFrom(lambda).toMatch({ diff --git a/private/aws-util-test/src/clients/Weather.integ.spec.ts b/private/aws-util-test/src/clients/Weather.integ.spec.ts index 2efeb8bcdd95e..56851f77d40ce 100644 --- a/private/aws-util-test/src/clients/Weather.integ.spec.ts +++ b/private/aws-util-test/src/clients/Weather.integ.spec.ts @@ -1,6 +1,5 @@ -import { test as it, describe, expect } from "vitest"; - import { Weather } from "@aws-sdk/weather"; +import { describe, expect, test as it } from "vitest"; import { requireRequestsFrom } from "../requests/test-http-handler"; @@ -12,6 +11,7 @@ describe(Weather.name, () => { secretAccessKey: "", }, endpoint: "https://localhost", + region: "us-west-2", }); requireRequestsFrom(client).toMatch({ diff --git a/private/aws-util-test/src/requests/test-http-handler.ts b/private/aws-util-test/src/requests/test-http-handler.ts index e27282cd780b8..f4bbe7126e2ff 100644 --- a/private/aws-util-test/src/requests/test-http-handler.ts +++ b/private/aws-util-test/src/requests/test-http-handler.ts @@ -28,14 +28,6 @@ export type HttpRequestMatcher = { log?: boolean; }; -/** - * @internal - */ -const MOCK_CREDENTIALS = { - accessKeyId: "MOCK_ACCESS_KEY_ID", - secretAccessKey: "MOCK_SECRET_ACCESS_KEY_ID", -}; - /** * Supplied to test clients to assert correct requests. * @internal @@ -47,7 +39,30 @@ export class TestHttpHandler implements HttpHandler { private client?: Client; private assertions = 0; - public constructor(public readonly matcher: HttpRequestMatcher) {} + public constructor(public readonly matcher: HttpRequestMatcher) { + const RESERVED_ENVIRONMENT_VARIABLES = { + AWS_DEFAULT_REGION: 1, + AWS_REGION: 1, + AWS_PROFILE: 1, + AWS_ACCESS_KEY_ID: 1, + AWS_SECRET_ACCESS_KEY: 1, + AWS_SESSION_TOKEN: 1, + AWS_CREDENTIAL_EXPIRATION: 1, + AWS_CREDENTIAL_SCOPE: 1, + AWS_EC2_METADATA_DISABLED: 1, + AWS_WEB_IDENTITY_TOKEN_FILE: 1, + AWS_ROLE_ARN: 1, + AWS_CONTAINER_CREDENTIALS_FULL_URI: 1, + AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: 1, + AWS_CONTAINER_AUTHORIZATION_TOKEN: 1, + AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE: 1, + }; + for (const key in RESERVED_ENVIRONMENT_VARIABLES) { + delete process.env[key]; + } + process.env.AWS_ACCESS_KEY_ID = "INTEGRATION_TEST_MOCK"; + process.env.AWS_SECRET_ACCESS_KEY = "INTEGRATION_TEST_MOCK"; + } /** * @param client - to watch for requests. @@ -58,26 +73,6 @@ export class TestHttpHandler implements HttpHandler { public watch(client: Client, matcher: HttpRequestMatcher = this.matcher) { this.client = client; this.originalRequestHandler = client.config.requestHandler; - // mock credentials to avoid default chain lookup. - client.config.credentials = async () => MOCK_CREDENTIALS; - client.config.credentialDefaultProvider = () => { - return async () => { - return MOCK_CREDENTIALS; - }; - }; - const signerProvider = client.config.signer; - if (typeof signerProvider === "function") { - client.config.signer = async () => { - const _signer = await signerProvider(); - if (typeof _signer.credentialProvider === "function") { - // signer is instance of SignatureV4 - _signer.credentialProvider = async () => { - return MOCK_CREDENTIALS; - }; - } - return _signer; - }; - } client.config.requestHandler = new TestHttpHandler(matcher); if (!(client as any)[TestHttpHandler.WATCHER]) { diff --git a/scripts/turbo/index.js b/scripts/turbo/index.js index 1bf55e046dc3e..b5640e799e7e4 100644 --- a/scripts/turbo/index.js +++ b/scripts/turbo/index.js @@ -28,10 +28,7 @@ const runTurbo = async (task, args, { apiSecret, apiEndpoint, apiSignatureKey } }); } catch (error) { console.error("Error running turbo:", error); - if (args?.length > 0) { - // Retry without additional filters - return await runTurbo(task, null, apiSecret, apiEndpoint); - } + throw error; } };