From 3317a92488ad5f6ab7a876cafa978b5d2943bc7b Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Thu, 19 Dec 2024 16:01:15 -0800 Subject: [PATCH] test: move flexible checksums integration tests (#6752) --- .../test/unit/flexibleChecksums.spec.ts | 170 ---------------- ...iddleware-flexible-checksums.integ.spec.ts | 190 +++++++++++------- 2 files changed, 118 insertions(+), 242 deletions(-) delete mode 100644 clients/client-s3/test/unit/flexibleChecksums.spec.ts diff --git a/clients/client-s3/test/unit/flexibleChecksums.spec.ts b/clients/client-s3/test/unit/flexibleChecksums.spec.ts deleted file mode 100644 index 710b513c1b30d..0000000000000 --- a/clients/client-s3/test/unit/flexibleChecksums.spec.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { ChecksumAlgorithm } from "@aws-sdk/middleware-flexible-checksums"; -import { HttpRequest } from "@smithy/protocol-http"; -import { BuildMiddleware } from "@smithy/types"; -import { Readable } from "stream"; -import { describe, expect, test as it } from "vitest"; - -import { ChecksumAlgorithm as Algo, S3 } from "../../src/index"; - -describe("Flexible Checksums", () => { - const testCases = [ - ["", ChecksumAlgorithm.CRC32, "AAAAAA=="], - ["abc", ChecksumAlgorithm.CRC32, "NSRBwg=="], - ["Hello world", ChecksumAlgorithm.CRC32, "i9aeUg=="], - - ["", ChecksumAlgorithm.CRC32C, "AAAAAA=="], - ["abc", ChecksumAlgorithm.CRC32C, "Nks/tw=="], - ["Hello world", ChecksumAlgorithm.CRC32C, "crUfeA=="], - - ["", ChecksumAlgorithm.SHA1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="], - ["abc", ChecksumAlgorithm.SHA1, "qZk+NkcGgWq6PiVxeFDCbJzQ2J0="], - ["Hello world", ChecksumAlgorithm.SHA1, "e1AsOh9IyGCa4hLN+2Od7jlnP14="], - - ["", ChecksumAlgorithm.SHA256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="], - ["abc", ChecksumAlgorithm.SHA256, "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="], - ["Hello world", ChecksumAlgorithm.SHA256, "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw="], - ]; - - describe("putObject", () => { - testCases.forEach(([body, checksumAlgorithm, checksumValue]) => { - const checksumHeader = `x-amz-checksum-${checksumAlgorithm.toLowerCase()}`; - - describe(`sets ${checksumHeader}="${checksumValue}"" for checksum="${checksumAlgorithm}"`, () => { - const getBodyAsReadableStream = (content: string) => { - const readableStream = new Readable(); - const separator = " "; - const wordsAsChunks = content.split(separator); - wordsAsChunks.forEach((word, index) => { - readableStream.push(word); - if (index !== wordsAsChunks.length - 1) { - readableStream.push(separator); - } - }); - readableStream.push(null); - return readableStream; - }; - - it(`when body is sent as a request`, async () => { - const requestChecksumValidator: BuildMiddleware = (next) => async (args) => { - // middleware intercept the request and return it early - const request = args.request as HttpRequest; - const { headers } = request; - expect(headers["x-amz-sdk-checksum-algorithm"]).to.equal(checksumAlgorithm); - expect(headers[checksumHeader]).to.equal(checksumValue); - return { output: {} as any, response: {} as any }; - }; - - const client = new S3({ - region: "us-west-2", - credentials: { - accessKeyId: "CLIENT_TEST", - secretAccessKey: "CLIENT_TEST", - }, - }); - client.middlewareStack.addRelativeTo(requestChecksumValidator, { - relation: "after", - toMiddleware: "flexibleChecksumsMiddleware", - }); - - return await client.putObject({ - Bucket: "bucket", - Key: "key", - Body: body, - ChecksumAlgorithm: checksumAlgorithm as Algo, - }); - }); - - it(`when body is sent as a stream`, async () => { - const requestChecksumValidator: BuildMiddleware = (next) => async (args) => { - // middleware intercept the request and return it early - const request = args.request as HttpRequest; - const { headers, body } = request; - expect(headers["content-length"]).to.be.undefined; - expect(headers["content-encoding"]).to.equal("aws-chunked"); - expect(headers["transfer-encoding"]).to.equal("chunked"); - expect(headers["x-amz-content-sha256"]).to.equal("STREAMING-UNSIGNED-PAYLOAD-TRAILER"); - expect(headers["x-amz-trailer"]).to.equal(checksumHeader); - body.on("data", (data: any) => { - const stringValue = data.toString(); - if (stringValue.startsWith(checksumHeader)) { - const receivedChecksum = stringValue.replace("\r\n", "").split(":")[1]; - expect(receivedChecksum).to.equal(checksumValue); - } - }); - return { output: {} as any, response: {} as any }; - }; - - const client = new S3({ - region: "us-west-2", - credentials: { - accessKeyId: "CLIENT_TEST", - secretAccessKey: "CLIENT_TEST", - }, - }); - client.middlewareStack.addRelativeTo(requestChecksumValidator, { - relation: "after", - toMiddleware: "flexibleChecksumsMiddleware", - }); - - const bodyStream = getBodyAsReadableStream(body); - await client.putObject({ - Bucket: "bucket", - Key: "key", - Body: bodyStream, - ChecksumAlgorithm: checksumAlgorithm as Algo, - }); - }); - }); - }); - }); - - describe("getObject", async () => { - testCases.forEach(([body, checksumAlgorithm, checksumValue]) => { - const checksumHeader = `x-amz-checksum-${checksumAlgorithm.toLowerCase()}`; - - it(`validates ${checksumHeader}="${checksumValue}"" set for checksum="${checksumAlgorithm}"`, async () => { - const responseBody = new Readable(); - responseBody.push(body); - responseBody.push(null); - const responseChecksumValidator: BuildMiddleware = (next, context) => async (args) => { - const request = args.request as HttpRequest; - return { - output: { - $metadata: { attempts: 0, httpStatusCode: 200 }, - request, - context, - Body: responseBody, - } as any, - response: { - body: responseBody, - headers: { - [checksumHeader]: checksumValue, - }, - } as any, - }; - }; - - const client = new S3({ - region: "us-west-2", - credentials: { - accessKeyId: "CLIENT_TEST", - secretAccessKey: "CLIENT_TEST", - }, - }); - client.middlewareStack.addRelativeTo(responseChecksumValidator, { - relation: "after", - toMiddleware: "flexibleChecksumsMiddleware", - }); - - const { Body } = await client.getObject({ - Bucket: "bucket", - Key: "key", - ChecksumMode: "ENABLED", - }); - (Body as Readable).on("data", (chunk) => { - expect(chunk.toString()).to.equal(body); - }); - }); - }); - }); -}); diff --git a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts index 5d304c5a467fe..1512a76b53f5f 100644 --- a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts +++ b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.integ.spec.ts @@ -1,5 +1,6 @@ import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3"; -import { Transform } from "stream"; +import { HttpHandler, HttpRequest, HttpResponse } from "@smithy/protocol-http"; +import { Readable, Transform } from "stream"; import { describe, expect, test as it } from "vitest"; import { requireRequestsFrom } from "../../../private/aws-util-test/src"; @@ -13,87 +14,134 @@ describe("middleware-flexible-checksums", () => { error() {}, }; + const testCases: [string, ChecksumAlgorithm, string][] = [ + ["", ChecksumAlgorithm.CRC32, "AAAAAA=="], + ["abc", ChecksumAlgorithm.CRC32, "NSRBwg=="], + ["Hello world", ChecksumAlgorithm.CRC32, "i9aeUg=="], + + ["", ChecksumAlgorithm.CRC32C, "AAAAAA=="], + ["abc", ChecksumAlgorithm.CRC32C, "Nks/tw=="], + ["Hello world", ChecksumAlgorithm.CRC32C, "crUfeA=="], + + ["", ChecksumAlgorithm.SHA1, "2jmj7l5rSw0yVb/vlWAYkK/YBwk="], + ["abc", ChecksumAlgorithm.SHA1, "qZk+NkcGgWq6PiVxeFDCbJzQ2J0="], + ["Hello world", ChecksumAlgorithm.SHA1, "e1AsOh9IyGCa4hLN+2Od7jlnP14="], + + ["", ChecksumAlgorithm.SHA256, "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="], + ["abc", ChecksumAlgorithm.SHA256, "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="], + ["Hello world", ChecksumAlgorithm.SHA256, "ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw="], + ]; + describe(S3.name, () => { - it("should set flexible checksums (SHA256)", async () => { - const client = new S3({ region: "us-west-2", logger }); + const client = new S3({ region: "us-west-2", logger }); - requireRequestsFrom(client).toMatch({ - method: "PUT", - hostname: "s3.us-west-2.amazonaws.com", - protocol: "https:", - path: "/b/k", - headers: { - "content-type": "application/octet-stream", - "x-amz-sdk-checksum-algorithm": "SHA256", - "content-length": "4", - Expect: "100-continue", - "x-amz-checksum-sha256": "iNQmb9TmM40TuEX88olXnSCciXgjuSF9o+Fhk28DFYk=", - host: "s3.us-west-2.amazonaws.com", - "x-amz-user-agent": /./, - "user-agent": /./, - "amz-sdk-invocation-id": /./, - "amz-sdk-request": /./, - "x-amz-date": /./, - "x-amz-content-sha256": "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", - authorization: /./, - }, - query: { - "x-id": "PutObject", - }, - }); + describe("putObject", () => { + testCases.forEach(([body, checksumAlgorithm, checksumValue]) => { + const checksumHeader = `x-amz-checksum-${checksumAlgorithm.toLowerCase()}`; - await client.putObject({ - Bucket: "b", - Key: "k", - Body: Buffer.from("abcd"), - ChecksumAlgorithm: "SHA256", - }); + it(`sets ${checksumHeader}="${checksumValue}"" for checksum="${checksumAlgorithm}"`, async () => { + requireRequestsFrom(client).toMatch({ + method: "PUT", + hostname: "s3.us-west-2.amazonaws.com", + protocol: "https:", + path: "/b/k", + headers: { + "content-type": "application/octet-stream", + ...(body.length + ? { + "content-length": body.length.toString(), + Expect: "100-continue", + } + : {}), + "x-amz-sdk-checksum-algorithm": checksumAlgorithm, + [checksumHeader]: checksumValue, + host: "s3.us-west-2.amazonaws.com", + "x-amz-user-agent": /./, + "user-agent": /./, + "amz-sdk-invocation-id": /./, + "amz-sdk-request": /./, + "x-amz-date": /./, + "x-amz-content-sha256": /./, + authorization: /./, + }, + query: { + "x-id": "PutObject", + }, + }); - expect.hasAssertions(); + await client.putObject({ + Bucket: "b", + Key: "k", + Body: body, + ChecksumAlgorithm: checksumAlgorithm, + }); + + expect.hasAssertions(); + }); + }); }); - it("should set flexible checksums (SHA1)", async () => { - const client = new S3({ region: "us-west-2", logger }); + describe("getObject", () => { + testCases.forEach(([body, checksumAlgorithm, checksumValue]) => { + const checksumHeader = `x-amz-checksum-${checksumAlgorithm.toLowerCase()}`; + + it(`validates ${checksumHeader}="${checksumValue}"" set for checksum="${checksumAlgorithm}"`, async () => { + const client = new S3({ + region: "us-west-2", + logger, + requestHandler: new (class implements HttpHandler { + async handle(request: HttpRequest): Promise { + expect(request).toMatchObject({ + method: "GET", + hostname: "s3.us-west-2.amazonaws.com", + protocol: "https:", + path: "/b/k", + headers: { + "x-amz-checksum-mode": "ENABLED", + host: "s3.us-west-2.amazonaws.com", + "x-amz-user-agent": /./, + "user-agent": /./, + "amz-sdk-invocation-id": /./, + "amz-sdk-request": /./, + "x-amz-date": /./, + "x-amz-content-sha256": /./, + authorization: /./, + }, + query: { + "x-id": "GetObject", + }, + }); + return { + response: new HttpResponse({ + statusCode: 200, + headers: { + "content-type": "application/octet-stream", + "content-length": body.length.toString(), + [checksumHeader]: checksumValue, + }, + body: Readable.from([body]), + }), + }; + } + updateHttpClientConfig(key: never, value: never): void {} + httpHandlerConfigs() { + return {}; + } + })(), + }); - requireRequestsFrom(client).toMatch({ - method: "PUT", - hostname: "s3.us-west-2.amazonaws.com", - protocol: "https:", - path: "/b/k", - headers: { - "content-type": "application/octet-stream", - "x-amz-sdk-checksum-algorithm": "SHA1", - "content-length": "4", - Expect: "100-continue", - "x-amz-checksum-sha1": "gf6L/odXbD7LIkJvjleEc4KRes8=", - host: "s3.us-west-2.amazonaws.com", - "x-amz-user-agent": /./, - "user-agent": /./, - "amz-sdk-invocation-id": /./, - "amz-sdk-request": /./, - "x-amz-date": /./, - authorization: /./, - // this is here even if algo is SHA1 - "x-amz-content-sha256": "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", - }, - query: { - "x-id": "PutObject", - }, - }); + const response = await client.getObject({ + Bucket: "b", + Key: "k", + ChecksumMode: "ENABLED", + }); - await client.putObject({ - Bucket: "b", - Key: "k", - Body: Buffer.from("abcd"), - ChecksumAlgorithm: "SHA1", + await expect(response.Body?.transformToString()).resolves.toEqual(body); + }); }); - - expect.hasAssertions(); }); it("should not set binary file content length", async () => { - const client = new S3({ region: "us-west-2", logger }); - requireRequestsFrom(client).toMatch({ method: "PUT", hostname: "s3.us-west-2.amazonaws.com", @@ -134,8 +182,6 @@ describe("middleware-flexible-checksums", () => { ["CRC32C", "V"], ].forEach(([algo, id]) => { it(`should feature-detect checksum ${algo}=${id}`, async () => { - const client = new S3({ region: "us-west-2", logger }); - requireRequestsFrom(client).toMatch({ headers: { "user-agent": new RegExp(`(.*?) m\/(.*?)${id}(.*?)$`),