fix(storage): Fix MD5 calculation (#13458)

packages/storage/__tests__/providers/s3/utils/md5.native.test.ts

@@ -0,0 +1,131 @@
+import { Buffer } from 'buffer';
+
+import { Md5 } from '@smithy/md5-js';
+
+import { calculateContentMd5 } from '../../../../src/providers/s3/utils/md5.native';
+import { toBase64 } from '../../../../src/providers/s3/utils/client/utils';
+
+jest.mock('@smithy/md5-js');
+jest.mock('../../../../src/providers/s3/utils/client/utils');
+jest.mock('buffer');
+
+interface MockFileReader {
+	error?: any;
+	result?: any;
+	onload?(): void;
+	onabort?(): void;
+	onerror?(): void;
+	readAsArrayBuffer?(): void;
+	readAsDataURL?(): void;
+}
+
+// The FileReader in React Native 0.71 did not support `readAsArrayBuffer`. This native implementation accomodates this
+// by attempting to use `readAsArrayBuffer` and changing the file reading strategy if it throws an error.
+// TODO: This file should be removable when we drop support for React Native 0.71
+describe('calculateContentMd5 (native)', () => {
+	const stringContent = 'string-content';
+	const base64data = 'base-64-data';
+	const fileReaderResult = new ArrayBuffer(8);
+	const fileReaderBase64Result = `data:foo/bar;base64,${base64data}`;
+	const fileReaderError = new Error();
+	// assert mocks
+	const mockBufferFrom = Buffer.from as jest.Mock;
+	const mockToBase64 = toBase64 as jest.Mock;
+	const mockMd5 = Md5 as jest.Mock;
+	// create mocks
+	const mockSuccessfulFileReader: MockFileReader = {
+		readAsArrayBuffer: jest.fn(() => {
+			mockSuccessfulFileReader.result = fileReaderResult;
+			mockSuccessfulFileReader.onload?.();
+		}),
+	};
+	const mockAbortedFileReader: MockFileReader = {
+		readAsArrayBuffer: jest.fn(() => {
+			mockAbortedFileReader.onabort?.();
+		}),
+	};
+	const mockFailedFileReader: MockFileReader = {
+		readAsArrayBuffer: jest.fn(() => {
+			mockFailedFileReader.error = fileReaderError;
+			mockFailedFileReader.onerror?.();
+		}),
+	};
+	const mockPartialFileReader: MockFileReader = {
+		readAsArrayBuffer: jest.fn(() => {
+			throw new Error('Not implemented');
+		}),
+		readAsDataURL: jest.fn(() => {
+			mockPartialFileReader.result = fileReaderBase64Result;
+			mockPartialFileReader.onload?.();
+		}),
+	};
+
+	beforeAll(() => {
+		mockBufferFrom.mockReturnValue(fileReaderResult);
+	});
+
+	afterEach(() => {
+		jest.clearAllMocks();
+		mockMd5.mockReset();
+	});
+
+	it('calculates MD5 for content type: string', async () => {
+		await calculateContentMd5(stringContent);
+		const [mockMd5Instance] = mockMd5.mock.instances;
+		expect(mockMd5Instance.update.mock.calls[0][0]).toBe(stringContent);
+		expect(mockToBase64).toHaveBeenCalled();
+	});
+
+	it.each([
+		{ type: 'ArrayBuffer view', content: new Uint8Array() },
+		{ type: 'ArrayBuffer', content: new ArrayBuffer(8) },
+		{ type: 'Blob', content: new Blob([stringContent]) },
+	])('calculates MD5 for content type: $type', async ({ content }) => {
+		Object.defineProperty(global, 'FileReader', {
+			writable: true,
+			value: jest.fn(() => mockSuccessfulFileReader),
+		});
+		await calculateContentMd5(content);
+		const [mockMd5Instance] = mockMd5.mock.instances;
+		expect(mockMd5Instance.update.mock.calls[0][0]).toBe(fileReaderResult);
+		expect(mockSuccessfulFileReader.readAsArrayBuffer).toHaveBeenCalled();
+		expect(mockToBase64).toHaveBeenCalled();
+	});
+
+	it('rejects on file reader abort', async () => {
+		Object.defineProperty(global, 'FileReader', {
+			writable: true,
+			value: jest.fn(() => mockAbortedFileReader),
+		});
+		await expect(
+			calculateContentMd5(new Blob([stringContent])),
+		).rejects.toThrow('Read aborted');
+		expect(mockAbortedFileReader.readAsArrayBuffer).toHaveBeenCalled();
+		expect(mockToBase64).not.toHaveBeenCalled();
+	});
+
+	it('rejects on file reader error', async () => {
+		Object.defineProperty(global, 'FileReader', {
+			writable: true,
+			value: jest.fn(() => mockFailedFileReader),
+		});
+		await expect(
+			calculateContentMd5(new Blob([stringContent])),
+		).rejects.toThrow(fileReaderError);
+		expect(mockFailedFileReader.readAsArrayBuffer).toHaveBeenCalled();
+		expect(mockToBase64).not.toHaveBeenCalled();
+	});
+
+	it('tries again using a different strategy if readAsArrayBuffer is unavailable', async () => {
+		Object.defineProperty(global, 'FileReader', {
+			writable: true,
+			value: jest.fn(() => mockPartialFileReader),
+		});
+		await calculateContentMd5(new Blob([stringContent]));
+		const [mockMd5Instance] = mockMd5.mock.instances;
+		expect(mockMd5Instance.update.mock.calls[0][0]).toBe(fileReaderResult);
+		expect(mockPartialFileReader.readAsDataURL).toHaveBeenCalled();
+		expect(mockBufferFrom).toHaveBeenCalledWith(base64data, 'base64');
+		expect(mockToBase64).toHaveBeenCalled();
+	});
+});

packages/storage/__tests__/providers/s3/utils/md5.test.ts

@@ -0,0 +1,95 @@
+import { Md5 } from '@smithy/md5-js';
+
+import { calculateContentMd5 } from '../../../../src/providers/s3/utils/md5';
+import { toBase64 } from '../../../../src/providers/s3/utils/client/utils';
+
+jest.mock('@smithy/md5-js');
+jest.mock('../../../../src/providers/s3/utils/client/utils');
+
+interface MockFileReader {
+	error?: any;
+	result?: any;
+	onload?(): void;
+	onabort?(): void;
+	onerror?(): void;
+	readAsArrayBuffer?(): void;
+}
+
+describe('calculateContentMd5', () => {
+	const stringContent = 'string-content';
+	const fileReaderResult = new ArrayBuffer(8);
+	const fileReaderError = new Error();
+	// assert mocks
+	const mockToBase64 = toBase64 as jest.Mock;
+	const mockMd5 = Md5 as jest.Mock;
+	// create mocks
+	const mockSuccessfulFileReader: MockFileReader = {
+		readAsArrayBuffer: jest.fn(() => {
+			mockSuccessfulFileReader.result = fileReaderResult;
+			mockSuccessfulFileReader.onload?.();
+		}),
+	};
+	const mockAbortedFileReader: MockFileReader = {
+		readAsArrayBuffer: jest.fn(() => {
+			mockAbortedFileReader.onabort?.();
+		}),
+	};
+	const mockFailedFileReader: MockFileReader = {
+		readAsArrayBuffer: jest.fn(() => {
+			mockFailedFileReader.error = fileReaderError;
+			mockFailedFileReader.onerror?.();
+		}),
+	};
+
+	afterEach(() => {
+		jest.clearAllMocks();
+		mockMd5.mockReset();
+	});
+
+	it('calculates MD5 for content type: string', async () => {
+		await calculateContentMd5(stringContent);
+		const [mockMd5Instance] = mockMd5.mock.instances;
+		expect(mockMd5Instance.update.mock.calls[0][0]).toBe(stringContent);
+		expect(mockToBase64).toHaveBeenCalled();
+	});
+
+	it.each([
+		{ type: 'ArrayBuffer view', content: new Uint8Array() },
+		{ type: 'ArrayBuffer', content: new ArrayBuffer(8) },
+		{ type: 'Blob', content: new Blob([stringContent]) },
+	])('calculates MD5 for content type: $type', async ({ content }) => {
+		Object.defineProperty(global, 'FileReader', {
+			writable: true,
+			value: jest.fn(() => mockSuccessfulFileReader),
+		});
+		await calculateContentMd5(content);
+		const [mockMd5Instance] = mockMd5.mock.instances;
+		expect(mockMd5Instance.update.mock.calls[0][0]).toBe(fileReaderResult);
+		expect(mockSuccessfulFileReader.readAsArrayBuffer).toHaveBeenCalled();
+		expect(mockToBase64).toHaveBeenCalled();
+	});
+
+	it('rejects on file reader abort', async () => {
+		Object.defineProperty(global, 'FileReader', {
+			writable: true,
+			value: jest.fn(() => mockAbortedFileReader),
+		});
+		await expect(
+			calculateContentMd5(new Blob([stringContent])),
+		).rejects.toThrow('Read aborted');
+		expect(mockAbortedFileReader.readAsArrayBuffer).toHaveBeenCalled();
+		expect(mockToBase64).not.toHaveBeenCalled();
+	});
+
+	it('rejects on file reader error', async () => {
+		Object.defineProperty(global, 'FileReader', {
+			writable: true,
+			value: jest.fn(() => mockFailedFileReader),
+		});
+		await expect(
+			calculateContentMd5(new Blob([stringContent])),
+		).rejects.toThrow(fileReaderError);
+		expect(mockFailedFileReader.readAsArrayBuffer).toHaveBeenCalled();
+		expect(mockToBase64).not.toHaveBeenCalled();
+	});
+});

packages/storage/src/providers/s3/utils/client/runtime/base64/index.browser.ts

@@ -7,13 +7,9 @@ function bytesToBase64(bytes: Uint8Array): string {
 	return btoa(base64Str);
 }
 
-export function utf8Encode(input: string): Uint8Array {
-	return new TextEncoder().encode(input);
-}
-
 export function toBase64(input: string | ArrayBufferView): string {
 	if (typeof input === 'string') {
-		return bytesToBase64(utf8Encode(input));
+		return bytesToBase64(new TextEncoder().encode(input));
 	}
 
 	return bytesToBase64(

packages/storage/src/providers/s3/utils/client/runtime/base64/index.native.ts

@@ -3,10 +3,6 @@
 
 import { Buffer } from 'buffer';
 
-export function utf8Encode(input: string): Uint8Array {
-	return Buffer.from(input, 'utf-8');
-}
-
 export function toBase64(input: string | ArrayBufferView): string {
 	if (typeof input === 'string') {
 		return Buffer.from(input, 'utf-8').toString('base64');

packages/storage/src/providers/s3/utils/client/runtime/index.browser.ts

@@ -10,4 +10,4 @@ export {
 } from './constants';
 export { s3TransferHandler } from './s3TransferHandler/xhr';
 export { parser } from './xmlParser/dom';
-export { toBase64, utf8Encode } from './base64/index.browser';
+export { toBase64 } from './base64/index.browser';

packages/storage/src/providers/s3/utils/client/runtime/index.native.ts

@@ -10,4 +10,4 @@ export {
 } from './constants';
 export { s3TransferHandler } from './s3TransferHandler/xhr';
 export { parser } from './xmlParser/pureJs';
-export { toBase64, utf8Encode } from './base64/index.native';
+export { toBase64 } from './base64/index.native';

packages/storage/src/providers/s3/utils/client/runtime/index.ts

@@ -11,4 +11,4 @@ export {
 } from './constants';
 export { s3TransferHandler } from './s3TransferHandler/fetch';
 export { parser } from './xmlParser/pureJs';
-export { toBase64, utf8Encode } from './index.native';
+export { toBase64 } from './index.native';

packages/storage/src/providers/s3/utils/client/utils/index.ts

@@ -9,7 +9,6 @@ export {
 	CANCELED_ERROR_MESSAGE,
 	CONTENT_SHA256_HEADER,
 	toBase64,
-	utf8Encode,
 } from '../runtime';
 export {
 	buildStorageServiceError,

packages/storage/src/providers/s3/utils/md5.native.ts

@@ -1,10 +1,15 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import { Buffer } from 'buffer';
+
 import { Md5 } from '@smithy/md5-js';
 
 import { toBase64 } from './client/utils';
 
+// The FileReader in React Native 0.71 did not support `readAsArrayBuffer`. This native implementation accomodates this
+// by attempting to use `readAsArrayBuffer` and changing the file reading strategy if it throws an error.
+// TODO: This file should be removable when we drop support for React Native 0.71
 export const calculateContentMd5 = async (
 	content: Blob | string | ArrayBuffer | ArrayBufferView,
 ): Promise<string> => {
@@ -24,20 +29,29 @@ export const calculateContentMd5 = async (
 	return toBase64(digest);
 };
 
-const readFile = (file: Blob): Promise<ArrayBuffer> => {
-	return new Promise((resolve, reject) => {
+const readFile = (file: Blob): Promise<ArrayBuffer> =>
+	new Promise((resolve, reject) => {
 		const reader = new FileReader();
-		reader.onloadend = () => {
-			if (reader.result) {
-				resolve(reader.result as ArrayBuffer);
-			}
-			reader.onabort = () => {
-				reject(new Error('Read aborted'));
-			};
-			reader.onerror = () => {
-				reject(reader.error);
-			};
+		reader.onload = () => {
+			resolve(reader.result as ArrayBuffer);
+		};
+		reader.onabort = () => {
+			reject(new Error('Read aborted'));
 		};
-		if (file !== undefined) reader.readAsArrayBuffer(file);
+		reader.onerror = () => {
+			reject(reader.error);
+		};
+
+		try {
+			reader.readAsArrayBuffer(file);
+		} catch (e) {
+			reader.onload = () => {
+				// reference: https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL
+				// response from readAsDataURL is always prepended with "data:*/*;base64,"
+				const [, base64Data] = (reader.result as string).split(',');
+				const arrayBuffer = Buffer.from(base64Data, 'base64');
+				resolve(arrayBuffer);
+			};
+			reader.readAsDataURL(file);
+		}
 	});
-};

packages/storage/src/providers/s3/utils/md5.ts

@@ -3,7 +3,7 @@
 
 import { Md5 } from '@smithy/md5-js';
 
-import { toBase64, utf8Encode } from './client/utils';
+import { toBase64 } from './client/utils';
 
 export const calculateContentMd5 = async (
 	content: Blob | string | ArrayBuffer | ArrayBufferView,
@@ -13,38 +13,28 @@ export const calculateContentMd5 = async (
 		hasher.update(content);
 	} else if (ArrayBuffer.isView(content) || content instanceof ArrayBuffer) {
 		const blob = new Blob([content]);
-		const buffer = await readFileToBase64(blob);
+		const buffer = await readFile(blob);
 		hasher.update(buffer);
 	} else {
-		const buffer = await readFileToBase64(content);
-		hasher.update(utf8Encode(buffer));
+		const buffer = await readFile(content);
+		hasher.update(buffer);
 	}
 	const digest = await hasher.digest();
 
 	return toBase64(digest);
 };
 
-const readFileToBase64 = (blob: Blob): Promise<string> => {
-	return new Promise((resolve, reject) => {
+const readFile = (file: Blob): Promise<ArrayBuffer> =>
+	new Promise((resolve, reject) => {
 		const reader = new FileReader();
-		reader.onloadend = () => {
-			// reference: https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readAsDataURL
-			// response from readAsDataURL is always prepended with "data:*/*;base64,"
-			// reference: https://developer.mozilla.org/en-US/docs/Web/API/FileReader/readyState
-			if (reader.readyState !== 2) {
-				reject(new Error('Reader aborted too early'));
-
-				return;
-			}
-			resolve((reader.result as string).split(',')[1]);
+		reader.onload = () => {
+			resolve(reader.result as ArrayBuffer);
 		};
 		reader.onabort = () => {
 			reject(new Error('Read aborted'));
 		};
 		reader.onerror = () => {
 			reject(reader.error);
 		};
-		// reader.readAsArrayBuffer is not available in RN
-		reader.readAsDataURL(blob);
+		reader.readAsArrayBuffer(file);
 	});
-};