Skip to content

Commit

Permalink
feat(@aws-amplify/storage): Adding download progress tracker for Stor…
Browse files Browse the repository at this point in the history 
…age.get (#8295)

* adding download progress tracker

* more consistent naming for upload progress

* ran prettier and a warning in progressCallback

* added unit tests for axios http handler

* adding axios http handler tests

* added unit tests for AWSS3Provider

* fix typo on axios http handler test

* removed unused lines from test

* fix progresscallback logging issue

* simplify logic of progressCallback check

* unit test fix for progressCallback

* add cleanup listeners

* unit test update removeAllListeners
  • Loading branch information
@jamesaucode
jamesaucode authored Jul 7, 2021
1 parent a84978d commit 8fe1853
Show file tree
Hide file tree
Showing 6 changed files with 178 additions and 113 deletions.
74 changes: 73 additions & 1 deletion packages/storage/__tests__/providers/AWSS3Provider-unit-test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import * as formatURL from '@aws-sdk/util-format-url';
import { S3Client, ListObjectsCommand } from '@aws-sdk/client-s3';
import { S3RequestPresigner } from '@aws-sdk/s3-request-presigner';
import * as events from 'events';

import { S3CopySource, S3CopyDestination } from '../../src/types';
/**
* NOTE - These test cases use Hub.dispatch but they should
Expand Down Expand Up @@ -229,6 +230,77 @@ describe('StorageProvider test', () => {
});
});

test('get object with download and progress tracker', async () => {
jest.spyOn(Credentials, 'get').mockImplementationOnce(() => {
return Promise.resolve(credentials);
});
const mockCallback = jest.fn();
const mockRemoveAllListeners = jest.fn();
const mockEventEmitter = {
emit: jest.fn(),
on: jest.fn(),
removeAllListeners: mockRemoveAllListeners,
};
jest
.spyOn(events, 'EventEmitter')
.mockImplementationOnce(() => mockEventEmitter);
const downloadOptionsWithProgressCallback = Object.assign({}, options, {
download: true,
progressCallback: mockCallback,
});
const storage = new StorageProvider();
storage.configure(downloadOptionsWithProgressCallback);
const spyon = jest
.spyOn(S3Client.prototype, 'send')
.mockImplementationOnce(async params => {
return { Body: [1, 2] };
});
expect(await storage.get('key', { download: true })).toEqual({
Body: [1, 2],
});
expect(mockEventEmitter.on).toBeCalledWith(
'sendDownloadProgress',
expect.any(Function)
);
// Get the anonymous function called by the emitter
const emitterOnFn = mockEventEmitter.on.mock.calls[0][1];
// Manully invoke it for testing
emitterOnFn('arg');
expect(mockCallback).toBeCalledWith('arg');
expect(mockRemoveAllListeners).toHaveBeenCalled();
});

test('get object with incorrect progressCallback type', async () => {
jest.spyOn(Credentials, 'get').mockImplementationOnce(() => {
return Promise.resolve(credentials);
});
const loggerSpy = jest.spyOn(Logger.prototype, '_log');
const mockEventEmitter = {
emit: jest.fn(),
on: jest.fn(),
removeAllListeners: jest.fn(),
};
jest
.spyOn(events, 'EventEmitter')
.mockImplementationOnce(() => mockEventEmitter);
const downloadOptionsWithProgressCallback = Object.assign({}, options);
const storage = new StorageProvider();
storage.configure(downloadOptionsWithProgressCallback);
jest
.spyOn(S3Client.prototype, 'send')
.mockImplementationOnce(async params => {
return { Body: [1, 2] };
});
await storage.get('key', {
download: true,
progressCallback: 'this is not a function',
});
expect(loggerSpy).toHaveBeenCalledWith(
'WARN',
'progressCallback should be a function, not a string'
);
});

test('get object with download with failure', async () => {
jest.spyOn(Credentials, 'get').mockImplementationOnce(() => {
return new Promise((res, rej) => {
Expand Down Expand Up @@ -604,7 +676,7 @@ describe('StorageProvider test', () => {
progressCallback: mockCallback,
});
expect(mockEventEmitter.on).toBeCalledWith(
'sendProgress',
'sendUploadProgress',
expect.any(Function)
);
const emitterOnFn = mockEventEmitter.on.mock.calls[0][1];
Expand Down
158 changes: 58 additions & 100 deletions packages/storage/__tests__/providers/AWSS3ProviderManagedUpload-unit-test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,7 @@
* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
import {
AWSS3ProviderManagedUpload,
Part,
} from '../../src/providers/AWSS3ProviderManagedUpload';
import { AWSS3ProviderManagedUpload, Part } from '../../src/providers/AWSS3ProviderManagedUpload';
import {
S3Client,
PutObjectCommand,
Expand All @@ -23,7 +20,7 @@ import {
AbortMultipartUploadCommand,
ListPartsCommand,
} from '@aws-sdk/client-s3';
import { Logger } from "@aws-amplify/core";
import { Logger } from '@aws-amplify/core';
import * as events from 'events';
import * as sinon from 'sinon';

Expand Down Expand Up @@ -65,7 +62,7 @@ class TestClass extends AWSS3ProviderManagedUpload {
await super.uploadParts(uploadId, parts);
// Now trigger some notifications from the event listeners
for (const part of parts) {
part.emitter.emit('sendProgress', {
part.emitter.emit('sendUploadProgress', {
// Assume that the notification is send when 100% of part is uploaded
loaded: (part.bodyPart as string).length,
});
Expand All @@ -80,64 +77,39 @@ afterEach(() => {

describe('single part upload tests', () => {
test('upload a string as body', async () => {
const putObjectSpyOn = jest
.spyOn(S3Client.prototype, 'send')
.mockImplementation(command => {
if (command instanceof PutObjectCommand)
return Promise.resolve(command.input.Key);
});
const uploader = new AWSS3ProviderManagedUpload(
testParams,
testOpts,
new events.EventEmitter()
);
const putObjectSpyOn = jest.spyOn(S3Client.prototype, 'send').mockImplementation(command => {
if (command instanceof PutObjectCommand) return Promise.resolve(command.input.Key);
});
const uploader = new AWSS3ProviderManagedUpload(testParams, testOpts, new events.EventEmitter());
const data = await uploader.upload();
expect(data).toBe(testParams.Key);
expect(putObjectSpyOn.mock.calls[0][0].input).toStrictEqual(testParams);
});

test('upload a javascript object as body', async () => {
const putObjectSpyOn = jest
.spyOn(S3Client.prototype, 'send')
.mockImplementation(command => {
if (command instanceof PutObjectCommand)
return Promise.resolve(command.input.Key);
});
const putObjectSpyOn = jest.spyOn(S3Client.prototype, 'send').mockImplementation(command => {
if (command instanceof PutObjectCommand) return Promise.resolve(command.input.Key);
});
const objectBody = { key1: 'value1', key2: 'value2' };
const testParamsWithObjectBody: any = Object.assign({}, testParams);
testParamsWithObjectBody.Body = objectBody;
const uploader = new AWSS3ProviderManagedUpload(
testParamsWithObjectBody,
testOpts,
new events.EventEmitter()
);
const uploader = new AWSS3ProviderManagedUpload(testParamsWithObjectBody, testOpts, new events.EventEmitter());
const data = await uploader.upload();
expect(data).toBe(testParamsWithObjectBody.Key);
expect(putObjectSpyOn.mock.calls[0][0].input).toStrictEqual(
testParamsWithObjectBody
);
expect(putObjectSpyOn.mock.calls[0][0].input).toStrictEqual(testParamsWithObjectBody);
});

test('upload a file as body', async () => {
const putObjectSpyOn = jest
.spyOn(S3Client.prototype, 'send')
.mockImplementation(command => {
if (command instanceof PutObjectCommand)
return Promise.resolve(command.input.Key);
});
const putObjectSpyOn = jest.spyOn(S3Client.prototype, 'send').mockImplementation(command => {
if (command instanceof PutObjectCommand) return Promise.resolve(command.input.Key);
});
const file = new File(['TestFileContent'], 'testFileName');
const testParamsWithFileBody: any = Object.assign({}, testParams);
testParamsWithFileBody.Body = file;
const uploader = new AWSS3ProviderManagedUpload(
testParamsWithFileBody,
testOpts,
new events.EventEmitter()
);
const uploader = new AWSS3ProviderManagedUpload(testParamsWithFileBody, testOpts, new events.EventEmitter());
const data = await uploader.upload();
expect(data).toBe(testParamsWithFileBody.Key);
expect(putObjectSpyOn.mock.calls[0][0].input).toStrictEqual(
testParamsWithFileBody
);
expect(putObjectSpyOn.mock.calls[0][0].input).toStrictEqual(testParamsWithFileBody);
});
});

Expand All @@ -146,22 +118,20 @@ describe('multi part upload tests', () => {
// setup event handling
const emitter = new events.EventEmitter();
const eventSpy = sinon.spy();
emitter.on('sendProgress', eventSpy);
emitter.on('sendUploadProgress', eventSpy);

// Setup Spy for S3 service calls
const s3ServiceCallSpy = jest
.spyOn(S3Client.prototype, 'send')
.mockImplementation(async command => {
if (command instanceof CreateMultipartUploadCommand) {
return Promise.resolve({ UploadId: testUploadId });
} else if (command instanceof UploadPartCommand) {
return Promise.resolve({
ETag: 'test_etag_' + command.input.PartNumber,
});
} else if (command instanceof CompleteMultipartUploadCommand) {
return Promise.resolve({ Key: testParams.Key });
}
});
const s3ServiceCallSpy = jest.spyOn(S3Client.prototype, 'send').mockImplementation(async command => {
if (command instanceof CreateMultipartUploadCommand) {
return Promise.resolve({ UploadId: testUploadId });
} else if (command instanceof UploadPartCommand) {
return Promise.resolve({
ETag: 'test_etag_' + command.input.PartNumber,
});
} else if (command instanceof CompleteMultipartUploadCommand) {
return Promise.resolve({ Key: testParams.Key });
}
});

// Now make calls
const uploader = new TestClass(testParams, testOpts, emitter);
Expand Down Expand Up @@ -230,36 +200,34 @@ describe('multi part upload tests', () => {
// setup event handling
const emitter = new events.EventEmitter();
const eventSpy = sinon.spy();
emitter.on('sendProgress', eventSpy);
emitter.on('sendUploadProgress', eventSpy);

// Setup Spy for S3 service calls and introduce a service failure
const s3ServiceCallSpy = jest
.spyOn(S3Client.prototype, 'send')
.mockImplementation(async command => {
if (command instanceof CreateMultipartUploadCommand) {
return Promise.resolve({ UploadId: testUploadId });
} else if (command instanceof UploadPartCommand) {
let promise = null;
if (command.input.PartNumber === 2) {
promise = new Promise((resolve, reject) => {
setTimeout(() => {
reject(new Error('Part 2 just going to fail in 100ms'));
}, 100);
});
} else {
promise = new Promise((resolve, reject) => {
setTimeout(() => {
resolve({
ETag: 'test_etag_' + command.input.PartNumber,
});
}, 200);
});
}
return promise;
} else if (command instanceof CompleteMultipartUploadCommand) {
return Promise.resolve({ Key: testParams.key });
const s3ServiceCallSpy = jest.spyOn(S3Client.prototype, 'send').mockImplementation(async command => {
if (command instanceof CreateMultipartUploadCommand) {
return Promise.resolve({ UploadId: testUploadId });
} else if (command instanceof UploadPartCommand) {
let promise = null;
if (command.input.PartNumber === 2) {
promise = new Promise((resolve, reject) => {
setTimeout(() => {
reject(new Error('Part 2 just going to fail in 100ms'));
}, 100);
});
} else {
promise = new Promise((resolve, reject) => {
setTimeout(() => {
resolve({
ETag: 'test_etag_' + command.input.PartNumber,
});
}, 200);
});
}
});
return promise;
} else if (command instanceof CompleteMultipartUploadCommand) {
return Promise.resolve({ Key: testParams.key });
}
});

// Now make calls
const uploader = new TestClass(testParams, testOpts, emitter);
Expand Down Expand Up @@ -341,14 +309,8 @@ describe('multi part upload tests', () => {
return Promise.resolve();
}
});
const uploader = new TestClass(
testParams,
testOpts,
new events.EventEmitter()
);
await expect(uploader.upload()).rejects.toThrow(
'Upload was cancelled. Multi Part upload clean up failed'
);
const uploader = new TestClass(testParams, testOpts, new events.EventEmitter());
await expect(uploader.upload()).rejects.toThrow('Upload was cancelled. Multi Part upload clean up failed');
});

test('error case: finish multipart upload failed', async () => {
Expand All @@ -364,16 +326,12 @@ describe('multi part upload tests', () => {
}
});
const loggerSpy = jest.spyOn(Logger.prototype, '_log');
const uploader = new TestClass(
testParams,
testOpts,
new events.EventEmitter()
);
const uploader = new TestClass(testParams, testOpts, new events.EventEmitter());
await uploader.upload();
expect(loggerSpy).toHaveBeenCalledWith(
'ERROR',
'error happened while finishing the upload. Cancelling the multipart upload',
'error'
)
);
});
});
13 changes: 11 additions & 2 deletions packages/storage/__tests__/providers/axios-http-handler.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import axios from 'axios';
import axios, { CancelTokenSource } from 'axios';
import * as events from 'events';

import {
AxiosHttpHandler,
Expand Down Expand Up @@ -123,11 +124,19 @@ describe('AxiosHttpHandler', () => {
responseType: 'blob',
url: 'http://localhost:3000/',
onUploadProgress: expect.any(Function),
onDownloadProgress: expect.any(Function),
});

// Invoke the request's onUploadProgress function manually
lastCall.onUploadProgress({ loaded: 10, total: 100 });
expect(mockEmit).toHaveBeenLastCalledWith('sendProgress', {
expect(mockEmit).toHaveBeenLastCalledWith('sendUploadProgress', {
loaded: 10,
total: 100,
});

// Invoke the request's onDownloadProgress function manually
lastCall.onDownloadProgress({ loaded: 10, total: 100 });
expect(mockEmit).toHaveBeenLastCalledWith('sendDownloadProgress', {
loaded: 10,
total: 100,
});
Expand Down
Loading

0 comments on commit 8fe1853

Please sign in to comment.