Create FileReadableStream class to stream large files for d&d uploads and add progress notification

src/vs/workbench/contrib/files/browser/views/explorerViewer.ts

@@ -50,7 +50,7 @@ import { Emitter, Event, EventMultiplexer } from 'vs/base/common/event';
 import { ITreeCompressionDelegate } from 'vs/base/browser/ui/tree/asyncDataTree';
 import { ICompressibleTreeRenderer } from 'vs/base/browser/ui/tree/objectTree';
 import { ICompressedTreeNode } from 'vs/base/browser/ui/tree/compressedObjectTreeModel';
-import { VSBuffer } from 'vs/base/common/buffer';
+import { VSBuffer, VSBufferReadableStream } from 'vs/base/common/buffer';
 import { ILabelService } from 'vs/platform/label/common/label';
 import { isNumber } from 'vs/base/common/types';
 import { domEvent } from 'vs/base/browser/event';
@@ -970,9 +970,20 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
 			entries.push(item.webkitGetAsEntry());
 		}
 
+		// Calculate files and bytes to upload
+		// const stats = { totalFiles: 0, totalSize: 0 };
+		// for (let entry of entries) {
+		// 	const itemStats = await this.scanFileEntity(entry);
+		// 	stats.totalFiles += itemStats.totalFiles;
+		// 	stats.totalSize += itemStats.totalSize;
+		// }
+
+		// Start upload
 		const results: { isFile: boolean, resource: URI }[] = [];
 		for (let entry of entries) {
-			const result = await this.doUploadWebFileEntry(entry, target.resource, target);
+			const result = await this.doUploadWebFileEntry(entry, target.resource, target, (bytes: number) => {
+				// update progress
+			});
 
 			if (result) {
 				results.push(result);
@@ -985,7 +996,37 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
 		}
 	}
 
-	private async doUploadWebFileEntry(entry: IWebkitDataTransferItemEntry, parentResource: URI, target: ExplorerItem | undefined): Promise<{ isFile: boolean, resource: URI } | undefined> {
+	private async scanFileEntity(entry: any): Promise<{ totalFiles: number, totalSize: number }> {
+		let stats = {
+			totalFiles: 0,
+			totalSize: 0
+		};
+		if (entry.isFile) {
+			const file = await new Promise<File>((resolve, reject) => entry.file(resolve, reject));
+			stats.totalFiles = 1;
+			stats.totalSize = file.size;
+		} else if (entry.isDirectory) {
+			// Recursive scan files in this directory
+			const dirReader = entry.createReader();
+			const childEntries = await new Promise<any[]>((resolve, reject) => {
+				dirReader.readEntries(resolve, reject);
+			});
+			for (let childEntry of childEntries) {
+				if (childEntry.isFile) {
+					const file = await new Promise<File>((resolve, reject) => childEntry.file(resolve, reject));
+					stats.totalFiles++;
+					stats.totalSize += file.size;
+				} else if (childEntry.isDirectory) {
+					let childStats = await this.scanFileEntity(childEntry);
+					stats.totalFiles += childStats.totalFiles;
+					stats.totalSize += childStats.totalSize;
+				}
+			}
+		}
+		return stats;
+	}
+
+	private async doUploadWebFileEntry(entry: IWebkitDataTransferItemEntry, parentResource: URI, target: ExplorerItem | undefined, progressCallback: (bytes: number) => void): Promise<{ isFile: boolean, resource: URI } | undefined> {
 		if (!entry.name || !entry.isFile && !entry.isDirectory) {
 			return undefined;
 		}
@@ -1003,25 +1044,12 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
 		// Handle file upload
 		if (entry.isFile) {
 			const file = await new Promise<File>((resolve, reject) => entry.file(resolve, reject));
-			await new Promise<void>((resolve, reject) => {
-				const reader = new FileReader();
-				reader.onload = async event => {
-					try {
-						if (event.target?.result instanceof ArrayBuffer) {
-							await this.fileService.writeFile(resource, VSBuffer.wrap(new Uint8Array(event.target.result)));
-						} else {
-							throw new Error('Could not read from dropped file.');
-						}
 
-						resolve();
-					} catch (error) {
-						reject(error);
-					}
-				};
-
-				// Start reading the file to trigger `onload`
-				reader.readAsArrayBuffer(file);
+			let readableStream = new FileReadableStream(file);
+			readableStream.on('progress', bytes => {
+				progressCallback(bytes);
 			});
+			await this.fileService.writeFile(resource, readableStream);
 
 			return { isFile: true, resource };
 		}
@@ -1038,7 +1066,7 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
 			});
 
 			for (let childEntry of childEntries) {
-				await this.doUploadWebFileEntry(childEntry, resource, folderTarget);
+				await this.doUploadWebFileEntry(childEntry, resource, folderTarget, progressCallback);
 			}
 
 			return { isFile: false, resource };
@@ -1312,3 +1340,97 @@ export class ExplorerCompressionDelegate implements ITreeCompressionDelegate<Exp
 		return stat.isRoot || !stat.isDirectory || stat instanceof NewExplorerItem || (!stat.parent || stat.parent.isRoot);
 	}
 }
+
+export class FileReadableStream implements VSBufferReadableStream {
+
+	private readonly dataCallbacks: Array<(data: VSBuffer) => void> = [];
+	private readonly errorCallbacks: Array<(err: Error) => void> = [];
+	private readonly progressCallbacks: Array<(bytes: number) => void> = [];
+	private readonly endCallbacks: Array<() => void> = [];
+	private readonly fileSize: number;
+	private readonly bufferSize: number = 1024;
+	private cursor: number = 0;
+	private paused: boolean = false;
+	private reading: boolean = false;
+
+	constructor(private file: any) {
+		this.fileSize = file.size;
+	}
+
+	public pause(): void {
+		this.paused = true;
+	}
+
+	public resume(): void {
+		this.paused = false;
+		if (!this.reading && this.cursor < this.fileSize) {
+			this._read();
+		}
+	}
+
+	public destroy(): void {
+		this.cursor = this.fileSize + 1;
+	}
+
+	private _read() {
+		if (this.reading) {
+			return;
+		}
+		this.reading = true;
+		const fileReader = new FileReader();
+		const blob = this.file.slice(this.cursor, this.cursor + this.bufferSize);
+		fileReader.onload = event => {
+			if (event.target?.result instanceof ArrayBuffer) {
+				let buffer = VSBuffer.wrap(new Uint8Array(event.target.result));
+				this.dataCallbacks.forEach(callback => callback(buffer));
+				this.progressCallbacks.forEach(callback => callback(buffer.byteLength));
+			}
+
+			this.reading = false;
+			this.cursor += this.bufferSize;
+			if (this.cursor >= this.fileSize) {
+				// End of file
+				this.endCallbacks.forEach(callback => callback());
+			} else {
+				// Read next chunk
+				if (!this.paused) {
+					this._read();
+				}
+			}
+		};
+		fileReader.onerror = () => {
+			this.reading = false;
+			let error = new Error(`Error loading file ${this.file.name}`);
+			this.errorCallbacks.forEach(callback => callback(error));
+		};
+		fileReader.readAsArrayBuffer(blob);
+	}
+
+	public on(event: 'data', callback: (data: VSBuffer) => void): void;
+	public on(event: 'error', callback: (err: Error) => void): void;
+	public on(event: 'end', callback: () => void): void;
+	public on(event: 'progress', callback: (bytes: number) => void): void;
+
+	public on(event: any, callback: any) {
+		switch (event) {
+			case 'data':
+				this.dataCallbacks.push(callback);
+				if (!this.reading) {
+					// Start reading after someone is listing
+					this._read();
+				}
+				break;
+			case 'error':
+				this.errorCallbacks.push(callback);
+				break;
+			case 'progress':
+				this.progressCallbacks.push(callback);
+				break;
+			case 'end':
+				this.endCallbacks.push(callback);
+				break;
+		}
+	}
+
+
+}