diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d1d3233e..26997b864 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -208,6 +208,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix possible prototype pollution vector in upload/download workers - (GL #1149) Fix headers no getting copied when replicating a container - (GL #1154) Fix session cookie not getting properly clear on invalidation +- (GL #1160) Fix large uploaded file (> 5 GiB) showing wrong size in the UI ### Removed diff --git a/swift_browser_ui/upload/cryptupload.py b/swift_browser_ui/upload/cryptupload.py index 20298f1ea..1339dc993 100644 --- a/swift_browser_ui/upload/cryptupload.py +++ b/swift_browser_ui/upload/cryptupload.py @@ -82,7 +82,6 @@ def __init__( self.total_segments: int = -(self.total // -SEGMENT_SIZE) self.remainder_segment: int = self.total % SEGMENT_SIZE self.total_chunks: int = -(self.total // -CHUNK_SIZE) - self.remainder_chunk: int = self.total % CHUNK_SIZE self.remainder_chunks: int = -(self.remainder_segment // -CHUNK_SIZE) self.q_cache: typing.List[asyncio.Queue] = [ @@ -183,12 +182,14 @@ async def slice_into_queue(self, segment: int, q: asyncio.Queue): """Slice a ~5GiB segment from queue.""" seg_start = segment * SEGMENT_CHUNKS seg_end = seg_start + SEGMENT_CHUNKS - if segment == self.total_segments - 1 and self.remainder_chunks: - LOGGER.debug( - f"Using {self.remainder_chunks} as chunk amount for last segment." - ) - seg_end = seg_start + self.remainder_chunks - + if segment == self.total_segments - 1 and self.total_chunks: + # In case of the object size > SEGMENT_SIZE, + # there are some variances between calculated variables + # which can cause some missing chunks. + # Using total_chunks for last segment's chunk amount + # is more accurate than remainder_chunks + LOGGER.debug(f"Using {self.total_chunks} as chunk amount for last segment.") + seg_end = self.total_chunks LOGGER.debug(f"Consuming chunks {seg_start} through {seg_end}") LOGGER.debug(f"Waiting until first chunk in segment {segment} is available.") diff --git a/swift_browser_ui_frontend/src/common/store.js b/swift_browser_ui_frontend/src/common/store.js index 4f6a2fd43..fceb763d4 100644 --- a/swift_browser_ui_frontend/src/common/store.js +++ b/swift_browser_ui_frontend/src/common/store.js @@ -495,7 +495,6 @@ const store = createStore({ projectID, container.name, marker, signal); } - if (objects.length > 0) { objects.forEach(obj => { obj.container = container.name; @@ -535,20 +534,44 @@ const store = createStore({ owner ? owner : "", ); - // Find the segments of an object and - // update the original objects size accordingly - for (let i = 0; i < newObjects.length; i++) { - if (segment_objects[i] && newObjects[i].bytes === 0) { - newObjects[i].bytes = segment_objects[i].bytes; - } - else if (!segment_objects[i] && state.isLoaderVisible) { + if (newObjects.length === segment_objects.length) { + // Find the segments of an object and + // update the original objects size accordingly + for (let i = 0; i < newObjects.length; i++) { + if (segment_objects[i] && newObjects[i].bytes === 0) { + newObjects[i].bytes = segment_objects[i].bytes; + } + else if (!segment_objects[i] && state.isLoaderVisible) { /* When cancelling the upload of large amount of files or big files sizes, the original folder could have more objects than segment folder which results in the last updated file has size 0 (segment folder doesn't have it) Therefore it's better to remove that file. */ - newObjects.splice(i, 1); + newObjects.splice(i, 1); + } + } + } else if (segment_objects.length > newObjects.length) { + /* + For uploaded objects having size > 5GiB, + their equivalent segment_objects are split off into + multiple segments (~5GiB/each) of which combined size + in total is roughly equal to the original uploaded file. + The regular objects are not separated into segments, hence + the number of segment_objects > regular objects. + */ + for (let i = 0; i < newObjects.length; i++) { + // Filter equivalent segment objects + const filteredSegmentObjects = segment_objects.filter(obj => + obj.name.includes(newObjects[i].name), + ); + // Calculate total size of equivalent segment objects + const totalSegmentSize = filteredSegmentObjects.reduce( + (totalSize, obj) => + obj.name.includes(newObjects[i].name) ? + totalSize + obj.bytes : null, 0, + ); + newObjects[i].bytes = totalSegmentSize; } } updateContainerLastmodified(projectID, container, newObjects); diff --git a/swift_browser_ui_frontend/src/components/DeleteModal.vue b/swift_browser_ui_frontend/src/components/DeleteModal.vue index f14eef651..75afeec7e 100644 --- a/swift_browser_ui_frontend/src/components/DeleteModal.vue +++ b/swift_browser_ui_frontend/src/components/DeleteModal.vue @@ -121,9 +121,11 @@ export default { this.owner ? this.owner : this.projectID, segment_container.name, ); - const segment_obj = segment_objects.filter(obj => - obj.name.includes(`${object.name}/`))[0]; - if (segment_obj) segments_to_remove.push(segment_obj.name); + const filtered_segment_objects = segment_objects.filter(obj => + obj.name.includes(`${object.name}/`)); + for (const segment_obj of filtered_segment_objects) { + segments_to_remove.push(segment_obj.name); + } } } else { //flag if user is trying to delete a subfolder