diff --git a/@xen-orchestra/backups/RemoteAdapter.js b/@xen-orchestra/backups/RemoteAdapter.js index 9d5e67919aa..f2d9f5a64a2 100644 --- a/@xen-orchestra/backups/RemoteAdapter.js +++ b/@xen-orchestra/backups/RemoteAdapter.js @@ -277,6 +277,12 @@ class RemoteAdapter { delta !== undefined && this.deleteDeltaVmBackups(delta), full !== undefined && this.deleteFullVmBackups(full), ]) + + const dirs = new Set(files.map(file => dirname(file))) + for (const dir of dirs) { + // don't merge in main process, unused VHDs will be merged in the next backup run + await this.cleanVm(dir, { remove: true, onLog: warn }) + } } #getCompressionType() { diff --git a/@xen-orchestra/fs/src/s3.js b/@xen-orchestra/fs/src/s3.js index 30dde87bbfe..a97fd521177 100644 --- a/@xen-orchestra/fs/src/s3.js +++ b/@xen-orchestra/fs/src/s3.js @@ -195,7 +195,7 @@ export default class S3Handler extends RemoteHandlerAbstract { Delimiter: '/', // will only return path until delimiters }) - if (result.isTruncated) { + if (result.IsTruncated) { const error = new Error('more than 1000 objects, unsupported in this implementation') error.dir = dir throw error @@ -285,7 +285,7 @@ export default class S3Handler extends RemoteHandlerAbstract { Prefix: this._dir + path + '/', ContinuationToken: NextContinuationToken, }) - NextContinuationToken = result.isTruncated ? result.NextContinuationToken : undefined + NextContinuationToken = result.IsTruncated ? result.NextContinuationToken : undefined await asyncEach( result.Contents, async ({ Key }) => { diff --git a/CHANGELOG.md b/CHANGELOG.md index ecd87713c84..1af6af64421 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ ### Bug fixes - [Dashboard/Health] Fix `error has occured` when a pool has no default SR -- [Delta Backup] Fix unnecessary full backup when not using S3 [Forum #5371](https://xcp-ng.org/forum/topic/5371/delta-backup-changes-in-5-66)d (PR [#6070](https://github.com/vatesfr/xen-orchestra/pull/6070)) +- [Delta Backup] Fix unnecessary full backup when not using S3 [Forum#5371](https://xcp-ng.org/forum/topic/5371/delta-backup-changes-in-5-66) (PR [#6070](https://github.com/vatesfr/xen-orchestra/pull/6070)) - [Backup] Fix incorrect warnings `incorrect size [...] instead of undefined` ### Released packages @@ -47,8 +47,8 @@ ### Bug fixes - [Tables/actions] Fix collapsed actions being clickable despite being disabled (PR [#6023](https://github.com/vatesfr/xen-orchestra/pull/6023)) -- [Backup] Remove incorrect size warning following a merge [Forum #5727](https://xcp-ng.org/forum/topic/4769/warnings-showing-in-system-logs-following-each-backup-job/4) (PR [#6010](https://github.com/vatesfr/xen-orchestra/pull/6010)) -- [Delta Backup] Preserve UEFI boot parameters [#6054](https://github.com/vatesfr/xen-orchestra/issues/6054) [Forum #5319](https://xcp-ng.org/forum/topic/5319/bug-uefi-boot-parameters-not-preserved-with-delta-backups) +- [Backup] Remove incorrect size warning following a merge [Forum#5727](https://xcp-ng.org/forum/topic/4769/warnings-showing-in-system-logs-following-each-backup-job/4) (PR [#6010](https://github.com/vatesfr/xen-orchestra/pull/6010)) +- [Delta Backup] Preserve UEFI boot parameters [#6054](https://github.com/vatesfr/xen-orchestra/issues/6054) [Forum#5319](https://xcp-ng.org/forum/topic/5319/bug-uefi-boot-parameters-not-preserved-with-delta-backups) ### Released packages @@ -83,9 +83,9 @@ ### Bug fixes -- [Backup] Fix `handler.rmTree` is not a function (Forum [5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) PR [#6041](https://github.com/vatesfr/xen-orchestra/pull/6041) ) -- [Backup] Fix `EEXIST` in logs when multiple merge tasks are created at the same time ([Forum #5301](https://xcp-ng.org/forum/topic/5301/warnings-errors-in-journalctl)) -- [Backup] Fix missing backup on restore (Forum [5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) (PR [#6048](https://github.com/vatesfr/xen-orchestra/pull/6048)) +- [Backup] Fix `handler.rmTree` is not a function [Forum#5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) (PR [#6041](https://github.com/vatesfr/xen-orchestra/pull/6041)) +- [Backup] Fix `EEXIST` in logs when multiple merge tasks are created at the same time [Forum#5301](https://xcp-ng.org/forum/topic/5301/warnings-errors-in-journalctl) +- [Backup] Fix missing backup on restore [Forum#5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) (PR [#6048](https://github.com/vatesfr/xen-orchestra/pull/6048)) ### Released packages @@ -99,10 +99,10 @@ ### Bug fixes -- [Delta Backup Restoration] Fix assertion error [Forum #5257](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/16) -- [Delta Backup Restoration] `TypeError: this disposable has already been disposed` [Forum #5257](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/20) -- [Backups] Fix: `Error: Chaining alias is forbidden xo-vm-backups/..alias.vhd to xo-vm-backups/....alias.vhd` when backuping a file to s3 [Forum #5226](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it) -- [Delta Backup Restoration] `VDI_IO_ERROR(Device I/O errors)` [Forum #5727](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/4) (PR [#6031](https://github.com/vatesfr/xen-orchestra/pull/6031)) +- [Delta Backup Restoration] Fix assertion error [Forum#5257](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/16) +- [Delta Backup Restoration] `TypeError: this disposable has already been disposed` [Forum#5257](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/20) +- [Backups] Fix: `Error: Chaining alias is forbidden xo-vm-backups/..alias.vhd to xo-vm-backups/....alias.vhd` when backuping a file to s3 [Forum#5226](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it) +- [Delta Backup Restoration] `VDI_IO_ERROR(Device I/O errors)` [Forum#5727](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/4) (PR [#6031](https://github.com/vatesfr/xen-orchestra/pull/6031)) - [Delta Backup] Fix `Cannot read property 'uuid' of undefined` when a VDI has been removed from a backed up VM (PR [#6034](https://github.com/vatesfr/xen-orchestra/pull/6034)) ### Released packages @@ -340,7 +340,7 @@ ### Bug fixes -- [SDN Controller] Private network creation failure when the tunnels were created on different devices [Forum #4620](https://xcp-ng.org/forum/topic/4620/no-pif-found-in-center) (PR [#5793](https://github.com/vatesfr/xen-orchestra/pull/5793)) +- [SDN Controller] Private network creation failure when the tunnels were created on different devices [Forum#4620](https://xcp-ng.org/forum/topic/4620/no-pif-found-in-center) (PR [#5793](https://github.com/vatesfr/xen-orchestra/pull/5793)) ### Released packages diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 68ca3133770..8f22626ad5b 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -7,12 +7,11 @@ > Users must be able to say: “Nice enhancement, I'm eager to test it” -- Limit number of concurrent VM migrations per pool to `3` [#6065](https://github.com/vatesfr/xen-orchestra/issues/6065) (PR [#6076](https://github.com/vatesfr/xen-orchestra/pull/6076)) - Can be changed in `xo-server`'s configuration file: `xapiOptions.vmMigrationConcurrency` -- [Proxy] Now ships a reverse proxy [PR#6072](https://github.com/vatesfr/xen-orchestra/pull/6072) -- [Delta Backup] When using S3 remote, retry uploading VHD parts on Internal Error to support [Blackblaze](https://www.backblaze.com/b2/docs/calling.html#error_handling) (PR [#6086](https://github.com/vatesfr/xen-orchestra/issues/6086)) (Forum [5397](https://xcp-ng.org/forum/topic/5397/delta-backups-failing-aws-s3-uploadpartcopy-cpu-too-busy/5)) -- [Backup] Add sanity check of aliases on S3 remotes (PR [6043](https://github.com/vatesfr/xen-orchestra/pull/6043)) -- [Export/Disks] Allow the export of disks in VMDK format(PR [#5982](https://github.com/vatesfr/xen-orchestra/pull/5982)) +- Limit number of concurrent VM migrations per pool to `3`. Can be changed in `xo-server`'s configuration file: `xapiOptions.vmMigrationConcurrency` [#6065](https://github.com/vatesfr/xen-orchestra/issues/6065) (PR [#6076](https://github.com/vatesfr/xen-orchestra/pull/6076)) +- [Proxy] Now ships a reverse proxy (PR [#6072](https://github.com/vatesfr/xen-orchestra/pull/6072)) +- [Delta Backup] When using S3 remote, retry uploading VHD parts on Internal Error to support [Blackblaze](https://www.backblaze.com/b2/docs/calling.html#error_handling) [Forum#5397](https://xcp-ng.org/forum/topic/5397/delta-backups-failing-aws-s3-uploadpartcopy-cpu-too-busy/5) (PR [#6086](https://github.com/vatesfr/xen-orchestra/issues/6086)) +- [Backup] Add sanity check of aliases on S3 remotes (PR [#6043](https://github.com/vatesfr/xen-orchestra/pull/6043)) +- [Export/Disks] Allow the export of disks in VMDK format (PR [#5982](https://github.com/vatesfr/xen-orchestra/pull/5982)) - [Rolling Pool Update] Automatically pause load balancer plugin during the update [#5711](https://github.com/vatesfr/xen-orchestra/issues/5711) - [Backup] Speedup merge and cleanup speed for S3 backup by a factor 10 (PR [#6100](https://github.com/vatesfr/xen-orchestra/pull/6100)) ### Bug fixes @@ -21,6 +20,10 @@ - [Backup] Detect and clear orphan merge states, fix `ENOENT` errors (PR [#6087](https://github.com/vatesfr/xen-orchestra/pull/6087)) - [Backup] Ensure merges are also executed after backup on S3, maintaining the size of the VHD chain under control [Forum#45743](https://xcp-ng.org/forum/post/45743) (PR [#6095](https://github.com/vatesfr/xen-orchestra/pull/6095)) +- [Backup] Delete backups immediately instead of waiting for the next backup (PR [#6081](https://github.com/vatesfr/xen-orchestra/pull/6081)) +- [Backup] Delete S3 backups completely, even if there are more than 1000 files (PR [#6103](https://github.com/vatesfr/xen-orchestra/pull/6103)) +- [Backup] Fix merge resuming (PR [#6099](https://github.com/vatesfr/xen-orchestra/pull/6099)) +- [Plugin/Audit] Fix `key cannot be 'null' or 'undefined'` error when no audit log in the database [#6040](https://github.com/vatesfr/xen-orchestra/issues/6040) (PR [#6071](https://github.com/vatesfr/xen-orchestra/pull/6071)) ### Packages to release @@ -45,5 +48,6 @@ - @xen-orchestra/backups minor - @xen-orchestra/backups-cli minor - @xen-orchestra/proxy minor +- xo-server-audit patch - xo-server minor - xo-web minor diff --git a/packages/vhd-lib/merge.js b/packages/vhd-lib/merge.js index 5e7083c27c1..c12054d663b 100644 --- a/packages/vhd-lib/merge.js +++ b/packages/vhd-lib/merge.js @@ -38,22 +38,15 @@ module.exports = limitConcurrency(2)(async function merge( const mergeStatePath = dirname(parentPath) + '/' + '.' + basename(parentPath) + '.merge.json' return await Disposable.use(async function* () { - let mergeState = await parentHandler - .readFile(mergeStatePath) - .then(content => { - const state = JSON.parse(content) - - // ensure the correct merge will be continued - assert.strictEqual(parentVhd.header.checksum, state.parent.header) - assert.strictEqual(childVhd.header.checksum, state.child.header) - - return state - }) - .catch(error => { - if (error.code !== 'ENOENT') { - warn('problem while checking the merge state', { error }) - } - }) + let mergeState + try { + const mergeStateContent = await parentHandler.readFile(mergeStatePath) + mergeState = JSON.parse(mergeStateContent) + } catch (error) { + if (error.code !== 'ENOENT') { + warn('problem while checking the merge state', { error }) + } + } // during merging, the end footer of the parent can be overwritten by new blocks // we should use it as a way to check vhd health @@ -70,6 +63,9 @@ module.exports = limitConcurrency(2)(async function merge( const parentDiskType = parentVhd.footer.diskType assert(parentDiskType === DISK_TYPES.DIFFERENCING || parentDiskType === DISK_TYPES.DYNAMIC) assert.strictEqual(childVhd.footer.diskType, DISK_TYPES.DIFFERENCING) + } else { + assert.strictEqual(parentVhd.header.checksum, mergeState.parent.header) + assert.strictEqual(childVhd.header.checksum, mergeState.child.header) } // Read allocation table of child/parent. diff --git a/packages/xo-server-audit/src/index.js b/packages/xo-server-audit/src/index.js index babb8afd18c..acd1edba0f6 100644 --- a/packages/xo-server-audit/src/index.js +++ b/packages/xo-server-audit/src/index.js @@ -396,12 +396,12 @@ class AuditXoPlugin { } async _generateFingerprint(props) { - const { oldest = NULL_ID, newest = await this._storage.getLastId() } = props + const { oldest = NULL_ID, newest = (await this._storage.getLastId()) ?? NULL_ID } = props try { return { fingerprint: `${oldest}|${newest}`, newest, - nValid: await this._checkIntegrity({ oldest, newest }), + nValid: newest !== NULL_ID ? await this._checkIntegrity({ oldest, newest }) : 0, oldest, } } catch (error) {