use chunk filters to store dedup

vatesfr · Jul 24, 2023 · 725b262 · 725b262
1 parent 50f378e
commit 725b262
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 9 deletions.
diff --git a/@xen-orchestra/backups/RemoteAdapter.mjs b/@xen-orchestra/backups/RemoteAdapter.mjs
@@ -663,7 +663,7 @@ export class RemoteAdapter {
   async writeVhd(path, input, { checksum = true, validator = noop, writeBlockConcurrency, dedup = false } = {}) {
     const handler = this._handler
     if (this.useVhdDirectory()) {
-      const dataPath = `${dirname(path)}/data/${uuidv4()}${dedup ? '.dedup' : ''}.vhd`
+      const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
       const size = await createVhdDirectoryFromStream(handler, dataPath, input, {
         concurrency: writeBlockConcurrency,
         compression: this.#getCompressionType(),

diff --git a/packages/vhd-lib/Vhd/VhdAbstract.integ.js b/packages/vhd-lib/Vhd/VhdAbstract.integ.js
@@ -104,7 +104,7 @@ describe('VhdAbstract', async () => {
   it('renames and unlink a VhdDirectory', async () => {
     const initalSize = 4
     const vhdDirectory = `${tempDir}/randomfile.dir`
-    await createRandomVhdDirectory(vhdDirectory, initalSize)
+    await createRandomVhdDirectory(vhdDirectory, initalSize, { dedup: true })
 
     await Disposable.use(async function* () {
       const handler = yield getSyncedHandler({ url: 'file:///' })
@@ -123,8 +123,8 @@ describe('VhdAbstract', async () => {
 
   it('unlinks a deduplicated VhdDirectory', async () => {
     const initalSize = 4
-    const vhdDirectory = `${tempDir}/random.dedup.vhd`
-    await createRandomVhdDirectory(vhdDirectory, initalSize)
+    const vhdDirectory = `${tempDir}/random.vhd`
+    await createRandomVhdDirectory(vhdDirectory, initalSize, { dedup: true })
 
     await Disposable.use(async function* () {
       const handler = yield getSyncedHandler({ url: 'file:///' })

diff --git a/packages/vhd-lib/Vhd/VhdAbstract.js b/packages/vhd-lib/Vhd/VhdAbstract.js
@@ -206,8 +206,14 @@ exports.VhdAbstract = class VhdAbstract {
       await handler.unlink(resolved)
     } catch (err) {
       if (err.code === 'EISDIR') {
-        const dedup = resolved.endsWith('.dedup.vhd')
-        await handler.rmtree(resolved, { dedup })
+        // @todo : should we open it ?
+        const chunkFilters = await handler.readFile(path + '/chunk-filters.json').then(JSON.parse, error => {
+          if (error.code === 'ENOENT') {
+            return []
+          }
+          throw error
+        })
+        await handler.rmtree(resolved, { dedup: chunkFilters[1] === true })
       } else {
         throw err
       }

diff --git a/packages/vhd-lib/Vhd/VhdDirectory.js b/packages/vhd-lib/Vhd/VhdDirectory.js
@@ -19,6 +19,7 @@ const NULL_COMPRESSOR = {
 }
 
 const COMPRESSORS = {
+  none: NULL_COMPRESSOR,
   gzip: {
     compress: (
       gzip => buffer =>

diff --git a/packages/vhd-lib/tests/utils.js b/packages/vhd-lib/tests/utils.js
@@ -62,7 +62,7 @@ exports.recoverRawContent = async function recoverRawContent(vhdName, rawName, o
 }
 
 // @ todo how can I call vhd-cli copy from here
-async function convertToVhdDirectory(rawFileName, vhdFileName, path) {
+async function convertToVhdDirectory(rawFileName, vhdFileName, path, { dedup }) {
   fs.mkdirp(path)
 
   const srcVhd = await fs.open(vhdFileName, 'r')
@@ -95,15 +95,17 @@ async function convertToVhdDirectory(rawFileName, vhdFileName, path) {
     await fs.read(srcRaw, blockData, 0, blockData.length, offset)
     await fs.writeFile(path + '/blocks/0/' + i, Buffer.concat([bitmap, blockData]))
   }
+
+  await fs.writeFile(path + '/chunk-filters.json', JSON.stringify(['none', dedup]))
   await fs.close(srcRaw)
 }
 exports.convertToVhdDirectory = convertToVhdDirectory
 
-exports.createRandomVhdDirectory = async function createRandomVhdDirectory(path, sizeMB) {
+exports.createRandomVhdDirectory = async function createRandomVhdDirectory(path, sizeMB, { dedup = false } = {}) {
   fs.mkdirp(path)
   const rawFileName = `${path}/temp.raw`
   await createRandomFile(rawFileName, sizeMB)
   const vhdFileName = `${path}/temp.vhd`
   await convertFromRawToVhd(rawFileName, vhdFileName)
-  await convertToVhdDirectory(rawFileName, vhdFileName, path)
+  await convertToVhdDirectory(rawFileName, vhdFileName, path, { dedup })
 }