feat: immutable backups

@xen-orchestra/backups/RemoteAdapter.mjs

@@ -35,6 +35,8 @@ export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 
 export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
 
+const IMMUTABILTY_METADATA_FILENAME = '/immutability.json'
+
 const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
 
 const compareTimestamp = (a, b) => a.timestamp - b.timestamp
@@ -749,10 +751,37 @@ export class RemoteAdapter {
   }
 
   async readVmBackupMetadata(path) {
+    let json
+    let isImmutable = false
+    let remoteIsImmutable = false
+    // if the remote is immutable, check if this metadatas are also immutables
+    try {
+      // this file is not encrypted
+      await this._handler._readFile(IMMUTABILTY_METADATA_FILENAME)
+      remoteIsImmutable = true
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw error
+      }
+    }
+
+    try {
+      // this will trigger an EPERM error if the file is immutable
+      json = await this.handler.readFile(path, { flag: 'r+' })
+      // s3 handler don't respect flags
+    } catch (err) {
+      // retry without triggerring immutbaility check ,only on immutable remote
+      if (err.code === 'EPERM' && remoteIsImmutable) {
+        isImmutable = true
+        json = await this._handler.readFile(path, { flag: 'r' })
+      } else {
+        throw err
+      }
+    }
     // _filename is a private field used to compute the backup id
     //
     // it's enumerable to make it cacheable
-    const metadata = { ...JSON.parse(await this._handler.readFile(path)), _filename: path }
+    const metadata = { ...JSON.parse(json), _filename: path, isImmutable }
 
     // backups created on XenServer < 7.1 via JSON in XML-RPC transports have boolean values encoded as integers, which make them unusable with more recent XAPIs
     if (typeof metadata.vm.is_a_template === 'number') {

@xen-orchestra/backups/formatVmBackups.mjs

@@ -31,6 +31,7 @@ function formatVmBackup(backup) {
           }),
 
     id: backup.id,
+    isImmutable: backup.isImmutable,
     jobId: backup.jobId,
     mode: backup.mode,
     scheduleId: backup.scheduleId,

@xen-orchestra/immutable-backups/.USAGE.md

@@ -0,0 +1,10 @@
+### make a remote immutable
+
+launch the `xo-immutable-remote` command. The configuration is stored in the config file.
+This script must be kept running to make file immutable reliably.
+
+### make file mutable
+
+launch the `xo-lift-remote-immutability` cli. The configuration is stored in the config file .
+
+If the config file have a `liftEvery`, this script will contiue to run and check regularly if there are files to update.

@xen-orchestra/immutable-backups/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@xen-orchestra/immutable-backups/README.md

@@ -0,0 +1,31 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/immutable-backups
+
+## Usage
+
+### make a remote immutable
+
+launch the `xo-immutable-remote` command. The configuration is stored in the config file.
+This script must be kept running to make file immutable reliably.
+
+### make file mutable
+
+launch the `xo-lift-remote-immutability` cli. The configuration is stored in the config file .
+
+If the config file have a `liftEvery`, this script will contiue to run and check regularly if there are files to update.
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@xen-orchestra/immutable-backups/_cleanXoCache.mjs

@@ -0,0 +1,10 @@
+import fs from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+import isBackupMetadata from './isBackupMetadata.mjs'
+
+export default async path => {
+  if (isBackupMetadata(path)) {
+    // snipe vm metadata cache to force XO to update it
+    await fs.unlink(join(dirname(path), 'cache.json.gz'))
+  }
+}

@xen-orchestra/immutable-backups/_isInVhdDirectory.mjs

@@ -0,0 +1,4 @@
+import { dirname } from 'node:path'
+
+// check if we are handling file directly under a vhd directory ( bat, headr, footer,..)
+export default path => dirname(path).endsWith('.vhd')

@xen-orchestra/immutable-backups/_loadConfig.mjs

@@ -0,0 +1,46 @@
+import { load } from 'app-conf'
+import { homedir } from 'os'
+import { join } from 'node:path'
+import ms from 'ms'
+
+const APP_NAME = 'xo-immutable-backups'
+const APP_DIR = new URL('.', import.meta.url).pathname
+
+export default async function loadConfig() {
+  const config = await load(APP_NAME, {
+    appDir: APP_DIR,
+    ignoreUnknownFormats: true,
+  })
+  if (config.remotes === undefined || config.remotes?.length < 1) {
+    throw new Error(
+      'No remotes are configured in the config file, please add at least one [remotes.<remoteid>]  with a root property pointing to the absolute path of the remote to watch'
+    )
+  }
+  if (config.liftEvery) {
+    config.liftEvery = ms(config.liftEvery)
+  }
+  for (const [remoteId, { indexPath, immutabilityDuration, root }] of Object.entries(config.remotes)) {
+    if (!root) {
+      throw new Error(
+        `Remote ${remoteId} don't have a root property,containing the absolute path to the root of a backup repository `
+      )
+    }
+    if (!immutabilityDuration) {
+      throw new Error(
+        `Remote ${remoteId} don't have a immutabilityDuration property to indicate the minimal duration the backups should be  protected by immutability `
+      )
+    }
+    if (ms(immutabilityDuration) < ms('1d')) {
+      throw new Error(
+        `Remote ${remoteId} immutability duration is smaller than the minimum allowed (1d), current : ${immutabilityDuration}`
+      )
+    }
+    if (!indexPath) {
+      const basePath = indexPath ?? process.env.XDG_DATA_HOME ?? join(homedir(), '.local', 'share')
+      const immutabilityIndexPath = join(basePath, APP_NAME, remoteId)
+      config.remotes[remoteId].indexPath = immutabilityIndexPath
+    }
+    config.remotes[remoteId].immutabilityDuration = ms(immutabilityDuration)
+  }
+  return config
+}

@xen-orchestra/immutable-backups/config.toml

@@ -0,0 +1,14 @@
+
+# how often does the lift immutability script will run to check if 
+# some files need to be made mutable 
+liftEvery =  1h
+
+# you can add as many remote as you want, if you change the id ( here : remote1)
+#[remotes.remote1]
+#root = "/mnt/ssd/vhdblock/" # the absolute path of the root of the backup repository
+#immutabilityDuration = 7d # mandatory
+# optional, default value is false will scan and update the index on start, can be expensive
+#rebuildIndexOnStart = true 
+
+# the index path is optional, default in XDG_DATA_HOME, or if this is not set, in ~/.local/share
+#indexPath = "/var/lib/" # will add automatically the application name immutable-backup 

@xen-orchestra/immutable-backups/directory.mjs

@@ -0,0 +1,21 @@
+import execa from 'execa'
+import { unindexFile, indexFile } from './fileIndex.mjs'
+
+export async function makeImmutable(dirPath, immutabilityCachePath) {
+  if (immutabilityCachePath) {
+    await indexFile(dirPath, immutabilityCachePath)
+  }
+  await execa('chattr', ['+i', '-R', dirPath])
+}
+
+export async function liftImmutability(dirPath, immutabilityCachePath) {
+  if (immutabilityCachePath) {
+    await unindexFile(dirPath, immutabilityCachePath)
+  }
+  await execa('chattr', ['-i', '-R', dirPath])
+}
+
+export async function isImmutable(path) {
+  const { stdout } = await execa('lsattr', ['-d', path])
+  return stdout[4] === 'i'
+}

@xen-orchestra/immutable-backups/directory.spec.mjs

@@ -0,0 +1,31 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { tmpdir } from 'node:os'
+import * as Directory from './directory.mjs'
+import { rimraf } from 'rimraf'
+
+describe('immutable-backups/file', async () => {
+  it('really lock a directory', async () => {
+    const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests'))
+    const dataDir = path.join(dir, 'data')
+    await fs.mkdir(dataDir)
+    const immutDir = path.join(dir, '.immutable')
+    const filePath = path.join(dataDir, 'test')
+    await fs.writeFile(filePath, 'data')
+    await Directory.makeImmutable(dataDir, immutDir)
+    await assert.rejects(() => fs.writeFile(filePath, 'data'))
+    await assert.rejects(() => fs.appendFile(filePath, 'data'))
+    await assert.rejects(() => fs.unlink(filePath))
+    await assert.rejects(() => fs.rename(filePath, filePath + 'copy'))
+    await assert.rejects(() => fs.writeFile(path.join(dataDir, 'test2'), 'data'))
+    await assert.rejects(() => fs.rename(dataDir, dataDir + 'copy'))
+    await Directory.liftImmutability(dataDir, immutDir)
+    await fs.writeFile(filePath, 'data')
+    await fs.appendFile(filePath, 'data')
+    await fs.unlink(filePath)
+    await fs.rename(dataDir, dataDir + 'copy')
+    await rimraf(dir)
+  })
+})

@xen-orchestra/immutable-backups/doc.md

@@ -0,0 +1,114 @@
+# Imutability
+
+the goal is to make a remote that XO can write, but not modify during the immutability duration set on the remote. That way, it's not possible for XO to delete or encrypt any backup during this period. It protects your backup agains ransomware, at least as long as the attacker does not have a root access to the remote server.
+
+We target `governance` type of immutability, **the local root account of the remote server will be able to lift immutability**.
+
+We use the file system capabilities, they are tested on the protection process start.
+
+It is compatible with encryption at rest made by XO.
+
+## Prerequisites
+
+The commands must be run as root on the remote, or by a user with the `CAP_LINUX_IMMUTABLE` capability . On start, the protect process writes into the remote `imutability.json` file its status and the immutability duration.
+
+the `chattr` and `lsattr` should be installed on the system
+
+## Configuring
+
+this package uses app-conf to store its config. The application name is `xo-immutable-backup`. A sample config file is provided in this package.
+
+## Making a file immutable
+
+when marking a file or a folder immutable, it create an alias file in the `<indexPath>/<DayOfFileCreation>/<sha256(fullpath)>`.
+
+`indexPath` can be defined in the config file, otherwise `XDG_HOME` is used. If not available it goes to `~/.local/share`
+
+This index is used when lifting the immutability of the remote, it will only look at the old enough `<indexPath>/<DayOfFileCreation>/` folders.
+
+## Real time protecting
+
+On start, the watcher will create the index if it does not exists.
+It will also do a checkup to ensure immutability could work on this remote and handle the easiest issues.
+
+The watching process depends on the backup type, since we don't want to make temporary files and cache immutable.
+
+It won't protect files during upload, only when the files have been completly written on disk. Real time, in this case, means "protecting critical files as soon as possible after they are uploaded"
+
+This can be alleviated by :
+
+- Coupling immutability with encryption to ensure the file is not modified
+- Making health check to ensure the data are exactly as the snapshot data
+
+List of protected files :
+
+```js
+const PATHS = [
+  // xo configuration backupq
+  'xo-config-backups/*/*/data',
+  'xo-config-backups/*/*/data.json',
+  'xo-config-backups/*/*/metadata.json',
+  // pool backupq
+  'xo-pool-metadata-backups/*/metadata.json',
+  'xo-pool-metadata-backups/*/data',
+  // vm backups , xo-vm-backups/<vmuuid>/
+  'xo-vm-backups/*/*.json',
+  'xo-vm-backups/*/*.xva',
+  'xo-vm-backups/*/*.xva.checksum',
+  // xo-vm-backups/<vmuuid>/vdis/<jobid>/<vdiUuid>
+  'xo-vm-backups/*/vdis/*/*/*.vhd', // can be an alias or a vhd file
+  // for vhd directory :
+  'xo-vm-backups/*/vdis/*/*/data/*.vhd/bat',
+  'xo-vm-backups/*/vdis/*/*/data/*.vhd/header',
+  'xo-vm-backups/*/vdis/*/*/data/*.vhd/footer',
+]
+```
+
+## Releasing protection on old enough files on a remote
+
+the watcher will periodically check if some file must by unlocked
+
+## Troubleshooting
+
+### some files are still locked
+
+add the `rebuildIndexOnStart` option to the config file
+
+### make remote fully mutable again
+
+- Update the immutability setting with a 0 duration
+- launch the `liftProtection` cli.
+- remove the `protectRemotes` service
+
+### increasing the immutability duration
+
+this will prolong immutable file, but won't protect files that are already out of immutability
+
+### reducing the immutability duration
+
+change the setting, and launch the `liftProtection` cli , or wait for next planed execution
+
+### why are my incremental backups not marked as protected in XO ?
+
+are not marked as protected in XO ?
+
+For incremental backups to be marked as protected in XO, the entire chain must be under protection. To ensure at least 7 days of backups are protected, you need to set the immutability duration and retention at 14 days, the full backup interval at 7 days
+
+That means that if the last backup chain is complete ( 7 backup ) it is completely under protection, and if not, the precedent chain is also under protection. K are key backups, and are delta
+
+```
+Kd Kdddddd Kdddddd K #  8 backups protected, 2 chains
+K Kdddddd Kdddddd Kd #  9 backups protected, 2 chains
+ Kdddddd Kdddddd Kdd # 10 backups protected, 2 chains
+ Kddddd Kdddddd Kddd # 11 backups protected, 2 chains
+ Kdddd Kdddddd Kdddd # 12 backups protected, 2 chains
+ Kddd Kdddddd Kddddd # 13 backups protected, 2 chains
+ Kdd Kdddddd Kdddddd #  7 backups protected, 1 chain since precedent full is now mutable
+Kd Kdddddd Kdddddd K #  8 backups protected, 2 chains
+```
+
+### Why doesn't the protect process start ?
+
+- it should be run as root or by a user with the `CAP_LINUX_IMMUTABLE` capability
+- the underlying file system should support immutability, especially the `chattr` and `lsattr` command
+- logs are in journalctl

@xen-orchestra/immutable-backups/file.mjs

@@ -0,0 +1,24 @@
+import execa from 'execa'
+import { unindexFile, indexFile } from './fileIndex.mjs'
+
+// this work only on linux like systems
+// this could work on windows : https://4sysops.com/archives/set-and-remove-the-read-only-file-attribute-with-powershell/
+
+export async function makeImmutable(path, immutabilityCachePath) {
+  if (immutabilityCachePath) {
+    await indexFile(path, immutabilityCachePath)
+  }
+  await execa('chattr', ['+i', path])
+}
+
+export async function liftImmutability(filePath, immutabilityCachePath) {
+  if (immutabilityCachePath) {
+    await unindexFile(filePath, immutabilityCachePath)
+  }
+  await execa('chattr', ['-i', filePath])
+}
+
+export async function isImmutable(path) {
+  const { stdout } = await execa('lsattr', ['-d', path])
+  return stdout[4] === 'i'
+}