Add site cloning API

.github/workflows/tests.yaml

@@ -4,13 +4,15 @@ on: [ push, pull_request ]
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     strategy:
       matrix:
         node-version: [19.9.x]
 
     steps:
+      - name: Install wget2 for cloning
+        run: sudo apt-get install -y wget2
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3

ansible/roles/distributed_press/tasks/main.yml

@@ -25,6 +25,13 @@
     groups: www-data
     append: yes
 
+- name: "Install wget2"
+  apt:
+    pkg:
+      - wget2
+    state: latest
+    update_cache: true
+
 - name: "Install git and ufw"
   apt:
     pkg:

api/publisher.ts

@@ -21,7 +21,7 @@ export const publisherRoutes = (_cfg: APIConfig, store: StoreI) => async (server
     },
     preHandler: server.auth([server.verifyAdmin])
   }, async (request, reply) => {
-    return await reply.send(await store.publisher.create(request.body))
+    return reply.send(await store.publisher.create(request.body))
   })
 
   server.post<{
@@ -44,7 +44,7 @@ export const publisherRoutes = (_cfg: APIConfig, store: StoreI) => async (server
     })
     const signed = await reply.jwtSign(newToken)
 
-    return await reply.send(signed)
+    return reply.send(signed)
   })
 
   server.get<{
@@ -67,7 +67,7 @@ export const publisherRoutes = (_cfg: APIConfig, store: StoreI) => async (server
     preHandler: server.auth([server.verifyAdmin, server.verifyPublisher])
   }, async (request, reply) => {
     const { id } = request.params
-    return await reply.send(await store.publisher.get(id))
+    return reply.send(await store.publisher.get(id))
   })
 
   server.get<{ Reply: string[] }>('/publisher', {
@@ -78,7 +78,7 @@ export const publisherRoutes = (_cfg: APIConfig, store: StoreI) => async (server
     },
     preHandler: server.auth([server.verifyAdmin])
   }, async (_request, reply) => {
-    return await reply.send(await store.publisher.keys())
+    return reply.send(await store.publisher.keys())
   })
 
   server.delete<{
@@ -95,6 +95,6 @@ export const publisherRoutes = (_cfg: APIConfig, store: StoreI) => async (server
   }, async (request, reply) => {
     const { id } = request.params
     await store.publisher.delete(id)
-    return await reply.code(200).send()
+    return reply.code(200).send()
   })
 }

api/sites.ts

@@ -58,12 +58,38 @@ export const siteRoutes = (cfg: APIConfig, store: StoreI) => async (server: Fast
     // Only register site to its owner if they are not an admin
     // Publishers need to track sites they own to ensure they can only modify/delete sites they own
     // This does *not* apply to admins as they effectively have 'wildcard' access to all sites
-    if (!request.user.capabilities.includes(CAPABILITIES.ADMIN)) {
+    if (request.user.capabilities.includes(CAPABILITIES.ADMIN) === false) {
       await store.publisher.registerSiteToPublisher(token.issuedTo, site.id)
     }
 
     await store.fs.makeFolder(site.id)
-    return await reply.send(site)
+    return reply.send(site)
+  })
+
+  server.post<{
+    Params: {
+      id: string
+    }
+    Reply: Static<typeof Site>
+  }>('/sites/:id/clone', {
+    schema: {
+      params: Type.Object({
+        id: Type.String()
+      }),
+      response: {
+        200: Site
+      },
+      description: 'Clone a website from its HTTPs version',
+      tags: ['site'],
+      security: [{ jwt: [] }]
+    },
+    preHandler: server.auth([server.verifyPublisher, server.verifyAdmin])
+  }, async (request, reply) => {
+    const { id } = request.params
+    await store.sites.get(id)
+    const path = store.fs.getPath(id)
+
+    await store.sites.clone(id, path)
   })
 
   server.get<{
@@ -86,7 +112,7 @@ export const siteRoutes = (cfg: APIConfig, store: StoreI) => async (server: Fast
     preHandler: server.auth([server.verifyPublisher, server.verifyAdmin])
   }, async (request, reply) => {
     const { id } = request.params
-    return await reply.send(await store.sites.get(id))
+    return reply.send(await store.sites.get(id))
   })
 
   server.get<{
@@ -110,10 +136,10 @@ export const siteRoutes = (cfg: APIConfig, store: StoreI) => async (server: Fast
   }, async (request, reply) => {
     const { id } = request.params
     // Logging the request initiation
-    request.log.info(`Fetching stats for site ID: ${id}`)
+    request.log.info(`Fetching stats for site ID: ${id as string}`)
     const stats = await store.sites.stats(id)
     // Logging the successful retrieval of stats
-    return await reply.send(stats)
+    return reply.send(stats)
   })
 
   if (cfg.useWebringDirectoryListing === true) {
@@ -127,10 +153,10 @@ export const siteRoutes = (cfg: APIConfig, store: StoreI) => async (server: Fast
       try {
         // admin case, safe to list all
         await verifyTokenCapabilities(request, store, [CAPABILITIES.ADMIN])
-        return await reply.send(await store.sites.listAll(false))
+        return reply.send(await store.sites.listAll(false))
       } catch {
         // no token
-        return await reply.send(await store.sites.listAll(true))
+        return reply.send(await store.sites.listAll(true))
       }
     })
   }
@@ -153,13 +179,13 @@ export const siteRoutes = (cfg: APIConfig, store: StoreI) => async (server: Fast
     const { id } = request.params
     const token = request.user
     if (!await checkOwnsSite(token, id)) {
-      return await reply.status(401).send('You must either own the site or be an admin to modify this resource')
+      return reply.status(401).send('You must either own the site or be an admin to modify this resource')
     }
 
     await store.sites.delete(id, { logger: request.log })
     await store.publisher.unregisterSiteFromAllPublishers(id)
     await store.fs.clear(id)
-    return await reply.send()
+    return reply.send()
   })
 
   server.post<{
@@ -182,7 +208,7 @@ export const siteRoutes = (cfg: APIConfig, store: StoreI) => async (server: Fast
     const { id } = request.params
     const token = request.user
     if (!await checkOwnsSite(token, id)) {
-      return await reply.status(401).send('You must either own the site or be an admin to modify this resource')
+      return reply.status(401).send('You must either own the site or be an admin to modify this resource')
     }
 
     // update config entry
@@ -191,7 +217,7 @@ export const siteRoutes = (cfg: APIConfig, store: StoreI) => async (server: Fast
     // sync files with protocols
     const path = store.fs.getPath(id)
     await store.sites.sync(id, path, { logger: request.log })
-    return await reply.code(200).send()
+    return reply.code(200).send()
   })
 
   server.put<{
@@ -211,7 +237,7 @@ export const siteRoutes = (cfg: APIConfig, store: StoreI) => async (server: Fast
     const { id } = request.params
     const token = request.user
     if (!await checkOwnsSite(token, id)) {
-      return await reply.status(401).send('You must either own the site or be an admin to modify this resource')
+      return reply.status(401).send('You must either own the site or be an admin to modify this resource')
     }
     return await processRequestFiles(request, reply, async (tarballPath) => {
       request.log.info('Deleting old files')
@@ -245,7 +271,7 @@ export const siteRoutes = (cfg: APIConfig, store: StoreI) => async (server: Fast
     const { id } = request.params
     const token = request.user
     if (!await checkOwnsSite(token, id)) {
-      return await reply.status(401).send('You must either own the site or be an admin to modify this resource')
+      return reply.status(401).send('You must either own the site or be an admin to modify this resource')
     }
     return await processRequestFiles(request, reply, async (tarballPath) => {
       // extract in place to existing directory

config/sites.test.ts

@@ -1,5 +1,9 @@
 import test from 'ava'
 import { exampleSiteConfig, newSiteConfigStore } from '../fixtures/siteConfig.js'
+import { tmpdir } from 'node:os'
+import { readdir } from 'node:fs/promises'
+import { join } from 'node:path'
+import rimraf from 'rimraf'
 
 test('create new siteconfig', async t => {
   const cfg = newSiteConfigStore()
@@ -83,3 +87,21 @@ test('listAll siteconfig', async t => {
   t.is((await cfg.listAll(true)).length, 2)
   t.is((await cfg.listAll(false)).length, 3)
 })
+
+test('clone a site', async (t) => {
+  const cfg = newSiteConfigStore()
+  const site = await cfg.create({ ...exampleSiteConfig, domain: 'staticpub.mauve.moe' })
+
+  const id = site.domain
+  const dir = join(tmpdir(), id)
+
+  try {
+    await cfg.clone(id, dir)
+
+    const files = await readdir(dir)
+
+    t.assert(files.length !== 0, 'Site got cloned')
+  } finally {
+    await rimraf(dir)
+  }
+})

config/sites.ts

@@ -6,6 +6,10 @@ import { ProtocolManager } from '../protocols/index.js'
 import { Ctx } from '../protocols/interfaces.js'
 import isValidHostname from 'is-valid-hostname'
 import createError from 'http-errors'
+import { promisify } from 'node:util'
+import child_process from 'node:child_process'
+import path from 'node:path'
+const exec = promisify(child_process.exec)
 
 export class SiteConfigStore extends Config<Static<typeof Site>> {
   protocols: ProtocolManager
@@ -30,6 +34,27 @@ export class SiteConfigStore extends Config<Static<typeof Site>> {
     return await this.db.put(id, obj).then(() => obj)
   }
 
+  async clone (siteId: string, filePath: string, ctx?: Ctx): Promise<void> {
+    const cwd = path.resolve(filePath, '..')
+    const destination = filePath.split(path.sep).at(-1) as string
+
+    await exec(`wget2 \
+  --random-wait \
+  --retry-on-http-error=503,504,429 \
+  --compression=identity,gzip,br \
+  --user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/119.0" \
+  --mirror \
+  --page-requisites \
+  --convert-links \
+  --adjust-extension \
+  --continue \
+  --no-host-directories \
+  --directory-prefix=${destination} \
+  "https://${siteId}"`, { cwd })
+
+    await this.sync(siteId, filePath)
+  }
+
   async sync (siteId: string, filePath: string, ctx?: Ctx): Promise<void> {
     const site = await this.get(siteId)