Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add site cloning API #85

Merged
merged 5 commits into from
Oct 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@ on: [ push, pull_request ]

jobs:
build:
runs-on: ubuntu-22.04
runs-on: ubuntu-latest

strategy:
matrix:
node-version: [19.9.x]

steps:
- name: Install wget2 for cloning
run: sudo apt-get install -y wget2
- uses: actions/checkout@v3
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
Expand Down
7 changes: 7 additions & 0 deletions ansible/roles/distributed_press/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,13 @@
groups: www-data
append: yes

- name: "Install wget2"
apt:
pkg:
- wget2
state: latest
update_cache: true

- name: "Install git and ufw"
apt:
pkg:
Expand Down
10 changes: 5 additions & 5 deletions api/publisher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export const publisherRoutes = (_cfg: APIConfig, store: StoreI) => async (server
},
preHandler: server.auth([server.verifyAdmin])
}, async (request, reply) => {
return await reply.send(await store.publisher.create(request.body))
return reply.send(await store.publisher.create(request.body))
})

server.post<{
Expand All @@ -44,7 +44,7 @@ export const publisherRoutes = (_cfg: APIConfig, store: StoreI) => async (server
})
const signed = await reply.jwtSign(newToken)

return await reply.send(signed)
return reply.send(signed)
})

server.get<{
Expand All @@ -67,7 +67,7 @@ export const publisherRoutes = (_cfg: APIConfig, store: StoreI) => async (server
preHandler: server.auth([server.verifyAdmin, server.verifyPublisher])
}, async (request, reply) => {
const { id } = request.params
return await reply.send(await store.publisher.get(id))
return reply.send(await store.publisher.get(id))
})

server.get<{ Reply: string[] }>('/publisher', {
Expand All @@ -78,7 +78,7 @@ export const publisherRoutes = (_cfg: APIConfig, store: StoreI) => async (server
},
preHandler: server.auth([server.verifyAdmin])
}, async (_request, reply) => {
return await reply.send(await store.publisher.keys())
return reply.send(await store.publisher.keys())
})

server.delete<{
Expand All @@ -95,6 +95,6 @@ export const publisherRoutes = (_cfg: APIConfig, store: StoreI) => async (server
}, async (request, reply) => {
const { id } = request.params
await store.publisher.delete(id)
return await reply.code(200).send()
return reply.code(200).send()
})
}
26 changes: 26 additions & 0 deletions api/sites.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,32 @@ export const siteRoutes = (cfg: APIConfig, store: StoreI) => async (server: Fast
return await reply.send(site)
})

server.post<{
Params: {
id: string
}
Reply: Static<typeof Site>
}>('/sites/:id/clone', {
schema: {
params: Type.Object({
id: Type.String()
}),
response: {
200: Site
},
description: 'Clone a website from its HTTPs version',
tags: ['site'],
security: [{ jwt: [] }]
},
preHandler: server.auth([server.verifyPublisher, server.verifyAdmin])
}, async (request, reply) => {
const { id } = request.params
await store.sites.get(id)
const path = store.fs.getPath(id)

await store.sites.clone(id, path)
})

server.get<{
Params: {
id: string
Expand Down
22 changes: 22 additions & 0 deletions config/sites.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import test from 'ava'
import { exampleSiteConfig, newSiteConfigStore } from '../fixtures/siteConfig.js'
import { tmpdir } from 'node:os'
import { readdir } from 'node:fs/promises'
import { join } from 'node:path'
import rimraf from 'rimraf'

test('create new siteconfig', async t => {
const cfg = newSiteConfigStore()
Expand Down Expand Up @@ -83,3 +87,21 @@ test('listAll siteconfig', async t => {
t.is((await cfg.listAll(true)).length, 2)
t.is((await cfg.listAll(false)).length, 3)
})

test('clone a site', async (t) => {
const cfg = newSiteConfigStore()
const site = await cfg.create({ ...exampleSiteConfig, domain: 'staticpub.mauve.moe' })

const id = site.domain
const dir = join(tmpdir(), id)

try {
await cfg.clone(id, dir)

const files = await readdir(dir)

t.assert(files.length !== 0, 'Site got cloned')
} finally {
await rimraf(dir)
}
})
24 changes: 24 additions & 0 deletions config/sites.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ import { ProtocolManager } from '../protocols/index.js'
import { Ctx } from '../protocols/interfaces.js'
import isValidHostname from 'is-valid-hostname'
import createError from 'http-errors'
import { promisify } from 'node:util'
import child_process from 'node:child_process'
import path from 'node:path'
const exec = promisify(child_process.exec)

export class SiteConfigStore extends Config<Static<typeof Site>> {
protocols: ProtocolManager
Expand All @@ -30,6 +34,26 @@ export class SiteConfigStore extends Config<Static<typeof Site>> {
return await this.db.put(id, obj).then(() => obj)
}

async clone (siteId: string, filePath: string, ctx?: Ctx): Promise<void> {
const cwd = path.resolve(filePath, '..')
const destination = filePath.split(path.sep).at(-1) as string

await exec(`wget2 \
--random-wait \
--compression=identity,gzip,br \
--user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/119.0" \
--mirror \
--page-requisites \
--convert-links \
--adjust-extension \
--continue \
--no-host-directories \
--directory-prefix=${destination} \
"https://${siteId}"`, { cwd })

await this.sync(siteId, filePath)
}

async sync (siteId: string, filePath: string, ctx?: Ctx): Promise<void> {
const site = await this.get(siteId)

Expand Down
Loading