From d61dc7be1b8503305f715cab803ddec39288f8fe Mon Sep 17 00:00:00 2001 From: caipira113 Date: Tue, 24 Oct 2023 03:38:12 +0900 Subject: [PATCH] external client assets --- .github/workflows/docker-deploy.yml | 92 ++++++++++++++++ deploy.Dockerfile | 100 ++++++++++++++++++ packages/backend/src/config.ts | 6 ++ .../src/server/web/ClientServerService.ts | 1 + packages/backend/src/server/web/boot.js | 4 +- .../backend/src/server/web/views/base.pug | 9 +- packages/frontend/vite.config.ts | 12 +++ 7 files changed, 218 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/docker-deploy.yml create mode 100644 deploy.Dockerfile diff --git a/.github/workflows/docker-deploy.yml b/.github/workflows/docker-deploy.yml new file mode 100644 index 0000000000..7c81495c38 --- /dev/null +++ b/.github/workflows/docker-deploy.yml @@ -0,0 +1,92 @@ +name: Publish Docker image (deploy) + +on: + push: + branches: + - develop + workflow_dispatch: + +jobs: + push_to_registry: + name: Push Docker image to Container Registry + runs-on: ubuntu-latest + + steps: + - name: Free Disk Space + uses: jlumbroso/free-disk-space@main + with: + # This might remove tools that are actually needed, if set to "true" but frees about 6 GB + tool-cache: false + # All of these default to true, but feel free to set to "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + - name: Check out the repo + uses: actions/checkout@v4.1.1 + - name: Set environment variables + run: | + echo "COMMIT_SHA=$(git rev-parse --short=7 HEAD)" >> $GITHUB_ENV + echo "CLIENT_ASSETS_BASE_URL=${{ secrets.CLIENT_ASSETS_BASE_URL }}" >> $GITHUB_ENV + echo "CLIENT_ASSETS_DIR=$(git show --no-patch --format='%at' HEAD)-$(git rev-parse --short=7 HEAD)" >> $GITHUB_ENV + - name: Add SSH key + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + run: | + mkdir -p ~/.ssh + ssh-keyscan -p ${{ secrets.ARM_NODE_PORT }} -H ${{ secrets.ARM_NODE_ADDR }} >> ~/.ssh/known_hosts + ssh-agent -a $SSH_AUTH_SOCK > /dev/null + echo "${{ secrets.SSH_PRIVATE_KEY }}" | ssh-add - + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3.0.0 + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + with: + endpoint: unix:///var/run/docker.sock + platforms: linux/amd64 + append: | + - endpoint: ssh://${{ secrets.ARM_NODE_USER }}@${{ secrets.ARM_NODE_ADDR }}:${{ secrets.ARM_NODE_PORT }} + platforms: linux/arm64 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }}:${{ env.COMMIT_SHA }}, ghcr.io/${{ github.repository }}:develop + - name: Log in to Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: pnpm/action-setup@v2 + - name: Install frontend dependencies + run: | + git submodule update --init + NODE_ENV=production pnpm --filter cherrypick-js install + NODE_ENV=production pnpm --filter frontend install + - name: Build frontend + run: | + sed -i "s/outDir: __dirname + '\/..\/..\/built\/_vite_'/outDir: __dirname + '\/..\/..\/vite'/" packages/frontend/vite.config.ts + NODE_ENV=production pnpm --filter cherrypick-js build + NODE_ENV=production pnpm --filter frontend build + sed -i "s/outDir: __dirname + '\/..\/..\/vite'/outDir: __dirname + '\/..\/..\/built\/_vite_'/" packages/frontend/vite.config.ts + - name: Deploy frontend + run: echo "${{ secrets.UPLOAD_SCRIPT }}" | base64 -d | node + - name: Build and Push to Container registry + uses: docker/build-push-action@v5 + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + with: + builder: ${{ steps.buildx.outputs.name }} + context: . + file: deploy.Dockerfile + push: true + platforms: ${{ steps.buildx.outputs.platforms }} + provenance: false + tags: ghcr.io/${{ github.repository }}:${{ env.COMMIT_SHA }}, ghcr.io/${{ github.repository }}:develop + labels: ${{ env.COMMIT_SHA }}, develop + build-args: | + CLIENT_ASSETS_BASE_URL=${{ env.CLIENT_ASSETS_BASE_URL }} + CLIENT_ASSETS_DIR=${{ env.CLIENT_ASSETS_DIR }} diff --git a/deploy.Dockerfile b/deploy.Dockerfile new file mode 100644 index 0000000000..986b0a1cea --- /dev/null +++ b/deploy.Dockerfile @@ -0,0 +1,100 @@ +# syntax = docker/dockerfile:1.4 + +ARG NODE_VERSION=20.5.1-bullseye +ARG CLIENT_ASSETS_BASE_URL +ARG CLIENT_ASSETS_DIR + +# build assets & compile TypeScript + +FROM --platform=$BUILDPLATFORM node:${NODE_VERSION} AS native-builder + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean \ + ; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && apt-get update \ + && apt-get install -yqq --no-install-recommends \ + build-essential + +RUN corepack enable + +WORKDIR /cherrypick + +COPY --link ["pnpm-lock.yaml", "pnpm-workspace.yaml", "package.json", "./"] +COPY --link ["scripts", "./scripts"] +COPY --link ["packages/backend/package.json", "./packages/backend/"] +COPY --link ["packages/frontend/package.json", "./packages/frontend/"] +COPY --link ["packages/sw/package.json", "./packages/sw/"] +COPY --link ["packages/cherrypick-js/package.json", "./packages/cherrypick-js/"] + +ARG NODE_ENV=production + +RUN --mount=type=cache,target=/root/.local/share/pnpm/store,sharing=locked \ + pnpm i --frozen-lockfile --aggregate-output --force + +COPY --link . ./ + +RUN git submodule update --init +RUN sed -i '/packages\/frontend/ s/^/# /' pnpm-workspace.yaml +RUN pnpm build +RUN sed -i '/packages\/frontend/ s/^# //' pnpm-workspace.yaml +RUN rm -rf .git/ + +# build native dependencies for target platform + +FROM --platform=$TARGETPLATFORM node:${NODE_VERSION} AS target-builder + +RUN apt-get update \ + && apt-get install -yqq --no-install-recommends \ + build-essential + +RUN corepack enable + +WORKDIR /cherrypick + +COPY --link ["pnpm-lock.yaml", "pnpm-workspace.yaml", "package.json", "./"] +COPY --link ["scripts", "./scripts"] +COPY --link ["packages/backend/package.json", "./packages/backend/"] + +ARG NODE_ENV=production + +RUN --mount=type=cache,target=/root/.local/share/pnpm/store,sharing=locked \ + pnpm i --frozen-lockfile --aggregate-output + +FROM --platform=$TARGETPLATFORM node:${NODE_VERSION}-slim AS runner + +ARG UID="991" +ARG GID="991" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ffmpeg tini curl libjemalloc-dev libjemalloc2 \ + && ln -s /usr/lib/$(uname -m)-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so \ + && corepack enable \ + && groupadd -g "${GID}" cherrypick \ + && useradd -l -u "${UID}" -g "${GID}" -m -d /cherrypick cherrypick \ + && find / -type d -path /proc -prune -o -type f -perm /u+s -ignore_readdir_race -exec chmod u-s {} \; \ + && find / -type d -path /proc -prune -o -type f -perm /g+s -ignore_readdir_race -exec chmod g-s {} \; \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists + +USER cherrypick +WORKDIR /cherrypick + +COPY --chown=cherrypick:cherrypick --from=target-builder /cherrypick/node_modules ./node_modules +COPY --chown=cherrypick:cherrypick --from=target-builder /cherrypick/packages/backend/node_modules ./packages/backend/node_modules +COPY --chown=cherrypick:cherrypick --from=native-builder /cherrypick/built ./built +COPY --chown=cherrypick:cherrypick --from=native-builder /cherrypick/packages/backend/built ./packages/backend/built +COPY --chown=cherrypick:cherrypick --from=native-builder /cherrypick/fluent-emojis /cherrypick/fluent-emojis +COPY --chown=cherrypick:cherrypick . ./ + +RUN mv ./vite ./built/_vite_ + +ENV LD_PRELOAD=/usr/local/lib/libjemalloc.so +ENV MALLOC_CONF=background_thread:true,metadata_thp:auto,dirty_decay_ms:30000,muzzy_decay_ms:30000 +ENV NODE_ENV=production +ENV CLIENT_ASSETS_BASE_URL=${CLIENT_ASSETS_BASE_URL} +ENV CLIENT_ASSETS_DIR=${CLIENT_ASSETS_DIR} +HEALTHCHECK --interval=5s --retries=20 CMD ["/bin/bash", "/cherrypick/healthcheck.sh"] +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["pnpm", "run", "migrateandstart:docker"] diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 1856600943..ca0df8f769 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -157,6 +157,7 @@ export type Config = { } apFileBaseUrl: string | undefined; + clientAssetsBaseUrl: string | undefined; proxyRemoteFiles: boolean | undefined; signToActivityPubGet: boolean | undefined; @@ -225,6 +226,10 @@ export function loadConfig(): Config { const internalMediaProxy = `${scheme}://${host}/proxy`; const redis = convertRedisOptions(config.redis, host); + const CLIENT_ASSETS_BASE_URL = process.env.CLIENT_ASSETS_BASE_URL; + const CLIENT_ASSETS_DIR = process.env.CLIENT_ASSETS_DIR; + const clientAssetsBaseUrl = CLIENT_ASSETS_BASE_URL && CLIENT_ASSETS_DIR ? `${CLIENT_ASSETS_BASE_URL}/${CLIENT_ASSETS_DIR}` : ""; + return { version, basedMisskeyVersion, @@ -269,6 +274,7 @@ export function loadConfig(): Config { proxyRemoteFiles: config.proxyRemoteFiles, signToActivityPubGet: config.signToActivityPubGet, apFileBaseUrl: config.apFileBaseUrl, + clientAssetsBaseUrl: clientAssetsBaseUrl, mediaProxy: externalMediaProxy ?? internalMediaProxy, externalMediaProxyEnabled: externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy, videoThumbnailGenerator: config.videoThumbnailGenerator ? diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index 6bbf2c48fc..b9f6a380f3 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -168,6 +168,7 @@ export class ClientServerService { @bindThis private generateCommonPugData(meta: MiMeta) { return { + clientAssetsBaseUrl: this.config.clientAssetsBaseUrl, instanceName: meta.name ?? 'CherryPick', icon: meta.iconUrl, appleTouchIcon: meta.app512IconUrl, diff --git a/packages/backend/src/server/web/boot.js b/packages/backend/src/server/web/boot.js index 2432ad2b1e..374f8f2a8f 100644 --- a/packages/backend/src/server/web/boot.js +++ b/packages/backend/src/server/web/boot.js @@ -78,7 +78,7 @@ lang = 'en-US'; } - const localRes = await window.fetch(`/assets/locales/${lang}.${v}.json`); + const localRes = await window.fetch(`${CLIENT_ASSETS_BASE_URL}/assets/locales/${lang}.${v}.json`); if (localRes.status === 200) { localStorage.setItem('lang', lang); localStorage.setItem('locale', await localRes.text()); @@ -92,7 +92,7 @@ //#region Script function importAppScript() { - import(`/vite/${CLIENT_ENTRY}`) + import(`${CLIENT_ASSETS_BASE_URL}/vite/${CLIENT_ENTRY}`) .catch(async e => { console.error(e); renderError('APP_IMPORT', e); diff --git a/packages/backend/src/server/web/views/base.pug b/packages/backend/src/server/web/views/base.pug index 904ab7c2d7..9ec9584442 100644 --- a/packages/backend/src/server/web/views/base.pug +++ b/packages/backend/src/server/web/views/base.pug @@ -35,15 +35,15 @@ html link(rel='prefetch' href=infoImageUrl) link(rel='prefetch' href=notFoundImageUrl) //- https://github.com/misskey-dev/misskey/issues/9842 - link(rel='stylesheet' href='/assets/tabler-icons/tabler-icons.min.css?v2.37.0') - link(rel='modulepreload' href=`/vite/${clientEntry.file}`) + link(rel='stylesheet' href=`${clientAssetsBaseUrl}/assets/tabler-icons/tabler-icons.min.css?v2.37.0`) + link(rel='modulepreload' href=`${clientAssetsBaseUrl}/vite/${clientEntry.file}`) if !config.clientManifestExists - script(type="module" src="/vite/@vite/client") + script(type="module" src=`${clientAssetsBaseUrl}/vite/@vite/client`) if Array.isArray(clientEntry.css) each href in clientEntry.css - link(rel='stylesheet' href=`/vite/${href}`) + link(rel='stylesheet' href=`${clientAssetsBaseUrl}/vite/${href}`) title block title @@ -66,6 +66,7 @@ html script. var VERSION = "#{version}"; var CLIENT_ENTRY = "#{clientEntry.file}"; + var CLIENT_ASSETS_BASE_URL = "#{clientAssetsBaseUrl}"; script include ../boot.js diff --git a/packages/frontend/vite.config.ts b/packages/frontend/vite.config.ts index c560e25a8e..8698cd77a2 100644 --- a/packages/frontend/vite.config.ts +++ b/packages/frontend/vite.config.ts @@ -49,6 +49,18 @@ export function getConfig(): UserConfig { port: 5173, }, + experimental: { + renderBuiltUrl(filename: string, { hostId, hostType, type }: { hostId: string, hostType: 'js' | 'css' | 'html', type: 'public' | 'asset' }) { + if (filename.includes('draw-blurhash') || filename.includes('test-webgl2')) { + return '/vite/' + filename + } else if (process.env.CLIENT_ASSETS_BASE_URL && process.env.CLIENT_ASSETS_DIR) { + return `${process.env.CLIENT_ASSETS_BASE_URL}/${process.env.CLIENT_ASSETS_DIR}/vite/${filename}` + } else { + return '/vite/' + filename + } + } + }, + plugins: [ pluginVue({ reactivityTransform: true,