From 17ad86d112026b5a9ce38e0a80e58d40d75f2c45 Mon Sep 17 00:00:00 2001 From: Gert Hengeveld Date: Mon, 9 Oct 2023 16:31:32 +0200 Subject: [PATCH 01/24] Pull upload functions out of upload task, for reusabililty --- node-src/lib/compress.test.ts | 11 +-- node-src/lib/compress.ts | 4 +- node-src/lib/upload.ts | 119 ++++++++++++++++++++++++++++ node-src/lib/uploadFiles.ts | 2 +- node-src/lib/utils.ts | 2 +- node-src/main.test.ts | 2 +- node-src/tasks/upload.ts | 141 ++++++---------------------------- 7 files changed, 151 insertions(+), 130 deletions(-) create mode 100644 node-src/lib/upload.ts diff --git a/node-src/lib/compress.test.ts b/node-src/lib/compress.test.ts index ca0869c52..6ea486e0b 100644 --- a/node-src/lib/compress.test.ts +++ b/node-src/lib/compress.test.ts @@ -13,11 +13,8 @@ afterEach(() => { mockFs.restore(); }); -const testContext = { - sourceDir: '/chromatic-tmp', - fileInfo: { paths: ['file1'] }, - log: new TestLogger(), -} as any; +const testContext = { sourceDir: '/chromatic-tmp', log: new TestLogger() } as any; +const fileInfo = { paths: ['file1'] }; describe('makeZipFile', () => { it('adds files to an archive', async () => { @@ -27,14 +24,14 @@ describe('makeZipFile', () => { }, }); - const result = await makeZipFile(testContext); + const result = await makeZipFile(testContext, fileInfo); expect(existsSync(result.path)).toBeTruthy(); expect(result.size).toBeGreaterThan(0); }); it('rejects on error signals', () => { - return expect(makeZipFile(testContext)).rejects.toThrow( + return expect(makeZipFile(testContext, fileInfo)).rejects.toThrow( `ENOENT: no such file or directory, open '/chromatic-tmp/file1'` ); }); diff --git a/node-src/lib/compress.ts b/node-src/lib/compress.ts index e34abe1bc..8a1144d39 100644 --- a/node-src/lib/compress.ts +++ b/node-src/lib/compress.ts @@ -4,11 +4,11 @@ import { join } from 'path'; import { file as tempFile } from 'tmp-promise'; import { Context } from '../types'; -export default async function makeZipFile(ctx: Context) { +export default async function makeZipFile(ctx: Context, fileInfo: { paths: string[] }) { const archive = archiver('zip', { zlib: { level: 9 } }); const tmp = await tempFile({ postfix: '.zip' }); const sink = createWriteStream(null, { fd: tmp.fd }); - const { paths } = ctx.fileInfo; + const { paths } = fileInfo; return new Promise<{ path: string; size: number }>((resolve, reject) => { sink.on('close', () => { diff --git a/node-src/lib/upload.ts b/node-src/lib/upload.ts new file mode 100644 index 000000000..04b7eb222 --- /dev/null +++ b/node-src/lib/upload.ts @@ -0,0 +1,119 @@ +import { join } from 'path'; + +import makeZipFile from './compress'; +import { Context } from '../types'; +import { uploadZip, waitForUnpack } from './uploadZip'; +import { uploadFiles } from './uploadFiles'; + +const GetUploadUrlsMutation = ` + mutation GetUploadUrlsMutation($buildId: ObjID, $paths: [String!]!) { + getUploadUrls(buildId: $buildId, paths: $paths) { + domain + urls { + path + url + contentType + } + } + } +`; +interface GetUploadUrlsMutationResult { + getUploadUrls: { + domain: string; + urls: { + path: string; + url: string; + contentType: string; + }[]; + }; +} + +const GetZipUploadUrlMutation = ` + mutation GetZipUploadUrlMutation($buildId: ObjID) { + getZipUploadUrl(buildId: $buildId) { + domain + url + sentinelUrl + } + } +`; +interface GetZipUploadUrlMutationResult { + getZipUploadUrl: { + domain: string; + url: string; + sentinelUrl: string; + }; +} + +export async function uploadAsIndividualFiles( + ctx: Context, + fileInfo: { + paths: string[]; + lengths: { + knownAs: string; + pathname: string; + contentLength: number; + }[]; + total: number; + }, + options: { + onStart?: () => void; + onProgress?: (progress: number, total: number) => void; + onComplete?: (uploadedBytes: number, domain?: string) => void; + onError?: (error: Error, path?: string) => void; + } = {} +) { + const { lengths, paths, total } = fileInfo; + const { getUploadUrls } = await ctx.client.runQuery( + GetUploadUrlsMutation, + { buildId: ctx.announcedBuild.id, paths } + ); + const { domain, urls } = getUploadUrls; + const files = urls.map(({ path, url, contentType }) => ({ + path: join(ctx.sourceDir, path), + url, + contentType, + contentLength: lengths.find(({ knownAs }) => knownAs === path).contentLength, + })); + + options.onStart?.(); + + try { + await uploadFiles(ctx, files, (progress) => options.onProgress?.(progress, total)); + } catch (e) { + return options.onError?.(e, files.some(({ path }) => path === e.message) && e.message); + } + + options.onComplete?.(total, domain); +} + +export async function uploadAsZipFile( + ctx: Context, + fileInfo: { paths: string[] }, + options: { + onStart?: () => void; + onProgress?: (progress: number, total: number) => void; + onComplete?: (uploadedBytes: number, domain?: string) => void; + onError?: (error: Error, path?: string) => void; + } = {} +) { + const zipped = await makeZipFile(ctx, fileInfo); + const { path, size: total } = zipped; + const { getZipUploadUrl } = await ctx.client.runQuery( + GetZipUploadUrlMutation, + { buildId: ctx.announcedBuild.id } + ); + const { domain, url, sentinelUrl } = getZipUploadUrl; + + options.onStart?.(); + + try { + await uploadZip(ctx, path, url, total, (progress) => options.onProgress?.(progress, total)); + } catch (e) { + return options.onError?.(e, path); + } + + await waitForUnpack(ctx, sentinelUrl); + + options.onComplete?.(total, domain); +} diff --git a/node-src/lib/uploadFiles.ts b/node-src/lib/uploadFiles.ts index f79576382..6ba5ce4d9 100644 --- a/node-src/lib/uploadFiles.ts +++ b/node-src/lib/uploadFiles.ts @@ -11,7 +11,7 @@ interface File { contentLength: number; } -export default async function uploadFiles( +export async function uploadFiles( ctx: Context, files: File[], onProgress: (progress: number) => void diff --git a/node-src/lib/utils.ts b/node-src/lib/utils.ts index 31717c2bc..eb1b7b50b 100644 --- a/node-src/lib/utils.ts +++ b/node-src/lib/utils.ts @@ -34,7 +34,7 @@ export const activityBar = (n = 0, size = 20) => { return `[${track.join('')}]`; }; -export const baseStorybookUrl = (url: string) => url.replace(/\/iframe\.html$/, ''); +export const baseStorybookUrl = (url: string) => url?.replace(/\/iframe\.html$/, ''); export const rewriteErrorMessage = (err: Error, message: string) => { try { diff --git a/node-src/main.test.ts b/node-src/main.test.ts index c1aee0d55..7b514a474 100644 --- a/node-src/main.test.ts +++ b/node-src/main.test.ts @@ -10,7 +10,7 @@ import * as git from './git/git'; import getEnv from './lib/getEnv'; import parseArgs from './lib/parseArgs'; import TestLogger from './lib/testLogger'; -import uploadFiles from './lib/uploadFiles'; +import { uploadFiles } from './lib/uploadFiles'; import { runAll } from '.'; import { runBuild } from './runBuild'; import { writeChromaticDiagnostics } from './lib/writeChromaticDiagnostics'; diff --git a/node-src/tasks/upload.ts b/node-src/tasks/upload.ts index 4678e3d67..de52a9911 100644 --- a/node-src/tasks/upload.ts +++ b/node-src/tasks/upload.ts @@ -5,10 +5,7 @@ import { URL } from 'url'; import { getDependentStoryFiles } from '../lib/getDependentStoryFiles'; import { createTask, transitionTo } from '../lib/tasks'; -import makeZipFile from '../lib/compress'; -import uploadFiles from '../lib/uploadFiles'; import { rewriteErrorMessage, throttle } from '../lib/utils'; -import { uploadZip, waitForUnpack } from '../lib/uploadZip'; import deviatingOutputDir from '../ui/messages/warnings/deviatingOutputDir'; import missingStatsFile from '../ui/messages/warnings/missingStatsFile'; import { @@ -30,46 +27,7 @@ import { readStatsFile } from './read-stats-file'; import bailFile from '../ui/messages/warnings/bailFile'; import { findChangedPackageFiles } from '../lib/findChangedPackageFiles'; import { findChangedDependencies } from '../lib/findChangedDependencies'; - -const GetUploadUrlsMutation = ` - mutation GetUploadUrlsMutation($buildId: ObjID, $paths: [String!]!) { - getUploadUrls(buildId: $buildId, paths: $paths) { - domain - urls { - path - url - contentType - } - } - } -`; -interface GetUploadUrlsMutationResult { - getUploadUrls: { - domain: string; - urls: { - path: string; - url: string; - contentType: string; - }[]; - }; -} - -const GetZipUploadUrlMutation = ` - mutation GetZipUploadUrlMutation($buildId: ObjID) { - getZipUploadUrl(buildId: $buildId) { - domain - url - sentinelUrl - } - } -`; -interface GetZipUploadUrlMutationResult { - getZipUploadUrl: { - domain: string; - url: string; - sentinelUrl: string; - }; -} +import { uploadAsIndividualFiles, uploadAsZipFile } from '../lib/upload'; interface PathSpec { pathname: string; @@ -214,93 +172,40 @@ export const traceChangedFiles = async (ctx: Context, task: Task) => { } }; -async function uploadAsIndividualFiles( - ctx: Context, - task: Task, - updateProgress: (progress: number, total: number) => void -) { - const { lengths, paths, total } = ctx.fileInfo; - const { getUploadUrls } = await ctx.client.runQuery( - GetUploadUrlsMutation, - { buildId: ctx.announcedBuild.id, paths } - ); - const { domain, urls } = getUploadUrls; - const files = urls.map(({ path, url, contentType }) => ({ - path: join(ctx.sourceDir, path), - url, - contentType, - contentLength: lengths.find(({ knownAs }) => knownAs === path).contentLength, - })); - - task.output = starting().output; - - try { - await uploadFiles(ctx, files, (progress) => updateProgress(progress, total)); - } catch (e) { - if (files.find(({ path }) => path === e.message)) { - throw new Error(failed({ path: e.message }).output); - } - throw e; - } - - ctx.uploadedBytes = total; - ctx.isolatorUrl = new URL('/iframe.html', domain).toString(); -} - -async function uploadAsZipFile( - ctx: Context, - task: Task, - updateProgress: (progress: number, total: number) => void -) { - const zipped = await makeZipFile(ctx); - const { path, size: total } = zipped; - const { getZipUploadUrl } = await ctx.client.runQuery( - GetZipUploadUrlMutation, - { buildId: ctx.announcedBuild.id } - ); - const { domain, url, sentinelUrl } = getZipUploadUrl; - - task.output = starting().output; - - try { - await uploadZip(ctx, path, url, total, (progress) => updateProgress(progress, total)); - } catch (e) { - if (path === e.message) { - throw new Error(failed({ path }).output); - } - throw e; - } - - ctx.uploadedBytes = total; - ctx.isolatorUrl = new URL('/iframe.html', domain).toString(); - - return waitForUnpack(ctx, sentinelUrl); -} - export const uploadStorybook = async (ctx: Context, task: Task) => { if (ctx.skip) return; transitionTo(preparing)(ctx, task); - const updateProgress = throttle( - (progress, total) => { - const percentage = Math.round((progress / total) * 100); - task.output = uploading({ percentage }).output; - - ctx.options.experimental_onTaskProgress?.({ ...ctx }, { progress, total, unit: 'bytes' }); + const options = { + onStart: () => (task.output = starting().output), + onProgress: throttle( + (progress, total) => { + const percentage = Math.round((progress / total) * 100); + task.output = uploading({ percentage }).output; + + ctx.options.experimental_onTaskProgress?.({ ...ctx }, { progress, total, unit: 'bytes' }); + }, + // Avoid spamming the logs with progress updates in non-interactive mode + ctx.options.interactive ? 100 : ctx.env.CHROMATIC_OUTPUT_INTERVAL + ), + onComplete: (uploadedBytes: number, domain: string) => { + ctx.uploadedBytes = uploadedBytes; + ctx.isolatorUrl = new URL('/iframe.html', domain).toString(); }, - // Avoid spamming the logs with progress updates in non-interactive mode - ctx.options.interactive ? 100 : ctx.env.CHROMATIC_OUTPUT_INTERVAL - ); + onError: (error: Error, path?: string) => { + throw path === error.message ? new Error(failed({ path }).output) : error; + }, + }; if (ctx.options.zip) { try { - await uploadAsZipFile(ctx, task, updateProgress); + await uploadAsZipFile(ctx, ctx.fileInfo, options); } catch (err) { ctx.log.debug({ err }, 'Error uploading zip file'); - await uploadAsIndividualFiles(ctx, task, updateProgress); + await uploadAsIndividualFiles(ctx, ctx.fileInfo, options); } } else { - await uploadAsIndividualFiles(ctx, task, updateProgress); + await uploadAsIndividualFiles(ctx, ctx.fileInfo, options); } }; From 980408f95dc058d68b1ca9577339259cdd4ec4c9 Mon Sep 17 00:00:00 2001 From: Gert Hengeveld Date: Wed, 11 Oct 2023 20:13:23 +0200 Subject: [PATCH 02/24] Refactor uploadAsIndividualFiles to accept a localPath and targetPath --- node-src/lib/upload.ts | 38 +++++++++++++++---------------------- node-src/lib/uploadFiles.ts | 22 +++++++++++---------- node-src/main.test.ts | 16 ++++++++-------- node-src/tasks/upload.ts | 13 +++++++++---- 4 files changed, 44 insertions(+), 45 deletions(-) diff --git a/node-src/lib/upload.ts b/node-src/lib/upload.ts index 04b7eb222..73d0ef4a7 100644 --- a/node-src/lib/upload.ts +++ b/node-src/lib/upload.ts @@ -1,5 +1,3 @@ -import { join } from 'path'; - import makeZipFile from './compress'; import { Context } from '../types'; import { uploadZip, waitForUnpack } from './uploadZip'; @@ -47,15 +45,11 @@ interface GetZipUploadUrlMutationResult { export async function uploadAsIndividualFiles( ctx: Context, - fileInfo: { - paths: string[]; - lengths: { - knownAs: string; - pathname: string; - contentLength: number; - }[]; - total: number; - }, + files: { + localPath: string; + targetPath: string; + contentLength?: number; + }[], options: { onStart?: () => void; onProgress?: (progress: number, total: number) => void; @@ -63,25 +57,23 @@ export async function uploadAsIndividualFiles( onError?: (error: Error, path?: string) => void; } = {} ) { - const { lengths, paths, total } = fileInfo; const { getUploadUrls } = await ctx.client.runQuery( GetUploadUrlsMutation, - { buildId: ctx.announcedBuild.id, paths } + { buildId: ctx.announcedBuild.id, paths: files.map(({ targetPath }) => targetPath) } ); const { domain, urls } = getUploadUrls; - const files = urls.map(({ path, url, contentType }) => ({ - path: join(ctx.sourceDir, path), - url, - contentType, - contentLength: lengths.find(({ knownAs }) => knownAs === path).contentLength, - })); + const targets = urls.map(({ path, url, contentType }) => { + const { localPath, contentLength } = files.find((f) => f.targetPath === path); + return { contentLength, contentType, localPath, targetUrl: url }; + }); + const total = targets.reduce((acc, { contentLength }) => acc + contentLength, 0); options.onStart?.(); try { - await uploadFiles(ctx, files, (progress) => options.onProgress?.(progress, total)); + await uploadFiles(ctx, targets, (progress) => options.onProgress?.(progress, total)); } catch (e) { - return options.onError?.(e, files.some(({ path }) => path === e.message) && e.message); + return options.onError?.(e, files.some((f) => f.localPath === e.message) && e.message); } options.onComplete?.(total, domain); @@ -89,7 +81,7 @@ export async function uploadAsIndividualFiles( export async function uploadAsZipFile( ctx: Context, - fileInfo: { paths: string[] }, + files: { localPath: string }[], options: { onStart?: () => void; onProgress?: (progress: number, total: number) => void; @@ -97,7 +89,7 @@ export async function uploadAsZipFile( onError?: (error: Error, path?: string) => void; } = {} ) { - const zipped = await makeZipFile(ctx, fileInfo); + const zipped = await makeZipFile(ctx, { paths: files.map((f) => f.localPath) }); const { path, size: total } = zipped; const { getZipUploadUrl } = await ctx.client.runQuery( GetZipUploadUrlMutation, diff --git a/node-src/lib/uploadFiles.ts b/node-src/lib/uploadFiles.ts index 6ba5ce4d9..9a8276383 100644 --- a/node-src/lib/uploadFiles.ts +++ b/node-src/lib/uploadFiles.ts @@ -5,8 +5,8 @@ import progress from 'progress-stream'; import { Context } from '../types'; interface File { - path: string; - url: string; + localPath: string; + targetUrl: string; contentType: string; contentLength: number; } @@ -21,10 +21,12 @@ export async function uploadFiles( let totalProgress = 0; await Promise.all( - files.map(({ path, url, contentType, contentLength }) => { + files.map(({ localPath, targetUrl, contentType, contentLength }) => { let fileProgress = 0; // The bytes uploaded for this this particular file - ctx.log.debug(`Uploading ${contentLength} bytes of ${contentType} for '${path}' to '${url}'`); + ctx.log.debug( + `Uploading ${contentLength} bytes of ${contentType} for '${localPath}' to '${targetUrl}'` + ); return limitConcurrency(() => retry( @@ -42,10 +44,10 @@ export async function uploadFiles( }); const res = await ctx.http.fetch( - url, + targetUrl, { method: 'PUT', - body: createReadStream(path).pipe(progressStream), + body: createReadStream(localPath).pipe(progressStream), headers: { 'content-type': contentType, 'content-length': contentLength.toString(), @@ -57,17 +59,17 @@ export async function uploadFiles( ); if (!res.ok) { - ctx.log.debug(`Uploading '${path}' failed: %O`, res); - throw new Error(path); + ctx.log.debug(`Uploading '${localPath}' failed: %O`, res); + throw new Error(localPath); } - ctx.log.debug(`Uploaded '${path}'.`); + ctx.log.debug(`Uploaded '${localPath}'.`); }, { retries: ctx.env.CHROMATIC_RETRIES, onRetry: (err: Error) => { totalProgress -= fileProgress; fileProgress = 0; - ctx.log.debug('Retrying upload %s, %O', url, err); + ctx.log.debug('Retrying upload %s, %O', targetUrl, err); onProgress(totalProgress); }, } diff --git a/node-src/main.test.ts b/node-src/main.test.ts index 7b514a474..8c45fdbab 100644 --- a/node-src/main.test.ts +++ b/node-src/main.test.ts @@ -440,14 +440,14 @@ it('calls out to npm build script passed and uploads files', async () => { { contentLength: 42, contentType: 'text/html', - path: expect.stringMatching(/\/iframe\.html$/), - url: 'https://cdn.example.com/iframe.html', + localPath: expect.stringMatching(/\/iframe\.html$/), + targetUrl: 'https://cdn.example.com/iframe.html', }, { contentLength: 42, contentType: 'text/html', - path: expect.stringMatching(/\/index\.html$/), - url: 'https://cdn.example.com/index.html', + localPath: expect.stringMatching(/\/index\.html$/), + targetUrl: 'https://cdn.example.com/index.html', }, ], expect.any(Function) @@ -465,14 +465,14 @@ it('skips building and uploads directly with storybook-build-dir', async () => { { contentLength: 42, contentType: 'text/html', - path: expect.stringMatching(/\/iframe\.html$/), - url: 'https://cdn.example.com/iframe.html', + localPath: expect.stringMatching(/\/iframe\.html$/), + targetUrl: 'https://cdn.example.com/iframe.html', }, { contentLength: 42, contentType: 'text/html', - path: expect.stringMatching(/\/index\.html$/), - url: 'https://cdn.example.com/index.html', + localPath: expect.stringMatching(/\/index\.html$/), + targetUrl: 'https://cdn.example.com/index.html', }, ], expect.any(Function) diff --git a/node-src/tasks/upload.ts b/node-src/tasks/upload.ts index de52a9911..a78a0bb5f 100644 --- a/node-src/tasks/upload.ts +++ b/node-src/tasks/upload.ts @@ -1,7 +1,6 @@ import { readdirSync, readFileSync, statSync } from 'fs'; import { join } from 'path'; import slash from 'slash'; -import { URL } from 'url'; import { getDependentStoryFiles } from '../lib/getDependentStoryFiles'; import { createTask, transitionTo } from '../lib/tasks'; @@ -197,15 +196,21 @@ export const uploadStorybook = async (ctx: Context, task: Task) => { }, }; + const files = ctx.fileInfo.paths.map((path) => ({ + localPath: join(ctx.sourceDir, path), + targetPath: path, + contentLength: ctx.fileInfo.lengths.find(({ knownAs }) => knownAs === path).contentLength, + })); + if (ctx.options.zip) { try { - await uploadAsZipFile(ctx, ctx.fileInfo, options); + await uploadAsZipFile(ctx, files, options); } catch (err) { ctx.log.debug({ err }, 'Error uploading zip file'); - await uploadAsIndividualFiles(ctx, ctx.fileInfo, options); + await uploadAsIndividualFiles(ctx, files, options); } } else { - await uploadAsIndividualFiles(ctx, ctx.fileInfo, options); + await uploadAsIndividualFiles(ctx, files, options); } }; From cbbcc511b46b14ec78c4417708c9cacb2f4d6c83 Mon Sep 17 00:00:00 2001 From: Gert Hengeveld Date: Wed, 11 Oct 2023 22:19:15 +0200 Subject: [PATCH 03/24] Write logs to chromatic.log file --- node-src/lib/LoggingRenderer.ts | 35 +++++++++++++++++++++++++++++++++ node-src/lib/NonTTYRenderer.ts | 1 - node-src/lib/log.ts | 25 +++++++++++++++++------ node-src/runBuild.ts | 13 ++++++++++-- 4 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 node-src/lib/LoggingRenderer.ts diff --git a/node-src/lib/LoggingRenderer.ts b/node-src/lib/LoggingRenderer.ts new file mode 100644 index 000000000..d430cee58 --- /dev/null +++ b/node-src/lib/LoggingRenderer.ts @@ -0,0 +1,35 @@ +import UpdateRenderer from 'listr-update-renderer'; + +export default class LoggingRenderer { + tasks; + options; + updateRenderer; + + constructor(tasks: any, options: any) { + this.tasks = tasks; + this.options = options; + this.updateRenderer = new UpdateRenderer(tasks, options); + } + + static get nonTTY() { + return false; + } + + render() { + this.updateRenderer.render(); + for (const task of this.tasks) { + let lastData; + task.subscribe((event) => { + if (event.type === 'TITLE') this.options.log.file(`${task.title}`); + if (event.type === 'DATA' && lastData !== event.data) { + lastData = event.data; + this.options.log.file(` → ${event.data}`); + } + }); + } + } + + end() { + this.updateRenderer.end(); + } +} diff --git a/node-src/lib/NonTTYRenderer.ts b/node-src/lib/NonTTYRenderer.ts index 7c5ee4add..f371c1d90 100644 --- a/node-src/lib/NonTTYRenderer.ts +++ b/node-src/lib/NonTTYRenderer.ts @@ -1,6 +1,5 @@ export default class NonTTYRenderer { tasks; - options; constructor(tasks: any, options: any) { diff --git a/node-src/lib/log.ts b/node-src/lib/log.ts index d9c6d62eb..09c0f087a 100644 --- a/node-src/lib/log.ts +++ b/node-src/lib/log.ts @@ -1,4 +1,5 @@ import debug from 'debug'; +import { createWriteStream, unlink } from 'fs'; import stripAnsi from 'strip-ansi'; import { format } from 'util'; @@ -8,8 +9,15 @@ const { DISABLE_LOGGING, LOG_LEVEL = '' } = process.env; const LOG_LEVELS = { silent: 0, error: 1, warn: 2, info: 3, debug: 4 }; const DEFAULT_LEVEL = 'info'; +export const CHROMATIC_LOG_FILE = 'chromatic.log'; + +unlink(CHROMATIC_LOG_FILE, () => {}); + +const stream = createWriteStream(CHROMATIC_LOG_FILE, { flags: 'a' }); +const appendToLogFile = (message: string) => stream.write(message + '\n'); + // Top-level promise rejection handler to deal with initialization errors -const handleRejection = (reason) => console.error('Unhandled promise rejection:', reason); +const handleRejection = (reason: string) => console.error('Unhandled promise rejection:', reason); process.on('unhandledRejection', handleRejection); // Omits any JSON metadata, returning only the message string @@ -30,6 +38,7 @@ export interface Logger { warn: LogFn; info: LogFn; log: LogFn; + file: LogFn; debug: LogFn; queue: () => void; flush: () => void; @@ -46,12 +55,15 @@ export const createLogger = () => { const queue = []; const log = - (type: LogType) => - (...args) => { + (type: LogType, logFileOnly?: boolean) => + (...args: any[]) => { if (LOG_LEVELS[level] < LOG_LEVELS[type]) return; - // Convert the messages to an appropriate format - const messages = interactive ? logInteractive(args) : logVerbose(type, args); + const logs = logVerbose(type, args); + logs.forEach(appendToLogFile); + if (logFileOnly) return; + + const messages = interactive ? logInteractive(args) : logs; if (!messages.length) return; // Queue up the logs or print them right away @@ -71,6 +83,7 @@ export const createLogger = () => { warn: log('warn'), info: log('info'), log: log('info'), + file: log('info', true), debug: log('debug'), queue: () => { enqueue = true; @@ -85,7 +98,7 @@ export const createLogger = () => { }, }; - debug.log = (...args) => logger.debug(format(...args)); + debug.log = (...args: any[]) => logger.debug(format(...args)); // Redirect unhandled promise rejections process.off('unhandledRejection', handleRejection); diff --git a/node-src/runBuild.ts b/node-src/runBuild.ts index 6f46b05f3..b406348ad 100644 --- a/node-src/runBuild.ts +++ b/node-src/runBuild.ts @@ -3,6 +3,7 @@ import Listr from 'listr'; import GraphQLClient from './io/GraphQLClient'; import { getConfiguration } from './lib/getConfiguration'; import getOptions from './lib/getOptions'; +import LoggingRenderer from './lib/LoggingRenderer'; import NonTTYRenderer from './lib/NonTTYRenderer'; import { exitCodes, setExitCode } from './lib/setExitCode'; import { rewriteErrorMessage } from './lib/utils'; @@ -49,8 +50,16 @@ export async function runBuild(ctx: Context) { try { ctx.log.info(''); - if (ctx.options.interactive) ctx.log.queue(); // queue up any log messages while Listr is running - const options = ctx.options.interactive ? {} : { renderer: NonTTYRenderer, log: ctx.log }; + const options = { + log: ctx.log, + renderer: NonTTYRenderer, + }; + if (ctx.options.interactive) { + // Use an enhanced version of Listr's default renderer, which also logs to a file + options.renderer = LoggingRenderer; + // Queue up any non-Listr log messages while Listr is running + ctx.log.queue(); + } await new Listr(getTasks(ctx.options), options).run(ctx); } catch (err) { endActivity(ctx); From a61d27ba1f049151f3423a8bf0d562e6c148ae15 Mon Sep 17 00:00:00 2001 From: Gert Hengeveld Date: Wed, 11 Oct 2023 22:26:43 +0200 Subject: [PATCH 04/24] Don't write log file if logging is disabled --- node-src/lib/log.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/node-src/lib/log.ts b/node-src/lib/log.ts index 09c0f087a..618541564 100644 --- a/node-src/lib/log.ts +++ b/node-src/lib/log.ts @@ -11,11 +11,6 @@ const DEFAULT_LEVEL = 'info'; export const CHROMATIC_LOG_FILE = 'chromatic.log'; -unlink(CHROMATIC_LOG_FILE, () => {}); - -const stream = createWriteStream(CHROMATIC_LOG_FILE, { flags: 'a' }); -const appendToLogFile = (message: string) => stream.write(message + '\n'); - // Top-level promise rejection handler to deal with initialization errors const handleRejection = (reason: string) => console.error('Unhandled promise rejection:', reason); process.on('unhandledRejection', handleRejection); @@ -49,11 +44,15 @@ export interface Logger { export const createLogger = () => { let level = (LOG_LEVEL.toLowerCase() as keyof typeof LOG_LEVELS) || DEFAULT_LEVEL; if (DISABLE_LOGGING === 'true') level = 'silent'; + if (level !== 'silent') unlink(CHROMATIC_LOG_FILE, () => {}); let interactive = !process.argv.slice(2).includes('--no-interactive'); let enqueue = false; const queue = []; + const stream = level !== 'silent' ? createWriteStream(CHROMATIC_LOG_FILE, { flags: 'a' }) : null; + const appendToLogFile = (message: string) => stream?.write(message + '\n'); + const log = (type: LogType, logFileOnly?: boolean) => (...args: any[]) => { From 15b2c28d6199822f70987ae2e9c425fc40dbec31 Mon Sep 17 00:00:00 2001 From: Gert Hengeveld Date: Thu, 12 Oct 2023 00:23:25 +0200 Subject: [PATCH 05/24] Upload metadata files to S3 --- node-src/index.ts | 9 +++- node-src/lib/getConfiguration.ts | 3 +- node-src/lib/getOptions.ts | 2 + node-src/lib/getStorybookMetadata.ts | 12 +++-- node-src/lib/parseArgs.ts | 2 + node-src/lib/upload.ts | 2 +- node-src/lib/uploadMetadataFiles.ts | 54 +++++++++++++++++++++++ node-src/lib/writeChromaticDiagnostics.ts | 7 +-- node-src/tasks/build.ts | 4 +- node-src/types.ts | 4 ++ 10 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 node-src/lib/uploadMetadataFiles.ts diff --git a/node-src/index.ts b/node-src/index.ts index b991145b9..0af5fb5c8 100644 --- a/node-src/index.ts +++ b/node-src/index.ts @@ -16,6 +16,7 @@ import invalidPackageJson from './ui/messages/errors/invalidPackageJson'; import noPackageJson from './ui/messages/errors/noPackageJson'; import { getBranch, getCommit, getSlug, getUserEmail, getUncommittedHash } from './git/git'; import { emailHash } from './lib/emailHash'; +import { uploadMetadataFiles } from './lib/uploadMetadataFiles'; /** Make keys of `T` outside of `R` optional. */ @@ -82,7 +83,7 @@ export async function run({ ctx.http = (ctx.http as HTTPClient) || new HTTPClient(ctx); ctx.extraOptions = options; - await runAll(ctx); + await runAll(ctx as Context); return { // Keep this in sync with the configured outputs in action.yml @@ -102,7 +103,7 @@ export async function run({ }; } -export async function runAll(ctx) { +export async function runAll(ctx: Context) { // Run these in parallel; neither should ever reject await Promise.all([runBuild(ctx), checkForUpdates(ctx)]); @@ -113,6 +114,10 @@ export async function runAll(ctx) { if (ctx.options.diagnostics) { await writeChromaticDiagnostics(ctx); } + + if (ctx.options.uploadMetadata) { + await uploadMetadataFiles(ctx); + } } export type GitInfo = { diff --git a/node-src/lib/getConfiguration.ts b/node-src/lib/getConfiguration.ts index ff84ddecc..02af513bf 100644 --- a/node-src/lib/getConfiguration.ts +++ b/node-src/lib/getConfiguration.ts @@ -15,7 +15,7 @@ const configurationSchema = z untraced: z.array(z.string()), externals: z.array(z.string()), debug: z.boolean(), - diagnostics: z.union([z.string(), z.boolean()]), + diagnostics: z.boolean(), junitReport: z.union([z.string(), z.boolean()]), zip: z.boolean(), autoAcceptChanges: z.union([z.string(), z.boolean()]), @@ -29,6 +29,7 @@ const configurationSchema = z storybookBuildDir: z.string(), storybookBaseDir: z.string(), storybookConfigDir: z.string(), + uploadMetadata: z.boolean(), }) .partial() .strict(); diff --git a/node-src/lib/getOptions.ts b/node-src/lib/getOptions.ts index 98fbfe282..0479160b4 100644 --- a/node-src/lib/getOptions.ts +++ b/node-src/lib/getOptions.ts @@ -77,6 +77,7 @@ export default function getOptions({ branchName: undefined, patchHeadRef: undefined, patchBaseRef: undefined, + uploadMetadata: undefined, }; const [patchHeadRef, patchBaseRef] = (flags.patchBuild || '').split('...').filter(Boolean); @@ -123,6 +124,7 @@ export default function getOptions({ branchName, patchHeadRef, patchBaseRef, + uploadMetadata: flags.uploadMetadata, }); const options: Options = { diff --git a/node-src/lib/getStorybookMetadata.ts b/node-src/lib/getStorybookMetadata.ts index bd74e814f..1ce4a3007 100644 --- a/node-src/lib/getStorybookMetadata.ts +++ b/node-src/lib/getStorybookMetadata.ts @@ -195,6 +195,13 @@ export const findBuilder = async (mainConfig, v7) => { ]); }; +export const findStorybookMainConfig = async (ctx: Context) => { + const configDir = ctx.options.storybookConfigDir ?? '.storybook'; + const files = await readdir(configDir); + const configFile = files.find((file) => file.startsWith('main.')) || null; + return join(configDir, configFile); +}; + export const getStorybookMetadata = async (ctx: Context) => { const configDir = ctx.options.storybookConfigDir ?? '.storybook'; const r = typeof __non_webpack_require__ !== 'undefined' ? __non_webpack_require__ : require; @@ -205,10 +212,7 @@ export const getStorybookMetadata = async (ctx: Context) => { mainConfig = await r(path.resolve(configDir, 'main')); } catch (storybookV6error) { try { - const files = await readdir(configDir); - const mainConfigFileName = files.find((file) => file.startsWith('main')) || null; - const mainConfigFilePath = join(configDir, mainConfigFileName); - mainConfig = await readConfig(mainConfigFilePath); + mainConfig = await readConfig(await findStorybookMainConfig(ctx)); v7 = true; } catch (storybookV7error) { mainConfig = null; diff --git a/node-src/lib/parseArgs.ts b/node-src/lib/parseArgs.ts index 7dc8750af..58dca63f0 100644 --- a/node-src/lib/parseArgs.ts +++ b/node-src/lib/parseArgs.ts @@ -48,6 +48,7 @@ export default function parseArgs(argv: string[]) { --list List available stories. This requires running a full build. --no-interactive Don't ask interactive questions about your setup and don't overwrite output. Always true in non-TTY environments. --trace-changed [mode] Print dependency trace for changed files to affected story files. Set to "expanded" to list individual modules. Requires --only-changed. + --upload-metadata Upload Chromatic metadata files as part of the published Storybook. Includes chromatic-diagnostics.json, chromatic.log, and storybook-build.log Deprecated options --app-code Renamed to --project-token. @@ -97,6 +98,7 @@ export default function parseArgs(argv: string[]) { list: { type: 'boolean' }, interactive: { type: 'boolean', default: true }, traceChanged: { type: 'string' }, + uploadMetadata: { type: 'boolean' }, // Deprecated options (for JSDOM and tunneled builds, among others) allowConsoleErrors: { type: 'boolean' }, diff --git a/node-src/lib/upload.ts b/node-src/lib/upload.ts index 73d0ef4a7..a1917d666 100644 --- a/node-src/lib/upload.ts +++ b/node-src/lib/upload.ts @@ -48,7 +48,7 @@ export async function uploadAsIndividualFiles( files: { localPath: string; targetPath: string; - contentLength?: number; + contentLength: number; }[], options: { onStart?: () => void; diff --git a/node-src/lib/uploadMetadataFiles.ts b/node-src/lib/uploadMetadataFiles.ts new file mode 100644 index 000000000..c07d50801 --- /dev/null +++ b/node-src/lib/uploadMetadataFiles.ts @@ -0,0 +1,54 @@ +import { stat, writeFileSync } from 'fs'; +import { basename } from 'path'; +import { withFile } from 'tmp-promise'; + +import { STORYBOOK_BUILD_LOG_FILE } from '../tasks/build'; +import { Context } from '../types'; +import { findStorybookMainConfig } from './getStorybookMetadata'; +import { CHROMATIC_LOG_FILE } from './log'; +import { uploadAsIndividualFiles } from './upload'; +import { CHROMATIC_DIAGNOSTICS_FILE } from './writeChromaticDiagnostics'; + +const fileSize = (path: string): Promise => + new Promise((resolve) => stat(path, (err, stats) => resolve(err ? 0 : stats.size))); + +export async function uploadMetadataFiles(ctx: Context) { + const metadataFiles = [ + CHROMATIC_DIAGNOSTICS_FILE, + CHROMATIC_LOG_FILE, + STORYBOOK_BUILD_LOG_FILE, + await findStorybookMainConfig(ctx).catch(() => null), + ctx.fileInfo.statsPath, + ].filter(Boolean); + + const files = await Promise.all( + metadataFiles.map(async (localPath) => { + const targetPath = `.chromatic/${basename(localPath)}`; + const contentLength = await fileSize(localPath); + return contentLength && { localPath, targetPath, contentLength }; + }) + ).then((files) => files.filter(Boolean)); + + await withFile(async ({ path }) => { + const html = ` + +
    + ${files + .map(({ targetPath }) => targetPath.replace(/^\.chromatic\//, '')) + .sort() + .map((path) => `
  • ${path}
    • `) + .join('')} +
+ `; + writeFileSync(path, html); + files.push({ + localPath: path, + targetPath: '.chromatic/index.html', + contentLength: html.length, + }); + + ctx.log.info('Uploading metadata files: \n- ' + files.map((f) => f.targetPath).join('\n- ')); + + await uploadAsIndividualFiles(ctx, files); + }); +} diff --git a/node-src/lib/writeChromaticDiagnostics.ts b/node-src/lib/writeChromaticDiagnostics.ts index 53de721be..b003bda8e 100644 --- a/node-src/lib/writeChromaticDiagnostics.ts +++ b/node-src/lib/writeChromaticDiagnostics.ts @@ -4,6 +4,8 @@ import wroteReport from '../ui/messages/info/wroteReport'; const { writeFile } = jsonfile; +export const CHROMATIC_DIAGNOSTICS_FILE = 'chromatic-diagnostics.json'; + export function getDiagnostics(ctx: Context) { const { argv, client, env, help, http, log, pkg, title, ...rest } = ctx; return Object.keys(rest) @@ -14,9 +16,8 @@ export function getDiagnostics(ctx: Context) { // Extract important information from ctx, sort it and output into a json file export async function writeChromaticDiagnostics(ctx: Context) { try { - const chromaticDiagnosticsPath = 'chromatic-diagnostics.json'; - await writeFile(chromaticDiagnosticsPath, getDiagnostics(ctx), { spaces: 2 }); - ctx.log.info(wroteReport(chromaticDiagnosticsPath, 'Chromatic diagnostics')); + await writeFile(CHROMATIC_DIAGNOSTICS_FILE, getDiagnostics(ctx), { spaces: 2 }); + ctx.log.info(wroteReport(CHROMATIC_DIAGNOSTICS_FILE, 'Chromatic diagnostics')); } catch (error) { ctx.log.error(error); } diff --git a/node-src/tasks/build.ts b/node-src/tasks/build.ts index de3d69ce1..23fb2b797 100644 --- a/node-src/tasks/build.ts +++ b/node-src/tasks/build.ts @@ -12,6 +12,8 @@ import buildFailed from '../ui/messages/errors/buildFailed'; import { failed, initial, pending, skipped, success } from '../ui/tasks/build'; import { getPackageManagerName, getPackageManagerRunCommand } from '../lib/getPackageManager'; +export const STORYBOOK_BUILD_LOG_FILE = 'build-storybook.log'; + const trimOutput = ({ stdout }) => stdout && stdout.toString().trim(); export const setSourceDir = async (ctx: Context) => { @@ -62,7 +64,7 @@ const timeoutAfter = (ms) => new Promise((resolve, reject) => setTimeout(reject, ms, new Error(`Operation timed out`))); export const buildStorybook = async (ctx: Context) => { - ctx.buildLogFile = path.resolve('./build-storybook.log'); + ctx.buildLogFile = path.resolve(STORYBOOK_BUILD_LOG_FILE); const logFile = createWriteStream(ctx.buildLogFile); await new Promise((resolve, reject) => { logFile.on('open', resolve); diff --git a/node-src/types.ts b/node-src/types.ts index 7acab5685..4f9e97ac6 100644 --- a/node-src/types.ts +++ b/node-src/types.ts @@ -41,6 +41,7 @@ export interface Flags { list?: boolean; interactive?: boolean; traceChanged?: string; + uploadMetadata?: boolean; // Deprecated options (for JSDOM and tunneled builds, among others) allowConsoleErrors?: boolean; @@ -65,8 +66,10 @@ export interface Options { dryRun: Flags['dryRun']; forceRebuild: boolean | string; debug: boolean; + diagnostics?: boolean; interactive: boolean; junitReport: boolean | string; + uploadMetadata?: Flags['uploadMetadata']; zip: Flags['zip']; autoAcceptChanges: boolean | string; @@ -211,6 +214,7 @@ export interface Context { packageName?: string; packageVersion?: string; }; + mainConfigFilePath?: string; }; spawnParams: { client: 'yarn' | 'npm'; From 9d72798fb2a61d0e298e9ca2cb17184cbe28cc30 Mon Sep 17 00:00:00 2001 From: Gert Hengeveld Date: Thu, 12 Oct 2023 00:34:10 +0200 Subject: [PATCH 06/24] Fix statsPath --- node-src/lib/uploadMetadataFiles.ts | 2 +- node-src/tasks/upload.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/node-src/lib/uploadMetadataFiles.ts b/node-src/lib/uploadMetadataFiles.ts index c07d50801..db9cdc4e9 100644 --- a/node-src/lib/uploadMetadataFiles.ts +++ b/node-src/lib/uploadMetadataFiles.ts @@ -18,7 +18,7 @@ export async function uploadMetadataFiles(ctx: Context) { CHROMATIC_LOG_FILE, STORYBOOK_BUILD_LOG_FILE, await findStorybookMainConfig(ctx).catch(() => null), - ctx.fileInfo.statsPath, + ctx.fileInfo?.statsPath, ].filter(Boolean); const files = await Promise.all( diff --git a/node-src/tasks/upload.ts b/node-src/tasks/upload.ts index a78a0bb5f..5f9773071 100644 --- a/node-src/tasks/upload.ts +++ b/node-src/tasks/upload.ts @@ -110,6 +110,8 @@ export const traceChangedFiles = async (ctx: Context, task: Task) => { transitionTo(tracing)(ctx, task); const statsPath = join(ctx.sourceDir, ctx.fileInfo.statsPath); + ctx.fileInfo.statsPath = statsPath; + const { changedFiles, packageManifestChanges } = ctx.git; try { const changedDependencyNames = await findChangedDependencies(ctx).catch((err) => { From ad3235483463a556b118f0a7ae551e2320d383a1 Mon Sep 17 00:00:00 2001 From: Gert Hengeveld Date: Thu, 12 Oct 2023 11:40:30 +0200 Subject: [PATCH 07/24] Generate nice metadata files index --- .storybook/preview.js | 4 ++ node-src/lib/upload.ts | 16 ++--- node-src/lib/uploadFiles.ts | 11 +-- node-src/lib/uploadMetadataFiles.ts | 21 +++--- node-src/types.ts | 11 +++ node-src/ui/content/metadata.html.stories.ts | 48 +++++++++++++ node-src/ui/content/metadata.html.ts | 74 ++++++++++++++++++++ package.json | 1 + yarn.lock | 5 ++ 9 files changed, 160 insertions(+), 31 deletions(-) create mode 100644 node-src/ui/content/metadata.html.stories.ts create mode 100644 node-src/ui/content/metadata.html.ts diff --git a/.storybook/preview.js b/.storybook/preview.js index e302a9846..00d424030 100644 --- a/.storybook/preview.js +++ b/.storybook/preview.js @@ -38,6 +38,10 @@ export const decorators = [ document.body.style.backgroundColor = '#16242c'; return ; } + if (kind.startsWith('HTML/')) { + document.body.style.backgroundColor = '#ffffff'; + return
; + } document.body.style.backgroundColor = 'paleturquoise'; return storyFn(); }, diff --git a/node-src/lib/upload.ts b/node-src/lib/upload.ts index a1917d666..7e6e8f4dd 100644 --- a/node-src/lib/upload.ts +++ b/node-src/lib/upload.ts @@ -1,5 +1,5 @@ import makeZipFile from './compress'; -import { Context } from '../types'; +import { Context, FileDesc, TargetedFile } from '../types'; import { uploadZip, waitForUnpack } from './uploadZip'; import { uploadFiles } from './uploadFiles'; @@ -45,11 +45,7 @@ interface GetZipUploadUrlMutationResult { export async function uploadAsIndividualFiles( ctx: Context, - files: { - localPath: string; - targetPath: string; - contentLength: number; - }[], + files: FileDesc[], options: { onStart?: () => void; onProgress?: (progress: number, total: number) => void; @@ -62,9 +58,9 @@ export async function uploadAsIndividualFiles( { buildId: ctx.announcedBuild.id, paths: files.map(({ targetPath }) => targetPath) } ); const { domain, urls } = getUploadUrls; - const targets = urls.map(({ path, url, contentType }) => { - const { localPath, contentLength } = files.find((f) => f.targetPath === path); - return { contentLength, contentType, localPath, targetUrl: url }; + const targets = urls.map(({ path, url, contentType }) => { + const file = files.find((f) => f.targetPath === path); + return { ...file, contentType, targetUrl: url }; }); const total = targets.reduce((acc, { contentLength }) => acc + contentLength, 0); @@ -81,7 +77,7 @@ export async function uploadAsIndividualFiles( export async function uploadAsZipFile( ctx: Context, - files: { localPath: string }[], + files: FileDesc[], options: { onStart?: () => void; onProgress?: (progress: number, total: number) => void; diff --git a/node-src/lib/uploadFiles.ts b/node-src/lib/uploadFiles.ts index 9a8276383..24dc4ddd8 100644 --- a/node-src/lib/uploadFiles.ts +++ b/node-src/lib/uploadFiles.ts @@ -2,18 +2,11 @@ import retry from 'async-retry'; import { createReadStream } from 'fs'; import pLimit from 'p-limit'; import progress from 'progress-stream'; -import { Context } from '../types'; - -interface File { - localPath: string; - targetUrl: string; - contentType: string; - contentLength: number; -} +import { Context, TargetedFile } from '../types'; export async function uploadFiles( ctx: Context, - files: File[], + files: TargetedFile[], onProgress: (progress: number) => void ) { const { experimental_abortSignal: signal } = ctx.options; diff --git a/node-src/lib/uploadMetadataFiles.ts b/node-src/lib/uploadMetadataFiles.ts index db9cdc4e9..cfc201fc8 100644 --- a/node-src/lib/uploadMetadataFiles.ts +++ b/node-src/lib/uploadMetadataFiles.ts @@ -3,7 +3,8 @@ import { basename } from 'path'; import { withFile } from 'tmp-promise'; import { STORYBOOK_BUILD_LOG_FILE } from '../tasks/build'; -import { Context } from '../types'; +import { Context, FileDesc } from '../types'; +import getMetadataHtml from '../ui/content/metadata.html'; import { findStorybookMainConfig } from './getStorybookMetadata'; import { CHROMATIC_LOG_FILE } from './log'; import { uploadAsIndividualFiles } from './upload'; @@ -13,6 +14,11 @@ const fileSize = (path: string): Promise => new Promise((resolve) => stat(path, (err, stats) => resolve(err ? 0 : stats.size))); export async function uploadMetadataFiles(ctx: Context) { + if (!ctx.announcedBuild) { + ctx.log.warn('No build announced, skipping metadata upload.'); + return; + } + const metadataFiles = [ CHROMATIC_DIAGNOSTICS_FILE, CHROMATIC_LOG_FILE, @@ -21,7 +27,7 @@ export async function uploadMetadataFiles(ctx: Context) { ctx.fileInfo?.statsPath, ].filter(Boolean); - const files = await Promise.all( + const files = await Promise.all( metadataFiles.map(async (localPath) => { const targetPath = `.chromatic/${basename(localPath)}`; const contentLength = await fileSize(localPath); @@ -30,16 +36,7 @@ export async function uploadMetadataFiles(ctx: Context) { ).then((files) => files.filter(Boolean)); await withFile(async ({ path }) => { - const html = ` - -
    - ${files - .map(({ targetPath }) => targetPath.replace(/^\.chromatic\//, '')) - .sort() - .map((path) => `
  • ${path}
    • `) - .join('')} -
- `; + const html = getMetadataHtml(ctx, files); writeFileSync(path, html); files.push({ localPath: path, diff --git a/node-src/types.ts b/node-src/types.ts index 4f9e97ac6..b20599309 100644 --- a/node-src/types.ts +++ b/node-src/types.ts @@ -350,3 +350,14 @@ export interface Module { export interface Stats { modules: Module[]; } + +export interface FileDesc { + contentLength: number; + localPath: string; + targetPath: string; +} + +export interface TargetedFile extends FileDesc { + contentType: string; + targetUrl: string; +} diff --git a/node-src/ui/content/metadata.html.stories.ts b/node-src/ui/content/metadata.html.stories.ts new file mode 100644 index 000000000..ea10efd83 --- /dev/null +++ b/node-src/ui/content/metadata.html.stories.ts @@ -0,0 +1,48 @@ +import metadataHtml from './metadata.html'; + +export default { + title: 'HTML/Metadata index', +}; + +const context = { announcedBuild: { number: 7801 } } as any; + +const files = [ + { + contentLength: 423, + localPath: 'main.ts', + targetPath: '.chromatic/main.ts', + }, + { + contentLength: 674, + localPath: 'chromatic.log', + targetPath: '.chromatic/chromatic.log', + }, + { + contentLength: 833, + localPath: 'build-storybook.log', + targetPath: '.chromatic/build-storybook.log', + }, + { + contentLength: 3645, + localPath: 'chromatic-diagnostics.json', + targetPath: '.chromatic/chromatic-diagnostics.json', + }, + { + contentLength: 5635, + localPath: 'preview-stats.json', + targetPath: '.chromatic/preview-stats.json', + }, +]; + +export const Default = () => metadataHtml(context, files); + +export const BuildUrl = () => + metadataHtml( + { + ...context, + build: { + webUrl: 'https://www.chromatic.com/build?appId=5d67dc0374b2e300209c41e7&number=7801', + }, + }, + files + ); diff --git a/node-src/ui/content/metadata.html.ts b/node-src/ui/content/metadata.html.ts new file mode 100644 index 000000000..c6d204289 --- /dev/null +++ b/node-src/ui/content/metadata.html.ts @@ -0,0 +1,74 @@ +import { filesize } from 'filesize'; +import { Context, FileDesc } from '../../types'; + +const linkIcon = + ''; + +export default ({ announcedBuild, build }: Context, files: FileDesc[]) => ` + + + + Build ${announcedBuild.number} metadata files + + + + +

Build ${announcedBuild.number} ${ + build ? `${linkIcon}` : '' +}

+ Metadata files +
    + ${files + .sort((a, b) => a.targetPath.localeCompare(b.targetPath, 'en', { numeric: true })) + .map(({ targetPath, contentLength }) => { + const path = targetPath.replace(/^\.chromatic\//, ''); + const size = filesize(contentLength); + return `
  • ${path} (${size})
    • `; + }) + .join('')} +
+ Generated on ${new Date().toLocaleString('en', { + timeStyle: 'medium', + dateStyle: 'full', + hourCycle: 'h24', + timeZone: 'UTC', + })} UTC + +`; diff --git a/package.json b/package.json index 09c2aa625..8f9827667 100644 --- a/package.json +++ b/package.json @@ -156,6 +156,7 @@ "esm": "^3.2.25", "execa": "^7.2.0", "fake-tag": "^2.0.0", + "filesize": "^10.1.0", "fs-extra": "^10.0.0", "https-proxy-agent": "^5.0.0", "husky": "^7.0.0", diff --git a/yarn.lock b/yarn.lock index b6fafab01..453fdf0f8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8084,6 +8084,11 @@ file-uri-to-path@1.0.0: resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== +filesize@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/filesize/-/filesize-10.1.0.tgz#846f5cd8d16e073c5d6767651a8264f6149183cd" + integrity sha512-GTLKYyBSDz3nPhlLVPjPWZCnhkd9TrrRArNcy8Z+J2cqScB7h2McAzR6NBX6nYOoWafql0roY8hrocxnZBv9CQ== + fill-range@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-4.0.0.tgz#d544811d428f98eb06a63dc402d2403c328c38f7" From 3883f19fbe24f018c5511eb90f339761e3f90bbf Mon Sep 17 00:00:00 2001 From: Gert Hengeveld Date: Thu, 12 Oct 2023 11:57:30 +0200 Subject: [PATCH 08/24] Include preview config file in metadata --- node-src/lib/getStorybookMetadata.ts | 8 ++++---- node-src/lib/uploadMetadataFiles.ts | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/node-src/lib/getStorybookMetadata.ts b/node-src/lib/getStorybookMetadata.ts index 1ce4a3007..d2eae25ca 100644 --- a/node-src/lib/getStorybookMetadata.ts +++ b/node-src/lib/getStorybookMetadata.ts @@ -195,11 +195,11 @@ export const findBuilder = async (mainConfig, v7) => { ]); }; -export const findStorybookMainConfig = async (ctx: Context) => { +export const findStorybookConfigFile = async (ctx: Context, pattern: RegExp) => { const configDir = ctx.options.storybookConfigDir ?? '.storybook'; const files = await readdir(configDir); - const configFile = files.find((file) => file.startsWith('main.')) || null; - return join(configDir, configFile); + const configFile = files.find((file) => pattern.test(file)); + return configFile && join(configDir, configFile); }; export const getStorybookMetadata = async (ctx: Context) => { @@ -212,7 +212,7 @@ export const getStorybookMetadata = async (ctx: Context) => { mainConfig = await r(path.resolve(configDir, 'main')); } catch (storybookV6error) { try { - mainConfig = await readConfig(await findStorybookMainConfig(ctx)); + mainConfig = await readConfig(await findStorybookConfigFile(ctx, /^main\.[jt]sx?$/)); v7 = true; } catch (storybookV7error) { mainConfig = null; diff --git a/node-src/lib/uploadMetadataFiles.ts b/node-src/lib/uploadMetadataFiles.ts index cfc201fc8..19c058dd0 100644 --- a/node-src/lib/uploadMetadataFiles.ts +++ b/node-src/lib/uploadMetadataFiles.ts @@ -5,7 +5,7 @@ import { withFile } from 'tmp-promise'; import { STORYBOOK_BUILD_LOG_FILE } from '../tasks/build'; import { Context, FileDesc } from '../types'; import getMetadataHtml from '../ui/content/metadata.html'; -import { findStorybookMainConfig } from './getStorybookMetadata'; +import { findStorybookConfigFile } from './getStorybookMetadata'; import { CHROMATIC_LOG_FILE } from './log'; import { uploadAsIndividualFiles } from './upload'; import { CHROMATIC_DIAGNOSTICS_FILE } from './writeChromaticDiagnostics'; @@ -23,7 +23,8 @@ export async function uploadMetadataFiles(ctx: Context) { CHROMATIC_DIAGNOSTICS_FILE, CHROMATIC_LOG_FILE, STORYBOOK_BUILD_LOG_FILE, - await findStorybookMainConfig(ctx).catch(() => null), + await findStorybookConfigFile(ctx, /^main\.[jt]sx?$/).catch(() => null), + await findStorybookConfigFile(ctx, /^preview\.[jt]sx?$/).catch(() => null), ctx.fileInfo?.statsPath, ].filter(Boolean); From 05be5c1a1d8721138b02d0625114d86f8bb4ef06 Mon Sep 17 00:00:00 2001 From: Gert Hengeveld Date: Thu, 12 Oct 2023 12:12:00 +0200 Subject: [PATCH 09/24] Nicer reporting --- node-src/lib/uploadMetadataFiles.ts | 8 +++- node-src/ui/content/metadata.html.stories.ts | 47 ++++---------------- 2 files changed, 15 insertions(+), 40 deletions(-) diff --git a/node-src/lib/uploadMetadataFiles.ts b/node-src/lib/uploadMetadataFiles.ts index 19c058dd0..932312e6f 100644 --- a/node-src/lib/uploadMetadataFiles.ts +++ b/node-src/lib/uploadMetadataFiles.ts @@ -9,6 +9,7 @@ import { findStorybookConfigFile } from './getStorybookMetadata'; import { CHROMATIC_LOG_FILE } from './log'; import { uploadAsIndividualFiles } from './upload'; import { CHROMATIC_DIAGNOSTICS_FILE } from './writeChromaticDiagnostics'; +import uploadingMetadata from '../ui/messages/info/uploadingMetadata'; const fileSize = (path: string): Promise => new Promise((resolve) => stat(path, (err, stats) => resolve(err ? 0 : stats.size))); @@ -36,6 +37,11 @@ export async function uploadMetadataFiles(ctx: Context) { }) ).then((files) => files.filter(Boolean)); + if (!files.length) { + ctx.log.warn('No metadata files found, skipping metadata upload.'); + return; + } + await withFile(async ({ path }) => { const html = getMetadataHtml(ctx, files); writeFileSync(path, html); @@ -45,7 +51,7 @@ export async function uploadMetadataFiles(ctx: Context) { contentLength: html.length, }); - ctx.log.info('Uploading metadata files: \n- ' + files.map((f) => f.targetPath).join('\n- ')); + ctx.log.info(uploadingMetadata(files)); await uploadAsIndividualFiles(ctx, files); }); diff --git a/node-src/ui/content/metadata.html.stories.ts b/node-src/ui/content/metadata.html.stories.ts index ea10efd83..0a1ed04f6 100644 --- a/node-src/ui/content/metadata.html.stories.ts +++ b/node-src/ui/content/metadata.html.stories.ts @@ -1,48 +1,17 @@ +import { files } from '../messages/info/uploadingMetadata.stories'; import metadataHtml from './metadata.html'; export default { title: 'HTML/Metadata index', }; -const context = { announcedBuild: { number: 7801 } } as any; +const announced: any = { announcedBuild: { number: 7801 } }; -const files = [ - { - contentLength: 423, - localPath: 'main.ts', - targetPath: '.chromatic/main.ts', - }, - { - contentLength: 674, - localPath: 'chromatic.log', - targetPath: '.chromatic/chromatic.log', - }, - { - contentLength: 833, - localPath: 'build-storybook.log', - targetPath: '.chromatic/build-storybook.log', - }, - { - contentLength: 3645, - localPath: 'chromatic-diagnostics.json', - targetPath: '.chromatic/chromatic-diagnostics.json', - }, - { - contentLength: 5635, - localPath: 'preview-stats.json', - targetPath: '.chromatic/preview-stats.json', - }, -]; +const build: any = { + ...announced, + build: { webUrl: 'https://www.chromatic.com/build?appId=5d67dc0374b2e300209c41e7&number=7801' }, +}; -export const Default = () => metadataHtml(context, files); +export const Default = () => metadataHtml(announced, files); -export const BuildUrl = () => - metadataHtml( - { - ...context, - build: { - webUrl: 'https://www.chromatic.com/build?appId=5d67dc0374b2e300209c41e7&number=7801', - }, - }, - files - ); +export const BuildUrl = () => metadataHtml(build, files); From 4025d0bdfbc4bf36a803f3e0881750dba7e75b2f Mon Sep 17 00:00:00 2001 From: Gert Hengeveld Date: Thu, 12 Oct 2023 12:54:10 +0200 Subject: [PATCH 10/24] Better reporting --- node-src/lib/uploadMetadataFiles.ts | 10 ++++- node-src/ui/content/metadata.html.stories.ts | 39 +++++++++++++++++-- node-src/ui/content/metadata.html.ts | 1 - .../info/uploadingMetadata.stories.ts | 10 +++++ .../ui/messages/info/uploadingMetadata.ts | 12 ++++++ 5 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 node-src/ui/messages/info/uploadingMetadata.stories.ts create mode 100644 node-src/ui/messages/info/uploadingMetadata.ts diff --git a/node-src/lib/uploadMetadataFiles.ts b/node-src/lib/uploadMetadataFiles.ts index 932312e6f..defbe0489 100644 --- a/node-src/lib/uploadMetadataFiles.ts +++ b/node-src/lib/uploadMetadataFiles.ts @@ -10,6 +10,7 @@ import { CHROMATIC_LOG_FILE } from './log'; import { uploadAsIndividualFiles } from './upload'; import { CHROMATIC_DIAGNOSTICS_FILE } from './writeChromaticDiagnostics'; import uploadingMetadata from '../ui/messages/info/uploadingMetadata'; +import { baseStorybookUrl } from './utils'; const fileSize = (path: string): Promise => new Promise((resolve) => stat(path, (err, stats) => resolve(err ? 0 : stats.size))); @@ -35,7 +36,11 @@ export async function uploadMetadataFiles(ctx: Context) { const contentLength = await fileSize(localPath); return contentLength && { localPath, targetPath, contentLength }; }) - ).then((files) => files.filter(Boolean)); + ).then((files) => + files + .filter(Boolean) + .sort((a, b) => a.targetPath.localeCompare(b.targetPath, 'en', { numeric: true })) + ); if (!files.length) { ctx.log.warn('No metadata files found, skipping metadata upload.'); @@ -51,7 +56,8 @@ export async function uploadMetadataFiles(ctx: Context) { contentLength: html.length, }); - ctx.log.info(uploadingMetadata(files)); + const directoryUrl = `${baseStorybookUrl(ctx.isolatorUrl)}/.chromatic/`; + ctx.log.info(uploadingMetadata(directoryUrl, files)); await uploadAsIndividualFiles(ctx, files); }); diff --git a/node-src/ui/content/metadata.html.stories.ts b/node-src/ui/content/metadata.html.stories.ts index 0a1ed04f6..ca05e2f33 100644 --- a/node-src/ui/content/metadata.html.stories.ts +++ b/node-src/ui/content/metadata.html.stories.ts @@ -1,15 +1,48 @@ -import { files } from '../messages/info/uploadingMetadata.stories'; import metadataHtml from './metadata.html'; export default { title: 'HTML/Metadata index', + includeStories: /^[A-Z]/, }; -const announced: any = { announcedBuild: { number: 7801 } }; +export const files = [ + { + contentLength: 833, + localPath: 'build-storybook.log', + targetPath: '.chromatic/build-storybook.log', + }, + { + contentLength: 674, + localPath: 'chromatic.log', + targetPath: '.chromatic/chromatic.log', + }, + { + contentLength: 3645, + localPath: 'chromatic-diagnostics.json', + targetPath: '.chromatic/chromatic-diagnostics.json', + }, + { + contentLength: 423, + localPath: 'main.ts', + targetPath: '.chromatic/main.ts', + }, + { + contentLength: 5635, + localPath: 'preview.tsx', + targetPath: '.chromatic/preview.tsx', + }, + { + contentLength: 5635, + localPath: 'preview-stats.json', + targetPath: '.chromatic/preview-stats.json', + }, +]; + +const announced: any = { announcedBuild: { number: 7805 } }; const build: any = { ...announced, - build: { webUrl: 'https://www.chromatic.com/build?appId=5d67dc0374b2e300209c41e7&number=7801' }, + build: { webUrl: 'https://www.chromatic.com/build?appId=5d67dc0374b2e300209c41e7&number=7805' }, }; export const Default = () => metadataHtml(announced, files); diff --git a/node-src/ui/content/metadata.html.ts b/node-src/ui/content/metadata.html.ts index c6d204289..4965ffcaa 100644 --- a/node-src/ui/content/metadata.html.ts +++ b/node-src/ui/content/metadata.html.ts @@ -56,7 +56,6 @@ export default ({ announcedBuild, build }: Context, files: FileDesc[]) => `Metadata files
    ${files - .sort((a, b) => a.targetPath.localeCompare(b.targetPath, 'en', { numeric: true })) .map(({ targetPath, contentLength }) => { const path = targetPath.replace(/^\.chromatic\//, ''); const size = filesize(contentLength); diff --git a/node-src/ui/messages/info/uploadingMetadata.stories.ts b/node-src/ui/messages/info/uploadingMetadata.stories.ts new file mode 100644 index 000000000..a91b1cb11 --- /dev/null +++ b/node-src/ui/messages/info/uploadingMetadata.stories.ts @@ -0,0 +1,10 @@ +import { files } from '../../content/metadata.html.stories'; +import uploadingMetadata from './uploadingMetadata'; + +export default { + title: 'CLI/Messages/Info', +}; + +const directoryUrl = 'https://5d67dc0374b2e300209c41e7-dlmmxasauj.chromatic.com/.chromatic/'; + +export const UploadingMetadata = () => uploadingMetadata(directoryUrl, files); diff --git a/node-src/ui/messages/info/uploadingMetadata.ts b/node-src/ui/messages/info/uploadingMetadata.ts new file mode 100644 index 000000000..d22300ab3 --- /dev/null +++ b/node-src/ui/messages/info/uploadingMetadata.ts @@ -0,0 +1,12 @@ +import chalk from 'chalk'; +import pluralize from 'pluralize'; + +import { info } from '../../components/icons'; +import { FileDesc } from '../../../types'; +import link from '../../components/link'; + +export default (directoryUrl: string, files: FileDesc[]) => { + const count = pluralize('metadata file', files.length, true); + const list = `- ${files.map((f) => f.targetPath.replace(/^\.chromatic\//, '')).join('\n- ')}`; + return chalk`${info} Uploading {bold ${count}} to ${link(directoryUrl)}\n${list}`; +}; From 1e9f59dd19981c4e973cffc2ffe6d5a890e7357b Mon Sep 17 00:00:00 2001 From: Gert Hengeveld Date: Thu, 12 Oct 2023 13:46:06 +0200 Subject: [PATCH 11/24] Move metadata content and fix Storybook styles --- .storybook/preview.js | 16 ++++++++++++---- node-src/lib/uploadMetadataFiles.ts | 2 +- .../{content => html}/metadata.html.stories.ts | 0 node-src/ui/{content => html}/metadata.html.ts | 1 - .../messages/info/uploadingMetadata.stories.ts | 2 +- 5 files changed, 14 insertions(+), 7 deletions(-) rename node-src/ui/{content => html}/metadata.html.stories.ts (100%) rename node-src/ui/{content => html}/metadata.html.ts (99%) diff --git a/.storybook/preview.js b/.storybook/preview.js index 00d424030..31d82685f 100644 --- a/.storybook/preview.js +++ b/.storybook/preview.js @@ -17,7 +17,7 @@ ansiHTML.setColors({ chalk.enabled = true; chalk.level = 3; -const style = { +const codeStyle = { display: 'inline-block', margin: 0, padding: '1rem', @@ -28,6 +28,14 @@ const style = { color: '#c0c4cd', }; +const htmlStyle = { + fontFamily: "'Nunito Sans', sans-serif", + fontSize: 14, + lineHeight: '1', + color: '#5C6870', + margin: 20, +}; + export const parameters = { layout: 'fullscreen', }; @@ -36,11 +44,11 @@ export const decorators = [ (storyFn, { kind }) => { if (kind.startsWith('CLI/')) { document.body.style.backgroundColor = '#16242c'; - return ; + return ; } if (kind.startsWith('HTML/')) { - document.body.style.backgroundColor = '#ffffff'; - return
    ; + document.body.style.backgroundColor = '#F6F9FC'; + return
    ; } document.body.style.backgroundColor = 'paleturquoise'; return storyFn(); diff --git a/node-src/lib/uploadMetadataFiles.ts b/node-src/lib/uploadMetadataFiles.ts index defbe0489..9224a5e91 100644 --- a/node-src/lib/uploadMetadataFiles.ts +++ b/node-src/lib/uploadMetadataFiles.ts @@ -4,7 +4,7 @@ import { withFile } from 'tmp-promise'; import { STORYBOOK_BUILD_LOG_FILE } from '../tasks/build'; import { Context, FileDesc } from '../types'; -import getMetadataHtml from '../ui/content/metadata.html'; +import getMetadataHtml from '../ui/html/metadata.html'; import { findStorybookConfigFile } from './getStorybookMetadata'; import { CHROMATIC_LOG_FILE } from './log'; import { uploadAsIndividualFiles } from './upload'; diff --git a/node-src/ui/content/metadata.html.stories.ts b/node-src/ui/html/metadata.html.stories.ts similarity index 100% rename from node-src/ui/content/metadata.html.stories.ts rename to node-src/ui/html/metadata.html.stories.ts diff --git a/node-src/ui/content/metadata.html.ts b/node-src/ui/html/metadata.html.ts similarity index 99% rename from node-src/ui/content/metadata.html.ts rename to node-src/ui/html/metadata.html.ts index 4965ffcaa..e33e7eecd 100644 --- a/node-src/ui/content/metadata.html.ts +++ b/node-src/ui/html/metadata.html.ts @@ -11,7 +11,6 @@ export default ({ announcedBuild, build }: Context, files: FileDesc[]) => `Build ${announcedBuild.number} metadata files