Merge pull request #14945 from storybookjs/feat/built-in-extract

Core: Built-in static `stories.json` support

examples/html-kitchen-sink/.storybook/main.js

@@ -24,4 +24,15 @@ module.exports = {
   core: {
     builder: 'webpack4',
   },
+  // Test code for built-in stories.json extraction
+  //
+  // refs: {
+  //   'react-ts': {
+  //     title: 'React TS',
+  //     // development
+  //     url: 'http://localhost:9011',
+  //     // production
+  //     // url: 'http://localhost:8080',
+  //   },
+  // },
 };

examples/react-ts/package.json

@@ -18,5 +18,8 @@
     "react-dom": "16.14.0",
     "typescript": "^3.9.7",
     "webpack": "4"
+  },
+  "devDependencies": {
+    "cross-env": "^7.0.3"
   }
 }

lib/core-common/src/types.ts

@@ -155,6 +155,7 @@ export interface BuilderOptions {
   docsMode: boolean;
   versionCheck?: VersionCheck;
   releaseNotesData?: ReleaseNotesData;
+  skipStoriesJson?: boolean;
 }
 
 export interface StorybookConfigOptions {

lib/core-server/package.json

@@ -49,6 +49,7 @@
     "@storybook/builder-webpack4": "6.3.0-alpha.29",
     "@storybook/core-client": "6.3.0-alpha.29",
     "@storybook/core-common": "6.3.0-alpha.29",
+    "@storybook/csf-tools": "6.3.0-alpha.29",
     "@storybook/node-logger": "6.3.0-alpha.29",
     "@storybook/semver": "^7.3.2",
     "@storybook/theming": "6.3.0-alpha.29",
@@ -77,6 +78,7 @@
     "find-up": "^5.0.0",
     "fs-extra": "^9.0.1",
     "global": "^4.4.0",
+    "globby": "^11.0.2",
     "html-webpack-plugin": "^4.0.0",
     "ip": "^1.1.5",
     "node-fetch": "^2.6.1",

lib/core-server/src/build-static.ts

@@ -21,6 +21,7 @@ import { getPrebuiltDir } from './utils/prebuilt-manager';
 import { cache } from './utils/cache';
 import { copyAllStaticFiles } from './utils/copy-all-static-files';
 import { getPreviewBuilder } from './utils/get-preview-builder';
+import { extractStoriesJson } from './utils/stories-json';
 
 export async function buildStaticStandalone(options: CLIOptions & LoadOptions & BuilderOptions) {
   /* eslint-disable no-param-reassign */
@@ -69,6 +70,15 @@ export async function buildStaticStandalone(options: CLIOptions & LoadOptions &
     presets,
   };
 
+  const storiesGlobs = (await presets.apply('stories')) as string[];
+  if (!options.skipStoriesJson) {
+    await extractStoriesJson(
+      path.join(options.outputDir, 'stories.json'),
+      storiesGlobs,
+      options.configDir
+    );
+  }
+
   const prebuiltDir = await getPrebuiltDir(fullOptions);
 
   const startTime = process.hrtime();

lib/core-server/src/core-presets.test.ts

@@ -133,6 +133,7 @@ describe.each([
     ...baseOptions,
     ...frameworkOptions,
     configDir: path.resolve(`${__dirname}/../../../examples/${example}/.storybook`),
+    skipStoriesJson: true,
   };
 
   describe('manager', () => {

lib/core-server/src/dev-server.ts

@@ -7,6 +7,7 @@ import { getMiddleware } from './utils/middleware';
 import { getServerAddresses } from './utils/server-address';
 import { getServer } from './utils/server-init';
 import { useStatics } from './utils/server-statics';
+import { useStoriesJson } from './utils/stories-json';
 
 import * as managerBuilder from './manager/builder';
 
@@ -35,6 +36,9 @@ export async function storybookDevServer(options: Options) {
 
   // User's own static files
   await useStatics(router, options);
+  if (!options.skipStoriesJson) {
+    await useStoriesJson(router, options);
+  }
 
   getMiddleware(options.configDir)(router);
   app.use(router);

lib/core-server/src/utils/stories-json.ts

@@ -0,0 +1,81 @@
+import path from 'path';
+import fs from 'fs-extra';
+import glob from 'globby';
+import { logger } from '@storybook/node-logger';
+import { resolvePathInStorybookCache, Options } from '@storybook/core-common';
+import { readCsf } from '@storybook/csf-tools';
+
+interface ExtractedStory {
+  id: string;
+  kind: string;
+  name: string;
+  parameters: Record<string, any>;
+}
+
+type ExtractedStories = Record<string, ExtractedStory>;
+
+export async function extractStoriesJson(
+  ouputFile: string,
+  storiesGlobs: string[],
+  configDir: string
+) {
+  if (!storiesGlobs) {
+    throw new Error('No stories glob');
+  }
+  const storyFiles: string[] = [];
+  await Promise.all(
+    storiesGlobs.map(async (storiesGlob) => {
+      const files = await glob(path.join(configDir, storiesGlob));
+      storyFiles.push(...files);
+    })
+  );
+  logger.info(`⚙️ Processing ${storyFiles.length} story files from ${storiesGlobs}`);
+
+  const stories: ExtractedStories = {};
+  await Promise.all(
+    storyFiles.map(async (fileName) => {
+      const ext = path.extname(fileName);
+      if (!['.js', '.jsx', '.ts', '.tsx'].includes(ext)) {
+        logger.info(`skipping ${fileName}`);
+        return;
+      }
+      try {
+        const csf = (await readCsf(fileName)).parse();
+        csf.stories.forEach((story) => {
+          stories[story.id] = {
+            ...story,
+            kind: csf.meta.title,
+            parameters: { ...story.parameters, fileName },
+          };
+        });
+      } catch (err) {
+        logger.error(`🚨 Extraction error on ${fileName}`);
+        throw err;
+      }
+    })
+  );
+  await fs.writeJson(ouputFile, { v: 3, stories });
+}
+
+const timeout = 30000; // 30s
+const step = 100; // .1s
+
+export async function useStoriesJson(router: any, options: Options) {
+  const storiesJson = resolvePathInStorybookCache('stories.json');
+  await fs.remove(storiesJson);
+  const storiesGlobs = (await options.presets.apply('stories')) as string[];
+  extractStoriesJson(storiesJson, storiesGlobs, options.configDir);
+  router.use('/stories.json', async (_req: any, res: any) => {
+    for (let i = 0; i < timeout / step; i += 1) {
+      if (fs.existsSync(storiesJson)) {
+        // eslint-disable-next-line no-await-in-loop
+        const json = await fs.readFile(storiesJson, 'utf-8');
+        res.header('Content-Type', 'application/json');
+        return res.send(json);
+      }
+      // eslint-disable-next-line no-await-in-loop
+      await new Promise((r: any) => setTimeout(r, step));
+    }
+    return res.status(408).send('stories.json timeout');
+  });
+}

lib/csf-tools/README.md

@@ -0,0 +1,11 @@
+# Storybook CSF Tools
+
+A library to read, analyze, transform, and write CSF programmatically.
+
+- Read - Parse a CSF file with Babel
+- Analyze - Extract its metadata & stories based on the Babel AST
+- Write - Write the AST back to a file
+
+Coming soon:
+
+- Transform - Update the AST to add/remove/modify stories & metadata (TODO)

lib/csf-tools/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@storybook/csf-tools",
+  "version": "6.3.0-alpha.29",
+  "description": "",
+  "keywords": [
+    "storybook"
+  ],
+  "homepage": "https://github.com/storybookjs/storybook/tree/master/lib/csf-tools",
+  "bugs": {
+    "url": "https://github.com/storybookjs/storybook/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/storybookjs/storybook.git",
+    "directory": "lib/csf-tools"
+  },
+  "funding": {
+    "type": "opencollective",
+    "url": "https://opencollective.com/storybook"
+  },
+  "license": "MIT",
+  "sideEffects": false,
+  "main": "dist/cjs/index.js",
+  "module": "dist/esm/index.js",
+  "types": "dist/ts3.9/index.d.ts",
+  "typesVersions": {
+    "<3.8": {
+      "*": [
+        "dist/ts3.4/*"
+      ]
+    }
+  },
+  "files": [
+    "dist/**/*",
+    "README.md",
+    "*.js",
+    "*.d.ts"
+  ],
+  "scripts": {
+    "prepare": "node ../../scripts/prepare.js"
+  },
+  "dependencies": {
+    "@babel/generator": "^7.12.11",
+    "@babel/parser": "^7.12.11",
+    "@babel/traverse": "^7.12.11",
+    "@babel/types": "^7.12.11",
+    "@storybook/csf": "^0.0.1",
+    "core-js": "^3.8.2",
+    "fs-extra": "^9.0.1"
+  },
+  "devDependencies": {
+    "@types/fs-extra": "^9.0.6",
+    "globby": "^11.0.2"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "gitHead": "a8870128bc9684f9b54083decbd10664bf26bcc4",
+  "sbmodern": "dist/modern/index.js"
+}

lib/csf-tools/src/CsfFile.test.ts

@@ -0,0 +1,24 @@
+import { addSerializer } from 'jest-specific-snapshot';
+import globby from 'globby';
+import path from 'path';
+
+import { readCsf } from './CsfFile';
+
+addSerializer({
+  print: (val: any) => JSON.stringify(val, null, 2),
+  test: (val) => typeof val !== 'string',
+});
+
+describe('csf extract', () => {
+  const fixturesDir = path.join(__dirname, '__testfixtures__');
+  const testFiles = globby
+    .sync(path.join(fixturesDir, '*.stories.*'))
+    .map((testFile) => [path.basename(testFile).split('.')[0], testFile]);
+
+  it.each(testFiles)('%s', async (testName, testFile) => {
+    const csf = (await readCsf(testFile)).parse();
+    expect({ meta: csf.meta, stories: csf.stories }).toMatchSpecificSnapshot(
+      path.join(fixturesDir, `${testName}.snapshot`)
+    );
+  });
+});