feat(cli): docs dev v2 with new OpenApi Parser (#5454)

* docs dev * correlate IDs properly, docs dev working * docs dev working for design customer * json schema * remove unused deps * fix issues with newer fdr version * linter * format * added fixture test for when fdr-sdk supports directory imports * address comments * ci/cd
fern-api · Jan 8, 2025 · 90912f3 · 90912f3
1 parent 41882ee
commit 90912f3
Show file tree

Hide file tree

Showing 44 changed files with 1,974 additions and 264 deletions.
diff --git a/docs-yml.schema.json b/docs-yml.schema.json
@@ -2258,6 +2258,16 @@
               "type": "null"
             }
           ]
+        },
+        "openapi-parser-v2": {
+          "oneOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "type": "null"
+            }
+          ]
         }
       },
       "additionalProperties": false

diff --git a/fern/apis/docs-yml/definition/docs.yml b/fern/apis/docs-yml/definition/docs.yml
@@ -974,6 +974,10 @@ types:
           If `disable-stream-toggle` is set to true, the stream toggle will be disabled.
 
           This behavior is unstable and may change in the future.
+      openapi-parser-v2:
+        type: optional<boolean>
+        docs: |
+          If `openapi-parser-v2` is set to true, the OpenAPI parser will be used directly, without Fern.
 
   PlaygroundSettings:
     properties:

diff --git a/packages/cli/cli/package.json b/packages/cli/cli/package.json
@@ -45,7 +45,6 @@
     "@fern-api/configuration-loader": "workspace:*",
     "@fern-api/core": "workspace:*",
     "@fern-api/core-utils": "workspace:*",
-    "@fern-api/docs-parsers": "^0.0.12",
     "@fern-api/docs-preview": "workspace:*",
     "@fern-api/docs-resolver": "workspace:*",
     "@fern-api/docs-validator": "workspace:*",
@@ -106,7 +105,6 @@
     "js-yaml": "^4.1.0",
     "latest-version": "^9.0.0",
     "lodash-es": "^4.17.21",
-    "openapi-types": "^12.1.3",
     "ora": "^7.0.1",
     "@trivago/prettier-plugin-sort-imports": "^5.2.1",
     "prettier": "^3.4.2",

diff --git a/...i/cli/src/commands/generate-openapi-fdr/generateOpenApiToFdrApiDefinitionForWorkspaces.ts b/...i/cli/src/commands/generate-openapi-fdr/generateOpenApiToFdrApiDefinitionForWorkspaces.ts
@@ -1,14 +1,8 @@
 import { writeFile } from "fs/promises";
-import { merge } from "lodash-es";
-import { OpenAPIV3_1 } from "openapi-types";
 import path from "path";
 
-// TODO: clean up imports
-import { OpenApiDocumentConverterNode } from "@fern-api/docs-parsers";
-import { ErrorCollector } from "@fern-api/docs-parsers";
-import { BaseOpenApiV3_1ConverterNodeContext } from "@fern-api/docs-parsers";
+import { generateFdrFromOpenApiWorkspace } from "@fern-api/docs-resolver";
 import { AbsoluteFilePath, stringifyLargeObject } from "@fern-api/fs-utils";
-import { LazyFernWorkspace, OSSWorkspace, OpenAPILoader, getAllOpenAPISpecs } from "@fern-api/lazy-fern-workspace";
 import { Project } from "@fern-api/project-loader";
 
 import { CliContext } from "../../cli-context/CliContext";
@@ -25,49 +19,11 @@ export async function generateOpenApiToFdrApiDefinitionForWorkspaces({
     await Promise.all(
         project.apiWorkspaces.map(async (workspace) => {
             await cliContext.runTaskForWorkspace(workspace, async (context) => {
-                await cliContext.runTaskForWorkspace(workspace, async (context) => {
-                    if (workspace instanceof LazyFernWorkspace) {
-                        context.logger.info("Skipping, API is specified as a Fern Definition.");
-                        return;
-                    } else if (!(workspace instanceof OSSWorkspace)) {
-                        return;
-                    }
+                const fdrApiDefinition = await generateFdrFromOpenApiWorkspace(workspace, context);
 
-                    const openApiLoader = new OpenAPILoader(workspace.absoluteFilePath);
-                    const openApiSpecs = await getAllOpenAPISpecs({ context, specs: workspace.specs });
-
-                    const openApiDocuments = await openApiLoader.loadDocuments({ context, specs: openApiSpecs });
-
-                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                    let fdrApiDefinition: any;
-                    for (const openApi of openApiDocuments) {
-                        if (openApi.type !== "openapi") {
-                            continue;
-                        }
-
-                        const oasContext: BaseOpenApiV3_1ConverterNodeContext = {
-                            document: openApi.value as OpenAPIV3_1.Document,
-                            logger: context.logger,
-                            errors: new ErrorCollector()
-                        };
-
-                        const openApiFdrJson = new OpenApiDocumentConverterNode({
-                            input: openApi.value as OpenAPIV3_1.Document,
-                            context: oasContext,
-                            accessPath: [],
-                            pathId: workspace.workspaceName ?? "openapi parser"
-                        });
-
-                        fdrApiDefinition = merge(fdrApiDefinition, openApiFdrJson.convert());
-                    }
-
-                    const resolvedOutputFilePath = path.resolve(outputFilepath);
-                    await writeFile(
-                        resolvedOutputFilePath,
-                        await stringifyLargeObject(fdrApiDefinition, { pretty: true })
-                    );
-                    context.logger.info(`Wrote FDR API definition to ${resolvedOutputFilePath}`);
-                });
+                const resolvedOutputFilePath = path.resolve(outputFilepath);
+                await writeFile(resolvedOutputFilePath, await stringifyLargeObject(fdrApiDefinition, { pretty: true }));
+                context.logger.info(`Wrote FDR API definition to ${resolvedOutputFilePath}`);
             });
         })
     );

diff --git a/packages/cli/cli/src/commands/generate/generateDocsWorkspace.ts b/packages/cli/cli/src/commands/generate/generateDocsWorkspace.ts
@@ -1,9 +1,12 @@
 import { createOrganizationIfDoesNotExist } from "@fern-api/auth";
+import { isNonNullish } from "@fern-api/core-utils";
+import { OSSWorkspace } from "@fern-api/lazy-fern-workspace";
 import { askToLogin } from "@fern-api/login";
 import { Project } from "@fern-api/project-loader";
 import { runRemoteGenerationForDocsWorkspace } from "@fern-api/remote-workspace-runner";
 
 import { CliContext } from "../../cli-context/CliContext";
+import { filterOssWorkspaces } from "../../utils/filterOssWorkspaces";
 import { validateDocsWorkspaceAndLogIssues } from "../validate/validateDocsWorkspaceAndLogIssues";
 
 export async function generateDocsWorkspace({
@@ -58,9 +61,12 @@ export async function generateDocsWorkspace({
             })
         );
 
+        const ossWorkspaces = await filterOssWorkspaces(project);
+
         await runRemoteGenerationForDocsWorkspace({
             organization: project.config.organization,
             fernWorkspaces,
+            ossWorkspaces,
             docsWorkspace,
             context,
             token,

diff --git a/packages/cli/cli/src/commands/write-docs-definition/writeDocsDefinitionForProject.ts b/packages/cli/cli/src/commands/write-docs-definition/writeDocsDefinitionForProject.ts
@@ -1,11 +1,14 @@
 import chalk from "chalk";
 import { writeFile } from "fs/promises";
 
+import { isNonNullish } from "@fern-api/core-utils";
 import { DocsDefinitionResolver } from "@fern-api/docs-resolver";
 import { AbsoluteFilePath } from "@fern-api/fs-utils";
+import { OSSWorkspace } from "@fern-api/lazy-fern-workspace";
 import { Project } from "@fern-api/project-loader";
 
 import { CliContext } from "../../cli-context/CliContext";
+import { filterOssWorkspaces } from "../../utils/filterOssWorkspaces";
 
 export async function writeDocsDefinitionForProject({
     project,
@@ -22,6 +25,8 @@ export async function writeDocsDefinitionForProject({
     }
 
     await cliContext.runTaskForWorkspace(docsWorkspace, async (context) => {
+        const ossWorkspaces = await filterOssWorkspaces(project);
+
         const fernWorkspaces = await Promise.all(
             project.apiWorkspaces.map(async (workspace) => {
                 return workspace.toFernWorkspace(
@@ -34,6 +39,7 @@ export async function writeDocsDefinitionForProject({
         const docsResolver = new DocsDefinitionResolver(
             docsWorkspace.config.instances[0]?.url ?? "http://localhost:8080",
             docsWorkspace,
+            ossWorkspaces,
             fernWorkspaces,
             context
         );

diff --git a/packages/cli/cli/src/utils/filterOssWorkspaces.ts b/packages/cli/cli/src/utils/filterOssWorkspaces.ts
@@ -0,0 +1,16 @@
+import { isNonNullish } from "@fern-api/core-utils";
+import { OSSWorkspace } from "@fern-api/lazy-fern-workspace";
+import { Project } from "@fern-api/project-loader";
+
+export async function filterOssWorkspaces(project: Project): Promise<OSSWorkspace[]> {
+    return (
+        await Promise.all(
+            project.apiWorkspaces.map(async (workspace) => {
+                if (workspace instanceof OSSWorkspace) {
+                    return workspace as OSSWorkspace;
+                }
+                return null;
+            })
+        )
+    ).filter(isNonNullish);
+}
diff --git a/packages/cli/configuration-loader/package.json b/packages/cli/configuration-loader/package.json
@@ -32,7 +32,7 @@
     "@fern-api/fs-utils": "workspace:*",
     "@fern-api/task-context": "workspace:*",
     "@fern-api/fern-definition-schema": "workspace:*",
-    "@fern-fern/fdr-cjs-sdk": "0.126.1-444264056",
+    "@fern-fern/fdr-cjs-sdk": "0.127.4-331678a74",
     "@fern-fern/fiddle-sdk": "0.0.584",
     "@fern-fern/generators-sdk": "0.114.0-5745f9e74",
     "find-up": "^6.3.0",

diff --git a/packages/cli/configuration/package.json b/packages/cli/configuration/package.json
@@ -30,7 +30,7 @@
     "@fern-api/core-utils": "workspace:*",
     "@fern-api/path-utils": "workspace:*",
     "@fern-api/fern-definition-schema": "workspace:*",
-    "@fern-fern/fdr-cjs-sdk": "0.126.1-444264056",
+    "@fern-fern/fdr-cjs-sdk": "0.127.4-331678a74",
     "@fern-fern/fiddle-sdk": "0.0.584",
     "zod": "^3.22.3"
   },

diff --git a/...cli/configuration/src/docs-yml/schemas/sdk/api/resources/docs/types/ExperimentalConfig.ts b/...cli/configuration/src/docs-yml/schemas/sdk/api/resources/docs/types/ExperimentalConfig.ts
@@ -14,4 +14,6 @@ export interface ExperimentalConfig {
      * This behavior is unstable and may change in the future.
      */
     disableStreamToggle?: boolean;
+    /** If `openapi-parser-v2` is set to true, the OpenAPI parser will be used directly, without Fern. */
+    openapiParserV2?: boolean;
 }
diff --git a/...uration/src/docs-yml/schemas/sdk/serialization/resources/docs/types/ExperimentalConfig.ts b/...uration/src/docs-yml/schemas/sdk/serialization/resources/docs/types/ExperimentalConfig.ts
@@ -15,11 +15,13 @@ export const ExperimentalConfig: core.serialization.ObjectSchema<
         core.serialization.list(core.serialization.string()).optional(),
     ),
     disableStreamToggle: core.serialization.property("disable-stream-toggle", core.serialization.boolean().optional()),
+    openapiParserV2: core.serialization.property("openapi-parser-v2", core.serialization.boolean().optional()),
 });
 
 export declare namespace ExperimentalConfig {
     export interface Raw {
         "mdx-components"?: string[] | null;
         "disable-stream-toggle"?: boolean | null;
+        "openapi-parser-v2"?: boolean | null;
     }
 }
diff --git a/packages/cli/docs-importers/commons/package.json b/packages/cli/docs-importers/commons/package.json
@@ -30,7 +30,7 @@
     "@fern-api/configuration": "workspace:*",
     "@fern-api/fs-utils": "workspace:*",
     "@fern-api/task-context": "workspace:*",
-    "@fern-fern/fdr-cjs-sdk": "0.126.1-444264056",
+    "@fern-fern/fdr-cjs-sdk": "0.127.4-331678a74",
     "js-yaml": "^4.1.0"
   },
   "devDependencies": {

diff --git a/packages/cli/docs-importers/mintlify/package.json b/packages/cli/docs-importers/mintlify/package.json
@@ -33,7 +33,7 @@
     "@fern-api/task-context": "workspace:*",
     "@fern-api/fs-utils": "workspace:*",
     "@fern-api/logger": "workspace:*",
-    "@fern-fern/fdr-cjs-sdk": "0.126.1-444264056",
+    "@fern-fern/fdr-cjs-sdk": "0.127.4-331678a74",
     "gray-matter": "^4.0.3"
   },
   "devDependencies": {

diff --git a/packages/cli/docs-importers/mintlify/src/convertMarkdown.ts b/packages/cli/docs-importers/mintlify/src/convertMarkdown.ts
@@ -2,7 +2,7 @@ import { readFile } from "fs/promises";
 import grayMatter from "gray-matter";
 
 import { FernDocsBuilder } from "@fern-api/docs-importer-commons";
-import { AbsoluteFilePath, RelativeFilePath, dirname, join, relativize } from "@fern-api/fs-utils";
+import { AbsoluteFilePath, RelativeFilePath, dirname, join } from "@fern-api/fs-utils";
 
 import { FernRegistry as CjsFdrSdk, FernRegistry } from "@fern-fern/fdr-cjs-sdk";
 
@@ -84,7 +84,9 @@ export async function convertMarkdown({
             "twitter:card": undefined,
             noindex: undefined,
             nofollow: undefined,
-            "jsonld:breadcrumb": undefined
+            "jsonld:breadcrumb": undefined,
+            logo: undefined,
+            keywords: undefined
         },
         content: transformedContent
     };

diff --git a/packages/cli/docs-markdown-utils/package.json b/packages/cli/docs-markdown-utils/package.json
@@ -29,7 +29,7 @@
   "dependencies": {
     "@fern-api/fs-utils": "workspace:*",
     "@fern-api/task-context": "workspace:*",
-    "@fern-fern/fdr-cjs-sdk": "0.126.1-444264056",
+    "@fern-fern/fdr-cjs-sdk": "0.127.4-331678a74",
     "gray-matter": "^4.0.3",
     "mdast-util-from-markdown": "^2.0.1",
     "mdast-util-mdx": "^3.0.0",

diff --git a/packages/cli/docs-preview/package.json b/packages/cli/docs-preview/package.json
@@ -27,10 +27,12 @@
     "depcheck": "depcheck"
   },
   "dependencies": {
+    "@fern-api/core-utils": "workspace:*",
     "@fern-api/docs-resolver": "workspace:*",
-    "@fern-api/fdr-sdk": "0.126.1-444264056",
+    "@fern-api/fdr-sdk": "0.127.4-331678a74",
     "@fern-api/fs-utils": "workspace:*",
     "@fern-api/ir-sdk": "workspace:*",
+    "@fern-api/lazy-fern-workspace": "workspace:*",
     "@fern-api/logger": "workspace:*",
     "@fern-api/project-loader": "workspace:*",
     "@fern-api/register": "workspace:*",