feat: optimize custom extensions (#6801)

packages/playground/optimize-deps/__tests__/optimize-deps.spec.ts

@@ -62,6 +62,10 @@ test('import * from optimized dep', async () => {
   expect(await page.textContent('.import-star')).toMatch(`[success]`)
 })
 
+test('import from dep with .notjs files', async () => {
+  expect(await page.textContent('.not-js')).toMatch(`[success]`)
+})
+
 test('dep with css import', async () => {
   expect(await getColor('h1')).toBe('red')
 })

packages/playground/optimize-deps/dep-not-js/foo.js

@@ -0,0 +1 @@
+export const foo = '[success] imported from .notjs file'

packages/playground/optimize-deps/dep-not-js/index.notjs

@@ -0,0 +1,4 @@
+<notjs>
+import { foo } from './foo'
+export const notjsValue = foo
+</notjs>

packages/playground/optimize-deps/dep-not-js/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "dep-not-js",
+  "private": true,
+  "version": "1.0.0",
+  "main": "index.notjs"
+}

packages/playground/optimize-deps/index.html

@@ -38,6 +38,9 @@ <h2>Optimizing force included dep even when it's linked</h2>
 <h2>import * as ...</h2>
 <div class="import-star"></div>
 
+<h2>Import from dependency with .notjs files</h2>
+<div class="not-js"></div>
+
 <h2>Dep w/ special file format supported via plugins</h2>
 <div class="plugin"></div>
 
@@ -73,6 +76,9 @@ <h2>Alias with colon</h2>
     text('.import-star', `[success] ${keys.join(', ')}`)
   }
 
+  import { notjsValue } from 'dep-not-js'
+  text('.not-js', notjsValue)
+
   import { createApp } from 'vue'
   import { createStore } from 'vuex'
   if (typeof createApp === 'function' && typeof createStore === 'function') {

packages/playground/optimize-deps/package.json

@@ -17,6 +17,7 @@
     "dep-esbuild-plugin-transform": "file:./dep-esbuild-plugin-transform",
     "dep-linked": "link:./dep-linked",
     "dep-linked-include": "link:./dep-linked-include",
+    "dep-not-js": "file:./dep-not-js",
     "lodash-es": "^4.17.21",
     "nested-exclude": "file:./nested-exclude",
     "phoenix": "^1.6.2",

packages/playground/optimize-deps/vite.config.js

@@ -1,3 +1,4 @@
+const fs = require('fs')
 const vue = require('@vitejs/plugin-vue')
 
 /**
@@ -39,6 +40,7 @@ module.exports = {
 
   plugins: [
     vue(),
+    notjs(),
     // for axios request test
     {
       name: 'mock',
@@ -51,3 +53,39 @@ module.exports = {
     }
   ]
 }
+
+// Handles .notjs file, basically remove wrapping <notjs> and </notjs> tags
+function notjs() {
+  return {
+    name: 'notjs',
+    config() {
+      return {
+        optimizeDeps: {
+          extensions: ['.notjs'],
+          esbuildOptions: {
+            plugins: [
+              {
+                name: 'esbuild-notjs',
+                setup(build) {
+                  build.onLoad({ filter: /\.notjs$/ }, ({ path }) => {
+                    let contents = fs.readFileSync(path, 'utf-8')
+                    contents = contents
+                      .replace('<notjs>', '')
+                      .replace('</notjs>', '')
+                    return { contents, loader: 'js' }
+                  })
+                }
+              }
+            ]
+          }
+        }
+      }
+    },
+    transform(code, id) {
+      if (id.endsWith('.notjs')) {
+        code = code.replace('<notjs>', '').replace('</notjs>', '')
+        return { code }
+      }
+    }
+  }
+}

packages/vite/src/node/optimizer/esbuildDepPlugin.ts

@@ -1,5 +1,5 @@
 import path from 'path'
-import type { Loader, Plugin, ImportKind } from 'esbuild'
+import type { Plugin, ImportKind } from 'esbuild'
 import { KNOWN_ASSET_TYPES } from '../constants'
 import type { ResolvedConfig } from '..'
 import {
@@ -40,6 +40,13 @@ export function esbuildDepPlugin(
   config: ResolvedConfig,
   ssr?: boolean
 ): Plugin {
+  // remove optimizable extensions from `externalTypes` list
+  const allExternalTypes = config.optimizeDeps.extensions
+    ? externalTypes.filter(
+        (type) => !config.optimizeDeps.extensions?.includes('.' + type)
+      )
+    : externalTypes
+
   // default resolver which prefers ESM
   const _resolve = config.createResolver({ asSrc: false })
 
@@ -74,7 +81,7 @@ export function esbuildDepPlugin(
       // externalize assets and commonly known non-js file types
       build.onResolve(
         {
-          filter: new RegExp(`\\.(` + externalTypes.join('|') + `)(\\?.*)?$`)
+          filter: new RegExp(`\\.(` + allExternalTypes.join('|') + `)(\\?.*)?$`)
         },
         async ({ path: id, importer, kind }) => {
           const resolved = await resolve(id, importer, kind)
@@ -181,10 +188,8 @@ export function esbuildDepPlugin(
           }
         }
 
-        let ext = path.extname(entryFile).slice(1)
-        if (ext === 'mjs') ext = 'js'
         return {
-          loader: ext as Loader,
+          loader: 'js',
           contents,
           resolveDir: root
         }

packages/vite/src/node/optimizer/index.ts

@@ -79,6 +79,16 @@ export interface DepOptimizationOptions {
    * @deprecated use `esbuildOptions.keepNames`
    */
   keepNames?: boolean
+  /**
+   * List of file extensions that can be optimized. A corresponding esbuild
+   * plugin must exist to handle the specific extension.
+   *
+   * By default, Vite can optimize `.mjs`, `.js`, and `.ts` files. This option
+   * allows specifying additional extensions.
+   *
+   * @experimental
+   */
+  extensions?: string[]
 }
 
 export interface DepOptimizationMetadata {
@@ -244,29 +254,43 @@ export async function optimizeDeps(
   for (const id in deps) {
     const flatId = flattenId(id)
     const filePath = (flatIdDeps[flatId] = deps[id])
-    const entryContent = fs.readFileSync(filePath, 'utf-8')
     let exportsData: ExportsData
-    try {
-      exportsData = parse(entryContent) as ExportsData
-    } catch {
-      debug(
-        `Unable to parse dependency: ${id}. Trying again with a JSX transform.`
-      )
-      const transformed = await transformWithEsbuild(entryContent, filePath, {
-        loader: 'jsx'
+    if (config.optimizeDeps.extensions?.some((ext) => filePath.endsWith(ext))) {
+      // For custom supported extensions, build the entry file to transform it into JS,
+      // and then parse with es-module-lexer. Note that the `bundle` option is not `true`,
+      // so only the entry file is being transformed.
+      const result = await build({
+        ...esbuildOptions,
+        plugins,
+        entryPoints: [filePath],
+        write: false,
+        format: 'esm'
       })
-      // Ensure that optimization won't fail by defaulting '.js' to the JSX parser.
-      // This is useful for packages such as Gatsby.
-      esbuildOptions.loader = {
-        '.js': 'jsx',
-        ...esbuildOptions.loader
+      exportsData = parse(result.outputFiles[0].text) as ExportsData
+    } else {
+      const entryContent = fs.readFileSync(filePath, 'utf-8')
+      try {
+        exportsData = parse(entryContent) as ExportsData
+      } catch {
+        debug(
+          `Unable to parse dependency: ${id}. Trying again with a JSX transform.`
+        )
+        const transformed = await transformWithEsbuild(entryContent, filePath, {
+          loader: 'jsx'
+        })
+        // Ensure that optimization won't fail by defaulting '.js' to the JSX parser.
+        // This is useful for packages such as Gatsby.
+        esbuildOptions.loader = {
+          '.js': 'jsx',
+          ...esbuildOptions.loader
+        }
+        exportsData = parse(transformed.code) as ExportsData
       }
-      exportsData = parse(transformed.code) as ExportsData
-    }
-    for (const { ss, se } of exportsData[0]) {
-      const exp = entryContent.slice(ss, se)
-      if (/export\s+\*\s+from/.test(exp)) {
-        exportsData.hasReExports = true
+      for (const { ss, se } of exportsData[0]) {
+        const exp = entryContent.slice(ss, se)
+        if (/export\s+\*\s+from/.test(exp)) {
+          exportsData.hasReExports = true
+        }
       }
     }
     idToExports[id] = exportsData

packages/vite/src/node/optimizer/scan.ts

@@ -181,6 +181,10 @@ function esbuildScanPlugin(
     '@vite/env'
   ]
 
+  const isOptimizable = (id: string) =>
+    OPTIMIZABLE_ENTRY_RE.test(id) ||
+    !!config.optimizeDeps.extensions?.some((ext) => id.endsWith(ext))
+
   const externalUnlessEntry = ({ path }: { path: string }) => ({
     path,
     external: !entries.includes(path)
@@ -218,8 +222,14 @@ function esbuildScanPlugin(
 
       // html types: extract script contents -----------------------------------
       build.onResolve({ filter: htmlTypesRE }, async ({ path, importer }) => {
+        const resolved = await resolve(path, importer)
+        if (!resolved) return
+        // It is possible for the scanner to scan html types in node_modules.
+        // If we can optimize this html type, skip it so it's handled by the
+        // bare import resolve, and recorded as optimization dep.
+        if (resolved.includes('node_modules') && isOptimizable(resolved)) return
         return {
-          path: await resolve(path, importer),
+          path: resolved,
           namespace: 'html'
         }
       })
@@ -340,17 +350,19 @@ function esbuildScanPlugin(
             }
             if (resolved.includes('node_modules') || include?.includes(id)) {
               // dependency or forced included, externalize and stop crawling
-              if (OPTIMIZABLE_ENTRY_RE.test(resolved)) {
+              if (isOptimizable(resolved)) {
                 depImports[id] = resolved
               }
               return externalUnlessEntry({ path: id })
-            } else {
+            } else if (isScannable(resolved)) {
               const namespace = htmlTypesRE.test(resolved) ? 'html' : undefined
               // linked package, keep crawling
               return {
                 path: path.resolve(resolved),
                 namespace
               }
+            } else {
+              return externalUnlessEntry({ path: id })
             }
           } else {
             missing[id] = normalizePath(importer)
@@ -396,7 +408,7 @@ function esbuildScanPlugin(
           // use vite resolver to support urls and omitted extensions
           const resolved = await resolve(id, importer)
           if (resolved) {
-            if (shouldExternalizeDep(resolved, id)) {
+            if (shouldExternalizeDep(resolved, id) || !isScannable(resolved)) {
               return externalUnlessEntry({ path: id })
             }
 
@@ -499,10 +511,7 @@ function extractImportPaths(code: string) {
   return js
 }
 
-export function shouldExternalizeDep(
-  resolvedId: string,
-  rawId: string
-): boolean {
+function shouldExternalizeDep(resolvedId: string, rawId: string): boolean {
   // not a valid file path
   if (!path.isAbsolute(resolvedId)) {
     return true
@@ -511,9 +520,9 @@ export function shouldExternalizeDep(
   if (resolvedId === rawId || resolvedId.includes('\0')) {
     return true
   }
-  // resolved is not a scannable type
-  if (!JS_TYPES_RE.test(resolvedId) && !htmlTypesRE.test(resolvedId)) {
-    return true
-  }
   return false
 }
+
+function isScannable(id: string): boolean {
+  return JS_TYPES_RE.test(id) || htmlTypesRE.test(id)
+}