diff --git a/packages/vite/src/node/plugins/importAnalysisBuild.ts b/packages/vite/src/node/plugins/importAnalysisBuild.ts
index 12c4325d1a01c9..3a99b310cd810a 100644
--- a/packages/vite/src/node/plugins/importAnalysisBuild.ts
+++ b/packages/vite/src/node/plugins/importAnalysisBuild.ts
@@ -403,9 +403,7 @@ export function buildImportAnalysisPlugin(config: ResolvedConfig): Plugin {
         return
       }
 
-      // If preload is not enabled, we parse through each imports and remove any imports to pure CSS chunks
-      // as they are removed from the bundle
-      if (!insertPreload) {
+      if (ssr || isWorker) {
         const removedPureCssFiles = removedPureCssFilesCache.get(config)
         if (removedPureCssFiles && removedPureCssFiles.size > 0) {
           for (const file in bundle) {
diff --git a/playground/lib/__tests__/lib.spec.ts b/playground/lib/__tests__/lib.spec.ts
index af144de44e9c46..021d7d1bc636b5 100644
--- a/playground/lib/__tests__/lib.spec.ts
+++ b/playground/lib/__tests__/lib.spec.ts
@@ -67,14 +67,6 @@ describe.runIf(isBuild)('build', () => {
     expect(code).toMatch(/await import\("\.\/message-[-\w]{8}.js"\)/)
   })
 
-  test('Library mode does not have any reference to pure CSS chunks', async () => {
-    const code = readFile('dist/lib/dynamic-import-message.es.mjs')
-
-    // Does not import pure CSS chunks and replaced by `Promise.resolve({})` instead
-    expect(code).not.toMatch(/await import\("\.\/dynamic-[-\w]{8}.js"\)/)
-    expect(code).toMatch(/await Promise.resolve\(\{.*\}\)/)
-  })
-
   test('@import hoist', async () => {
     serverLogs.forEach((log) => {
       // no warning from esbuild css minifier