diff --git a/crates/swc/src/config/mod.rs b/crates/swc/src/config/mod.rs
index 0dbb836cd71b..fc8eb4b75385 100644
--- a/crates/swc/src/config/mod.rs
+++ b/crates/swc/src/config/mod.rs
@@ -1587,13 +1587,14 @@ impl ModuleConfig {
         config: Option<ModuleConfig>,
         available_features: FeatureFlag,
     ) -> Box<dyn swc_ecma_visit::Fold + 'cmt> {
+        let skip_resolver = base_url.as_os_str().is_empty() && paths.is_empty();
+
         let base = match base {
-            FileName::Real(v) if !paths.is_empty() => {
+            FileName::Real(v) if !skip_resolver => {
                 FileName::Real(v.canonicalize().unwrap_or_else(|_| v.to_path_buf()))
             }
             _ => base.clone(),
         };
-        let skip_resolver = base_url.as_os_str().is_empty() && paths.is_empty();
 
         match config {
             None => {