feat(query): provide ast for files and depth for dependencies (#9285)

### Description

Some more requests. Add the serialized AST from SWC and a depth limit
for dependency tracing.

### Testing Instructions

Added some tests to `turbo-trace.t`

Cargo.toml

@@ -146,6 +146,10 @@ smallvec = { version = "1.13.1", features = [
   "union",
   "const_new",
 ] }
+swc_common = "0.37.5"
+swc_ecma_ast = "0.118.2"
+swc_ecma_parser = "0.149.1"
+swc_ecma_visit = "0.104.8"
 syn = "1.0.107"
 tempfile = "3.3.0"
 test-case = "3.0.0"

crates/turbo-trace/Cargo.toml

@@ -9,10 +9,10 @@ camino.workspace = true
 clap = { version = "4.5.17", features = ["derive"] }
 miette = { workspace = true, features = ["fancy"] }
 oxc_resolver = "1.11.0"
-swc_common = "0.37.5"
-swc_ecma_ast = "0.118.2"
-swc_ecma_parser = "0.149.1"
-swc_ecma_visit = "0.104.8"
+swc_common = { workspace = true }
+swc_ecma_ast = { workspace = true }
+swc_ecma_parser = { workspace = true }
+swc_ecma_visit = { workspace = true }
 thiserror = { workspace = true }
 turbopath = { workspace = true }
 

crates/turbo-trace/src/main.rs

@@ -13,6 +13,8 @@ struct Args {
     #[clap(long)]
     ts_config: Option<Utf8PathBuf>,
     files: Vec<Utf8PathBuf>,
+    #[clap(long)]
+    depth: Option<usize>,
 }
 
 fn main() -> Result<(), PathError> {
@@ -32,15 +34,15 @@ fn main() -> Result<(), PathError> {
 
     let tracer = Tracer::new(abs_cwd, files, args.ts_config);
 
-    let result = tracer.trace();
+    let result = tracer.trace(args.depth);
 
     if !result.errors.is_empty() {
         for error in &result.errors {
             eprintln!("error: {}", error);
         }
         std::process::exit(1);
     } else {
-        for file in &result.files {
+        for file in result.files.keys() {
             println!("{}", file);
         }
     }

crates/turbo-trace/src/tracer.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashSet, fs, rc::Rc};
+use std::{collections::HashMap, fs, rc::Rc};
 
 use camino::Utf8PathBuf;
 use miette::{Diagnostic, NamedSource, SourceSpan};
@@ -14,9 +14,13 @@ use turbopath::{AbsoluteSystemPathBuf, PathError};
 
 use crate::import_finder::ImportFinder;
 
+#[derive(Default)]
+pub struct SeenFile {
+    pub ast: Option<swc_ecma_ast::Module>,
+}
+
 pub struct Tracer {
-    files: Vec<AbsoluteSystemPathBuf>,
-    seen: HashSet<AbsoluteSystemPathBuf>,
+    files: Vec<(AbsoluteSystemPathBuf, usize)>,
     ts_config: Option<AbsoluteSystemPathBuf>,
     source_map: Rc<SourceMap>,
 }
@@ -40,7 +44,7 @@ pub enum TraceError {
 
 pub struct TraceResult {
     pub errors: Vec<TraceError>,
-    pub files: HashSet<AbsoluteSystemPathBuf>,
+    pub files: HashMap<AbsoluteSystemPathBuf, SeenFile>,
 }
 
 impl Tracer {
@@ -52,22 +56,22 @@ impl Tracer {
         let ts_config =
             ts_config.map(|ts_config| AbsoluteSystemPathBuf::from_unknown(&cwd, ts_config));
 
-        let seen = HashSet::new();
+        let files = files.into_iter().map(|file| (file, 0)).collect::<Vec<_>>();
 
         Self {
             files,
-            seen,
             ts_config,
             source_map: Rc::new(SourceMap::default()),
         }
     }
 
-    pub fn trace(mut self) -> TraceResult {
+    pub fn trace(mut self, max_depth: Option<usize>) -> TraceResult {
         let mut options = ResolveOptions::default()
             .with_builtin_modules(true)
             .with_force_extension(EnforceExtension::Disabled)
             .with_extension(".ts")
             .with_extension(".tsx");
+
         if let Some(ts_config) = self.ts_config.take() {
             options.tsconfig = Some(TsconfigOptions {
                 config_file: ts_config.into(),
@@ -77,17 +81,24 @@ impl Tracer {
 
         let resolver = Resolver::new(options);
         let mut errors = vec![];
+        let mut seen: HashMap<AbsoluteSystemPathBuf, SeenFile> = HashMap::new();
 
-        while let Some(file_path) = self.files.pop() {
+        while let Some((file_path, file_depth)) = self.files.pop() {
             if matches!(file_path.extension(), Some("json") | Some("css")) {
                 continue;
             }
 
-            if self.seen.contains(&file_path) {
+            if seen.contains_key(&file_path) {
                 continue;
             }
 
-            self.seen.insert(file_path.clone());
+            if let Some(max_depth) = max_depth {
+                if file_depth > max_depth {
+                    continue;
+                }
+            }
+
+            let entry = seen.entry(file_path.clone()).or_default();
 
             // Read the file content
             let Ok(file_content) = fs::read_to_string(&file_path) else {
@@ -135,6 +146,8 @@ impl Tracer {
             let mut finder = ImportFinder::default();
             module.visit_with(&mut finder);
 
+            entry.ast = Some(module);
+
             // Convert found imports/requires to absolute paths and add them to files to
             // visit
             for (import, span) in finder.imports() {
@@ -144,7 +157,7 @@ impl Tracer {
                 };
                 match resolver.resolve(file_dir, import) {
                     Ok(resolved) => match resolved.into_path_buf().try_into() {
-                        Ok(path) => self.files.push(path),
+                        Ok(path) => self.files.push((path, file_depth + 1)),
                         Err(err) => {
                             errors.push(TraceError::PathEncoding(err));
                         }
@@ -163,7 +176,7 @@ impl Tracer {
         }
 
         TraceResult {
-            files: self.seen,
+            files: seen,
             errors,
         }
     }

crates/turborepo-lib/Cargo.toml

@@ -99,6 +99,9 @@ sha2 = { workspace = true }
 shared_child = "1.0.0"
 struct_iterable = "0.1.1"
 svix-ksuid = { version = "0.7.0", features = ["serde"] }
+swc_common = { workspace = true }
+swc_ecma_ast = { workspace = true, features = ["serde-impl"] }
+swc_ecma_parser = { workspace = true }
 sysinfo = "0.27.7"
 tabwriter = "1.3.0"
 thiserror = "1.0.38"

crates/turborepo-lib/src/query/file.rs

@@ -2,6 +2,8 @@ use std::sync::Arc;
 
 use async_graphql::{Object, SimpleObject};
 use itertools::Itertools;
+use swc_ecma_ast::EsVersion;
+use swc_ecma_parser::{EsSyntax, Syntax, TsSyntax};
 use turbo_trace::Tracer;
 use turbopath::AbsoluteSystemPathBuf;
 
@@ -13,11 +15,56 @@ use crate::{
 pub struct File {
     run: Arc<Run>,
     path: AbsoluteSystemPathBuf,
+    ast: Option<swc_ecma_ast::Module>,
 }
 
 impl File {
     pub fn new(run: Arc<Run>, path: AbsoluteSystemPathBuf) -> Self {
-        Self { run, path }
+        Self {
+            run,
+            path,
+            ast: None,
+        }
+    }
+
+    pub fn with_ast(mut self, ast: Option<swc_ecma_ast::Module>) -> Self {
+        self.ast = ast;
+
+        self
+    }
+
+    fn parse_file(&self) -> Result<swc_ecma_ast::Module, Error> {
+        let contents = self.path.read_to_string()?;
+        let source_map = swc_common::SourceMap::default();
+        let file = source_map.new_source_file(
+            swc_common::FileName::Custom(self.path.to_string()).into(),
+            contents.clone(),
+        );
+        let syntax = if self.path.extension() == Some("ts") || self.path.extension() == Some("tsx")
+        {
+            Syntax::Typescript(TsSyntax {
+                tsx: self.path.extension() == Some("tsx"),
+                decorators: true,
+                ..Default::default()
+            })
+        } else {
+            Syntax::Es(EsSyntax {
+                jsx: self.path.ends_with(".jsx"),
+                ..Default::default()
+            })
+        };
+        let comments = swc_common::comments::SingleThreadedComments::default();
+        let mut errors = Vec::new();
+        let module = swc_ecma_parser::parse_file_as_module(
+            &file,
+            syntax,
+            EsVersion::EsNext,
+            Some(&comments),
+            &mut errors,
+        )
+        .map_err(Error::Parse)?;
+
+        Ok(module)
     }
 }
 
@@ -73,8 +120,8 @@ impl TraceResult {
             files: result
                 .files
                 .into_iter()
-                .sorted()
-                .map(|path| File::new(run.clone(), path))
+                .sorted_by(|a, b| a.0.cmp(&b.0))
+                .map(|(path, file)| File::new(run.clone(), path).with_ast(file.ast))
                 .collect(),
             errors: result.errors.into_iter().map(|e| e.into()).collect(),
         }
@@ -100,16 +147,24 @@ impl File {
         Ok(self.path.to_string())
     }
 
-    async fn dependencies(&self) -> TraceResult {
+    async fn dependencies(&self, depth: Option<usize>) -> TraceResult {
         let tracer = Tracer::new(
             self.run.repo_root().to_owned(),
             vec![self.path.clone()],
             None,
         );
 
-        let mut result = tracer.trace();
+        let mut result = tracer.trace(depth);
         // Remove the file itself from the result
         result.files.remove(&self.path);
         TraceResult::new(result, self.run.clone())
     }
+
+    async fn ast(&self) -> Option<serde_json::Value> {
+        if let Some(ast) = &self.ast {
+            serde_json::to_value(ast).ok()
+        } else {
+            serde_json::to_value(&self.parse_file().ok()?).ok()
+        }
+    }
 }

crates/turborepo-lib/src/query/mod.rs

@@ -49,6 +49,8 @@ pub enum Error {
     #[error(transparent)]
     #[diagnostic(transparent)]
     Resolution(#[from] crate::run::scope::filter::ResolutionError),
+    #[error("failed to parse file: {0:?}")]
+    Parse(swc_ecma_parser::error::Error),
 }
 
 pub struct RepositoryQuery {

turborepo-tests/integration/fixtures/turbo_trace/bar.js

@@ -0,0 +1,3 @@
+export default function bar() {
+  console.log("bar");
+}

turborepo-tests/integration/fixtures/turbo_trace/foo.js

@@ -1,3 +1,5 @@
+import { bar } from "./bar";
+
 export default function foo() {
   if (!process.env.IS_CI) {
     return "bar";