Merge pull request #785 from tweag/nb/coverage

Introduce `coverage` subcommand

CHANGELOG.md

@@ -44,6 +44,9 @@ This name should be decided amongst the team before the release.
 
 [Full list of changes](https://github.com/tweag/topiary/compare/v0.5.1...HEAD)
 
+### Added
+- [#785](https://github.com/tweag/topiary/pull/785) Added the `coverage` command, that checks how much of the query file is used by the input.
+
 ### Changed
 - [#780](https://github.com/tweag/topiary/pull/780) Measuring scopes are now independent from captures order
 

README.md

@@ -185,6 +185,7 @@ Commands:
   visualise   Visualise the input's Tree-sitter parse tree
   config      Print the current configuration
   prefetch    Prefetch all languages in the configuration
+  coverage    Checks how much of the tree-sitter query is used
   completion  Generate shell completion script
   help        Print this message or the help of the given subcommand(s)
 
@@ -366,6 +367,48 @@ Options:
 ```
 <!-- usage:end:prefetch -->
 
+#### Coverage
+
+This subcommand checks how much of the language query file is used to process the input.
+Specifically, it checks the percentage of queries in the query file that match the given input,
+And prints the queries that don't matched anything.
+
+<!-- DO NOT REMOVE THE "usage" COMMENTS -->
+<!-- usage:start:coverage-->
+```
+Checks how much of the tree-sitter query is used
+
+Usage: topiary coverage [OPTIONS] <--language <LANGUAGE>|FILE>
+
+Arguments:
+  [FILE]
+          Input file (omit to read from stdin)
+
+          Language detection and query selection is automatic, mapped from file extensions defined
+          in the Topiary configuration.
+
+Options:
+  -l, --language <LANGUAGE>
+          Topiary language identifier (for formatting stdin)
+
+  -q, --query <QUERY>
+          Topiary query file override (when formatting stdin)
+
+  -C, --configuration <CONFIGURATION>
+          Configuration file
+
+          [env: TOPIARY_CONFIG_FILE]
+
+  -v, --verbose...
+          Logging verbosity (increased per occurrence)
+
+  -h, --help
+          Print help (see a summary with '-h')
+```
+<!-- usage:end:coverage -->
+
+The `coverage` subcommand will exit with error code `1` if the coverage is less than 100%.
+
 #### Logging
 
 By default, the Topiary CLI will only output error messages. You can
@@ -387,6 +430,7 @@ formatting. Otherwise, the following exit codes are defined:
 
 | Reason                       | Code |
 | :--------------------------- | ---: |
+| Negative result              |    1 |
 | CLI argument parsing error   |    2 |
 | I/O error                    |    3 |
 | Topiary query error          |    4 |
@@ -397,6 +441,9 @@ formatting. Otherwise, the following exit codes are defined:
 | Multiple errors              |    9 |
 | Unspecified error            |   10 |
 
+Negative results with error code `1` only happen when Topiary is called
+with the `coverage` sub-command, if the input does not cover 100% of the query.
+
 When given multiple inputs, Topiary will do its best to process them
 all, even in the presence of errors. Should _any_ errors occur, Topiary
 will return a non-zero exit code. For more details on the nature of

bin/verify-documented-usage.sh

@@ -40,7 +40,7 @@ diff-usage() {
 }
 
 main() {
-  local -a subcommands=(ROOT format visualise config completion prefetch)
+  local -a subcommands=(ROOT format visualise config completion coverage prefetch)
 
   local _diff
   local _subcommand

topiary-cli/src/cli.rs

@@ -144,6 +144,13 @@ pub enum Commands {
     #[command(display_order = 4)]
     Prefetch,
 
+    /// Checks how much of the tree-sitter query is used
+    #[command(display_order = 5)]
+    Coverage {
+        #[command(flatten)]
+        input: ExactlyOneInput,
+    },
+
     /// Generate shell completion script
     #[command(display_order = 100)]
     Completion {

topiary-cli/src/error.rs

@@ -68,6 +68,9 @@ impl error::Error for TopiaryError {
 impl From<TopiaryError> for ExitCode {
     fn from(e: TopiaryError) -> Self {
         let exit_code = match e {
+            // Things went well but Topiary needs to answer 'false' in a clean way: Exit 1
+            _ if e.benign() => 1,
+
             // Multiple errors: Exit 9
             TopiaryError::Bin(_, Some(CLIError::Multiple)) => 9,
 
@@ -92,9 +95,6 @@ impl From<TopiaryError> for ExitCode {
             // Bad arguments: Exit 2
             // (Handled by clap: https://github.com/clap-rs/clap/issues/3426)
 
-            // Things went well but Topiary needs to answer 'false' in a clean way: Exit 1
-            // (Not used at the moment)
-
             // Anything else: Exit 10
             _ => 10,
         };
@@ -167,3 +167,19 @@ impl From<tokio::task::JoinError> for TopiaryError {
         )
     }
 }
+
+// Tells whether an error should raise a message on stderr,
+// or if it's an "expected" error.
+pub trait Benign {
+    fn benign(&self) -> bool;
+}
+
+impl Benign for TopiaryError {
+    #[allow(clippy::match_like_matches_macro)]
+    fn benign(&self) -> bool {
+        match self {
+            TopiaryError::Lib(FormatterError::PatternDoesNotMatch) => true,
+            _ => false,
+        }
+    }
+}

topiary-cli/src/main.rs

@@ -10,7 +10,8 @@ use std::{
     process::ExitCode,
 };
 
-use topiary_core::{formatter, Operation};
+use error::Benign;
+use topiary_core::{coverage, formatter, Operation};
 
 use crate::{
     cli::Commands,
@@ -22,7 +23,9 @@ use crate::{
 #[tokio::main]
 async fn main() -> ExitCode {
     if let Err(e) = run().await {
-        print_error(&e);
+        if !e.benign() {
+            print_error(&e)
+        }
         return e.into();
     }
 
@@ -147,6 +150,27 @@ async fn run() -> CLIResult<()> {
             config.prefetch_languages()?;
         }
 
+        Commands::Coverage { input } => {
+            // We are guaranteed (by clap) to have exactly one input, so it's safe to unwrap
+            let input = Inputs::new(&config, &input).next().unwrap()?;
+            let output = OutputFile::Stdout;
+
+            // We don't need a `LanguageDefinitionCache` when there's only one input,
+            // which saves us the thread-safety overhead
+            let language = input.to_language().await?;
+
+            log::info!(
+                "Checking query coverage of {}, as {}",
+                input.source(),
+                input.language().name,
+            );
+
+            let mut buf_input = BufReader::new(input);
+            let mut buf_output = BufWriter::new(output);
+
+            coverage(&mut buf_input, &mut buf_output, &language)?
+        }
+
         Commands::Completion { shell } => {
             // The CLI parser fails if no shell is provided/detected, so it's safe to unwrap here
             cli::completion(shell.unwrap());

topiary-core/src/error.rs

@@ -28,8 +28,7 @@ pub enum FormatterError {
     },
 
     /// The query contains a pattern that had no match in the input file.
-    /// Should only be raised in the test suite.
-    PatternDoesNotMatch(String),
+    PatternDoesNotMatch,
 
     /// There was an error in the query file. If this happened using our
     /// provided query files, it is a bug. Please log an issue.
@@ -81,10 +80,10 @@ impl fmt::Display for FormatterError {
                 write!(f, "Parsing error between line {start_line}, column {start_column} and line {end_line}, column {end_column}")
             }
 
-            Self::PatternDoesNotMatch(pattern_content) => {
+            Self::PatternDoesNotMatch => {
                 write!(
                     f,
-                    "The following pattern matches nothing in the input:\n{pattern_content}"
+                    "The query contains a pattern that does not match the input"
                 )
             }
 
@@ -102,7 +101,7 @@ impl Error for FormatterError {
         match self {
             Self::Idempotence
             | Self::Parsing { .. }
-            | Self::PatternDoesNotMatch(_)
+            | Self::PatternDoesNotMatch
             | Self::Io(IoError::Generic(_, None)) => None,
             Self::Internal(_, source) => source.as_ref().map(Deref::deref),
             Self::Query(_, source) => source.as_ref().map(|e| e as &dyn Error),

topiary-core/src/lib.rs

@@ -19,7 +19,7 @@ use tree_sitter::Position;
 pub use crate::{
     error::{FormatterError, IoError},
     language::Language,
-    tree_sitter::{apply_query, SyntaxNode, TopiaryQuery, Visualisation},
+    tree_sitter::{apply_query, CoverageData, SyntaxNode, TopiaryQuery, Visualisation},
 };
 
 mod atom_collection;
@@ -240,7 +240,6 @@ pub fn formatter(
                 &language.query,
                 &language.grammar,
                 tolerate_parsing_errors,
-                false,
             )?;
 
             // Various post-processing of whitespace
@@ -276,6 +275,43 @@ pub fn formatter(
     Ok(())
 }
 
+pub fn coverage(
+    input: &mut impl io::Read,
+    output: &mut impl io::Write,
+    language: &Language,
+) -> FormatterResult<()> {
+    let content = read_input(input).map_err(|e| {
+        FormatterError::Io(IoError::Filesystem(
+            "Failed to read input contents".into(),
+            e,
+        ))
+    })?;
+
+    let res = tree_sitter::check_query_coverage(&content, &language.query, &language.grammar)?;
+
+    let queries_string = if res.missing_patterns.is_empty() {
+        "All queries are matched".into()
+    } else {
+        format!(
+            "Unmatched queries:\n{}",
+            &res.missing_patterns[..].join("\n"),
+        )
+    };
+
+    write!(
+        output,
+        "Query coverage: {:.2}%\n{}\n",
+        res.cover_percentage * 100.0,
+        queries_string,
+    )?;
+
+    if res.cover_percentage == 1.0 {
+        Ok(())
+    } else {
+        Err(FormatterError::PatternDoesNotMatch)
+    }
+}
+
 /// Simple helper function to read the full content of an io Read stream
 fn read_input(input: &mut dyn io::Read) -> Result<String, io::Error> {
     let mut content = String::new();