Skip to content

Commit

Permalink
feat: support typst query (#35)
Browse files Browse the repository at this point in the history
* feat: support typst query

* style: use Literal to improve type hint

* Apply suggestions from code review

* Merge branch 'main' into query

---------

Co-authored-by: messense <[email protected]>
  • Loading branch information
OrangeX4 and messense authored May 21, 2024
1 parent 2ddaa30 commit c7c9604
Show file tree
Hide file tree
Showing 6 changed files with 254 additions and 7 deletions.
3 changes: 3 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ fontdb = "0.17.0"
pathdiff = "0.2"
pyo3 = { version = "0.21.2", features = ["abi3-py37"] }
same-file = "1"
serde = { version = "1.0.184", features = ["derive"] }
serde_json = "1"
serde_yaml = "0.9"
siphasher = "1.0"
tar = "0.4"
typst ="0.11.1"
Expand Down
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ images = typst.compile("hello.typ", output="hello{n}.png", format="png")
# Or use Compiler class to avoid reinitialization
compiler = typst.Compiler("hello.typ")
compiler.compile(format="png", ppi=144.0)

# Query something
import json

values = json.loads(typst.query("hello.typ", "<note>", field="value", one=True))
```

## License
Expand Down
51 changes: 46 additions & 5 deletions python/typst/__init__.pyi
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import pathlib
from typing import List, Optional, TypeVar, overload, Dict, Union
from typing import List, Optional, TypeVar, overload, Dict, Union, Literal

PathLike = TypeVar("PathLike", str, pathlib.Path)

Expand All @@ -22,7 +22,7 @@ class Compiler:
def compile(
self,
output: Optional[PathLike] = None,
format: Optional[str] = None,
format: Optional[Literal["pdf", "svg", "png"]] = None,
ppi: Optional[float] = None,
) -> Optional[Union[bytes, List[bytes]]]:
"""Compile a Typst project.
Expand All @@ -36,13 +36,30 @@ class Compiler:
Optional[Union[bytes, List[bytes]]]: Return the compiled file as `bytes` if output is `None`.
"""

def query(
self,
selector: str,
field: Optional[str] = None,
one: bool = False,
format: Optional[Literal["json", "yaml"]] = None,
) -> str:
"""Query a Typst document.
Args:
selector (str): Typst selector like `<label>`.
field (Optional[str], optional): Field to query.
one (bool, optional): Query only one element.
format (Optional[str]): Output format, `json` or `yaml`.
Returns:
str: Return the query result.
"""

@overload
def compile(
input: PathLike,
output: PathLike,
root: Optional[PathLike] = None,
font_paths: List[PathLike] = [],
format: Optional[str] = None,
format: Optional[Literal["pdf", "svg", "png"]] = None,
ppi: Optional[float] = None,
sys_inputs: Dict[str, str] = {},
) -> None: ...
Expand All @@ -52,7 +69,7 @@ def compile(
output: None = None,
root: Optional[PathLike] = None,
font_paths: List[PathLike] = [],
format: Optional[str] = None,
format: Optional[Literal["pdf", "svg", "png"]] = None,
ppi: Optional[float] = None,
sys_inputs: Dict[str, str] = {},
) -> bytes: ...
Expand All @@ -61,7 +78,7 @@ def compile(
output: Optional[PathLike] = None,
root: Optional[PathLike] = None,
font_paths: List[PathLike] = [],
format: Optional[str] = None,
format: Optional[Literal["pdf", "svg", "png"]] = None,
ppi: Optional[float] = None,
sys_inputs: Dict[str, str] = {},
) -> Optional[Union[bytes, List[bytes]]]:
Expand All @@ -79,3 +96,27 @@ def compile(
Returns:
Optional[Union[bytes, List[bytes]]]: Return the compiled file as `bytes` if output is `None`.
"""

def query(
input: PathLike,
selector: str,
field: Optional[str] = None,
one: bool = False,
format: Optional[Literal["json", "yaml"]] = None,
root: Optional[PathLike] = None,
font_paths: List[PathLike] = [],
sys_inputs: Dict[str, str] = {},
) -> str:
"""Query a Typst document.
Args:
input (PathLike): Project's main .typ file.
selector (str): Typst selector like `<label>`.
field (Optional[str], optional): Field to query.
one (bool, optional): Query only one element.
format (Optional[str]): Output format, `json` or `yaml`.
root (Optional[PathLike], optional): Root path for the Typst project.
font_paths (List[PathLike]): Folders with fonts.
sys_inputs (Dict[str, str]): string key-value pairs to be passed to the document via sys.inputs
Returns:
str: Return the query result.
"""
68 changes: 66 additions & 2 deletions src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
use std::path::PathBuf;

use pyo3::exceptions::{PyIOError, PyRuntimeError};
use pyo3::exceptions::{PyIOError, PyRuntimeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyList};
use pyo3::types::{PyBytes, PyList, PyString};

use query::{query as typst_query, QueryCommand, SerializationFormat};
use std::collections::HashMap;
use typst::foundations::{Dict, Value};
use world::SystemWorld;
Expand All @@ -12,6 +13,7 @@ mod compiler;
mod download;
mod fonts;
mod package;
mod query;
mod world;

mod output_template {
Expand Down Expand Up @@ -84,6 +86,34 @@ impl Compiler {
.map_err(|msg| PyRuntimeError::new_err(msg.to_string()))?;
Ok(buffer)
}

fn query(
&mut self,
selector: &str,
field: Option<&str>,
one: bool,
format: Option<&str>,
) -> PyResult<String> {
let format = format.unwrap_or("json");
let format = match format {
"json" => SerializationFormat::Json,
"yaml" => SerializationFormat::Yaml,
_ => return Err(PyValueError::new_err("unsupported serialization format")),
};
let result = typst_query(
&mut self.world,
&QueryCommand {
selector: selector.into(),
field: field.map(Into::into),
one,
format,
},
);
match result {
Ok(data) => Ok(data),
Err(msg) => Err(PyRuntimeError::new_err(msg.to_string())),
}
}
}

#[pymethods]
Expand Down Expand Up @@ -195,6 +225,20 @@ impl Compiler {
}
}
}

/// Query a typst document
#[pyo3(name = "query", signature = (selector, field = None, one = false, format = None))]
fn py_query(
&mut self,
py: Python<'_>,
selector: &str,
field: Option<&str>,
one: bool,
format: Option<&str>,
) -> PyResult<PyObject> {
py.allow_threads(|| self.query(selector, field, one, format))
.map(|s| PyString::new(py, &s).into())
}
}

/// Compile a typst document to PDF
Expand All @@ -215,11 +259,31 @@ fn compile(
compiler.py_compile(py, output, format, ppi)
}

/// Query a typst document
#[pyfunction]
#[pyo3(name = "query", signature = (input, selector, field = None, one = false, format = None, root = None, font_paths = Vec::new(), sys_inputs = HashMap::new()))]
#[allow(clippy::too_many_arguments)]
fn py_query(
py: Python<'_>,
input: PathBuf,
selector: &str,
field: Option<&str>,
one: bool,
format: Option<&str>,
root: Option<PathBuf>,
font_paths: Vec<PathBuf>,
sys_inputs: HashMap<String, String>,
) -> PyResult<PyObject> {
let mut compiler = Compiler::new(input, root, font_paths, sys_inputs)?;
compiler.py_query(py, selector, field, one, format)
}

/// Python binding to typst
#[pymodule]
fn _typst(_py: Python, m: &PyModule) -> PyResult<()> {
m.add("__version__", env!("CARGO_PKG_VERSION"))?;
m.add_class::<Compiler>()?;
m.add_function(wrap_pyfunction!(compile, m)?)?;
m.add_function(wrap_pyfunction!(py_query, m)?)?;
Ok(())
}
131 changes: 131 additions & 0 deletions src/query.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
use comemo::Track;
use ecow::{eco_format, EcoString};
use serde::Serialize;
use typst::diag::{bail, StrResult};
use typst::eval::{eval_string, EvalMode, Tracer};
use typst::foundations::{Content, IntoValue, LocatableSelector, Scope};
use typst::model::Document;
use typst::syntax::Span;
use typst::World;

use crate::world::SystemWorld;

/// Processes an input file to extract provided metadata
#[derive(Debug, Clone)]
pub struct QueryCommand {
/// Defines which elements to retrieve
pub selector: String,

/// Extracts just one field from all retrieved elements
pub field: Option<String>,

/// Expects and retrieves exactly one element
pub one: bool,

/// The format to serialize in
pub format: SerializationFormat,
}

// Output file format for query command
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum SerializationFormat {
Json,
Yaml,
}

/// Execute a query command.
pub fn query(world: &mut SystemWorld, command: &QueryCommand) -> StrResult<String> {
// Reset everything and ensure that the main file is present.
world.reset();
world.source(world.main()).map_err(|err| err.to_string())?;

let mut tracer = Tracer::new();
let result = typst::compile(world, &mut tracer);
let warnings = tracer.warnings();

match result {
// Retrieve and print query results.
Ok(document) => {
let data = retrieve(world, command, &document)?;
let serialized = format(data, command)?;
Ok(serialized)
}
// Print errors and warnings.
Err(errors) => {
let mut message = EcoString::from("failed to compile document");
for (i, error) in errors.into_iter().enumerate() {
message.push_str(if i == 0 { ": " } else { ", " });
message.push_str(&error.message);
}
for warning in warnings {
message.push_str(": ");
message.push_str(&warning.message);
}
Err(message)
}
}
}

/// Retrieve the matches for the selector.
fn retrieve(
world: &dyn World,
command: &QueryCommand,
document: &Document,
) -> StrResult<Vec<Content>> {
let selector = eval_string(
world.track(),
&command.selector,
Span::detached(),
EvalMode::Code,
Scope::default(),
)
.map_err(|errors| {
let mut message = EcoString::from("failed to evaluate selector");
for (i, error) in errors.into_iter().enumerate() {
message.push_str(if i == 0 { ": " } else { ", " });
message.push_str(&error.message);
}
message
})?
.cast::<LocatableSelector>()?;

Ok(document
.introspector
.query(&selector.0)
.into_iter()
.collect::<Vec<_>>())
}

/// Format the query result in the output format.
fn format(elements: Vec<Content>, command: &QueryCommand) -> StrResult<String> {
if command.one && elements.len() != 1 {
bail!("expected exactly one element, found {}", elements.len());
}

let mapped: Vec<_> = elements
.into_iter()
.filter_map(|c| match &command.field {
Some(field) => c.get_by_name(field),
_ => Some(c.into_value()),
})
.collect();

if command.one {
let Some(value) = mapped.first() else {
bail!("no such field found for element");
};
serialize(value, command.format)
} else {
serialize(&mapped, command.format)
}
}

/// Serialize data to the output format.
fn serialize(data: &impl Serialize, format: SerializationFormat) -> StrResult<String> {
match format {
SerializationFormat::Json => {
serde_json::to_string_pretty(data).map_err(|e| eco_format!("{e}"))
}
SerializationFormat::Yaml => serde_yaml::to_string(&data).map_err(|e| eco_format!("{e}")),
}
}

0 comments on commit c7c9604

Please sign in to comment.