diff --git a/Cargo.lock b/Cargo.lock index 5248ebd5..f8ec5de7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -513,6 +513,8 @@ dependencies = [ "glob", "js-sys", "log", + "pyo3", + "pyo3-stub-gen", "regex", "serde", "serde_json", @@ -689,6 +691,8 @@ dependencies = [ "lazy_static", "log", "pretty_assertions", + "pyo3", + "pyo3-stub-gen", "rand", "regex", "serde", @@ -807,9 +811,12 @@ dependencies = [ name = "context-py" version = "0.1.0" dependencies = [ + "bundle", "context", + "futures-io", "pyo3", "pyo3-stub-gen", + "tokio", ] [[package]] diff --git a/bundle/Cargo.toml b/bundle/Cargo.toml index 63c1854a..e44d386e 100644 --- a/bundle/Cargo.toml +++ b/bundle/Cargo.toml @@ -17,6 +17,7 @@ serde = { version = "1.0.215", default-features = false, features = ["derive"] } serde_json = "1.0.133" tsify-next = { version = "0.5.4", optional = true } wasm-bindgen = { version = "0.2.95", optional = true } +pyo3-stub-gen = { version = "0.6.0", optional = true } # For encoding async-compression = { version = "0.4.17", features = ["futures-io", "zstd"] } @@ -27,6 +28,14 @@ tar = { version = "0.4.30", default-features = false } uuid = { version = "1.10.0", features = ["v5"] } zstd = { version = "0.13.0", default-features = false } +[target.'cfg(target_os = "linux")'.dependencies] +pyo3 = { version = "0.22.5", optional = true, features = [ + "abi3-py39", + "extension-module", +] } + +[target.'cfg(target_os = "macos")'.dependencies] +pyo3 = { version = "0.22.5", optional = true, features = ["abi3-py39"] } [features] bindings = [] @@ -38,3 +47,10 @@ wasm = [ "codeowners/wasm", "context/wasm", ] +pyo3 = [ + "bindings", + "dep:pyo3", + "dep:pyo3-stub-gen", + "codeowners/pyo3", + "context/pyo3", +] diff --git a/bundle/src/bundle_meta.rs b/bundle/src/bundle_meta.rs index 5ddd2970..004b92bb 100644 --- a/bundle/src/bundle_meta.rs +++ b/bundle/src/bundle_meta.rs @@ -5,6 +5,10 @@ use codeowners::CodeOwners; use context::repo::BundleRepo; +#[cfg(feature = "pyo3")] +use pyo3::prelude::*; +#[cfg(feature = "pyo3")] +use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pyclass_enum}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; #[cfg(feature = "wasm")] @@ -16,7 +20,8 @@ use crate::{files::FileSet, CustomTag, Test}; pub const META_VERSION: &str = "1"; // 0.5.29 was first version to include bundle_upload_id and serves as the base -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass(get_all))] #[cfg_attr(feature = "wasm", derive(Tsify))] pub struct BundleMetaBaseProps { pub version: String, @@ -33,21 +38,24 @@ pub struct BundleMetaBaseProps { pub quarantined_tests: Vec, pub codeowners: Option, } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass(get_all))] #[cfg_attr(feature = "wasm", derive(Tsify))] pub struct BundleMetaV0_5_29 { #[serde(flatten)] pub base_props: BundleMetaBaseProps, } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass(get_all))] #[cfg_attr(feature = "wasm", derive(Tsify))] pub struct BundleMetaJunitProps { pub num_files: usize, pub num_tests: usize, } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass(get_all))] #[cfg_attr(feature = "wasm", derive(Tsify))] pub struct BundleMetaV0_5_34 { #[serde(flatten)] @@ -56,7 +64,8 @@ pub struct BundleMetaV0_5_34 { pub junit_props: BundleMetaJunitProps, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "pyo3", gen_stub_pyclass_enum, pyclass(eq))] #[cfg_attr(feature = "wasm", derive(Tsify))] #[cfg_attr(feature = "wasm", tsify(into_wasm_abi, from_wasm_abi))] #[serde(tag = "schema")] diff --git a/bundle/src/files.rs b/bundle/src/files.rs index 0300d893..f082d3dc 100644 --- a/bundle/src/files.rs +++ b/bundle/src/files.rs @@ -1,13 +1,17 @@ use std::{format, time::SystemTime}; -#[cfg(feature = "wasm")] -use tsify_next::Tsify; -#[cfg(feature = "wasm")] -use wasm_bindgen::prelude::*; use codeowners::{CodeOwners, Owners, OwnersOfPath}; use constants::ALLOW_LIST; +#[cfg(feature = "pyo3")] +use pyo3::prelude::*; +#[cfg(feature = "pyo3")] +use pyo3_stub_gen::derive::gen_stub_pyclass; use regex::Regex; use serde::{Deserialize, Serialize}; +#[cfg(feature = "wasm")] +use tsify_next::Tsify; +#[cfg(feature = "wasm")] +use wasm_bindgen::prelude::*; use crate::types::{BundledFile, FileSetType}; @@ -28,7 +32,8 @@ impl FileSetCounter { } } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass(get_all))] #[cfg_attr(feature = "wasm", derive(Tsify))] pub struct FileSet { pub file_set_type: FileSetType, diff --git a/bundle/src/types.rs b/bundle/src/types.rs index 64bfb8c0..60d3ba0e 100644 --- a/bundle/src/types.rs +++ b/bundle/src/types.rs @@ -1,11 +1,14 @@ +use context::repo::BundleRepo; +#[cfg(feature = "pyo3")] +use pyo3::prelude::*; +#[cfg(feature = "pyo3")] +use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pyclass_enum}; +use serde::{Deserialize, Serialize}; #[cfg(feature = "wasm")] use tsify_next::Tsify; #[cfg(feature = "wasm")] use wasm_bindgen::prelude::*; -use context::repo::BundleRepo; -use serde::{Deserialize, Serialize}; - pub struct RunResult { pub exit_code: i32, pub failures: Vec, @@ -17,7 +20,8 @@ pub struct QuarantineRunResult { pub quarantine_status: QuarantineBulkTestStatus, } -#[derive(Debug, Serialize, Clone, Deserialize)] +#[derive(Debug, Serialize, Clone, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass(get_all))] #[cfg_attr(feature = "wasm", derive(Tsify))] pub struct Test { pub name: String, @@ -85,7 +89,8 @@ pub struct BundleUploader { pub org_slug: String, } -#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq)] +#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq)] +#[cfg_attr(feature = "pyo3", gen_stub_pyclass_enum, pyclass(eq, eq_int))] #[cfg_attr(feature = "wasm", derive(Tsify))] pub enum FileSetType { #[default] @@ -94,7 +99,8 @@ pub enum FileSetType { #[cfg(feature = "wasm")] // u128 will be supported in the next release after 0.2.95 -#[derive(Debug, Serialize, Deserialize, Clone, Default)] +#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq)] +#[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass(get_all))] #[cfg_attr(feature = "wasm", derive(Tsify))] pub struct BundledFile { pub original_path: String, @@ -131,6 +137,7 @@ impl BundledFile { /// Custom tags defined by the user. /// #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass(get_all))] #[cfg_attr(feature = "wasm", derive(Tsify))] pub struct CustomTag { pub key: String, diff --git a/codeowners/Cargo.toml b/codeowners/Cargo.toml index 5c730aa0..ecfdce74 100644 --- a/codeowners/Cargo.toml +++ b/codeowners/Cargo.toml @@ -24,7 +24,18 @@ serde = { version = "1.0.215", default-features = false, features = ["derive"] } serde_json = "1.0.133" tsify-next = { version = "0.5.4", optional = true } wasm-bindgen = { version = "0.2.95", optional = true } +pyo3-stub-gen = { version = "0.6.0", optional = true } + +[target.'cfg(target_os = "linux")'.dependencies] +pyo3 = { version = "0.22.5", optional = true, features = [ + "abi3-py39", + "extension-module", +] } + +[target.'cfg(target_os = "macos")'.dependencies] +pyo3 = { version = "0.22.5", optional = true, features = ["abi3-py39"] } [features] bindings = [] wasm = ["bindings", "dep:wasm-bindgen", "dep:js-sys", "dep:tsify-next"] +pyo3 = ["bindings", "dep:pyo3", "dep:pyo3-stub-gen"] diff --git a/codeowners/src/codeowners.rs b/codeowners/src/codeowners.rs index 34cbf9fd..70c88e8c 100644 --- a/codeowners/src/codeowners.rs +++ b/codeowners/src/codeowners.rs @@ -1,9 +1,14 @@ -use constants::CODEOWNERS_LOCATIONS; -use serde::{Deserialize, Serialize}; use std::{ fs::File, path::{Path, PathBuf}, }; + +use constants::CODEOWNERS_LOCATIONS; +#[cfg(feature = "pyo3")] +use pyo3::prelude::*; +#[cfg(feature = "pyo3")] +use pyo3_stub_gen::derive::gen_stub_pyclass; +use serde::{Deserialize, Serialize}; #[cfg(feature = "wasm")] use tsify_next::Tsify; #[cfg(feature = "wasm")] @@ -12,7 +17,8 @@ use wasm_bindgen::prelude::*; use crate::{github::GitHubOwners, gitlab::GitLabOwners, traits::FromReader}; // TODO(TRUNK-13628): Implement serializing and deserializing for CodeOwners -#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq)] +#[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass)] #[cfg_attr(feature = "wasm", derive(Tsify))] pub struct CodeOwners { pub path: PathBuf, @@ -61,7 +67,7 @@ impl CodeOwners { } } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum Owners { GitHubOwners(GitHubOwners), GitLabOwners(GitLabOwners), diff --git a/codeowners/src/github.rs b/codeowners/src/github.rs index 0b667483..63de7621 100644 --- a/codeowners/src/github.rs +++ b/codeowners/src/github.rs @@ -24,7 +24,7 @@ use crate::{FromPath, FromReader, OwnersOfPath}; /// raw /// ); /// ``` -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Eq)] pub enum GitHubOwner { /// Owner in the form @username Username(String), @@ -67,7 +67,7 @@ impl FromStr for GitHubOwner { } /// Mappings of GitHub owners to path patterns -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Eq)] pub struct GitHubOwners { paths: Vec<(Pattern, Vec)>, } diff --git a/codeowners/src/gitlab/error.rs b/codeowners/src/gitlab/error.rs index e3bb61da..caafc77e 100644 --- a/codeowners/src/gitlab/error.rs +++ b/codeowners/src/gitlab/error.rs @@ -19,7 +19,7 @@ pub enum ErrorType { InvalidSectionFormat, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Error { message: String, line_number: usize, diff --git a/codeowners/src/gitlab/file.rs b/codeowners/src/gitlab/file.rs index 4fecbefa..a47f49d3 100644 --- a/codeowners/src/gitlab/file.rs +++ b/codeowners/src/gitlab/file.rs @@ -11,7 +11,7 @@ use super::{Entry, Error, Section, SectionParser}; pub type ParsedData = IndexMap>; /// Reference: https://gitlab.com/gitlab-org/gitlab/-/blob/master/ee/lib/gitlab/code_owners/file.rb -#[derive(Debug, Clone, Default, PartialEq)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct File { path: PathBuf, errors: Vec, diff --git a/codeowners/src/gitlab/mod.rs b/codeowners/src/gitlab/mod.rs index 01cb0af2..7d7f9ab9 100644 --- a/codeowners/src/gitlab/mod.rs +++ b/codeowners/src/gitlab/mod.rs @@ -38,7 +38,7 @@ impl fmt::Display for GitLabOwner { } } -#[derive(Debug, Clone, Default, PartialEq)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct GitLabOwners { file: File, } diff --git a/context-py/Cargo.toml b/context-py/Cargo.toml index b4cb4900..00c6b8c9 100644 --- a/context-py/Cargo.toml +++ b/context-py/Cargo.toml @@ -13,8 +13,11 @@ name = "stub_gen" path = "bin/stub_gen.rs" [dependencies] +bundle = { path = "../bundle", default-features = false, features = ["pyo3"] } context = { path = "../context", features = ["git-access", "pyo3"] } pyo3-stub-gen = "0.6.0" +futures-io = "0.3.31" +tokio = { version = "*", default-features = false, features = ["rt"] } [target.'cfg(target_os = "linux")'.dependencies] pyo3 = { version = "0.22.5", features = ["abi3-py39", "extension-module"] } diff --git a/context-py/src/lib.rs b/context-py/src/lib.rs index 09908ef2..9bfcd303 100644 --- a/context-py/src/lib.rs +++ b/context-py/src/lib.rs @@ -1,9 +1,14 @@ use std::{collections::HashMap, io::BufReader}; +use bundle::{parse_meta_from_tarball as parse_tarball, VersionedBundle}; use context::{env, junit, repo}; use pyo3::{exceptions::PyTypeError, prelude::*}; use pyo3_stub_gen::{define_stub_info_gatherer, derive::gen_stub_pyfunction}; +mod py_bytes_read; + +use py_bytes_read::PyBytesReader; + define_stub_info_gatherer!(stub_info); #[gen_stub_pyfunction] @@ -68,6 +73,15 @@ fn repo_validate(bundle_repo: repo::BundleRepo) -> repo::validator::RepoValidati repo::validator::validate(&bundle_repo) } +#[gen_stub_pyfunction] +#[pyfunction] +pub fn parse_meta_from_tarball(py: Python<'_>, reader: PyObject) -> PyResult { + let py_bytes_reader = PyBytesReader::new(reader.into_bound(py))?; + let rt = tokio::runtime::Runtime::new()?; + rt.block_on(parse_tarball(py_bytes_reader)) + .map_err(|err| PyTypeError::new_err(err.to_string())) +} + #[pymodule] fn context_py(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; diff --git a/context-py/src/py_bytes_read.rs b/context-py/src/py_bytes_read.rs new file mode 100644 index 00000000..c13b3028 --- /dev/null +++ b/context-py/src/py_bytes_read.rs @@ -0,0 +1,75 @@ +use std::{ + cmp, io, + pin::Pin, + task::{Context, Poll}, +}; + +use futures_io::{AsyncBufRead, AsyncRead}; +use pyo3::{prelude::*, types::PyBytes}; + +pub struct PyBytesReader<'py> { + inner: Bound<'py, PyAny>, + content_length: usize, + content_length_read: usize, + inner_buffer: Vec, +} + +impl<'py> PyBytesReader<'py> { + pub fn new(py_bytes_reader: Bound<'py, PyAny>) -> PyResult { + let content_length = py_bytes_reader + .getattr("_content_length")? + .extract::()?; + Ok(Self { + inner: py_bytes_reader, + content_length, + content_length_read: 0, + inner_buffer: Vec::with_capacity(0), + }) + } + + pub fn content_length_remaining(&self) -> usize { + self.content_length - self.content_length_read + } +} + +impl AsyncRead for PyBytesReader<'_> { + fn poll_read( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + let self_mut = self.get_mut(); + let amt = cmp::min(buf.len(), self_mut.content_length_remaining()); + let read = self_mut.inner.call_method1("read", (amt,))?; + let bytes = read + .downcast::() + .map_err(|err| io::Error::new(io::ErrorKind::Other, err.to_string()))? + .as_bytes(); + buf[..amt].copy_from_slice(&bytes[..amt]); + self_mut.content_length_read += amt; + Poll::Ready(Ok(amt)) + } +} + +impl AsyncBufRead for PyBytesReader<'_> { + fn poll_fill_buf(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + let self_mut = self.get_mut(); + let amt = self_mut.content_length_remaining(); + let read = self_mut.inner.call_method1("read", (amt,))?; + let bytes = read + .downcast::() + .map_err(|err| io::Error::new(io::ErrorKind::Other, err.to_string()))? + .as_bytes(); + self_mut.inner_buffer = Vec::from(bytes); + self_mut.content_length_read += amt; + Poll::Ready(Ok(&self_mut.inner_buffer)) + } + + fn consume(self: Pin<&mut Self>, amt: usize) { + let self_mut = self.get_mut(); + let amt = cmp::min(amt, self_mut.content_length_remaining()); + if self_mut.inner.call_method1("read", (amt,)).is_ok() { + self_mut.content_length_read += amt; + } + } +} diff --git a/context/src/repo/mod.rs b/context/src/repo/mod.rs index a6d08353..148adcaa 100644 --- a/context/src/repo/mod.rs +++ b/context/src/repo/mod.rs @@ -32,7 +32,7 @@ struct BundleRepoOptions { #[cfg_attr(feature = "pyo3", gen_stub_pyclass, pyclass(get_all))] #[cfg_attr(feature = "wasm", wasm_bindgen(getter_with_clone))] -#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct BundleRepo { pub repo: RepoUrlParts, pub repo_root: String,