parse_meta_from_tarball python bindings #176
Conversation
dfrankland
force-pushed
the
dylan/parse_meta_from_tarball-python-bindings
branch
from
54eba5c to
1251bff
Compare
dfrankland
force-pushed
the
dylan/parse_meta_from_tarball-python-bindings
branch
2 times, most recently
from
ef5b7aa to
c7483d4
Compare
dfrankland
force-pushed
the
dylan/parse_meta_from_tarball-python-bindings
branch
from
c7483d4 to
65c56a5
Compare
| raw_stream = StreamingBody( | ||
| io.BytesIO(meta_tarball_compressed), len(meta_tarball_compressed) | ||
| ) |
There was a problem hiding this comment.
StreamingBody is the type returned by S3 when fetching an object
| inner: py_bytes_reader, | ||
| content_length, | ||
| content_length_read: 0, | ||
| inner_buffer: Vec::with_capacity(0), |
There was a problem hiding this comment.
fun fact: there's no allocation when a Vec is created with 0 capacity 👍
| } | ||
|
|
||
| impl<'py> PyBytesReader<'py> { | ||
| const DEFAULT_CHUNK_SIZE: usize = 1024; |
There was a problem hiding this comment.
This matches StreamingBody's default chunk size
| let rt = tokio::runtime::Builder::new_current_thread() | ||
| .enable_all() | ||
| .build()?; | ||
| let versioned_bundle = rt | ||
| .block_on(parse_tarball(py_bytes_reader)) | ||
| .map_err(|err| PyTypeError::new_err(err.to_string()))?; |
There was a problem hiding this comment.
Even though this is not async, reading bytes still happens as needed
| #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] | ||
| #[cfg(feature = "pyo3")] | ||
| #[gen_stub_pyclass] | ||
| #[pyclass] | ||
| pub struct BindingsVersionedBundle(pub VersionedBundle); | ||
|
|
||
| #[cfg(feature = "pyo3")] | ||
| #[gen_stub_pymethods] | ||
| #[pymethods] | ||
| impl BindingsVersionedBundle { | ||
| pub fn get_v0_5_29(&self) -> Option<BundleMetaV0_5_29> { | ||
| match &self.0 { | ||
| VersionedBundle::V0_5_29(bundle_meta) => Some(bundle_meta.clone()), | ||
| _ => None, | ||
| } | ||
| } | ||
| pub fn get_v0_5_34(&self) -> Option<BundleMetaV0_5_34> { | ||
| match &self.0 { | ||
| VersionedBundle::V0_5_34(bundle_meta) => Some(bundle_meta.clone()), | ||
| _ => None, | ||
| } | ||
| } | ||
| pub fn get_v0_6_2(&self) -> Option<BundleMetaV0_6_2> { | ||
| match &self.0 { | ||
| VersionedBundle::V0_6_2(bundle_meta) => Some(bundle_meta.clone()), | ||
| _ => None, | ||
| } | ||
| } | ||
| } |
There was a problem hiding this comment.
Using gen_stub_pymethodsdoesn't seem to work on enums yet:
Jij-Inc/pyo3-stub-gen#69
All that's needed is a new type here to get around it
Also, the Python bindings aren't using serde, so VersionedBundle doesn't end up creating a nice dict—we can just use methods to get the meta.json values
| @@ -276,3 +276,113 @@ def test_repo_validate(): | |||
| assert repo_validation.max_level() == RepoValidationLevel.Valid, "\n" + "\n".join( | |||
| [issue.error_message for issue in repo_validation.issues_flat()] | |||
| ) | |||
|
|
|||
|
|
|||
| def test_parse_meta_from_tarball(): | |||
There was a problem hiding this comment.
nit/optional: I made this a separate ts test file just to make these more digestible. Would be better to do that here too for consistency imo
dfrankland
force-pushed
the
dylan/parse_meta_from_tarball-python-bindings
branch
from
cc4c675 to
ca88274
Compare
Depends on #181
Will be used to filter bundles without completely downloading them