diff --git a/Cargo.lock b/Cargo.lock index 67ff15539..8b065ff76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -384,6 +384,17 @@ dependencies = [ "syn 2.0.43", ] +[[package]] +name = "async-trait" +version = "0.1.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.43", +] + [[package]] name = "atoi" version = "2.0.0" @@ -711,6 +722,16 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -912,6 +933,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "dotenvy" version = "0.15.7" @@ -937,6 +964,15 @@ dependencies = [ "serde", ] +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + [[package]] name = "env_logger" version = "0.10.1" @@ -1040,10 +1076,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b7d486286d0fc946e00b0133fa2ded824e08f03412450e5d9a1f446abcd589d" dependencies = [ "byteorder", + "bytes", "fallible-streaming-iterator", "flatbuffers", "geozero", + "http-range-client", "log", + "reqwest", "tempfile", ] @@ -1070,6 +1109,27 @@ dependencies = [ "spin 0.9.8", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1260,8 +1320,10 @@ dependencies = [ "arrow-ipc", "arrow-schema", "async-stream", + "async-trait", "bumpalo", "byteorder", + "bytes", "chrono", "criterion", "flatgeobuf", @@ -1271,9 +1333,11 @@ dependencies = [ "geodesy", "geos", "geozero", + "http-range-client", "indexmap", "itertools 0.12.0", "num_enum", + "object_store", "parquet", "phf", "proj", @@ -1389,6 +1453,25 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "h2" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "1.8.2" @@ -1514,12 +1597,97 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "http" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-range-client" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c724cf94381bf0f09a60c980ae732d1f2859ee2a98a0d6bce68ae509f915785c" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "read-logger", + "reqwest", + "thiserror", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -1569,6 +1737,12 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "is-terminal" version = "0.4.9" @@ -1801,6 +1975,12 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1827,6 +2007,24 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.27.1" @@ -1982,6 +2180,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d139f545f64630e2e3688fd9f81c470888ab01edeb72d13b4e86c566f1130000" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "humantime", + "itertools 0.12.0", + "parking_lot", + "percent-encoding", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -1994,6 +2213,50 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl" +version = "0.10.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" +dependencies = [ + "bitflags 2.4.1", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.43", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -2316,6 +2579,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "read-logger" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7f715a23c7db804b71eb9162a9cf210b89e99db9c3649a2a038d13b7594a99" +dependencies = [ + "log", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -2365,6 +2637,46 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "reqwest" +version = "0.11.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.17.7" @@ -2506,6 +2818,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2528,6 +2849,29 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.20" @@ -2571,6 +2915,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2630,6 +2986,28 @@ version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +[[package]] +name = "snafu" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" +dependencies = [ + "doc-comment", + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "snap" version = "1.1.1" @@ -2955,6 +3333,33 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tar" version = "0.4.40" @@ -3081,6 +3486,16 @@ dependencies = [ "syn 2.0.43", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -3092,6 +3507,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml_datetime" version = "0.6.3" @@ -3109,6 +3538,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.40" @@ -3141,6 +3576,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "twox-hash" version = "1.6.3" @@ -3250,6 +3691,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -3281,6 +3731,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.89" @@ -3525,6 +3987,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wkt" version = "0.10.3" diff --git a/Cargo.toml b/Cargo.toml index 98b7e33f3..37ee6129a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,13 @@ rust-version = "1.75" [features] csv = ["dep:geozero", "geozero/with-csv"] flatgeobuf = ["dep:flatgeobuf", "geozero"] +flatgeobuf_async = [ + "flatgeobuf/http", + "dep:async-trait", + "dep:bytes", + "dep:http-range-client", + "dep:object_store", +] gdal = ["dep:gdal"] geos = ["dep:geos"] geozero = ["dep:geozero"] @@ -40,8 +47,10 @@ arrow-data = "50" arrow-ipc = "50" arrow-schema = "50" async-stream = { version = "0.3", optional = true } +async-trait = { version = "0.1", optional = true } bumpalo = { version = "3", features = ["collections"] } byteorder = "1" +bytes = { version = "*", optional = true } chrono = "0.4" # Set default-features = false because async not working in wasm right now flatgeobuf = { version = "4.1.0", optional = true, default-features = false } @@ -51,9 +60,11 @@ geo = "0.28" geodesy = { version = "0.12", optional = true } geos = { version = "8.3", features = ["v3_10_0", "geo"], optional = true } geozero = { version = "0.12", features = ["with-wkb"], optional = true } +http-range-client = { version = "0.7.2", optional = true } indexmap = "2" itertools = "0.12" num_enum = "0.7" +object_store = { version = "0.9.0", optional = true } parquet = { version = "50", optional = true, default-features = false, features = [ "arrow", ] } diff --git a/python/core/Cargo.lock b/python/core/Cargo.lock index f2aa012bd..746a9b3d2 100644 --- a/python/core/Cargo.lock +++ b/python/core/Cargo.lock @@ -330,6 +330,17 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "async-trait" +version = "0.1.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "atoi" version = "2.0.0" @@ -479,6 +490,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-targets 0.52.0", ] @@ -509,6 +521,16 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" @@ -648,6 +670,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "dotenvy" version = "0.15.7" @@ -673,6 +701,15 @@ dependencies = [ "serde", ] +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -751,10 +788,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b7d486286d0fc946e00b0133fa2ded824e08f03412450e5d9a1f446abcd589d" dependencies = [ "byteorder", + "bytes", "fallible-streaming-iterator", "flatbuffers", "geozero", + "http-range-client", "log", + "reqwest", "tempfile", ] @@ -775,6 +815,27 @@ dependencies = [ "spin 0.9.8", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -936,16 +997,20 @@ dependencies = [ "arrow-ipc", "arrow-schema", "async-stream", + "async-trait", "bumpalo", "byteorder", + "bytes", "chrono", "flatgeobuf", "futures", "geo", "geozero", + "http-range-client", "indexmap", "itertools 0.12.1", "num_enum", + "object_store", "parquet", "phf", "rayon", @@ -969,12 +1034,15 @@ dependencies = [ "geoarrow", "geozero", "numpy", + "object_store", "parquet", "pyo3", "pyo3-asyncio", + "reqwest", "sqlx", "thiserror", "tokio", + "url", ] [[package]] @@ -1031,6 +1099,25 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "h2" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.3.1" @@ -1150,6 +1237,111 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "http" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-range-client" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c724cf94381bf0f09a60c980ae732d1f2859ee2a98a0d6bce68ae509f915785c" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "read-logger", + "reqwest", + "thiserror", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -1211,6 +1403,12 @@ version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767" +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "itertools" version = "0.11.0" @@ -1406,6 +1604,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1432,6 +1636,24 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndarray" version = "0.15.6" @@ -1614,12 +1836,96 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d139f545f64630e2e3688fd9f81c470888ab01edeb72d13b4e86c566f1130000" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "futures", + "humantime", + "hyper", + "itertools 0.12.1", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand", + "reqwest", + "ring", + "rustls-pemfile 2.1.0", + "serde", + "serde_json", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + [[package]] name = "once_cell" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.4.2", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-src" +version = "300.1.3+3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd2c101a165fff9935e34def4669595ab1c7847943c42be86e21503e482be107" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -1888,6 +2194,16 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.35" @@ -1953,6 +2269,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "read-logger" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7f715a23c7db804b71eb9162a9cf210b89e99db9c3649a2a038d13b7594a99" +dependencies = [ + "log", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -1991,6 +2316,52 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "reqwest" +version = "0.11.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.17.7" @@ -2093,11 +2464,24 @@ version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ + "log", "ring", "rustls-webpki", "sct", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -2107,6 +2491,22 @@ dependencies = [ "base64", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c333bb734fcdedcea57de1602543590f545f127dc8b533324318fd492c5c70b" +dependencies = [ + "base64", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048a63e5b3ac996d78d402940b5fa47973d2d080c6c6fffa1d0f19c4445310b7" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -2123,6 +2523,24 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2145,6 +2563,29 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.21" @@ -2188,6 +2629,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2241,6 +2694,28 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +[[package]] +name = "snafu" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" +dependencies = [ + "doc-comment", + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "snap" version = "1.1.1" @@ -2348,7 +2823,7 @@ dependencies = [ "paste", "percent-encoding", "rustls", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "sha2", @@ -2560,6 +3035,33 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "target-lexicon" version = "0.12.13" @@ -2646,9 +3148,41 @@ dependencies = [ "num_cpus", "pin-project-lite", "socket2", + "tokio-macros", "windows-sys 0.48.0", ] +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -2660,6 +3194,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml_datetime" version = "0.6.5" @@ -2677,6 +3225,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.40" @@ -2709,6 +3263,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "twox-hash" version = "1.6.3" @@ -2799,6 +3359,25 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2830,6 +3409,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.91" @@ -2859,6 +3450,29 @@ version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" +[[package]] +name = "wasm-streams" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.25.4" @@ -2871,6 +3485,37 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.52.0" @@ -3021,6 +3666,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wkt" version = "0.10.3" diff --git a/python/core/Cargo.toml b/python/core/Cargo.toml index af42d440c..55f2de07f 100644 --- a/python/core/Cargo.toml +++ b/python/core/Cargo.toml @@ -22,6 +22,7 @@ arrow-buffer = "50" arrow = { version = "50", features = ["ffi"] } bytes = "1" flatgeobuf = { version = "4.1.0", default-features = false } +object_store = { version = "0.9.0", features = ["aws", "azure", "gcp", "http"] } parquet = "50" pyo3 = { version = "0.20.0", features = [ "abi3-py38", @@ -35,6 +36,7 @@ geo = "0.28" geoarrow = { path = "../../", features = [ "csv", "flatgeobuf", + "flatgeobuf_async", "geozero", "ipc_compression", "parquet_compression", @@ -47,3 +49,8 @@ numpy = "0.20" sqlx = { version = "0.7", default-features = false, features = ["postgres"] } thiserror = "1" tokio = { version = "1.9", features = ["rt"] } +url = "2.5" + +# reqwest is pulled in by object store, but not used by python binding itself +# for binary wheel best practice, statically link openssl +reqwest = { version = "*", features = ["native-tls-vendored"] } diff --git a/python/core/python/geoarrow/rust/core/_rust.pyi b/python/core/python/geoarrow/rust/core/_rust.pyi index c6b20994e..a28bab866 100644 --- a/python/core/python/geoarrow/rust/core/_rust.pyi +++ b/python/core/python/geoarrow/rust/core/_rust.pyi @@ -3,6 +3,7 @@ from __future__ import annotations from pathlib import Path from typing import ( BinaryIO, + Dict, List, Optional, Self, @@ -1240,6 +1241,9 @@ def read_csv( def read_flatgeobuf( file: Union[str, Path, BinaryIO], batch_size: int = 65536 ) -> GeoTable: ... +async def read_flatgeobuf_async( + url: str, *, batch_size: int = 65536, options: Dict[str, str] | None = None +) -> GeoTable: ... def read_geojson( file: Union[str, Path, BinaryIO], batch_size: int = 65536 ) -> GeoTable: ... diff --git a/python/core/src/io/flatgeobuf.rs b/python/core/src/io/flatgeobuf.rs index b39dc7706..5c7312010 100644 --- a/python/core/src/io/flatgeobuf.rs +++ b/python/core/src/io/flatgeobuf.rs @@ -1,10 +1,16 @@ -use crate::error::PyGeoArrowResult; +use std::collections::HashMap; + +use crate::error::{PyGeoArrowError, PyGeoArrowResult}; use crate::io::file::{BinaryFileReader, BinaryFileWriter}; use crate::table::GeoTable; use flatgeobuf::FgbWriterOptions; use geoarrow::io::flatgeobuf::read_flatgeobuf as _read_flatgeobuf; +use geoarrow::io::flatgeobuf::read_flatgeobuf_async as _read_flatgeobuf_async; use geoarrow::io::flatgeobuf::write_flatgeobuf_with_options as _write_flatgeobuf; +use object_store::{parse_url, parse_url_opts}; +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; +use url::Url; /// Read a FlatGeobuf file from a path on disk into a GeoTable. /// @@ -26,6 +32,39 @@ pub fn read_flatgeobuf( Ok(GeoTable(table)) } +/// Read a FlatGeobuf file from a url into a GeoTable. +/// +/// Returns: +/// Table from FlatGeobuf file. +#[pyfunction] +#[pyo3(signature = (url, *, batch_size=65536, options=None))] +pub fn read_flatgeobuf_async( + py: Python, + url: String, + batch_size: usize, + options: Option>, +) -> PyGeoArrowResult { + let fut = pyo3_asyncio::tokio::future_into_py(py, async move { + let url = Url::parse(&url).map_err(|err| PyValueError::new_err(err.to_string()))?; + let (reader, location) = if let Some(options) = options { + parse_url_opts(&url, options) + } else { + parse_url(&url) + } + .map_err(|err| PyValueError::new_err(err.to_string()))?; + // dbg!(&reader); + // dbg!(&location); + + let table = + _read_flatgeobuf_async(reader, location, Default::default(), Some(batch_size), None) + .await + .map_err(PyGeoArrowError::GeoArrowError)?; + + Ok(GeoTable(table)) + })?; + Ok(fut.into()) +} + /// Write a GeoTable to a FlatGeobuf file on disk. /// /// Args: diff --git a/python/core/src/io/postgis.rs b/python/core/src/io/postgis.rs index 1d15a07e1..ca1545c30 100644 --- a/python/core/src/io/postgis.rs +++ b/python/core/src/io/postgis.rs @@ -28,21 +28,6 @@ pub fn read_postgis(connection_url: String, sql: String) -> PyGeoArrowResult PyResult<()> { m.add_function(wrap_pyfunction!(crate::io::csv::read_csv, m)?)?; m.add_function(wrap_pyfunction!(crate::io::flatgeobuf::read_flatgeobuf, m)?)?; + m.add_function(wrap_pyfunction!( + crate::io::flatgeobuf::read_flatgeobuf_async, + m + )?)?; m.add_function(wrap_pyfunction!(crate::io::geojson::read_geojson, m)?)?; m.add_function(wrap_pyfunction!( crate::io::geojson_lines::read_geojson_lines, diff --git a/python/docs/source/api/core/io.md b/python/docs/source/api/core/io.md index ca42fd485..e46e5db02 100644 --- a/python/docs/source/api/core/io.md +++ b/python/docs/source/api/core/io.md @@ -9,6 +9,7 @@ Read and write to files on disk and databases like PostGIS. members: - read_csv - read_flatgeobuf + - read_flatgeobuf_async - read_geojson - read_geojson_lines - read_ipc diff --git a/src/error.rs b/src/error.rs index c0caad730..157891387 100644 --- a/src/error.rs +++ b/src/error.rs @@ -46,6 +46,10 @@ pub enum GeoArrowError { #[error(transparent)] GeosError(#[from] geos::Error), + #[cfg(feature = "flatgeobuf_async")] + #[error(transparent)] + ObjectStoreError(#[from] object_store::Error), + #[cfg(feature = "parquet")] #[error(transparent)] ParquetError(#[from] parquet::errors::ParquetError), diff --git a/src/io/flatgeobuf/mod.rs b/src/io/flatgeobuf/mod.rs index 535e6bba2..1765aebf4 100644 --- a/src/io/flatgeobuf/mod.rs +++ b/src/io/flatgeobuf/mod.rs @@ -4,4 +4,6 @@ mod reader; mod writer; pub use reader::read_flatgeobuf; +#[cfg(feature = "flatgeobuf_async")] +pub use reader::read_flatgeobuf_async; pub use writer::{write_flatgeobuf, write_flatgeobuf_with_options}; diff --git a/src/io/flatgeobuf/reader/async.rs b/src/io/flatgeobuf/reader/async.rs new file mode 100644 index 000000000..50d0701d7 --- /dev/null +++ b/src/io/flatgeobuf/reader/async.rs @@ -0,0 +1,145 @@ +use std::sync::Arc; + +use flatgeobuf::{GeometryType, HttpFgbReader}; +use http_range_client::AsyncBufferedHttpRangeClient; +use object_store::path::Path; +use object_store::ObjectStore; + +use crate::algorithm::native::Downcast; +use crate::array::*; +use crate::error::{GeoArrowError, Result}; +use crate::io::flatgeobuf::reader::common::infer_schema; +use crate::io::flatgeobuf::reader::object_store_reader::ObjectStoreWrapper; +use crate::io::geozero::array::MixedGeometryStreamBuilder; +use crate::io::geozero::table::{GeoTableBuilder, GeoTableBuilderOptions}; +use crate::table::GeoTable; + +pub async fn read_flatgeobuf_async( + reader: T, + location: Path, + coord_type: CoordType, + batch_size: Option, + bbox: Option<(f64, f64, f64, f64)>, +) -> Result { + let head = reader.head(&location).await?; + + let object_store_wrapper = ObjectStoreWrapper { + reader, + location, + size: head.size, + }; + let async_client = AsyncBufferedHttpRangeClient::with(object_store_wrapper, ""); + + let reader = HttpFgbReader::new(async_client).await.unwrap(); + + let header = reader.header(); + if header.has_m() | header.has_t() | header.has_tm() | header.has_z() { + return Err(GeoArrowError::General( + "Only XY dimensions are supported".to_string(), + )); + } + + let schema = infer_schema(header); + let geometry_type = header.geometry_type(); + + let mut selection = if let Some((min_x, min_y, max_x, max_y)) = bbox { + reader.select_bbox(min_x, min_y, max_x, max_y).await? + } else { + reader.select_all().await? + }; + + let features_count = selection.features_count(); + + // TODO: propagate CRS + let options = GeoTableBuilderOptions::new( + coord_type, + true, + batch_size, + Some(Arc::new(schema.finish())), + features_count, + Default::default(), + ); + + match geometry_type { + GeometryType::Point => { + let mut builder = GeoTableBuilder::::new_with_options(options); + selection.process_features(&mut builder).await?; + builder.finish() + } + GeometryType::LineString => { + let mut builder = GeoTableBuilder::>::new_with_options(options); + selection.process_features(&mut builder).await?; + builder.finish() + } + GeometryType::Polygon => { + let mut builder = GeoTableBuilder::>::new_with_options(options); + selection.process_features(&mut builder).await?; + builder.finish() + } + GeometryType::MultiPoint => { + let mut builder = GeoTableBuilder::>::new_with_options(options); + selection.process_features(&mut builder).await?; + builder.finish() + } + GeometryType::MultiLineString => { + let mut builder = + GeoTableBuilder::>::new_with_options(options); + selection.process_features(&mut builder).await?; + builder.finish() + } + GeometryType::MultiPolygon => { + let mut builder = + GeoTableBuilder::>::new_with_options(options); + selection.process_features(&mut builder).await?; + builder.finish() + } + GeometryType::Unknown => { + let mut builder = + GeoTableBuilder::>::new_with_options(options); + selection.process_features(&mut builder).await?; + let table = builder.finish()?; + table.downcast(true) + } + // TODO: Parse into a GeometryCollection array and then downcast to a single-typed array if possible. + geom_type => Err(GeoArrowError::NotYetImplemented(format!( + "Parsing FlatGeobuf from {:?} geometry type not yet supported", + geom_type + ))), + } +} + +#[cfg(test)] +mod test { + use std::env::current_dir; + + use super::*; + use object_store::local::LocalFileSystem; + + #[tokio::test] + async fn test_countries() { + let fs = LocalFileSystem::new_with_prefix(current_dir().unwrap()).unwrap(); + let _table = read_flatgeobuf_async( + fs, + Path::from("fixtures/flatgeobuf/countries.fgb"), + Default::default(), + None, + None, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn test_nz_buildings() { + let fs = LocalFileSystem::new_with_prefix(current_dir().unwrap()).unwrap(); + let _table = read_flatgeobuf_async( + fs, + Path::from("fixtures/flatgeobuf/nz-building-outlines-small.fgb"), + Default::default(), + None, + None, + ) + .await + .unwrap(); + } +} diff --git a/src/io/flatgeobuf/reader/common.rs b/src/io/flatgeobuf/reader/common.rs new file mode 100644 index 000000000..6eb703090 --- /dev/null +++ b/src/io/flatgeobuf/reader/common.rs @@ -0,0 +1,37 @@ +use arrow_schema::{DataType, Field, SchemaBuilder, TimeUnit}; +use flatgeobuf::{ColumnType, Header}; + +pub(super) fn infer_schema(header: Header<'_>) -> SchemaBuilder { + let columns = header.columns().unwrap(); + let mut schema = SchemaBuilder::with_capacity(columns.len()); + + for col in columns.into_iter() { + let field = match col.type_() { + ColumnType::Bool => Field::new(col.name(), DataType::Boolean, col.nullable()), + ColumnType::Byte => Field::new(col.name(), DataType::Int8, col.nullable()), + ColumnType::UByte => Field::new(col.name(), DataType::UInt8, col.nullable()), + ColumnType::Short => Field::new(col.name(), DataType::Int16, col.nullable()), + ColumnType::UShort => Field::new(col.name(), DataType::UInt16, col.nullable()), + ColumnType::Int => Field::new(col.name(), DataType::Int32, col.nullable()), + ColumnType::UInt => Field::new(col.name(), DataType::UInt32, col.nullable()), + ColumnType::Long => Field::new(col.name(), DataType::Int64, col.nullable()), + ColumnType::ULong => Field::new(col.name(), DataType::UInt64, col.nullable()), + ColumnType::Float => Field::new(col.name(), DataType::Float32, col.nullable()), + ColumnType::Double => Field::new(col.name(), DataType::Float64, col.nullable()), + ColumnType::String => Field::new(col.name(), DataType::Utf8, col.nullable()), + ColumnType::Json => Field::new(col.name(), DataType::Utf8, col.nullable()), + ColumnType::DateTime => Field::new( + col.name(), + DataType::Timestamp(TimeUnit::Microsecond, None), + col.nullable(), + ), + ColumnType::Binary => Field::new(col.name(), DataType::Binary, col.nullable()), + // ColumnType is actually a struct, not an enum, so the rust compiler doesn't know + // we've matched all types + _ => unreachable!(), + }; + schema.push(field); + } + + schema +} diff --git a/src/io/flatgeobuf/reader/mod.rs b/src/io/flatgeobuf/reader/mod.rs new file mode 100644 index 000000000..908d19150 --- /dev/null +++ b/src/io/flatgeobuf/reader/mod.rs @@ -0,0 +1,10 @@ +#[cfg(feature = "flatgeobuf_async")] +mod r#async; +mod common; +#[cfg(feature = "flatgeobuf_async")] +mod object_store_reader; +mod sync; + +#[cfg(feature = "flatgeobuf_async")] +pub use r#async::read_flatgeobuf_async; +pub use sync::read_flatgeobuf; diff --git a/src/io/flatgeobuf/reader/object_store_reader.rs b/src/io/flatgeobuf/reader/object_store_reader.rs new file mode 100644 index 000000000..339a506b5 --- /dev/null +++ b/src/io/flatgeobuf/reader/object_store_reader.rs @@ -0,0 +1,52 @@ +use async_trait::async_trait; +use bytes::Bytes; +use http_range_client::{AsyncHttpRangeClient, Result as HTTPRangeClientResult}; +use object_store::path::Path; +use object_store::ObjectStore; + +pub struct ObjectStoreWrapper { + pub location: Path, + pub reader: T, + pub size: usize, +} + +#[async_trait] +impl AsyncHttpRangeClient for ObjectStoreWrapper { + /// Send a GET range request + async fn get_range(&self, _url: &str, range: &str) -> HTTPRangeClientResult { + assert!(range.starts_with("bytes=")); + + let split_range = range[6..].split('-').collect::>(); + let start_range = split_range[0].parse::().unwrap(); + + // Add one to the range because HTTP range strings are end-inclusive (I think) + let end_range = split_range[1].parse::().unwrap() + 1; + + // Flatgeobuf will sometimes overfetch, but not all object store backends support + // overfetches (e.g. this errors on a LocalFileSystem) + // See https://github.com/flatgeobuf/flatgeobuf/issues/338 + let end_range = end_range.min(self.size); + + let bytes = self + .reader + .get_range(&self.location, start_range..end_range) + .await + .unwrap(); + Ok(bytes) + } + + /// Send a HEAD request and return response header value + async fn head_response_header( + &self, + _url: &str, + header: &str, + ) -> HTTPRangeClientResult> { + // This is a massive hack to align APIs + if header == "content-length" { + let meta = self.reader.head(&self.location).await.unwrap(); + Ok(Some(meta.size.to_string())) + } else { + Ok(None) + } + } +} diff --git a/src/io/flatgeobuf/reader.rs b/src/io/flatgeobuf/reader/sync.rs similarity index 70% rename from src/io/flatgeobuf/reader.rs rename to src/io/flatgeobuf/reader/sync.rs index 85f6e9dac..241c8e1a7 100644 --- a/src/io/flatgeobuf/reader.rs +++ b/src/io/flatgeobuf/reader/sync.rs @@ -22,12 +22,11 @@ use crate::algorithm::native::Downcast; use crate::array::*; use crate::error::{GeoArrowError, Result}; +use crate::io::flatgeobuf::reader::common::infer_schema; use crate::io::geozero::array::MixedGeometryStreamBuilder; use crate::io::geozero::table::{GeoTableBuilder, GeoTableBuilderOptions}; use crate::table::GeoTable; -use arrow_schema::{DataType, Field, SchemaBuilder, TimeUnit}; -use flatgeobuf::{ColumnType, GeometryType}; -use flatgeobuf::{FgbReader, Header}; +use flatgeobuf::{FgbReader, GeometryType}; use std::io::{Read, Seek}; use std::sync::Arc; @@ -108,41 +107,6 @@ pub fn read_flatgeobuf( } } -fn infer_schema(header: Header<'_>) -> SchemaBuilder { - let columns = header.columns().unwrap(); - let mut schema = SchemaBuilder::with_capacity(columns.len()); - - for col in columns.into_iter() { - let field = match col.type_() { - ColumnType::Bool => Field::new(col.name(), DataType::Boolean, col.nullable()), - ColumnType::Byte => Field::new(col.name(), DataType::Int8, col.nullable()), - ColumnType::UByte => Field::new(col.name(), DataType::UInt8, col.nullable()), - ColumnType::Short => Field::new(col.name(), DataType::Int16, col.nullable()), - ColumnType::UShort => Field::new(col.name(), DataType::UInt16, col.nullable()), - ColumnType::Int => Field::new(col.name(), DataType::Int32, col.nullable()), - ColumnType::UInt => Field::new(col.name(), DataType::UInt32, col.nullable()), - ColumnType::Long => Field::new(col.name(), DataType::Int64, col.nullable()), - ColumnType::ULong => Field::new(col.name(), DataType::UInt64, col.nullable()), - ColumnType::Float => Field::new(col.name(), DataType::Float32, col.nullable()), - ColumnType::Double => Field::new(col.name(), DataType::Float64, col.nullable()), - ColumnType::String => Field::new(col.name(), DataType::Utf8, col.nullable()), - ColumnType::Json => Field::new(col.name(), DataType::Utf8, col.nullable()), - ColumnType::DateTime => Field::new( - col.name(), - DataType::Timestamp(TimeUnit::Microsecond, None), - col.nullable(), - ), - ColumnType::Binary => Field::new(col.name(), DataType::Binary, col.nullable()), - // ColumnType is actually a struct, not an enum, so the rust compiler doesn't know - // we've matched all types - _ => unreachable!(), - }; - schema.push(field); - } - - schema -} - #[cfg(test)] mod test { use std::fs::File;