diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index 17e06b3d8b..0000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,70 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: Rust - -on: - push: - branches: - - 'master' - - 'branch-*' - tags: - - '*' - pull_request: - workflow_dispatch: - -env: - CARGO_TERM_COLOR: always - -jobs: - build: - strategy: - matrix: - features: - - "" - - "memory-prof" - runs-on: ubuntu-latest - steps: - - - name: Install native libs - run: sudo apt-get install -y libkrb5-dev libgsasl-dev - - - name: Install Protoc - uses: arduino/setup-protoc@v2 - with: - version: "23.2" - - uses: actions/checkout@v3 - - name: Build - working-directory: ./rust/experimental/server - run: cargo build --features hdfs --verbose - if: ${{ matrix.features == '' }} - - name: Build with memory-prof - working-directory: ./rust/experimental/server - run: cargo build --verbose --features memory-prof - if: ${{ matrix.features == 'memory-prof' }} - - name: Code style check - working-directory: ./rust/experimental/server - run: cargo fmt --check - if: ${{ matrix.features == '' }} - - name: Run tests - working-directory: ./rust/experimental/server - run: cargo test --verbose -- --test-threads=1 - if: ${{ matrix.features == '' }} - - name: Run tests with memory-prof - working-directory: ./rust/experimental/server - run: cargo test --verbose --features memory-prof -- --test-threads=1 - if: ${{ matrix.features == 'memory-prof' }} diff --git a/rust/experimental/server/.cargo/config.toml b/rust/experimental/server/.cargo/config.toml deleted file mode 100644 index beb0d61b29..0000000000 --- a/rust/experimental/server/.cargo/config.toml +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[build] -rustflags = ["-A", "dead-code", "-D", "warnings", "--cfg", "tokio_unstable"] diff --git a/rust/experimental/server/Cargo.lock b/rust/experimental/server/Cargo.lock deleted file mode 100644 index 1ea617918b..0000000000 --- a/rust/experimental/server/Cargo.lock +++ /dev/null @@ -1,4315 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "addr2line" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "ahash" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" -dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "version_check", -] - -[[package]] -name = "aho-corasick" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" -dependencies = [ - "memchr", -] - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anyhow" -version = "1.0.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" - -[[package]] -name = "arrayvec" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" - -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - -[[package]] -name = "async-compat" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f68a707c1feb095d8c07f8a65b9f506b117d30af431cab89374357de7c11461b" -dependencies = [ - "futures-core", - "futures-io", - "once_cell", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "async-trait" -version = "0.1.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "await-tree" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325bcfc4b87d4aa36f1319b806bacc40fcefcaf43a12bd85a5a2f44fc14ce9de" -dependencies = [ - "coarsetime", - "derive_builder", - "flexstr", - "indextree", - "itertools", - "parking_lot", - "pin-project 1.1.3", - "tokio", - "tracing", -] - -[[package]] -name = "awaitility" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46ee60785cbb3a23bde2c462098564a6337432b9fa032a62bb7ddb5e734c135d" - -[[package]] -name = "axum" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" -dependencies = [ - "async-trait", - "axum-core", - "bitflags 1.3.2", - "bytes 1.5.0", - "futures-util", - "http", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" -dependencies = [ - "async-trait", - "bytes 1.5.0", - "futures-util", - "http", - "http-body", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "backon" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a6197b2120bb2185a267f6515038558b019e92b832bb0320e96d66268dcf9" -dependencies = [ - "fastrand 1.9.0", - "futures-core", - "pin-project 1.1.3", - "tokio", -] - -[[package]] -name = "backtrace" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" -dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" - -[[package]] -name = "base64ct" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" - -[[package]] -name = "bindgen" -version = "0.64.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" -dependencies = [ - "bitflags 1.3.2", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "log", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 1.0.109", - "which", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bumpalo" -version = "3.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" - -[[package]] -name = "bytemuck" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" - -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - -[[package]] -name = "bytes" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" - -[[package]] -name = "bytes" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" - -[[package]] -name = "cap" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f125eb85b84a24c36b02ed1d22c9dd8632f53b3cde6e4d23512f94021030003" - -[[package]] -name = "cassowary" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" - -[[package]] -name = "cc" -version = "1.0.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] - -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "chrono" -version = "0.4.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-targets 0.48.5", -] - -[[package]] -name = "clang-sys" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" -dependencies = [ - "glob", - "libc", - "libloading", -] - -[[package]] -name = "clap" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" -dependencies = [ - "atty", - "bitflags 1.3.2", - "clap_derive", - "clap_lex", - "indexmap 1.9.3", - "once_cell", - "strsim", - "termcolor", - "textwrap", -] - -[[package]] -name = "clap_complete" -version = "3.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f7a2e0a962c45ce25afce14220bc24f9dade0a1787f185cecf96bfba7847cd8" -dependencies = [ - "clap", -] - -[[package]] -name = "clap_derive" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" -dependencies = [ - "heck", - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] - -[[package]] -name = "coarsetime" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99280f81a35511dda7d44f7c943491b41d3ac6fd0b54aea92498bec8612a2423" -dependencies = [ - "libc", - "once_cell", - "wasi", - "wasm-bindgen", -] - -[[package]] -name = "color-eyre" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204" -dependencies = [ - "backtrace", - "color-spantrace", - "eyre", - "indenter", - "once_cell", - "owo-colors", - "tracing-error", - "url", -] - -[[package]] -name = "color-spantrace" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba75b3d9449ecdccb27ecbc479fdc0b87fa2dd43d2f8298f9bf0e59aacc8dce" -dependencies = [ - "once_cell", - "owo-colors", - "tracing-core", - "tracing-error", -] - -[[package]] -name = "concurrent-queue" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "console-api" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e" -dependencies = [ - "prost 0.11.9", - "prost-types 0.11.9", - "tonic", - "tracing-core", -] - -[[package]] -name = "console-subscriber" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4cf42660ac07fcebed809cfe561dd8730bcd35b075215e6479c516bcd0d11cb" -dependencies = [ - "console-api", - "crossbeam-channel", - "crossbeam-utils", - "futures", - "hdrhistogram", - "humantime", - "prost-types 0.11.9", - "serde", - "serde_json", - "thread_local", - "tokio", - "tokio-stream", - "tonic", - "tracing", - "tracing-core", - "tracing-subscriber", -] - -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - -[[package]] -name = "const-random" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" -dependencies = [ - "const-random-macro", -] - -[[package]] -name = "const-random-macro" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" -dependencies = [ - "getrandom", - "once_cell", - "tiny-keccak", -] - -[[package]] -name = "core-foundation" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" - -[[package]] -name = "cpp_demangle" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "cpufeatures" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" -dependencies = [ - "libc", -] - -[[package]] -name = "crc" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - -[[package]] -name = "crc32fast" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "croaring" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb8264aab40cc6593ec5cbb4ba98c785f323de053d77406c814640312f14774a" -dependencies = [ - "byteorder", - "croaring-sys", -] - -[[package]] -name = "croaring-sys" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfcf3afb25740bb232303d3a61eef6a2b5c369ca7767f10a207f05014147cf3b" -dependencies = [ - "cc", -] - -[[package]] -name = "crossbeam" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" -dependencies = [ - "cfg-if", - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" -dependencies = [ - "cfg-if", - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" -dependencies = [ - "autocfg", - "cfg-if", - "crossbeam-utils", - "memoffset", - "scopeguard", -] - -[[package]] -name = "crossbeam-queue" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossterm" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" -dependencies = [ - "bitflags 1.3.2", - "crossterm_winapi", - "futures-core", - "libc", - "mio", - "parking_lot", - "signal-hook", - "signal-hook-mio", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" -dependencies = [ - "winapi", -] - -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 1.0.109", -] - -[[package]] -name = "darling_macro" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" -dependencies = [ - "darling_core", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "dashmap" -version = "5.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" -dependencies = [ - "cfg-if", - "hashbrown 0.14.0", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "debugid" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" -dependencies = [ - "uuid", -] - -[[package]] -name = "der" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" -dependencies = [ - "const-oid", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "deranged" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" - -[[package]] -name = "derive_builder" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "derive_builder_macro" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" -dependencies = [ - "derive_builder_core", - "syn 1.0.109", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", - "subtle", -] - -[[package]] -name = "dirs" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - -[[package]] -name = "dlv-list" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" -dependencies = [ - "const-random", -] - -[[package]] -name = "either" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" - -[[package]] -name = "encoding_rs" -version = "0.8.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "env_logger" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" -dependencies = [ - "humantime", - "is-terminal", - "log", - "regex", - "termcolor", -] - -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - -[[package]] -name = "errno" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - -[[package]] -name = "eyre" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb" -dependencies = [ - "indenter", - "once_cell", -] - -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - -[[package]] -name = "fastrand" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" - -[[package]] -name = "findshlibs" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" -dependencies = [ - "cc", - "lazy_static", - "libc", - "winapi", -] - -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - -[[package]] -name = "flagset" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a7e408202050813e6f1d9addadcaafef3dca7530c7ddfb005d4081cce6779" - -[[package]] -name = "flate2" -version = "1.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "flexstr" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d50aef14619d336a54fca5a592d952eb39037b1a1e7e6afd9f91c892ac7ef65" -dependencies = [ - "static_assertions", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "form_urlencoded" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - -[[package]] -name = "futures" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" - -[[package]] -name = "futures-executor" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" - -[[package]] -name = "futures-macro" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "futures-sink" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" - -[[package]] -name = "futures-task" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" - -[[package]] -name = "futures-util" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "futures_codec" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce54d63f8b0c75023ed920d46fd71d0cbbb830b0ee012726b5b4f506fb6dea5b" -dependencies = [ - "bytes 0.5.6", - "futures", - "memchr", - "pin-project 0.4.30", -] - -[[package]] -name = "g2gen" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2c7625b2fc250dd90b63f7887a6bb0f7ec1d714c8278415bea2669ef20820e" -dependencies = [ - "g2poly", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "g2p" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc36d9bdc3d2da057775a9f4fa7d7b09edab3e0eda7a92cc353358fa63b8519e" -dependencies = [ - "g2gen", - "g2poly", -] - -[[package]] -name = "g2poly" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6a86e750338603ea2c14b1c0bfe58cd61f87ca67a0021d9334996024608e12" - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi", - "wasm-bindgen", -] - -[[package]] -name = "gimli" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" - -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes 1.5.0", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap 2.0.0", - "slab", - "tokio", - "tokio-util 0.7.9", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" - -[[package]] -name = "hdfs-native" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7068d52978481fb74f1ac1522c0823d180a6e112c459169cf72f3cd08b207b0d" -dependencies = [ - "base64 0.21.4", - "bytes 1.5.0", - "crc", - "futures", - "g2p", - "libc", - "libgssapi", - "log", - "num-traits", - "prost 0.12.3", - "prost-types 0.12.3", - "roxmltree", - "socket2 0.5.4", - "thiserror", - "tokio", - "url", - "users", - "uuid", -] - -[[package]] -name = "hdrhistogram" -version = "7.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" -dependencies = [ - "base64 0.13.1", - "byteorder", - "flate2", - "nom", - "num-traits", -] - -[[package]] -name = "headers" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" -dependencies = [ - "base64 0.21.4", - "bytes 1.5.0", - "headers-core", - "http", - "httpdate", - "mime", - "sha1", -] - -[[package]] -name = "headers-core" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" -dependencies = [ - "http", -] - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys 0.48.0", -] - -[[package]] -name = "http" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" -dependencies = [ - "bytes 1.5.0", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" -dependencies = [ - "bytes 1.5.0", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "hyper" -version = "0.14.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" -dependencies = [ - "bytes 1.5.0", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.4.9", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http", - "hyper", - "rustls 0.21.7", - "tokio", - "tokio-rustls 0.24.1", -] - -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes 1.5.0", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.59" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "indenter" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - -[[package]] -name = "indexmap" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" -dependencies = [ - "equivalent", - "hashbrown 0.14.0", -] - -[[package]] -name = "indextree" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c40411d0e5c63ef1323c3d09ce5ec6d84d71531e18daed0743fccea279d7deb6" - -[[package]] -name = "inferno" -version = "0.11.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50453ec3a6555fad17b1cd1a80d16af5bc7cb35094f64e429fd46549018c6a3" -dependencies = [ - "ahash", - "indexmap 2.0.0", - "is-terminal", - "itoa", - "log", - "num-format", - "once_cell", - "quick-xml 0.26.0", - "rgb", - "str_stack", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi 0.3.3", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "ipnet" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" - -[[package]] -name = "is-terminal" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" -dependencies = [ - "hermit-abi 0.3.3", - "rustix 0.38.14", - "windows-sys 0.48.0", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" - -[[package]] -name = "js-sys" -version = "0.3.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "jsonwebtoken" -version = "9.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" -dependencies = [ - "base64 0.21.4", - "js-sys", - "pem", - "ring 0.17.7", - "serde", - "serde_json", - "simple_asn1", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -dependencies = [ - "spin 0.5.2", -] - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - -[[package]] -name = "libc" -version = "0.2.148" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" - -[[package]] -name = "libgssapi" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcfb7f77cbefc242a46ea667491c4f1129712f563cd368623d3f1b261a90e5f" -dependencies = [ - "bitflags 2.4.0", - "bytes 1.5.0", - "lazy_static", - "libgssapi-sys", -] - -[[package]] -name = "libgssapi-sys" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efdcdd31923aa6280d41ff2636fd93a18cc60fe25983b24887d1a8d24478cbfb" -dependencies = [ - "bindgen", -] - -[[package]] -name = "libloading" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - -[[package]] -name = "libm" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" - -[[package]] -name = "linux-raw-sys" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" - -[[package]] -name = "linux-raw-sys" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" - -[[package]] -name = "lock_api" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" - -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata 0.1.10", -] - -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - -[[package]] -name = "memchr" -version = "2.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" - -[[package]] -name = "memmap2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" -dependencies = [ - "libc", -] - -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" -dependencies = [ - "adler", -] - -[[package]] -name = "mio" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" -dependencies = [ - "libc", - "log", - "wasi", - "windows-sys 0.48.0", -] - -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "nix" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - -[[package]] -name = "num-bigint" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-bigint-dig" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" -dependencies = [ - "byteorder", - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand 0.8.5", - "smallvec", - "zeroize", -] - -[[package]] -name = "num-format" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" -dependencies = [ - "arrayvec", - "itoa", -] - -[[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi 0.3.3", - "libc", -] - -[[package]] -name = "object" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" - -[[package]] -name = "opendal" -version = "0.44.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c32736a48ef08a5d2212864e2295c8e54f4d6b352b7f49aa0c29a12fc410ff66" -dependencies = [ - "anyhow", - "async-compat", - "async-trait", - "backon", - "base64 0.21.4", - "bytes 1.5.0", - "chrono", - "flagset", - "futures", - "getrandom", - "http", - "log", - "md-5", - "once_cell", - "parking_lot", - "percent-encoding", - "pin-project 1.1.3", - "quick-xml 0.30.0", - "reqsign", - "reqwest", - "serde", - "serde_json", - "sha2", - "tokio", - "uuid", -] - -[[package]] -name = "openssl" -version = "0.10.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" -dependencies = [ - "bitflags 2.4.0", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "openssl-sys" -version = "0.9.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "ordered-multimap" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4d6a8c22fc714f0c2373e6091bf6f5e9b37b1bc0b1184874b7e0a4e303d318f" -dependencies = [ - "dlv-list", - "hashbrown 0.14.0", -] - -[[package]] -name = "os_str_bytes" -version = "6.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" - -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - -[[package]] -name = "owo-colors" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.3.5", - "smallvec", - "windows-targets 0.48.5", -] - -[[package]] -name = "paste" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - -[[package]] -name = "pem" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" -dependencies = [ - "base64 0.21.4", - "serde", -] - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - -[[package]] -name = "percent-encoding" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" - -[[package]] -name = "petgraph" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" -dependencies = [ - "fixedbitset", - "indexmap 2.0.0", -] - -[[package]] -name = "pin-project" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ef0f924a5ee7ea9cbcea77529dba45f8a9ba9f622419fe3386ca581a3ae9d5a" -dependencies = [ - "pin-project-internal 0.4.30", -] - -[[package]] -name = "pin-project" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" -dependencies = [ - "pin-project-internal 1.1.3", -] - -[[package]] -name = "pin-project-internal" -version = "0.4.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "851c8d0ce9bebe43790dedfc86614c23494ac9f423dd618d3a61fc693eafe61e" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der", - "pkcs8", - "spki", -] - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - -[[package]] -name = "pkg-config" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" - -[[package]] -name = "poem" -version = "1.3.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc7ae19f3e791ae8108b08801abb3708d64d3a16490c720e0b81040cae87b5d" -dependencies = [ - "async-trait", - "bytes 1.5.0", - "futures-util", - "headers", - "http", - "hyper", - "mime", - "parking_lot", - "percent-encoding", - "pin-project-lite", - "poem-derive", - "regex", - "rfc7239", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "smallvec", - "sse-codec", - "thiserror", - "tokio", - "tokio-rustls 0.24.1", - "tokio-stream", - "tokio-util 0.7.9", - "tracing", -] - -[[package]] -name = "poem-derive" -version = "1.3.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2550a0bce7273b278894ef3ccc5a6869e7031b6870042f3cc6826ed9faa980a6" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "pprof" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196ded5d4be535690899a4631cc9f18cdc41b7ebf24a79400f46f48e49a11059" -dependencies = [ - "backtrace", - "cfg-if", - "findshlibs", - "inferno", - "libc", - "log", - "nix", - "once_cell", - "parking_lot", - "protobuf", - "protobuf-codegen-pure", - "smallvec", - "symbolic-demangle", - "tempfile", - "thiserror", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - -[[package]] -name = "proc-macro-crate" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" -dependencies = [ - "once_cell", - "toml_edit", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - -[[package]] -name = "proc-macro2" -version = "1.0.67" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "procfs" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de8dacb0873f77e6aefc6d71e044761fcc68060290f5b1089fcdf84626bb69" -dependencies = [ - "bitflags 1.3.2", - "byteorder", - "hex", - "lazy_static", - "rustix 0.36.15", -] - -[[package]] -name = "prometheus" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" -dependencies = [ - "cfg-if", - "fnv", - "lazy_static", - "libc", - "memchr", - "parking_lot", - "procfs", - "protobuf", - "reqwest", - "thiserror", -] - -[[package]] -name = "prost" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" -dependencies = [ - "bytes 1.5.0", - "prost-derive 0.11.9", -] - -[[package]] -name = "prost" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" -dependencies = [ - "bytes 1.5.0", - "prost-derive 0.12.3", -] - -[[package]] -name = "prost-build" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" -dependencies = [ - "bytes 1.5.0", - "heck", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost 0.11.9", - "prost-types 0.11.9", - "regex", - "syn 1.0.109", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "prost-derive" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "prost-types" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" -dependencies = [ - "prost 0.11.9", -] - -[[package]] -name = "prost-types" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" -dependencies = [ - "prost 0.12.3", -] - -[[package]] -name = "protobuf" -version = "2.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" - -[[package]] -name = "protobuf-codegen" -version = "2.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "033460afb75cf755fcfc16dfaed20b86468082a2ea24e05ac35ab4a099a017d6" -dependencies = [ - "protobuf", -] - -[[package]] -name = "protobuf-codegen-pure" -version = "2.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a29399fc94bcd3eeaa951c715f7bea69409b2445356b00519740bcd6ddd865" -dependencies = [ - "protobuf", - "protobuf-codegen", -] - -[[package]] -name = "quick-xml" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" -dependencies = [ - "memchr", -] - -[[package]] -name = "quick-xml" -version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quick-xml" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quote" -version = "1.0.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" -dependencies = [ - "fuchsia-cprng", - "libc", - "rand_core 0.3.1", - "rdrand", - "winapi", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "ratatui" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc0d032bccba900ee32151ec0265667535c230169f5a011154cdcd984e16829" -dependencies = [ - "bitflags 1.3.2", - "cassowary", - "crossterm", - "unicode-segmentation", - "unicode-width", -] - -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom", - "redox_syscall 0.2.16", - "thiserror", -] - -[[package]] -name = "regex" -version = "1.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata 0.3.8", - "regex-syntax 0.7.5", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", -] - -[[package]] -name = "regex-automata" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax 0.7.5", -] - -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - -[[package]] -name = "regex-syntax" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - -[[package]] -name = "reqsign" -version = "0.14.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce87f66ba6c6acef277a729f989a0eca946cb9ce6a15bcc036bda0f72d4b9fd" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.21.4", - "chrono", - "form_urlencoded", - "getrandom", - "hex", - "hmac", - "home", - "http", - "jsonwebtoken", - "log", - "once_cell", - "percent-encoding", - "quick-xml 0.31.0", - "rand 0.8.5", - "reqwest", - "rsa", - "rust-ini", - "serde", - "serde_json", - "sha1", - "sha2", - "tokio", -] - -[[package]] -name = "reqwest" -version = "0.11.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" -dependencies = [ - "base64 0.21.4", - "bytes 1.5.0", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-rustls", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls 0.21.7", - "rustls-native-certs", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-native-tls", - "tokio-rustls 0.24.1", - "tokio-util 0.7.9", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "winreg", -] - -[[package]] -name = "rfc7239" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "087317b3cf7eb481f13bd9025d729324b7cd068d6f470e2d76d049e191f5ba47" -dependencies = [ - "uncased", -] - -[[package]] -name = "rgb" -version = "0.8.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20ec2d3e3fc7a92ced357df9cebd5a10b6fb2aa1ee797bf7e9ce2f17dffc8f59" -dependencies = [ - "bytemuck", -] - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - -[[package]] -name = "ring" -version = "0.17.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" -dependencies = [ - "cc", - "getrandom", - "libc", - "spin 0.9.8", - "untrusted 0.9.0", - "windows-sys 0.48.0", -] - -[[package]] -name = "roxmltree" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862340e351ce1b271a378ec53f304a5558f7db87f3769dc655a8f6ecbb68b302" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "rsa" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" -dependencies = [ - "const-oid", - "digest", - "num-bigint-dig", - "num-integer", - "num-traits", - "pkcs1", - "pkcs8", - "rand_core 0.6.4", - "signature", - "spki", - "subtle", - "zeroize", -] - -[[package]] -name = "rust-ini" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0698206bcb8882bf2a9ecb4c1e7785db57ff052297085a6efd4fe42302068a" -dependencies = [ - "cfg-if", - "ordered-multimap", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustix" -version = "0.36.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c37f1bd5ef1b5422177b7646cba67430579cfe2ace80f284fee876bca52ad941" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.1.4", - "windows-sys 0.45.0", -] - -[[package]] -name = "rustix" -version = "0.38.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" -dependencies = [ - "bitflags 2.4.0", - "errno", - "libc", - "linux-raw-sys 0.4.7", - "windows-sys 0.48.0", -] - -[[package]] -name = "rustls" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" -dependencies = [ - "base64 0.13.1", - "log", - "ring 0.16.20", - "sct 0.6.1", - "webpki", -] - -[[package]] -name = "rustls" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" -dependencies = [ - "log", - "ring 0.16.20", - "rustls-webpki", - "sct 0.7.0", -] - -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" -dependencies = [ - "base64 0.21.4", -] - -[[package]] -name = "rustls-webpki" -version = "0.101.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" -dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", -] - -[[package]] -name = "rustversion" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" - -[[package]] -name = "ryu" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" - -[[package]] -name = "schannel" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" -dependencies = [ - "windows-sys 0.48.0", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "sct" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" -dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", -] - -[[package]] -name = "sct" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" -dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", -] - -[[package]] -name = "security-framework" -version = "2.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "serde" -version = "1.0.188" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.188" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "serde_json" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_spanned" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sharded-slab" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "shlex" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" - -[[package]] -name = "signal-hook" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" -dependencies = [ - "libc", - "signal-hook-registry", -] - -[[package]] -name = "signal-hook-mio" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" -dependencies = [ - "libc", - "mio", - "signal-hook", -] - -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest", - "rand_core 0.6.4", -] - -[[package]] -name = "simple_asn1" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror", - "time", -] - -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "smallvec" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" - -[[package]] -name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "socket2" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" -dependencies = [ - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "sse-codec" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a59f811350c44b4a037aabeb72dc6a9591fc22aa95a036db9a96297c58085a" -dependencies = [ - "bytes 0.5.6", - "futures-io", - "futures_codec", - "memchr", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "str_stack" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "subtle" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" - -[[package]] -name = "symbolic-common" -version = "10.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b55cdc318ede251d0957f07afe5fed912119b8c1bc5a7804151826db999e737" -dependencies = [ - "debugid", - "memmap2", - "stable_deref_trait", - "uuid", -] - -[[package]] -name = "symbolic-demangle" -version = "10.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79be897be8a483a81fff6a3a4e195b4ac838ef73ca42d348b3f722da9902e489" -dependencies = [ - "cpp_demangle", - "rustc-demangle", - "symbolic-common", -] - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "tempdir" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -dependencies = [ - "rand 0.4.6", - "remove_dir_all", -] - -[[package]] -name = "tempfile" -version = "3.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" -dependencies = [ - "cfg-if", - "fastrand 2.0.0", - "redox_syscall 0.3.5", - "rustix 0.38.14", - "windows-sys 0.48.0", -] - -[[package]] -name = "termcolor" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "textwrap" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" - -[[package]] -name = "thiserror" -version = "1.0.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "thread_local" -version = "1.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" -dependencies = [ - "cfg-if", - "once_cell", -] - -[[package]] -name = "tikv-jemalloc-ctl" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "619bfed27d807b54f7f776b9430d4f8060e66ee138a28632ca898584d462c31c" -dependencies = [ - "libc", - "paste", - "tikv-jemalloc-sys", -] - -[[package]] -name = "tikv-jemalloc-sys" -version = "0.5.4+5.3.0-patched" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "tikv-jemallocator" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" -dependencies = [ - "libc", - "tikv-jemalloc-sys", -] - -[[package]] -name = "time" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" -dependencies = [ - "deranged", - "itoa", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" - -[[package]] -name = "time-macros" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" -dependencies = [ - "time-core", -] - -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" -dependencies = [ - "backtrace", - "bytes 1.5.0", - "libc", - "mio", - "num_cpus", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2 0.5.4", - "tokio-macros", - "tracing", - "windows-sys 0.48.0", -] - -[[package]] -name = "tokio-console" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d8d44c50f1b17838c6044119701900e4242dbc0e8a3792f6fbf512b489b3dbf" -dependencies = [ - "atty", - "clap", - "clap_complete", - "color-eyre", - "console-api", - "crossterm", - "dirs", - "futures", - "h2", - "hdrhistogram", - "humantime", - "once_cell", - "prost-types 0.11.9", - "ratatui", - "regex", - "serde", - "tokio", - "toml 0.5.11", - "tonic", - "tower", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-macros" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" -dependencies = [ - "rustls 0.19.1", - "tokio", - "webpki", -] - -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.7", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util 0.7.9", -] - -[[package]] -name = "tokio-util" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes 1.5.0", - "futures-core", - "futures-io", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" -dependencies = [ - "bytes 1.5.0", - "futures-core", - "futures-io", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "toml" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.19.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.0.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - -[[package]] -name = "tonic" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" -dependencies = [ - "async-trait", - "axum", - "base64 0.21.4", - "bytes 1.5.0", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project 1.1.3", - "prost 0.11.9", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-build" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project 1.1.3", - "pin-project-lite", - "rand 0.8.5", - "slab", - "tokio", - "tokio-util 0.7.9", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - -[[package]] -name = "tracing" -version = "0.1.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" -dependencies = [ - "cfg-if", - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-appender" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d48f71a791638519505cefafe162606f706c25592e4bde4d97600c0195312e" -dependencies = [ - "crossbeam-channel", - "time", - "tracing-subscriber", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - -[[package]] -name = "tracing-core" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-error" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" -dependencies = [ - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "tracing-log" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" -dependencies = [ - "lazy_static", - "log", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", -] - -[[package]] -name = "try-lock" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" - -[[package]] -name = "typenum" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" - -[[package]] -name = "uncased" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b9bc53168a4be7402ab86c3aad243a84dd7381d09be0eddc81280c1da95ca68" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-segmentation" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" - -[[package]] -name = "unicode-width" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" - -[[package]] -name = "uniffle-x" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-channel", - "async-trait", - "await-tree", - "awaitility", - "bytes 1.5.0", - "cap", - "clap", - "console-subscriber", - "crc32fast", - "croaring", - "crossbeam", - "crossbeam-channel", - "dashmap", - "env_logger", - "fs2", - "futures", - "hdfs-native", - "hyper", - "log", - "once_cell", - "opendal", - "pin-project-lite", - "poem", - "pprof", - "prometheus", - "prost 0.11.9", - "prost-build", - "serde", - "signal-hook", - "socket2 0.4.9", - "spin 0.9.8", - "tempdir", - "tempfile", - "thiserror", - "tikv-jemalloc-ctl", - "tikv-jemalloc-sys", - "tikv-jemallocator", - "tokio", - "tokio-console", - "tokio-rustls 0.22.0", - "tokio-stream", - "tokio-util 0.6.10", - "toml 0.7.8", - "tonic", - "tonic-build", - "tower", - "tracing", - "tracing-appender", - "tracing-subscriber", - "url", -] - -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "users" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24cc0f6d6f267b73e5a2cadf007ba8f9bc39c6a6f9666f8cf25ea809a153b032" -dependencies = [ - "libc", -] - -[[package]] -name = "uuid" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" -dependencies = [ - "getrandom", - "serde", -] - -[[package]] -name = "valuable" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.37", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" - -[[package]] -name = "wasm-streams" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "web-sys" -version = "0.3.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", -] - -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.14", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" -dependencies = [ - "winapi", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets 0.52.0", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" -dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" - -[[package]] -name = "winnow" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" -dependencies = [ - "memchr", -] - -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - -[[package]] -name = "xmlparser" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" - -[[package]] -name = "zeroize" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/rust/experimental/server/Cargo.toml b/rust/experimental/server/Cargo.toml deleted file mode 100644 index 860ad43202..0000000000 --- a/rust/experimental/server/Cargo.toml +++ /dev/null @@ -1,133 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -name = "uniffle-x" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[[bin]] -name="uniffle-worker" -path= "src/main.rs" - -[lib] -name = "uniffle_worker" -path = "src/lib.rs" - -[features] -# whether to use jemalloc as the global allocator or not -jemalloc = [ - "dep:tikv-jemalloc-ctl", - "dep:tikv-jemalloc-sys", - "dep:tikv-jemallocator", -] - -# whether to enable memory profiling or not, once enabled, it implies using jemalloc as the global allocator -memory-prof = [ - "jemalloc", - "tikv-jemallocator/profiling" -] - -hdfs = [ - "dep:hdfs-native" -] - -[dependencies] -anyhow = "1" -tokio = { version = "1.28.2", features = ["full" ] } -tokio-rustls = "0.22" -tokio-stream = { version = "0.1", features = ["sync"] } -tokio-util = { version = "0.6", features = ["compat"]} -toml = "0.7.4" -tracing = "0.1" -tracing-appender = "0.2" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -tonic = "0.9.1" -prost = "0.11.9" -bytes = "1" -tonic-build = "0.9.1" -thiserror = "1" -dashmap = "5.4.0" -log = "0.4.17" -env_logger = "0.10.0" -crossbeam = "0.8.2" -crossbeam-channel = "0.5" -tempdir = "0.3.7" -async-trait = "0.1.68" -futures = "0.3" -serde = { version = "1", features = ["derive"] } -async-channel = "1.8.0" -croaring = "0.8.1" -prometheus = { version = "0.13", features = ["process", "push"] } -crc32fast = "1.3.2" -fs2 = "0.4.3" -url = "2.4.0" -await-tree = "0.1.1" -poem = { version = "1.3.56", features = ["rustls", "test"]} -pprof = { version = "0.11.1", features = [ - "flamegraph", - "protobuf-codec", - "protobuf", -] } -tempfile = "3.7.0" -once_cell = "1.18.0" -tower = { version = "0.4", features = ["util", "load-shed"] } -hyper = "0.14" -tokio-console = "0.1.8" -console-subscriber = "0.1.9" -pin-project-lite = "0.2.8" -signal-hook = "0.3.17" -clap = "3.0.14" -socket2 = { version="0.4", features = ["all"]} -cap = "0.1.2" -spin = "0.9.8" -opendal = { version = "0.44.0", features = ["services-fs"]} - -[dependencies.hdfs-native] -version = "0.7.0" -optional = true -features = ["kerberos"] - -# jemalloc related optional dependencies -[dependencies.tikv-jemalloc-ctl] -version = "0.5.0" -optional = true - -[dependencies.tikv-jemalloc-sys] -version = "0.5.0" -optional = true -features = ["stats"] - -[dependencies.tikv-jemallocator] -version = "0.5.0" -optional = true -features = ["stats"] - -[build-dependencies] -tonic-build = "0.9.1" -prost-build = "0.11.9" - -[dev-dependencies] -env_logger = "0.10.0" -awaitility = "0.3.1" - -[profile.dev] -# re-enable debug assertions when pprof-rs fixed the reports for misaligned pointer dereferences -# https://github.com/rust-lang/rust/pull/98112/ -debug-assertions = false diff --git a/rust/experimental/server/README.md b/rust/experimental/server/README.md deleted file mode 100644 index 7c83cc9243..0000000000 --- a/rust/experimental/server/README.md +++ /dev/null @@ -1,188 +0,0 @@ - - -Another implementation of Apache Uniffle shuffle server (Single binary, no extra dependencies) - -## Benchmark report - -#### Environment - -_Software_: Uniffle 0.8.0 / Hadoop 3.2.2 / Spark 3.1.2 - -_Hardware_: Machine 96 cores, 512G memory, 1T * 4 SSD, network bandwidth 8GB/s - -_Hadoop Yarn Cluster_: 1 * ResourceManager + 40 * NodeManager, every machine 4T * 4 HDD - -_Uniffle Cluster_: 1 * Coordinator + 1 * Shuffle Server, every machine 1T * 4 NVME SSD - -#### Configuration - -spark's conf -```yaml -spark.executor.instances 400 -spark.executor.cores 1 -spark.executor.memory 2g -spark.shuffle.manager org.apache.spark.shuffle.RssShuffleManager -spark.rss.storage.type MEMORY_LOCALFILE -``` - -uniffle grpc-based server's conf -``` yaml -JVM XMX=30g - -# JDK11 + G1 - -rss.server.buffer.capacity 10g -rss.server.read.buffer.capacity 10g -rss.server.flush.thread.alive 10 -rss.server.flush.threadPool.size 50 -rss.server.high.watermark.write 80 -rss.server.low.watermark.write 70 -``` - -Rust-based shuffle-server conf -``` -store_type = "MEMORY_LOCALFILE" - -[memory_store] -capacity = "10G" - -[localfile_store] -data_paths = ["/data1/uniffle", "/data2/uniffle", "/data3/uniffle", "/data4/uniffle"] -healthy_check_min_disks = 0 - -[hybrid_store] -memory_spill_high_watermark = 0.8 -memory_spill_low_watermark = 0.7 -``` - -#### TeraSort cost times -| type/buffer capacity | 250G (compressed) | comment | -|--------------------------------------|:------------------:|:------------------------------------------------------------------------:| -| vanilla spark ess | 5.0min (2.2/2.8) | ess use 400 nodes but uniffle only one. But the rss speed is still fast! | -| vanilla uniffle (grpc-based) / 10g | 5.3min (2.3m/3m) | 1.9G/s | -| vanilla uniffle (grpc-based) / 300g | 5.6min (3.7m/1.9m) | GC occurs frequently / 2.5G/s | -| vanilla uniffle (netty-based) / 10g | / | read failed. 2.5G/s (write is better due to zero copy) | -| vanilla uniffle (netty-based) / 300g | / | app hang | -| rust based shuffle server / 10g | 4.6min (2.2m/2.4m) | 2.4 G/s | -| rust based shuffle server / 300g | 4min (1.5m/2.5m) | 3.5 G/s | - - -Compared with grpc based server, rust-based server has less memory footprint and stable performance. - -And Netty is still not stable for production env. - -In the future, rust-based server will use io_uring mechanism to improve writing performance. - -## Build - -`cargo build --release --features hdfs,jemalloc` - -Uniffle-x currently treats all compiler warnings as error, with some dead-code warning excluded. When you are developing -and really want to ignore the warnings for now, you can use `ccargo --config 'build.rustflags=["-W", "warnings"]' build` -to restore the default behavior. However, before submit your pr, you should fix all the warnings. - - -## Config - -config.toml as follows: - -``` -store_type = "MEMORY_LOCALFILE" -grpc_port = 21100 -coordinator_quorum = ["xxxxxxx1", "xxxxxxx2] -tags = ["uniffle-worker"] - -[memory_store] -capacity = "100G" - -[localfile_store] -data_paths = ["/data1/uniffle", "/data2/uniffle"] -healthy_check_min_disks = 0 - -[hdfs_store] -max_concurrency = 10 - -[hybrid_store] -memory_spill_high_watermark = 0.8 -memory_spill_low_watermark = 0.2 -memory_single_buffer_max_spill_size = "256M" - -[metrics] -http_port = 19998 -push_gateway_endpoint = "http://xxxxxxxxxxxxxx/pushgateway" -``` - -## Run - -`WORKER_IP={ip} RUST_LOG=info WORKER_CONFIG_PATH=./config.toml ./uniffle-worker` - -### HDFS Setup - -Benefit from the hdfs-native crate, there is no need to setup the JAVA_HOME and relative dependencies. -If HDFS store is valid, the spark client must specify the conf of `spark.rss.client.remote.storage.useLocalConfAsDefault=true` - -```shell -cargo build --features hdfs --release -``` - -```shell -# configure the kerberos and conf env -HADOOP_CONF_DIR=/etc/hadoop/conf KRB5_CONFIG=/etc/krb5.conf KRB5CCNAME=/tmp/krb5cc_2002 LOG=info ./uniffle-worker -``` - -## Profiling - -### Tokio console -1. build with unstable tokio for uniffle-worker binary (this has been enabled by default) - ```shell - cargo build - ``` -2. worker run with tokio-console. the log level of `trace` must be enabled - ```shell - WORKER_IP={ip} RUST_LOG=trace ./uniffle-worker -c ./config.toml - ``` -3. tokio-console client side connect - ```shell - tokio-console http://{uniffle-worker-host}:21002 - ``` - -### Heap profiling -1. build with profile support - ```shell - cargo build --release --features memory-prof - ``` -2. Start with profile - ```shell - _RJEM_MALLOC_CONF=prof:true,prof_prefix:jeprof.out ./uniffle-worker - ``` - -### CPU Profiling -1. build with jemalloc feature - ```shell - cargo build --release --features jemalloc - ``` -2. Paste following command to get cpu profile flamegraph - ```shell - go tool pprof -http="0.0.0.0:8081" http://{remote_ip}:8080/debug/pprof/profile?seconds=30 - ``` - - localhost:8080: riffle server. - - remote_ip: pprof server address. - - seconds=30: Profiling lasts for 30 seconds. - - Then open the URL :8081/ui/flamegraph in your browser to view the flamegraph: - \ No newline at end of file diff --git a/rust/experimental/server/build.rs b/rust/experimental/server/build.rs deleted file mode 100644 index 74195cf8c7..0000000000 --- a/rust/experimental/server/build.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::fs; -use std::path::Path; - -fn main() -> Result<(), Box> { - // generate the uniffle code for grpc server - let mut config = prost_build::Config::new(); - config.bytes(&["."]); - - tonic_build::configure() - .build_server(true) - .out_dir("src/proto") - .compile_with_config(config, &["src/proto/uniffle.proto"], &["."])?; - - // rename the generated filename to uniffle.rs - rename_file("src/proto/rss.common.rs", "src/proto/uniffle.rs"); - - Ok(()) -} - -fn rename_file(file_path: impl AsRef, renamed_path: impl AsRef) { - let f = file_path.as_ref(); - if !f.exists() || !f.is_file() { - panic!("The file is missing or not a file."); - } - fs::rename(&f, renamed_path).expect("Errors on renaming file."); -} diff --git a/rust/experimental/server/rust-toolchain.toml b/rust/experimental/server/rust-toolchain.toml deleted file mode 100644 index 8d16c17df5..0000000000 --- a/rust/experimental/server/rust-toolchain.toml +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[toolchain] -channel = "nightly-2023-05-31" -components = ["rustfmt", "clippy", "rust-analyzer"] \ No newline at end of file diff --git a/rust/experimental/server/rustfmt.toml b/rust/experimental/server/rustfmt.toml deleted file mode 100644 index f4c3b57da4..0000000000 --- a/rust/experimental/server/rustfmt.toml +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -edition = "2021" -reorder_imports = true - -# format_code_in_doc_comments = true -# group_imports = "StdExternalCrate" -# imports_granularity = "Item" -# overflow_delimited_expr = true -# trailing_comma = "Vertical" -# where_single_line = true diff --git a/rust/experimental/server/src/app.rs b/rust/experimental/server/src/app.rs deleted file mode 100644 index 57e79b5521..0000000000 --- a/rust/experimental/server/src/app.rs +++ /dev/null @@ -1,943 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::config::Config; -use crate::error::WorkerError; -use crate::metric::{ - GAUGE_APP_NUMBER, GAUGE_TOPN_APP_RESIDENT_DATA_SIZE, TOTAL_APP_NUMBER, - TOTAL_HUGE_PARTITION_REQUIRE_BUFFER_FAILED, TOTAL_READ_DATA, TOTAL_READ_DATA_FROM_LOCALFILE, - TOTAL_READ_DATA_FROM_MEMORY, TOTAL_RECEIVED_DATA, TOTAL_REQUIRE_BUFFER_FAILED, -}; - -use crate::readable_size::ReadableSize; -use crate::runtime::manager::RuntimeManager; -use crate::store::hybrid::HybridStore; -use crate::store::memory::MemorySnapshot; -use crate::store::{ - PartitionedDataBlock, RequireBufferResponse, ResponseData, ResponseDataIndex, Store, - StoreProvider, -}; -use crate::util::current_timestamp_sec; -use anyhow::{anyhow, Result}; -use bytes::Bytes; -use croaring::treemap::JvmSerializer; -use croaring::Treemap; - -use dashmap::DashMap; -use log::{debug, error, info}; - -use std::collections::hash_map::DefaultHasher; -use std::collections::{HashMap, HashSet}; - -use std::hash::{Hash, Hasher}; - -use std::str::FromStr; - -use crate::proto::uniffle::RemoteStorage; -use std::sync::atomic::Ordering::SeqCst; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::RwLock; -use std::sync::{Arc, OnceLock}; -use std::time::Duration; - -pub static SHUFFLE_SERVER_ID: OnceLock = OnceLock::new(); - -#[derive(Debug, Clone)] -pub enum DataDistribution { - NORMAL, - #[allow(non_camel_case_types)] - LOCAL_ORDER, -} - -pub const MAX_CONCURRENCY_PER_PARTITION_TO_WRITE: i32 = 20; - -#[derive(Debug, Clone)] -pub struct AppConfigOptions { - pub data_distribution: DataDistribution, - pub max_concurrency_per_partition_to_write: i32, - pub remote_storage_config_option: Option, -} - -impl AppConfigOptions { - pub fn new( - data_distribution: DataDistribution, - max_concurrency_per_partition_to_write: i32, - remote_storage_config_option: Option, - ) -> Self { - Self { - data_distribution, - max_concurrency_per_partition_to_write, - remote_storage_config_option, - } - } -} - -impl Default for AppConfigOptions { - fn default() -> Self { - AppConfigOptions { - data_distribution: DataDistribution::LOCAL_ORDER, - max_concurrency_per_partition_to_write: 20, - remote_storage_config_option: None, - } - } -} - -// ============================================================= - -#[derive(Clone, Debug)] -pub struct RemoteStorageConfig { - pub root: String, - pub configs: HashMap, -} - -impl From for RemoteStorageConfig { - fn from(remote_conf: RemoteStorage) -> Self { - let root = remote_conf.path; - let mut confs = HashMap::new(); - for kv in remote_conf.remote_storage_conf { - confs.insert(kv.key, kv.value); - } - - Self { - root, - configs: confs, - } - } -} - -// ============================================================= - -pub struct App { - app_id: String, - // key: shuffleId, value: partitionIds - partitions: DashMap>, - app_config_options: AppConfigOptions, - latest_heartbeat_time: AtomicU64, - store: Arc, - // key: (shuffle_id, partition_id) - bitmap_of_blocks: DashMap<(i32, i32), PartitionedMeta>, - huge_partition_marked_threshold: Option, - huge_partition_memory_max_available_size: Option, - - total_received_data_size: AtomicU64, - total_resident_data_size: AtomicU64, -} - -#[derive(Clone)] -struct PartitionedMeta { - inner: Arc>, -} - -struct PartitionedMetaInner { - blocks_bitmap: Treemap, - total_size: u64, -} - -impl PartitionedMeta { - fn new() -> Self { - PartitionedMeta { - inner: Arc::new(RwLock::new(PartitionedMetaInner { - blocks_bitmap: Treemap::default(), - total_size: 0, - })), - } - } - - fn get_data_size(&self) -> Result { - let meta = self.inner.read().unwrap(); - Ok(meta.total_size) - } - - fn incr_data_size(&mut self, data_size: i32) -> Result<()> { - let mut meta = self.inner.write().unwrap(); - meta.total_size += data_size as u64; - Ok(()) - } - - fn get_block_ids_bytes(&self) -> Result { - let meta = self.inner.read().unwrap(); - let serialized_data = meta.blocks_bitmap.serialize()?; - Ok(Bytes::from(serialized_data)) - } - - fn report_block_ids(&mut self, ids: Vec) -> Result<()> { - let mut meta = self.inner.write().unwrap(); - for id in ids { - meta.blocks_bitmap.add(id as u64); - } - Ok(()) - } -} - -impl App { - fn from( - app_id: String, - config_options: AppConfigOptions, - store: Arc, - huge_partition_marked_threshold: Option, - huge_partition_memory_max_available_size: Option, - runtime_manager: RuntimeManager, - ) -> Self { - // todo: should throw exception if register failed. - let copy_app_id = app_id.to_string(); - let app_options = config_options.clone(); - let cloned_store = store.clone(); - let register_result = futures::executor::block_on(async move { - runtime_manager - .default_runtime - .spawn(async move { - cloned_store - .register_app(RegisterAppContext { - app_id: copy_app_id, - app_config_options: app_options, - }) - .await - }) - .await - }); - if register_result.is_err() { - error!( - "Errors on registering app to store: {:#?}", - register_result.err() - ); - } - - App { - app_id, - partitions: DashMap::new(), - app_config_options: config_options, - latest_heartbeat_time: AtomicU64::new(current_timestamp_sec()), - store, - bitmap_of_blocks: DashMap::new(), - huge_partition_marked_threshold, - huge_partition_memory_max_available_size, - total_received_data_size: Default::default(), - total_resident_data_size: Default::default(), - } - } - - fn get_latest_heartbeat_time(&self) -> u64 { - self.latest_heartbeat_time.load(Ordering::SeqCst) - } - - pub fn heartbeat(&self) -> Result<()> { - let timestamp = current_timestamp_sec(); - self.latest_heartbeat_time.swap(timestamp, Ordering::SeqCst); - Ok(()) - } - - pub fn register_shuffle(&self, shuffle_id: i32) -> Result<()> { - self.partitions - .entry(shuffle_id) - .or_insert_with(|| HashSet::new()); - Ok(()) - } - - pub async fn insert(&self, ctx: WritingViewContext) -> Result { - let len: i32 = ctx.data_blocks.iter().map(|block| block.length).sum(); - self.get_underlying_partition_bitmap(ctx.uid.clone()) - .incr_data_size(len)?; - TOTAL_RECEIVED_DATA.inc_by(len as u64); - self.total_received_data_size.fetch_add(len as u64, SeqCst); - self.total_resident_data_size.fetch_add(len as u64, SeqCst); - - let ctx = match self.is_huge_partition(&ctx.uid).await { - Ok(true) => WritingViewContext::new(ctx.uid.clone(), ctx.data_blocks, true), - _ => ctx, - }; - - self.store.insert(ctx).await?; - Ok(len) - } - - pub async fn select(&self, ctx: ReadingViewContext) -> Result { - let response = self.store.get(ctx).await; - response.map(|data| { - match &data { - ResponseData::Local(local_data) => { - let length = local_data.data.len() as u64; - TOTAL_READ_DATA_FROM_LOCALFILE.inc_by(length); - TOTAL_READ_DATA.inc_by(length); - } - ResponseData::Mem(mem_data) => { - let length = mem_data.data.len() as u64; - TOTAL_READ_DATA_FROM_MEMORY.inc_by(length); - TOTAL_READ_DATA.inc_by(length); - } - }; - - data - }) - } - - pub async fn list_index( - &self, - ctx: ReadingIndexViewContext, - ) -> Result { - self.store.get_index(ctx).await - } - - async fn is_huge_partition(&self, uid: &PartitionedUId) -> Result { - let huge_partition_threshold_option = &self.huge_partition_marked_threshold; - let huge_partition_memory_used_option = &self.huge_partition_memory_max_available_size; - if huge_partition_threshold_option.is_none() || huge_partition_memory_used_option.is_none() - { - return Ok(false); - } - - let huge_partition_threshold = &huge_partition_threshold_option.unwrap(); - - let meta = self.get_underlying_partition_bitmap(uid.clone()); - let data_size = meta.get_data_size()?; - if data_size > *huge_partition_threshold { - return Ok(true); - } - - return Ok(false); - } - - async fn is_backpressure_for_huge_partition(&self, uid: &PartitionedUId) -> Result { - if !self.is_huge_partition(uid).await? { - return Ok(false); - } - let huge_partition_memory_used = &self.huge_partition_memory_max_available_size; - let huge_partition_memory = &huge_partition_memory_used.unwrap(); - - if self - .store - .get_hot_store_memory_partitioned_buffer_size(uid) - .await? - > *huge_partition_memory - { - info!( - "[{:?}] with huge partition, it has been writing speed limited", - uid - ); - TOTAL_HUGE_PARTITION_REQUIRE_BUFFER_FAILED.inc(); - Ok(true) - } else { - Ok(false) - } - } - - pub async fn free_allocated_memory_size(&self, size: i64) -> Result { - self.store.free_hot_store_allocated_memory_size(size).await - } - - pub async fn require_buffer( - &self, - ctx: RequireBufferContext, - ) -> Result { - if self.is_backpressure_for_huge_partition(&ctx.uid).await? { - TOTAL_REQUIRE_BUFFER_FAILED.inc(); - return Err(WorkerError::MEMORY_USAGE_LIMITED_BY_HUGE_PARTITION); - } - - self.store.require_buffer(ctx).await.map_err(|err| { - TOTAL_REQUIRE_BUFFER_FAILED.inc(); - err - }) - } - - pub async fn release_buffer(&self, ticket_id: i64) -> Result { - self.store - .release_buffer(ReleaseBufferContext::from(ticket_id)) - .await - } - - fn get_underlying_partition_bitmap(&self, uid: PartitionedUId) -> PartitionedMeta { - let shuffle_id = uid.shuffle_id; - let partition_id = uid.partition_id; - let partitioned_meta = self - .bitmap_of_blocks - .entry((shuffle_id, partition_id)) - .or_insert_with(|| PartitionedMeta::new()); - partitioned_meta.clone() - } - - pub fn get_block_ids(&self, ctx: GetBlocksContext) -> Result { - debug!("get blocks: {:?}", ctx.clone()); - let partitioned_meta = self.get_underlying_partition_bitmap(ctx.uid); - partitioned_meta.get_block_ids_bytes() - } - - pub async fn report_block_ids(&self, ctx: ReportBlocksContext) -> Result<()> { - debug!("Report blocks: {:?}", ctx.clone()); - let mut partitioned_meta = self.get_underlying_partition_bitmap(ctx.uid); - partitioned_meta.report_block_ids(ctx.blocks)?; - - Ok(()) - } - - pub async fn purge(&self, app_id: String, shuffle_id: Option) -> Result<()> { - let removed_size = self - .store - .purge(PurgeDataContext::new(app_id, shuffle_id)) - .await?; - self.total_resident_data_size - .fetch_sub(removed_size as u64, SeqCst); - Ok(()) - } - - pub fn total_received_data_size(&self) -> u64 { - self.total_received_data_size.load(SeqCst) - } - - pub fn total_resident_data_size(&self) -> u64 { - self.total_resident_data_size.load(SeqCst) - } -} - -#[derive(Debug, Clone)] -pub struct PurgeDataContext { - pub(crate) app_id: String, - pub(crate) shuffle_id: Option, -} - -impl PurgeDataContext { - pub fn new(app_id: String, shuffle_id: Option) -> PurgeDataContext { - PurgeDataContext { app_id, shuffle_id } - } -} - -impl From<&str> for PurgeDataContext { - fn from(app_id_ref: &str) -> Self { - PurgeDataContext { - app_id: app_id_ref.to_string(), - shuffle_id: None, - } - } -} - -#[derive(Debug, Clone)] -pub struct ReportBlocksContext { - pub(crate) uid: PartitionedUId, - pub(crate) blocks: Vec, -} - -#[derive(Debug, Clone)] -pub struct GetBlocksContext { - pub(crate) uid: PartitionedUId, -} - -#[derive(Debug, Clone)] -pub struct WritingViewContext { - pub uid: PartitionedUId, - pub data_blocks: Vec, - pub owned_by_huge_partition: bool, -} - -impl WritingViewContext { - pub fn from(uid: PartitionedUId, data_blocks: Vec) -> Self { - WritingViewContext { - uid, - data_blocks, - owned_by_huge_partition: false, - } - } - - pub fn new( - uid: PartitionedUId, - data_blocks: Vec, - owned_by_huge_partition: bool, - ) -> Self { - WritingViewContext { - uid, - data_blocks, - owned_by_huge_partition, - } - } -} - -#[derive(Debug, Clone)] -pub struct ReadingViewContext { - pub uid: PartitionedUId, - pub reading_options: ReadingOptions, - pub serialized_expected_task_ids_bitmap: Option, -} - -pub struct ReadingIndexViewContext { - pub partition_id: PartitionedUId, -} - -#[derive(Debug, Clone)] -pub struct RequireBufferContext { - pub uid: PartitionedUId, - pub size: i64, -} - -#[derive(Debug, Clone)] -pub struct RegisterAppContext { - pub app_id: String, - pub app_config_options: AppConfigOptions, -} - -#[derive(Debug, Clone)] -pub struct ReleaseBufferContext { - pub(crate) ticket_id: i64, -} - -impl From for ReleaseBufferContext { - fn from(value: i64) -> Self { - Self { ticket_id: value } - } -} - -impl RequireBufferContext { - pub fn new(uid: PartitionedUId, size: i64) -> Self { - Self { uid, size } - } -} - -#[derive(Debug, Clone)] -pub enum ReadingOptions { - #[allow(non_camel_case_types)] - MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE(i64, i64), - #[allow(non_camel_case_types)] - FILE_OFFSET_AND_LEN(i64, i64), -} - -// ========================================================== - -#[derive(Debug, Clone)] -#[allow(non_camel_case_types)] -pub enum PurgeEvent { - // app_id - HEART_BEAT_TIMEOUT(String), - // app_id + shuffle_id - APP_PARTIAL_SHUFFLES_PURGE(String, i32), - // app_id - APP_PURGE(String), -} - -pub type AppManagerRef = Arc; - -pub struct AppManager { - // key: app_id - apps: DashMap>, - receiver: async_channel::Receiver, - sender: async_channel::Sender, - store: Arc, - app_heartbeat_timeout_min: u32, - config: Config, - runtime_manager: RuntimeManager, -} - -impl AppManager { - fn new(runtime_manager: RuntimeManager, config: Config) -> Self { - let (sender, receiver) = async_channel::unbounded(); - let app_heartbeat_timeout_min = config.app_heartbeat_timeout_min.unwrap_or(10); - let store = Arc::new(StoreProvider::get(runtime_manager.clone(), config.clone())); - store.clone().start(); - let manager = AppManager { - apps: DashMap::new(), - receiver, - sender, - store, - app_heartbeat_timeout_min, - config, - runtime_manager: runtime_manager.clone(), - }; - manager - } -} - -impl AppManager { - pub fn get_ref(runtime_manager: RuntimeManager, config: Config) -> AppManagerRef { - let app_ref = Arc::new(AppManager::new(runtime_manager.clone(), config)); - let app_manager_ref_cloned = app_ref.clone(); - - runtime_manager.default_runtime.spawn(async move { - info!("Starting app heartbeat checker..."); - loop { - // task1: find out heartbeat timeout apps - tokio::time::sleep(Duration::from_secs(120)).await; - - let _current_timestamp = current_timestamp_sec(); - for item in app_manager_ref_cloned.apps.iter() { - let (key, app) = item.pair(); - let last_time = app.get_latest_heartbeat_time(); - - if current_timestamp_sec() - last_time - > (app_manager_ref_cloned.app_heartbeat_timeout_min * 60) as u64 - { - if app_manager_ref_cloned - .sender - .send(PurgeEvent::HEART_BEAT_TIMEOUT(key.clone())) - .await - .is_err() - { - error!( - "Errors on sending purge event when app: {} heartbeat timeout", - key - ); - } - } - } - } - }); - - // calculate topN app shuffle data size - let app_manager_ref = app_ref.clone(); - runtime_manager.default_runtime.spawn(async move { - info!("Starting calculating topN app shuffle data size..."); - loop { - tokio::time::sleep(Duration::from_secs(10)).await; - - let view = app_manager_ref.apps.clone().into_read_only(); - let mut apps: Vec<_> = view.values().collect(); - apps.sort_by_key(|x| 0 - x.total_resident_data_size()); - - let top_n = 10; - let limit = if apps.len() > top_n { - top_n - } else { - apps.len() - }; - for idx in 0..limit { - GAUGE_TOPN_APP_RESIDENT_DATA_SIZE - .with_label_values(&[&apps[idx].app_id]) - .set(apps[idx].total_resident_data_size() as i64); - } - } - }); - - let app_manager_cloned = app_ref.clone(); - runtime_manager.default_runtime.spawn(async move { - info!("Starting purge event handler..."); - while let Ok(event) = app_manager_cloned.receiver.recv().await { - GAUGE_APP_NUMBER.dec(); - let _ = match event { - PurgeEvent::HEART_BEAT_TIMEOUT(app_id) => { - info!( - "The app:[{}]'s data will be purged due to heartbeat timeout", - &app_id - ); - app_manager_cloned.purge_app_data(app_id, None).await - } - PurgeEvent::APP_PURGE(app_id) => { - info!( - "The app:[{}] has been finished, its data will be purged.", - &app_id - ); - app_manager_cloned.purge_app_data(app_id, None).await - } - PurgeEvent::APP_PARTIAL_SHUFFLES_PURGE(app_id, shuffle_id) => { - info!("The app:[{:?}] with shuffleId: [{:?}] will be purged due to unregister grpc interface", &app_id, shuffle_id); - app_manager_cloned.purge_app_data(app_id, Some(shuffle_id)).await - } - } - .map_err(|err| error!("Errors on purging data. error: {:?}", err)); - } - }); - - app_ref - } - - pub async fn store_is_healthy(&self) -> Result { - self.store.is_healthy().await - } - - pub async fn store_memory_snapshot(&self) -> Result { - self.store.get_hot_store_memory_snapshot().await - } - - pub fn store_memory_spill_event_num(&self) -> Result { - self.store.memory_spill_event_num() - } - - async fn purge_app_data(&self, app_id: String, shuffle_id_option: Option) -> Result<()> { - let app = self.get_app(&app_id).ok_or(anyhow!(format!( - "App:{} don't exist when purging data, this should not happen", - &app_id - )))?; - app.purge(app_id.clone(), shuffle_id_option).await?; - - if shuffle_id_option.is_none() { - self.apps.remove(&app_id); - } - - Ok(()) - } - - pub fn get_app(&self, app_id: &str) -> Option> { - self.apps.get(app_id).map(|v| v.value().clone()) - } - - pub fn register( - &self, - app_id: String, - shuffle_id: i32, - app_config_options: AppConfigOptions, - ) -> Result<()> { - info!( - "Accepted registry. app_id: {}, shuffle_id: {}", - app_id.clone(), - shuffle_id - ); - let app_ref = self.apps.entry(app_id.clone()).or_insert_with(|| { - TOTAL_APP_NUMBER.inc(); - GAUGE_APP_NUMBER.inc(); - - let capacity = - ReadableSize::from_str(&self.config.memory_store.clone().unwrap().capacity) - .unwrap() - .as_bytes(); - let huge_partition_max_available_size = Some( - (self - .config - .huge_partition_memory_max_used_percent - .unwrap_or(1.0) - * capacity as f64) as u64, - ); - - let threshold = match &self.config.huge_partition_marked_threshold { - Some(v) => Some( - ReadableSize::from_str(v.clone().as_str()) - .unwrap() - .as_bytes(), - ), - _ => None, - }; - - Arc::new(App::from( - app_id, - app_config_options, - self.store.clone(), - threshold, - huge_partition_max_available_size, - self.runtime_manager.clone(), - )) - }); - app_ref.register_shuffle(shuffle_id) - } - - pub async fn unregister_shuffle(&self, app_id: String, shuffle_id: i32) -> Result<()> { - self.sender - .send(PurgeEvent::APP_PARTIAL_SHUFFLES_PURGE(app_id, shuffle_id)) - .await?; - Ok(()) - } - - pub async fn unregister_app(&self, app_id: String) -> Result<()> { - self.sender.send(PurgeEvent::APP_PURGE(app_id)).await?; - Ok(()) - } -} - -#[derive(Ord, PartialOrd, Eq, PartialEq, Default, Debug, Hash, Clone)] -pub struct PartitionedUId { - pub app_id: String, - pub shuffle_id: i32, - pub partition_id: i32, -} - -impl PartitionedUId { - pub fn from(app_id: String, shuffle_id: i32, partition_id: i32) -> PartitionedUId { - PartitionedUId { - app_id, - shuffle_id, - partition_id, - } - } - - pub fn get_hash(uid: &PartitionedUId) -> u64 { - let mut hasher = DefaultHasher::new(); - - uid.hash(&mut hasher); - let hash_value = hasher.finish(); - - hash_value - } -} - -#[cfg(test)] -mod test { - use crate::app::{ - AppManager, GetBlocksContext, PartitionedUId, ReadingOptions, ReadingViewContext, - ReportBlocksContext, WritingViewContext, - }; - use crate::config::{Config, HybridStoreConfig, LocalfileStoreConfig, MemoryStoreConfig}; - - use crate::runtime::manager::RuntimeManager; - use crate::store::{PartitionedDataBlock, ResponseData}; - use croaring::treemap::JvmSerializer; - use croaring::Treemap; - use dashmap::DashMap; - - #[test] - fn test_uid_hash() { - let uid = PartitionedUId::from("a".to_string(), 1, 1); - let hash_value = PartitionedUId::get_hash(&uid); - println!("{}", hash_value); - } - - fn mock_config() -> Config { - let temp_dir = tempdir::TempDir::new("test_local_store").unwrap(); - let temp_path = temp_dir.path().to_str().unwrap().to_string(); - println!("init local file path: {}", temp_path); - - let mut config = Config::default(); - config.memory_store = Some(MemoryStoreConfig::new((1024 * 1024).to_string())); - config.localfile_store = Some(LocalfileStoreConfig::new(vec![temp_path])); - config.hybrid_store = Some(HybridStoreConfig::default()); - config - } - - #[test] - fn app_put_get_purge_test() { - let app_id = "app_put_get_purge_test-----id"; - - let runtime_manager: RuntimeManager = Default::default(); - let app_manager_ref = AppManager::get_ref(runtime_manager.clone(), mock_config()).clone(); - app_manager_ref - .register(app_id.clone().into(), 1, Default::default()) - .unwrap(); - - if let Some(app) = app_manager_ref.get_app("app_id".into()) { - let writing_ctx = WritingViewContext::from( - PartitionedUId { - app_id: app_id.clone().into(), - shuffle_id: 1, - partition_id: 0, - }, - vec![ - PartitionedDataBlock { - block_id: 0, - length: 10, - uncompress_length: 20, - crc: 10, - data: Default::default(), - task_attempt_id: 0, - }, - PartitionedDataBlock { - block_id: 1, - length: 20, - uncompress_length: 30, - crc: 0, - data: Default::default(), - task_attempt_id: 0, - }, - ], - ); - - // case1: put - let f = app.insert(writing_ctx); - if runtime_manager.wait(f).is_err() { - panic!() - } - - let reading_ctx = ReadingViewContext { - uid: Default::default(), - reading_options: ReadingOptions::MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE(-1, 1000000), - serialized_expected_task_ids_bitmap: Default::default(), - }; - - // case2: get - let f = app.select(reading_ctx); - let result = runtime_manager.wait(f); - if result.is_err() { - panic!() - } - - match result.unwrap() { - ResponseData::Mem(data) => { - assert_eq!(2, data.shuffle_data_block_segments.len()); - } - _ => todo!(), - } - - // check the data size - assert_eq!(30, app.total_received_data_size()); - assert_eq!(30, app.total_resident_data_size()); - - // case3: purge - runtime_manager - .wait(app_manager_ref.purge_app_data(app_id.to_string(), None)) - .expect(""); - - assert_eq!(false, app_manager_ref.get_app(app_id).is_none()); - - // check the data size again after the data has been removed - assert_eq!(30, app.total_received_data_size()); - assert_eq!(0, app.total_resident_data_size()); - } - } - - #[test] - fn app_manager_test() { - let app_manager_ref = AppManager::get_ref(Default::default(), mock_config()).clone(); - app_manager_ref - .register("app_id".into(), 1, Default::default()) - .unwrap(); - if let Some(app) = app_manager_ref.get_app("app_id".into()) { - assert_eq!("app_id", app.app_id); - } - } - - #[test] - fn test_get_or_put_block_ids() { - let app_id = "test_get_or_put_block_ids-----id".to_string(); - - let runtime_manager: RuntimeManager = Default::default(); - let app_manager_ref = AppManager::get_ref(runtime_manager.clone(), mock_config()).clone(); - app_manager_ref - .register(app_id.clone().into(), 1, Default::default()) - .unwrap(); - - let app = app_manager_ref.get_app(app_id.as_ref()).unwrap(); - runtime_manager - .wait(app.report_block_ids(ReportBlocksContext { - uid: PartitionedUId { - app_id: app_id.clone(), - shuffle_id: 1, - partition_id: 0, - }, - blocks: vec![123, 124], - })) - .expect("TODO: panic message"); - - let data = app - .get_block_ids(GetBlocksContext { - uid: PartitionedUId { - app_id, - shuffle_id: 1, - partition_id: 0, - }, - }) - .expect("TODO: panic message"); - - let deserialized = Treemap::deserialize(&data).unwrap(); - assert_eq!(deserialized, Treemap::from_iter(vec![123, 124])); - } - - #[test] - fn test_dashmap_values() { - let dashmap = DashMap::new(); - dashmap.insert(1, 3); - dashmap.insert(2, 2); - dashmap.insert(3, 8); - - let cloned = dashmap.clone().into_read_only(); - let mut vals: Vec<_> = cloned.values().collect(); - vals.sort_by_key(|x| -(*x)); - assert_eq!(vec![&8, &3, &2], vals); - - let apps = vec![0, 1, 2, 3]; - println!("{:#?}", &apps[0..2]); - } -} diff --git a/rust/experimental/server/src/await_tree.rs b/rust/experimental/server/src/await_tree.rs deleted file mode 100644 index ee9299e221..0000000000 --- a/rust/experimental/server/src/await_tree.rs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use await_tree::{Registry, TreeRoot}; - -use once_cell::sync::Lazy; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; - -type AwaitTreeRegistryRef = Arc>>; - -pub static AWAIT_TREE_REGISTRY: Lazy = Lazy::new(|| AwaitTreeInner::new()); - -#[derive(Clone)] -pub struct AwaitTreeInner { - inner: AwaitTreeRegistryRef, - next_id: Arc, -} - -impl AwaitTreeInner { - fn new() -> Self { - Self { - inner: Arc::new(Mutex::new(Registry::new(await_tree::Config::default()))), - next_id: Arc::new(Default::default()), - } - } - - pub async fn register(&self, msg: String) -> TreeRoot { - let id = self.next_id.fetch_add(1, Ordering::SeqCst); - let msg = format!("actor=[{}], {}", id, msg); - self.inner.lock().unwrap().register(id, msg) - } - - pub fn get_inner(&self) -> AwaitTreeRegistryRef { - self.inner.clone() - } -} diff --git a/rust/experimental/server/src/config.rs b/rust/experimental/server/src/config.rs deleted file mode 100644 index d197649321..0000000000 --- a/rust/experimental/server/src/config.rs +++ /dev/null @@ -1,304 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use serde::{Deserialize, Serialize}; -use std::fs; -use std::path::Path; - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub struct MemoryStoreConfig { - pub capacity: String, - pub buffer_ticket_timeout_sec: Option, -} - -impl MemoryStoreConfig { - pub fn new(capacity: String) -> Self { - Self { - capacity, - buffer_ticket_timeout_sec: Some(5 * 60), - } - } - - pub fn from(capacity: String, buffer_ticket_timeout_sec: i64) -> Self { - Self { - capacity, - buffer_ticket_timeout_sec: Some(buffer_ticket_timeout_sec), - } - } -} - -// ========================================================= - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Default)] -pub struct HdfsStoreConfig { - pub max_concurrency: Option, -} - -// ========================================================= - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub struct LocalfileStoreConfig { - pub data_paths: Vec, - pub healthy_check_min_disks: Option, - pub disk_high_watermark: Option, - pub disk_low_watermark: Option, - pub disk_max_concurrency: Option, -} - -impl LocalfileStoreConfig { - pub fn new(data_paths: Vec) -> Self { - LocalfileStoreConfig { - data_paths, - healthy_check_min_disks: None, - disk_high_watermark: None, - disk_low_watermark: None, - disk_max_concurrency: None, - } - } -} - -// ========================================================= - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -#[serde(default)] -pub struct RuntimeConfig { - pub read_thread_num: usize, - pub write_thread_num: usize, - pub grpc_thread_num: usize, - pub http_thread_num: usize, - pub default_thread_num: usize, -} - -impl Default for RuntimeConfig { - fn default() -> Self { - RuntimeConfig { - read_thread_num: 10, - write_thread_num: 40, - grpc_thread_num: 100, - http_thread_num: 5, - default_thread_num: 5, - } - } -} - -// ========================================================= - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub struct HybridStoreConfig { - pub memory_spill_high_watermark: f32, - pub memory_spill_low_watermark: f32, - pub memory_single_buffer_max_spill_size: Option, - pub memory_spill_to_cold_threshold_size: Option, - - pub memory_spill_max_concurrency: Option, -} - -impl HybridStoreConfig { - pub fn new( - memory_spill_high_watermark: f32, - memory_spill_low_watermark: f32, - memory_single_buffer_max_spill_size: Option, - ) -> Self { - HybridStoreConfig { - memory_spill_high_watermark, - memory_spill_low_watermark, - memory_single_buffer_max_spill_size, - memory_spill_to_cold_threshold_size: None, - memory_spill_max_concurrency: None, - } - } -} - -impl Default for HybridStoreConfig { - fn default() -> Self { - HybridStoreConfig { - memory_spill_high_watermark: 0.8, - memory_spill_low_watermark: 0.7, - memory_single_buffer_max_spill_size: None, - memory_spill_to_cold_threshold_size: None, - memory_spill_max_concurrency: None, - } - } -} - -fn as_default_runtime_config() -> RuntimeConfig { - RuntimeConfig::default() -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Default)] -pub struct Config { - pub memory_store: Option, - pub localfile_store: Option, - pub hybrid_store: Option, - pub hdfs_store: Option, - - pub store_type: Option, - - #[serde(default = "as_default_runtime_config")] - pub runtime_config: RuntimeConfig, - - pub metrics: Option, - - pub grpc_port: Option, - pub coordinator_quorum: Vec, - pub tags: Option>, - - pub log: Option, - - pub app_heartbeat_timeout_min: Option, - - pub huge_partition_marked_threshold: Option, - pub huge_partition_memory_max_used_percent: Option, - - pub http_monitor_service_port: Option, -} - -// ========================================================= -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub struct MetricsConfig { - pub push_gateway_endpoint: Option, - pub push_interval_sec: Option, -} - -// ========================================================= - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub struct LogConfig { - pub path: String, - pub rotation: RotationConfig, -} - -impl Default for LogConfig { - fn default() -> Self { - LogConfig { - path: "/tmp/".to_string(), - rotation: RotationConfig::Hourly, - } - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub enum RotationConfig { - Hourly, - Daily, - Never, -} - -// ========================================================= - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Copy)] -#[allow(non_camel_case_types)] -pub enum StorageType { - MEMORY = 1, - LOCALFILE = 2, - MEMORY_LOCALFILE = 3, - HDFS = 4, - MEMORY_HDFS = 5, - MEMORY_LOCALFILE_HDFS = 7, -} - -impl StorageType { - pub fn contains_localfile(storage_type: &StorageType) -> bool { - let val = *storage_type as u8; - val & *&StorageType::LOCALFILE as u8 != 0 - } - - pub fn contains_memory(storage_type: &StorageType) -> bool { - let val = *storage_type as u8; - val & *&StorageType::MEMORY as u8 != 0 - } - - pub fn contains_hdfs(storage_type: &StorageType) -> bool { - let val = *storage_type as u8; - val & *&StorageType::HDFS as u8 != 0 - } -} - -const CONFIG_FILE_PATH_KEY: &str = "WORKER_CONFIG_PATH"; - -impl Config { - pub fn from(cfg_path: &str) -> Self { - let path = Path::new(cfg_path); - - // Read the file content as a string - let file_content = fs::read_to_string(path).expect("Failed to read file"); - - toml::from_str(&file_content).unwrap() - } - - pub fn create_from_env() -> Config { - let path = match std::env::var(CONFIG_FILE_PATH_KEY) { - Ok(val) => val, - _ => panic!( - "config path must be set in env args. key: {}", - CONFIG_FILE_PATH_KEY - ), - }; - - Config::from(&path) - } -} - -#[cfg(test)] -mod test { - use crate::config::{Config, RuntimeConfig, StorageType}; - use crate::readable_size::ReadableSize; - use std::str::FromStr; - - #[test] - fn storage_type_test() { - let stype = StorageType::MEMORY_LOCALFILE; - assert_eq!(true, StorageType::contains_localfile(&stype)); - - let stype = StorageType::MEMORY_LOCALFILE; - assert_eq!(true, StorageType::contains_memory(&stype)); - assert_eq!(false, StorageType::contains_hdfs(&stype)); - - let stype = StorageType::MEMORY_LOCALFILE_HDFS; - assert_eq!(true, StorageType::contains_hdfs(&stype)); - } - - #[test] - fn config_test() { - let toml_str = r#" - store_type = "MEMORY_LOCALFILE" - coordinator_quorum = ["xxxxxxx"] - - [memory_store] - capacity = "1024M" - - [localfile_store] - data_paths = ["/data1/uniffle"] - - [hybrid_store] - memory_spill_high_watermark = 0.8 - memory_spill_low_watermark = 0.2 - memory_single_buffer_max_spill_size = "256M" - "#; - - let decoded: Config = toml::from_str(toml_str).unwrap(); - println!("{:#?}", decoded); - - let capacity = ReadableSize::from_str(&decoded.memory_store.unwrap().capacity).unwrap(); - assert_eq!(1024 * 1024 * 1024, capacity.as_bytes()); - - assert_eq!( - decoded.runtime_config.read_thread_num, - RuntimeConfig::default().read_thread_num - ); - } -} diff --git a/rust/experimental/server/src/error.rs b/rust/experimental/server/src/error.rs deleted file mode 100644 index 7a70408825..0000000000 --- a/rust/experimental/server/src/error.rs +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use anyhow::Error; - -use log::error; -use poem::error::ParseQueryError; -use thiserror::Error; -use tokio::sync::AcquireError; - -#[derive(Error, Debug)] -#[allow(non_camel_case_types)] -pub enum WorkerError { - #[error("There is no available disks in local file store")] - NO_AVAILABLE_LOCAL_DISK, - - #[error("Internal error, it should not happen")] - INTERNAL_ERROR, - - #[error("Partial data has been lost, corrupted path: {0}")] - PARTIAL_DATA_LOST(String), - - #[error("Local disk:[{0}] is not healthy")] - LOCAL_DISK_UNHEALTHY(String), - - #[error("Local disk:[{0}] owned by current partition has been corrupted")] - LOCAL_DISK_OWNED_BY_PARTITION_CORRUPTED(String), - - #[error("No enough memory to be allocated.")] - NO_ENOUGH_MEMORY_TO_BE_ALLOCATED, - - #[error("The memory usage is limited by huge partition mechanism")] - MEMORY_USAGE_LIMITED_BY_HUGE_PARTITION, - - #[error(transparent)] - Other(#[from] anyhow::Error), - - #[error("Http request failed. {0}")] - HTTP_SERVICE_ERROR(String), - - #[error("Ticket id: {0} not exist")] - TICKET_ID_NOT_EXIST(i64), - - #[error("Hdfs native client not found for app: {0}")] - HDFS_NATIVE_CLIENT_NOT_FOUND(String), - - #[error("Data should be read from hdfs in client side instead of from server side")] - NOT_READ_HDFS_DATA_FROM_SERVER, - - #[error("Spill event has been retried exceed the max limit for app: {0}")] - SPILL_EVENT_EXCEED_RETRY_MAX_LIMIT(String), -} - -impl From for WorkerError { - fn from(error: AcquireError) -> Self { - WorkerError::Other(Error::new(error)) - } -} - -impl From for WorkerError { - fn from(error: ParseQueryError) -> Self { - WorkerError::Other(Error::new(error)) - } -} - -#[cfg(test)] -mod tests { - - use anyhow::Result; - - #[test] - pub fn error_test() -> Result<()> { - // bail macro means it will return directly. - // bail!(WorkerError::APP_PURGE_EVENT_SEND_ERROR("error_test_app_id".into(), None)); - Ok(()) - } -} diff --git a/rust/experimental/server/src/grpc.rs b/rust/experimental/server/src/grpc.rs deleted file mode 100644 index 1ff4bd400c..0000000000 --- a/rust/experimental/server/src/grpc.rs +++ /dev/null @@ -1,876 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::app::{ - AppConfigOptions, AppManagerRef, DataDistribution, GetBlocksContext, PartitionedUId, - ReadingIndexViewContext, ReadingOptions, ReadingViewContext, RemoteStorageConfig, - ReportBlocksContext, RequireBufferContext, WritingViewContext, -}; -use crate::proto::uniffle::shuffle_server_server::ShuffleServer; -use crate::proto::uniffle::{ - AppHeartBeatRequest, AppHeartBeatResponse, FinishShuffleRequest, FinishShuffleResponse, - GetLocalShuffleDataRequest, GetLocalShuffleDataResponse, GetLocalShuffleIndexRequest, - GetLocalShuffleIndexResponse, GetMemoryShuffleDataRequest, GetMemoryShuffleDataResponse, - GetShuffleResultForMultiPartRequest, GetShuffleResultForMultiPartResponse, - GetShuffleResultRequest, GetShuffleResultResponse, ReportShuffleResultRequest, - ReportShuffleResultResponse, RequireBufferRequest, RequireBufferResponse, - SendShuffleDataRequest, SendShuffleDataResponse, ShuffleCommitRequest, ShuffleCommitResponse, - ShuffleRegisterRequest, ShuffleRegisterResponse, ShuffleUnregisterByAppIdRequest, - ShuffleUnregisterByAppIdResponse, ShuffleUnregisterRequest, ShuffleUnregisterResponse, -}; -use crate::store::{PartitionedData, ResponseDataIndex}; -use await_tree::InstrumentAwait; -use bytes::{BufMut, BytesMut}; -use croaring::treemap::JvmSerializer; -use croaring::Treemap; -use std::collections::HashMap; - -use log::{debug, error, info, warn}; - -use crate::metric::{ - GRPC_BUFFER_REQUIRE_PROCESS_TIME, GRPC_GET_LOCALFILE_DATA_PROCESS_TIME, - GRPC_GET_MEMORY_DATA_PROCESS_TIME, GRPC_GET_MEMORY_DATA_TRANSPORT_TIME, - GRPC_SEND_DATA_PROCESS_TIME, GRPC_SEND_DATA_TRANSPORT_TIME, -}; -use crate::util; -use tonic::{Request, Response, Status}; - -/// Use the maximum value for HTTP/2 connection window size to avoid deadlock among multiplexed -/// streams on the same connection. -pub const MAX_CONNECTION_WINDOW_SIZE: u32 = (1 << 31) - 1; -/// Use a large value for HTTP/2 stream window size to improve the performance of remote exchange, -/// as we don't rely on this for back-pressure. -pub const STREAM_WINDOW_SIZE: u32 = 32 * 1024 * 1024; // 32 MB - -#[allow(non_camel_case_types)] -enum StatusCode { - SUCCESS = 0, - DOUBLE_REGISTER = 1, - NO_BUFFER = 2, - INVALID_STORAGE = 3, - NO_REGISTER = 4, - NO_PARTITION = 5, - INTERNAL_ERROR = 6, - TIMEOUT = 7, - NO_BUFFER_FOR_HUGE_PARTITION = 8, -} - -impl Into for StatusCode { - fn into(self) -> i32 { - self as i32 - } -} - -pub struct DefaultShuffleServer { - app_manager_ref: AppManagerRef, -} - -impl DefaultShuffleServer { - pub fn from(app_manager_ref: AppManagerRef) -> DefaultShuffleServer { - DefaultShuffleServer { app_manager_ref } - } -} - -#[tonic::async_trait] -impl ShuffleServer for DefaultShuffleServer { - async fn register_shuffle( - &self, - request: Request, - ) -> Result, Status> { - let inner = request.into_inner(); - // todo: fast fail when hdfs is enabled but empty remote storage info. - let remote_storage_info = inner.remote_storage.map(|x| RemoteStorageConfig::from(x)); - // todo: add more options: huge_partition_threshold. and so on... - let app_config_option = AppConfigOptions::new( - DataDistribution::LOCAL_ORDER, - inner.max_concurrency_per_partition_to_write, - remote_storage_info, - ); - let status = self - .app_manager_ref - .register(inner.app_id, inner.shuffle_id, app_config_option) - .map_or(StatusCode::INTERNAL_ERROR, |_| StatusCode::SUCCESS) - .into(); - Ok(Response::new(ShuffleRegisterResponse { - status, - ret_msg: "".to_string(), - })) - } - - async fn unregister_shuffle( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - let shuffle_id = request.shuffle_id; - let app_id = request.app_id; - - info!( - "Accepted unregister shuffle info for [app:{:?}, shuffle_id:{:?}]", - &app_id, shuffle_id - ); - let status_code = self - .app_manager_ref - .unregister_shuffle(app_id, shuffle_id) - .await - .map_or_else(|_e| StatusCode::INTERNAL_ERROR, |_| StatusCode::SUCCESS); - - Ok(Response::new(ShuffleUnregisterResponse { - status: status_code.into(), - ret_msg: "".to_string(), - })) - } - - // Once unregister app accepted, the data could be purged. - async fn unregister_shuffle_by_app_id( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - let app_id = request.app_id; - - info!("Accepted unregister app rpc. app_id: {:?}", &app_id); - - let code = self - .app_manager_ref - .unregister_app(app_id) - .await - .map_or_else(|_e| StatusCode::INTERNAL_ERROR, |_| StatusCode::SUCCESS); - - Ok(Response::new(ShuffleUnregisterByAppIdResponse { - status: code.into(), - ret_msg: "".to_string(), - })) - } - - async fn send_shuffle_data( - &self, - request: Request, - ) -> Result, Status> { - let timer = GRPC_SEND_DATA_PROCESS_TIME.start_timer(); - let req = request.into_inner(); - let app_id = req.app_id; - let shuffle_id: i32 = req.shuffle_id; - let ticket_id = req.require_buffer_id; - - GRPC_SEND_DATA_TRANSPORT_TIME - .observe((util::current_timestamp_ms() - req.timestamp as u128) as f64); - - let app_option = self.app_manager_ref.get_app(&app_id); - - if app_option.is_none() { - return Ok(Response::new(SendShuffleDataResponse { - status: StatusCode::NO_REGISTER.into(), - ret_msg: "The app is not found".to_string(), - })); - } - - let app = app_option.unwrap(); - - let release_result = app.release_buffer(ticket_id).await; - if release_result.is_err() { - return Ok(Response::new(SendShuffleDataResponse { - status: StatusCode::NO_BUFFER.into(), - ret_msg: "No such buffer ticket id, it may be discarded due to timeout".to_string(), - })); - } - - let ticket_required_size = release_result.unwrap(); - - let mut blocks_map = HashMap::new(); - for shuffle_data in req.shuffle_data { - let data: PartitionedData = shuffle_data.into(); - let partitioned_blocks = data.blocks; - - let partition_id = data.partition_id; - let blocks = blocks_map.entry(partition_id).or_insert_with(|| vec![]); - blocks.extend(partitioned_blocks); - } - - let mut inserted_failure_occurs = false; - let mut inserted_failure_error = None; - let mut inserted_total_size = 0; - - for (partition_id, blocks) in blocks_map.into_iter() { - if inserted_failure_occurs { - continue; - } - - let uid = PartitionedUId { - app_id: app_id.clone(), - shuffle_id, - partition_id, - }; - let ctx = WritingViewContext::from(uid.clone(), blocks); - - let inserted = app - .insert(ctx) - .instrument_await(format!("insert data for app. uid: {:?}", &uid)) - .await; - if inserted.is_err() { - let err = format!( - "Errors on putting data. app_id: {}, err: {:?}", - &app_id, - inserted.err() - ); - error!("{}", &err); - - inserted_failure_error = Some(err); - inserted_failure_occurs = true; - continue; - } - - let inserted_size = inserted.unwrap(); - inserted_total_size += inserted_size as i64; - } - - let unused_allocated_size = ticket_required_size - inserted_total_size; - if unused_allocated_size != 0 { - debug!("The required buffer size:[{:?}] has remaining allocated size:[{:?}] of unused, this should not happen", - ticket_required_size, unused_allocated_size); - if let Err(e) = app.free_allocated_memory_size(unused_allocated_size).await { - warn!( - "Errors on free allocated size: {:?} for app: {:?}. err: {:#?}", - unused_allocated_size, &app_id, e - ); - } - } - - if inserted_failure_occurs { - return Ok(Response::new(SendShuffleDataResponse { - status: StatusCode::INTERNAL_ERROR.into(), - ret_msg: inserted_failure_error.unwrap(), - })); - } - - timer.observe_duration(); - - Ok(Response::new(SendShuffleDataResponse { - status: StatusCode::SUCCESS.into(), - ret_msg: "".to_string(), - })) - } - - async fn get_local_shuffle_index( - &self, - request: Request, - ) -> Result, Status> { - let req = request.into_inner(); - let app_id = req.app_id; - let shuffle_id: i32 = req.shuffle_id; - let partition_id = req.partition_id; - let _partition_num = req.partition_num; - let _partition_per_range = req.partition_num_per_range; - - let app_option = self.app_manager_ref.get_app(&app_id); - - if app_option.is_none() { - return Ok(Response::new(GetLocalShuffleIndexResponse { - index_data: Default::default(), - status: StatusCode::NO_PARTITION.into(), - ret_msg: "Partition not found".to_string(), - data_file_len: 0, - })); - } - - let app = app_option.unwrap(); - - let partition_id = PartitionedUId::from(app_id.to_string(), shuffle_id, partition_id); - let data_index_wrapper = app - .list_index(ReadingIndexViewContext { - partition_id: partition_id.clone(), - }) - .instrument_await(format!( - "get index from localfile. uid: {:?}", - &partition_id - )) - .await; - - if data_index_wrapper.is_err() { - let error_msg = data_index_wrapper.err(); - error!( - "Errors on getting localfile data index for app:[{}], error: {:?}", - &app_id, error_msg - ); - return Ok(Response::new(GetLocalShuffleIndexResponse { - index_data: Default::default(), - status: StatusCode::INTERNAL_ERROR.into(), - ret_msg: format!("{:?}", error_msg), - data_file_len: 0, - })); - } - - match data_index_wrapper.unwrap() { - ResponseDataIndex::Local(data_index) => { - Ok(Response::new(GetLocalShuffleIndexResponse { - index_data: data_index.index_data, - status: StatusCode::SUCCESS.into(), - ret_msg: "".to_string(), - data_file_len: data_index.data_file_len, - })) - } - } - } - - async fn get_local_shuffle_data( - &self, - request: Request, - ) -> Result, Status> { - let timer = GRPC_GET_LOCALFILE_DATA_PROCESS_TIME.start_timer(); - let req = request.into_inner(); - let app_id = req.app_id; - let shuffle_id: i32 = req.shuffle_id; - let partition_id = req.partition_id; - - GRPC_GET_MEMORY_DATA_TRANSPORT_TIME - .observe((util::current_timestamp_ms() - req.timestamp as u128) as f64); - - let app = self.app_manager_ref.get_app(&app_id); - if app.is_none() { - return Ok(Response::new(GetLocalShuffleDataResponse { - data: Default::default(), - status: StatusCode::NO_REGISTER.into(), - ret_msg: "No such app in this shuffle server".to_string(), - })); - } - - let partition_id = PartitionedUId { - app_id: app_id.to_string(), - shuffle_id, - partition_id, - }; - let data_fetched_result = app - .unwrap() - .select(ReadingViewContext { - uid: partition_id.clone(), - reading_options: ReadingOptions::FILE_OFFSET_AND_LEN(req.offset, req.length as i64), - serialized_expected_task_ids_bitmap: Default::default(), - }) - .instrument_await(format!( - "select data from localfile. uid: {:?}", - &partition_id - )) - .await; - - if data_fetched_result.is_err() { - let err_msg = data_fetched_result.err(); - error!( - "Errors on getting localfile index for app:[{}], error: {:?}", - &app_id, err_msg - ); - return Ok(Response::new(GetLocalShuffleDataResponse { - data: Default::default(), - status: StatusCode::INTERNAL_ERROR.into(), - ret_msg: format!("{:?}", err_msg), - })); - } - - timer.observe_duration(); - - Ok(Response::new(GetLocalShuffleDataResponse { - data: data_fetched_result.unwrap().from_local(), - status: StatusCode::SUCCESS.into(), - ret_msg: "".to_string(), - })) - } - - async fn get_memory_shuffle_data( - &self, - request: Request, - ) -> Result, Status> { - let timer = GRPC_GET_MEMORY_DATA_PROCESS_TIME.start_timer(); - let req = request.into_inner(); - let app_id = req.app_id; - let shuffle_id: i32 = req.shuffle_id; - let partition_id = req.partition_id; - - GRPC_GET_MEMORY_DATA_TRANSPORT_TIME - .observe((util::current_timestamp_ms() - req.timestamp as u128) as f64); - - let app = self.app_manager_ref.get_app(&app_id); - if app.is_none() { - return Ok(Response::new(GetMemoryShuffleDataResponse { - shuffle_data_block_segments: Default::default(), - data: Default::default(), - status: StatusCode::NO_REGISTER.into(), - ret_msg: "No such app in this shuffle server".to_string(), - })); - } - - let partition_id = PartitionedUId { - app_id: app_id.to_string(), - shuffle_id, - partition_id, - }; - - let serialized_expected_task_ids_bitmap = - if !req.serialized_expected_task_ids_bitmap.is_empty() { - match Treemap::deserialize(&req.serialized_expected_task_ids_bitmap) { - Ok(filter) => Some(filter), - Err(e) => { - error!("Failed to deserialize: {}", e); - None - } - } - } else { - None - }; - - let data_fetched_result = app - .unwrap() - .select(ReadingViewContext { - uid: partition_id.clone(), - reading_options: ReadingOptions::MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE( - req.last_block_id, - req.read_buffer_size as i64, - ), - serialized_expected_task_ids_bitmap, - }) - .instrument_await(format!("select data from memory. uid: {:?}", &partition_id)) - .await; - - if data_fetched_result.is_err() { - let error_msg = data_fetched_result.err(); - error!( - "Errors on getting data from memory for [{}], error: {:?}", - &app_id, error_msg - ); - return Ok(Response::new(GetMemoryShuffleDataResponse { - shuffle_data_block_segments: vec![], - data: Default::default(), - status: StatusCode::INTERNAL_ERROR.into(), - ret_msg: format!("{:?}", error_msg), - })); - } - - let data = data_fetched_result.unwrap().from_memory(); - - timer.observe_duration(); - - Ok(Response::new(GetMemoryShuffleDataResponse { - shuffle_data_block_segments: data - .shuffle_data_block_segments - .into_iter() - .map(|x| x.into()) - .collect(), - data: data.data, - status: StatusCode::SUCCESS.into(), - ret_msg: "".to_string(), - })) - } - - async fn commit_shuffle_task( - &self, - _request: Request, - ) -> Result, Status> { - warn!("It has not been supported of committing shuffle data"); - Ok(Response::new(ShuffleCommitResponse { - commit_count: 0, - status: StatusCode::INTERNAL_ERROR.into(), - ret_msg: "Not supported".to_string(), - })) - } - - async fn report_shuffle_result( - &self, - request: Request, - ) -> Result, Status> { - let req = request.into_inner(); - let app_id = req.app_id; - let shuffle_id = req.shuffle_id; - let partition_to_block_ids = req.partition_to_block_ids; - - let app = self.app_manager_ref.get_app(&app_id); - if app.is_none() { - return Ok(Response::new(ReportShuffleResultResponse { - status: StatusCode::NO_REGISTER.into(), - ret_msg: "No such app in this shuffle server".to_string(), - })); - } - let app = app.unwrap(); - - for partition_to_block_id in partition_to_block_ids { - let partition_id = partition_to_block_id.partition_id; - debug!( - "Reporting partition:{} {} blocks", - partition_id, - partition_to_block_id.block_ids.len() - ); - let ctx = ReportBlocksContext { - uid: PartitionedUId { - app_id: app_id.clone(), - shuffle_id, - partition_id, - }, - blocks: partition_to_block_id.block_ids, - }; - - match app.report_block_ids(ctx).await { - Err(e) => { - return Ok(Response::new(ReportShuffleResultResponse { - status: StatusCode::INTERNAL_ERROR.into(), - ret_msg: e.to_string(), - })) - } - _ => (), - } - } - - Ok(Response::new(ReportShuffleResultResponse { - status: StatusCode::SUCCESS.into(), - ret_msg: "".to_string(), - })) - } - - async fn get_shuffle_result( - &self, - request: Request, - ) -> Result, Status> { - let req = request.into_inner(); - let app_id = req.app_id; - let shuffle_id = req.shuffle_id; - let partition_id = req.partition_id; - - let app = self.app_manager_ref.get_app(&app_id); - if app.is_none() { - return Ok(Response::new(GetShuffleResultResponse { - status: StatusCode::NO_REGISTER.into(), - ret_msg: "No such app in this shuffle server".to_string(), - serialized_bitmap: Default::default(), - })); - } - - let partition_id = PartitionedUId { - app_id: app_id.to_string(), - shuffle_id, - partition_id, - }; - let block_ids_result = app.unwrap().get_block_ids(GetBlocksContext { - uid: partition_id.clone(), - }); - - if block_ids_result.is_err() { - let err_msg = block_ids_result.err(); - error!( - "Errors on getting shuffle block ids for app:[{}], error: {:?}", - &app_id, err_msg - ); - return Ok(Response::new(GetShuffleResultResponse { - status: StatusCode::INTERNAL_ERROR.into(), - ret_msg: format!("{:?}", err_msg), - serialized_bitmap: Default::default(), - })); - } - - Ok(Response::new(GetShuffleResultResponse { - status: StatusCode::SUCCESS.into(), - ret_msg: "".to_string(), - serialized_bitmap: block_ids_result.unwrap(), - })) - } - - async fn get_shuffle_result_for_multi_part( - &self, - request: Request, - ) -> Result, Status> { - let req = request.into_inner(); - let app_id = req.app_id; - let shuffle_id = req.shuffle_id; - - let app = self.app_manager_ref.get_app(&app_id); - if app.is_none() { - return Ok(Response::new(GetShuffleResultForMultiPartResponse { - status: StatusCode::NO_REGISTER.into(), - ret_msg: "No such app in this shuffle server".to_string(), - serialized_bitmap: Default::default(), - })); - } - let app = app.unwrap(); - - let mut bytes_mut = BytesMut::new(); - for partition_id in req.partitions { - let block_ids_result = app.get_block_ids(GetBlocksContext { - uid: PartitionedUId { - app_id: app_id.clone(), - shuffle_id, - partition_id, - }, - }); - if block_ids_result.is_err() { - let err_msg = block_ids_result.err(); - error!( - "Errors on getting shuffle block ids by multipart way of app:[{}], error: {:?}", - &app_id, err_msg - ); - return Ok(Response::new(GetShuffleResultForMultiPartResponse { - status: StatusCode::INTERNAL_ERROR.into(), - ret_msg: format!("{:?}", err_msg), - serialized_bitmap: Default::default(), - })); - } - bytes_mut.put(block_ids_result.unwrap()); - } - - Ok(Response::new(GetShuffleResultForMultiPartResponse { - status: 0, - ret_msg: "".to_string(), - serialized_bitmap: bytes_mut.freeze(), - })) - } - - async fn finish_shuffle( - &self, - _request: Request, - ) -> Result, Status> { - info!("Accepted unregister shuffle info...."); - Ok(Response::new(FinishShuffleResponse { - status: StatusCode::SUCCESS.into(), - ret_msg: "".to_string(), - })) - } - - async fn require_buffer( - &self, - request: Request, - ) -> Result, Status> { - let timer = GRPC_BUFFER_REQUIRE_PROCESS_TIME.start_timer(); - let req = request.into_inner(); - let app_id = req.app_id; - let shuffle_id = req.shuffle_id; - - let app = self.app_manager_ref.get_app(&app_id); - if app.is_none() { - return Ok(Response::new(RequireBufferResponse { - require_buffer_id: 0, - status: StatusCode::NO_REGISTER.into(), - ret_msg: "No such app in this shuffle server".to_string(), - })); - } - - let partition_id = PartitionedUId { - app_id, - shuffle_id, - // ignore this. - partition_id: 1, - }; - let app = app - .unwrap() - .require_buffer(RequireBufferContext { - uid: partition_id.clone(), - size: req.require_size as i64, - }) - .instrument_await(format!("require buffer. uid: {:?}", &partition_id)) - .await; - - let res = match app { - Ok(required_buffer_res) => ( - StatusCode::SUCCESS, - required_buffer_res.ticket_id, - "".to_string(), - ), - Err(err) => (StatusCode::NO_BUFFER, -1i64, format!("{:?}", err)), - }; - - timer.observe_duration(); - - Ok(Response::new(RequireBufferResponse { - require_buffer_id: res.1, - status: res.0.into(), - ret_msg: res.2, - })) - } - - async fn app_heartbeat( - &self, - request: Request, - ) -> Result, Status> { - let app_id = request.into_inner().app_id; - info!("Accepted heartbeat for app: {:#?}", &app_id); - - let app = self.app_manager_ref.get_app(&app_id); - if app.is_none() { - return Ok(Response::new(AppHeartBeatResponse { - status: StatusCode::NO_REGISTER.into(), - ret_msg: "No such app in this shuffle server".to_string(), - })); - } - - let app = app.unwrap(); - let _ = app.heartbeat(); - - Ok(Response::new(AppHeartBeatResponse { - status: StatusCode::SUCCESS.into(), - ret_msg: "".to_string(), - })) - } -} - -pub mod metrics_middleware { - use crate::metric::{GAUGE_GRPC_REQUEST_QUEUE_SIZE, TOTAL_GRPC_REQUEST}; - use hyper::service::Service; - use hyper::Body; - use prometheus::HistogramVec; - use std::task::{Context, Poll}; - use tower::Layer; - - #[derive(Clone)] - pub struct MetricsMiddlewareLayer { - metric: HistogramVec, - } - - impl MetricsMiddlewareLayer { - pub fn new(metric: HistogramVec) -> Self { - Self { metric } - } - } - - impl Layer for MetricsMiddlewareLayer { - type Service = MetricsMiddleware; - - fn layer(&self, service: S) -> Self::Service { - MetricsMiddleware { - inner: service, - metric: self.metric.clone(), - } - } - } - - #[derive(Clone)] - pub struct MetricsMiddleware { - inner: S, - metric: HistogramVec, - } - - impl Service> for MetricsMiddleware - where - S: Service> + Clone + Send + 'static, - S::Future: Send + 'static, - { - type Response = S::Response; - type Error = S::Error; - type Future = futures::future::BoxFuture<'static, Result>; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.inner.poll_ready(cx) - } - - fn call(&mut self, req: hyper::Request) -> Self::Future { - TOTAL_GRPC_REQUEST.inc(); - GAUGE_GRPC_REQUEST_QUEUE_SIZE.inc(); - - // This is necessary because tonic internally uses `tower::buffer::Buffer`. - // See https://github.com/tower-rs/tower/issues/547#issuecomment-767629149 - // for details on why this is necessary - let clone = self.inner.clone(); - let mut inner = std::mem::replace(&mut self.inner, clone); - - let metrics = self.metric.clone(); - - Box::pin(async move { - let path = req.uri().path(); - let timer = metrics.with_label_values(&[path]).start_timer(); - - let response = inner.call(req).await?; - - timer.observe_duration(); - - GAUGE_GRPC_REQUEST_QUEUE_SIZE.dec(); - - Ok(response) - }) - } - } -} - -pub mod await_tree_middleware { - use std::task::{Context, Poll}; - - use crate::await_tree::AwaitTreeInner; - use futures::Future; - use hyper::Body; - use tower::layer::util::Identity; - use tower::util::Either; - use tower::{Layer, Service}; - - /// Manages the await-trees of `gRPC` requests that are currently served by the compute node. - - #[derive(Clone)] - pub struct AwaitTreeMiddlewareLayer { - manager: AwaitTreeInner, - } - pub type OptionalAwaitTreeMiddlewareLayer = Either; - - impl AwaitTreeMiddlewareLayer { - pub fn new(manager: AwaitTreeInner) -> Self { - Self { manager } - } - - pub fn new_optional(optional: Option) -> OptionalAwaitTreeMiddlewareLayer { - if let Some(manager) = optional { - Either::A(Self::new(manager)) - } else { - Either::B(Identity::new()) - } - } - } - - impl Layer for AwaitTreeMiddlewareLayer { - type Service = AwaitTreeMiddleware; - - fn layer(&self, service: S) -> Self::Service { - AwaitTreeMiddleware { - inner: service, - manager: self.manager.clone(), - } - } - } - - #[derive(Clone)] - pub struct AwaitTreeMiddleware { - inner: S, - manager: AwaitTreeInner, - } - - impl Service> for AwaitTreeMiddleware - where - S: Service> + Clone + Send + 'static, - S::Future: Send + 'static, - { - type Response = S::Response; - type Error = S::Error; - - type Future = impl Future>; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.inner.poll_ready(cx) - } - - fn call(&mut self, req: hyper::Request) -> Self::Future { - // This is necessary because tonic internally uses `tower::buffer::Buffer`. - // See https://github.com/tower-rs/tower/issues/547#issuecomment-767629149 - // for details on why this is necessary - let clone = self.inner.clone(); - let mut inner = std::mem::replace(&mut self.inner, clone); - - let manager = self.manager.clone(); - async move { - let root = manager.register(format!("{}", req.uri().path())).await; - root.instrument(inner.call(req)).await - } - } - } -} diff --git a/rust/experimental/server/src/http/await_tree.rs b/rust/experimental/server/src/http/await_tree.rs deleted file mode 100644 index e8f788d0f8..0000000000 --- a/rust/experimental/server/src/http/await_tree.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::await_tree::AWAIT_TREE_REGISTRY; -use crate::http::Handler; -use poem::endpoint::make; -use poem::{get, RouteMethod}; - -pub struct AwaitTreeHandler {} - -impl Default for AwaitTreeHandler { - fn default() -> Self { - Self {} - } -} - -impl Handler for AwaitTreeHandler { - fn get_route_method(&self) -> RouteMethod { - get(make(|_| async { - let registry_cloned = AWAIT_TREE_REGISTRY.clone().get_inner(); - let registry = registry_cloned.lock().unwrap(); - let mut dynamic_string = String::new(); - for (_, tree) in registry.iter() { - let raw_tree = format!("{}", tree); - dynamic_string.push_str(raw_tree.as_str()); - dynamic_string.push('\n'); - } - dynamic_string - })) - } - - fn get_route_path(&self) -> String { - "/await-tree".to_string() - } -} diff --git a/rust/experimental/server/src/http/http_service.rs b/rust/experimental/server/src/http/http_service.rs deleted file mode 100644 index 5e83dcd5eb..0000000000 --- a/rust/experimental/server/src/http/http_service.rs +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::error::WorkerError; - -use poem::endpoint::make_sync; -use poem::error::ResponseError; -use poem::http::StatusCode; -use poem::listener::TcpListener; -use poem::{get, Route, RouteMethod, Server}; - -use std::sync::Mutex; - -use crate::http::{HTTPServer, Handler}; -use crate::runtime::manager::RuntimeManager; -impl ResponseError for WorkerError { - fn status(&self) -> StatusCode { - StatusCode::BAD_REQUEST - } -} - -struct IndexPageHandler {} -impl Handler for IndexPageHandler { - fn get_route_method(&self) -> RouteMethod { - get(make_sync(|_| "Hello uniffle server")) - } - - fn get_route_path(&self) -> String { - "/".to_string() - } -} - -pub struct PoemHTTPServer { - handlers: Mutex>>, -} - -unsafe impl Send for PoemHTTPServer {} -unsafe impl Sync for PoemHTTPServer {} - -impl PoemHTTPServer { - pub fn new() -> Self { - let handlers: Vec> = vec![Box::new(IndexPageHandler {})]; - Self { - handlers: Mutex::new(handlers), - } - } -} - -impl HTTPServer for PoemHTTPServer { - fn start(&self, runtime_manager: RuntimeManager, port: u16) { - let mut app = Route::new(); - let handlers = self.handlers.lock().unwrap(); - for handler in handlers.iter() { - app = app.at(handler.get_route_path(), handler.get_route_method()); - } - runtime_manager.http_runtime.spawn(async move { - let _ = Server::new(TcpListener::bind(format!("0.0.0.0:{}", port))) - .name("uniffle-server-http-service") - .run(app) - .await; - }); - } - - fn register_handler(&self, handler: impl Handler + 'static) { - let mut handlers = self.handlers.lock().unwrap(); - handlers.push(Box::new(handler)); - } -} diff --git a/rust/experimental/server/src/http/jeprof.rs b/rust/experimental/server/src/http/jeprof.rs deleted file mode 100644 index 57e44eff8d..0000000000 --- a/rust/experimental/server/src/http/jeprof.rs +++ /dev/null @@ -1,144 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use log::error; -use serde::{Deserialize, Serialize}; -use std::time::Duration; - -use poem::error::ResponseError; -use poem::http::StatusCode; -use poem::{IntoResponse, Request, RouteMethod}; -use tempfile::Builder; -use tokio::time::sleep as delay_for; - -use super::Handler; -use crate::mem_allocator::error::ProfError; -use crate::mem_allocator::error::ProfError::IoError; -use crate::mem_allocator::*; - -#[derive(Deserialize, Serialize)] -#[serde(default)] -pub struct JeProfRequest { - pub(crate) duration: u64, - pub(crate) keep_profiling: bool, -} - -impl Default for JeProfRequest { - fn default() -> Self { - JeProfRequest { - duration: 15, - keep_profiling: false, - } - } -} - -// converts profiling error to http error -impl ResponseError for ProfError { - fn status(&self) -> StatusCode { - StatusCode::INTERNAL_SERVER_ERROR - } -} - -#[poem::handler] -async fn jeprof_handler(req: &Request) -> poem::Result { - let req = req.params::()?; - // when memory-prof feature is not enabled, return error - if !is_prof_enabled() { - return Err(ProfError::MemProfilingNotEnabled.into()); - } - // todo: maybe we can use a global lock to prevent multiple profiling at the same time - // enables heap profiling - let _ = activate_prof().map_err(|e| { - let msg = format!("could not start profiling: {:?}", e); - error!("{}", msg); - e - })?; - // create tmp_file - let tmp_file = Builder::new() - .prefix("heap_dump_") - .suffix(".prof") - .tempfile() - .map_err(|e| IoError(e))?; - let path = tmp_file.path(); - delay_for(Duration::from_secs(req.duration)).await; - // dump heap profile - let buf: Vec = dump_prof(&path.to_string_lossy()).map_err(|e| { - let msg = format!("could not dump heap profile: {:?}", e); - error!("{}", msg); - e - })?; - let len = buf.len(); - if !req.keep_profiling { - // disables heap profiling - let _ = deactivate_prof().map_err(|e| { - let msg = format!("could not stop profiling: {:?}", e); - error!("{}", msg); - e - })?; - } - Ok(buf - .with_header("content-length", len) - .with_header( - "Content-Disposition", - format!("attachment; filename=\"{}\"", &path.to_string_lossy()), - ) - .into_response()) -} - -pub struct JeProfHandler {} - -impl Default for JeProfHandler { - fn default() -> Self { - JeProfHandler {} - } -} - -impl Handler for JeProfHandler { - fn get_route_method(&self) -> RouteMethod { - RouteMethod::new().get(jeprof_handler) - } - - fn get_route_path(&self) -> String { - "/debug/heap/profile".to_string() - } -} - -#[cfg(test)] -mod test { - use crate::http::jeprof::JeProfHandler; - use crate::http::Handler; - use poem::test::TestClient; - use poem::Route; - use tonic::codegen::http::StatusCode; - - #[tokio::test] - async fn test_router() { - let handler = JeProfHandler::default(); - let app = Route::new().at(handler.get_route_path(), handler.get_route_method()); - let cli = TestClient::new(app); - let resp = cli - .get("/debug/heap/profile") - .query("seconds", &10) - .send() - .await; - - #[cfg(feature = "mem-profiling")] - resp.assert_status_is_ok(); - #[cfg(not(feature = "mem-profiling"))] - resp.assert_status(StatusCode::INTERNAL_SERVER_ERROR); - } -} diff --git a/rust/experimental/server/src/http/metrics.rs b/rust/experimental/server/src/http/metrics.rs deleted file mode 100644 index 6de760af0f..0000000000 --- a/rust/experimental/server/src/http/metrics.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::Handler; -use crate::metric::REGISTRY; -use log::error; -use poem::endpoint::make_sync; -use poem::{get, RouteMethod}; - -pub struct MetricsHTTPHandler {} - -impl Default for MetricsHTTPHandler { - fn default() -> Self { - Self {} - } -} - -impl Handler for MetricsHTTPHandler { - fn get_route_method(&self) -> RouteMethod { - get(make_sync(|_| { - use prometheus::Encoder; - let encoder = prometheus::TextEncoder::new(); - - let mut buffer = Vec::new(); - if let Err(e) = encoder.encode(®ISTRY.gather(), &mut buffer) { - error!("could not encode custom metrics: {:?}", e); - }; - let mut res = match String::from_utf8(buffer.clone()) { - Ok(v) => v, - Err(e) => { - error!("custom metrics could not be from_utf8'd: {:?}", e); - String::default() - } - }; - buffer.clear(); - - let mut buffer = Vec::new(); - if let Err(e) = encoder.encode(&prometheus::gather(), &mut buffer) { - error!("could not encode prometheus metrics: {:?}", e); - }; - let res_custom = match String::from_utf8(buffer.clone()) { - Ok(v) => v, - Err(e) => { - error!("prometheus metrics could not be from_utf8'd: {}", e); - String::default() - } - }; - buffer.clear(); - - res.push_str(&res_custom); - res - })) - } - - fn get_route_path(&self) -> String { - "/metrics".to_string() - } -} diff --git a/rust/experimental/server/src/http/mod.rs b/rust/experimental/server/src/http/mod.rs deleted file mode 100644 index a5e1219723..0000000000 --- a/rust/experimental/server/src/http/mod.rs +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod await_tree; -mod http_service; -mod jeprof; -mod metrics; -mod pprof; - -use crate::http::await_tree::AwaitTreeHandler; -use crate::http::http_service::PoemHTTPServer; -use crate::http::jeprof::JeProfHandler; -use crate::http::metrics::MetricsHTTPHandler; -use crate::http::pprof::PProfHandler; -use crate::runtime::manager::RuntimeManager; -use once_cell::sync::Lazy; -use poem::RouteMethod; -pub static HTTP_SERVICE: Lazy> = Lazy::new(|| new_server()); - -/// Implement the own handlers for concrete components -pub trait Handler { - fn get_route_method(&self) -> RouteMethod; - fn get_route_path(&self) -> String; -} - -pub trait HTTPServer: Send + Sync { - fn start(&self, runtime_manager: RuntimeManager, port: u16); - fn register_handler(&self, handler: impl Handler + 'static); -} - -fn new_server() -> Box { - let server = PoemHTTPServer::new(); - server.register_handler(PProfHandler::default()); - server.register_handler(MetricsHTTPHandler::default()); - server.register_handler(AwaitTreeHandler::default()); - server.register_handler(JeProfHandler::default()); - Box::new(server) -} diff --git a/rust/experimental/server/src/http/pprof.rs b/rust/experimental/server/src/http/pprof.rs deleted file mode 100644 index 1d486a6ac0..0000000000 --- a/rust/experimental/server/src/http/pprof.rs +++ /dev/null @@ -1,113 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::error::WorkerError; -use crate::http::Handler; -use log::error; - -use poem::{handler, Request, RouteMethod}; -use pprof::protos::Message; -use pprof::ProfilerGuard; -use serde::{Deserialize, Serialize}; -use std::num::NonZeroI32; -use std::time::Duration; -use tokio::time::sleep as delay_for; - -#[derive(Deserialize, Serialize)] -#[serde(default)] -pub struct PProfRequest { - pub(crate) seconds: u64, - pub(crate) frequency: NonZeroI32, -} - -impl Default for PProfRequest { - fn default() -> Self { - PProfRequest { - seconds: 5, - frequency: NonZeroI32::new(100).unwrap(), - } - } -} - -#[handler] -async fn pprof_handler(req: &Request) -> poem::Result, WorkerError> { - let req = req.params::()?; - let mut body: Vec = Vec::new(); - - let guard = ProfilerGuard::new(req.frequency.into()).map_err(|e| { - let msg = format!("could not start profiling: {:?}", e); - error!("{}", msg); - WorkerError::HTTP_SERVICE_ERROR(msg) - })?; - delay_for(Duration::from_secs(req.seconds)).await; - let report = guard.report().build().map_err(|e| { - let msg = format!("could not build profiling report: {:?}", e); - error!("{}", msg); - WorkerError::HTTP_SERVICE_ERROR(msg) - })?; - let profile = report.pprof().map_err(|e| { - let msg = format!("could not get pprof profile: {:?}", e); - error!("{}", msg); - WorkerError::HTTP_SERVICE_ERROR(msg) - })?; - profile.write_to_vec(&mut body).map_err(|e| { - let msg = format!("could not write pprof profile: {:?}", e); - error!("{}", msg); - WorkerError::HTTP_SERVICE_ERROR(msg) - })?; - Ok(body) -} - -pub struct PProfHandler {} - -impl Default for PProfHandler { - fn default() -> Self { - Self {} - } -} - -impl Handler for PProfHandler { - fn get_route_method(&self) -> RouteMethod { - RouteMethod::new().get(pprof_handler) - } - - fn get_route_path(&self) -> String { - "/debug/pprof/profile".to_string() - } -} - -#[cfg(test)] -mod tests { - use crate::http::pprof::PProfHandler; - use crate::http::Handler; - use poem::test::TestClient; - use poem::Route; - - #[tokio::test] - async fn test_router() { - let handler = PProfHandler::default(); - let app = Route::new().at(handler.get_route_path(), handler.get_route_method()); - let cli = TestClient::new(app); - let resp = cli - .get("/debug/pprof/profile") - .query("seconds", &4) - .query("frequency", &100) - .send() - .await; - resp.assert_status_is_ok(); - } -} diff --git a/rust/experimental/server/src/lib.rs b/rust/experimental/server/src/lib.rs deleted file mode 100644 index 1219285b1e..0000000000 --- a/rust/experimental/server/src/lib.rs +++ /dev/null @@ -1,279 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#![feature(impl_trait_in_assoc_type)] - -pub mod app; -pub mod await_tree; -pub mod config; -pub mod error; -pub mod grpc; -pub mod http; -mod mem_allocator; -pub mod metric; -pub mod proto; -pub mod readable_size; -pub mod runtime; -pub mod signal; -pub mod store; -pub mod util; - -use crate::app::AppManager; -use crate::grpc::DefaultShuffleServer; -use crate::http::{HTTPServer, HTTP_SERVICE}; -use crate::metric::init_metric_service; -use crate::proto::uniffle::shuffle_server_client::ShuffleServerClient; -use crate::proto::uniffle::shuffle_server_server::ShuffleServerServer; -use crate::proto::uniffle::{ - GetLocalShuffleDataRequest, GetLocalShuffleIndexRequest, GetMemoryShuffleDataRequest, - GetShuffleResultRequest, PartitionToBlockIds, ReportShuffleResultRequest, RequireBufferRequest, - SendShuffleDataRequest, ShuffleBlock, ShuffleData, ShuffleRegisterRequest, -}; -use crate::runtime::manager::RuntimeManager; -use crate::util::gen_worker_uid; -use anyhow::Result; -use bytes::{Buf, Bytes, BytesMut}; -use croaring::treemap::JvmSerializer; -use croaring::Treemap; -use log::info; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::time::Duration; -use tokio::signal::unix::{signal, SignalKind}; -use tokio::sync::oneshot; -use tonic::transport::{Channel, Server}; - -pub async fn start_uniffle_worker(config: config::Config) -> Result<()> { - let rpc_port = config.grpc_port.unwrap_or(19999); - let worker_uid = gen_worker_uid(rpc_port); - let metric_config = config.metrics.clone(); - - let runtime_manager = RuntimeManager::from(config.runtime_config.clone()); - - init_metric_service(runtime_manager.clone(), &metric_config, worker_uid.clone()); - // start the http monitor service - let http_port = config.http_monitor_service_port.unwrap_or(20010); - HTTP_SERVICE.start(runtime_manager.clone(), http_port); - - let (tx, rx) = oneshot::channel::<()>(); - - // implement server startup - let cloned_runtime_manager = runtime_manager.clone(); - runtime_manager.grpc_runtime.spawn(async move { - let app_manager_ref = AppManager::get_ref(cloned_runtime_manager, config.clone()); - let rpc_port = config.grpc_port.unwrap_or(19999); - info!("Starting GRpc server with port:[{}] ......", rpc_port); - let shuffle_server = DefaultShuffleServer::from(app_manager_ref); - let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), rpc_port as u16); - let service = ShuffleServerServer::new(shuffle_server) - .max_decoding_message_size(usize::MAX) - .max_encoding_message_size(usize::MAX); - let _ = Server::builder() - .add_service(service) - .serve_with_shutdown(addr, async { - rx.await.expect("graceful_shutdown fail"); - println!("Successfully received the shutdown signal."); - }) - .await; - }); - - runtime_manager.default_runtime.spawn(async move { - let _ = signal(SignalKind::terminate()) - .expect("Failed to register signal handlers") - .recv() - .await; - - let _ = tx.send(()); - }); - - Ok(()) -} - -pub async fn write_read_for_one_time(mut client: ShuffleServerClient) -> Result<()> { - let app_id = "write_read_test-app-id".to_string(); - - let register_response = client - .register_shuffle(ShuffleRegisterRequest { - app_id: app_id.clone(), - shuffle_id: 0, - partition_ranges: vec![], - remote_storage: None, - user: "".to_string(), - shuffle_data_distribution: 1, - max_concurrency_per_partition_to_write: 10, - }) - .await? - .into_inner(); - assert_eq!(register_response.status, 0); - - let mut all_bytes_data = BytesMut::new(); - let mut block_ids = vec![]; - - let batch_size = 3000; - - for idx in 0..batch_size { - block_ids.push(idx as i64); - - let data = b"hello world"; - let len = data.len(); - - all_bytes_data.extend_from_slice(data); - - let buffer_required_resp = client - .require_buffer(RequireBufferRequest { - require_size: len as i32, - app_id: app_id.clone(), - shuffle_id: 0, - partition_ids: vec![], - }) - .await? - .into_inner(); - - assert_eq!(0, buffer_required_resp.status); - - let response = client - .send_shuffle_data(SendShuffleDataRequest { - app_id: app_id.clone(), - shuffle_id: 0, - require_buffer_id: buffer_required_resp.require_buffer_id, - shuffle_data: vec![ShuffleData { - partition_id: idx, - block: vec![ShuffleBlock { - block_id: idx as i64, - length: len as i32, - uncompress_length: 0, - crc: 0, - data: Bytes::copy_from_slice(data), - task_attempt_id: 0, - }], - }], - timestamp: 0, - }) - .await?; - - let response = response.into_inner(); - assert_eq!(0, response.status); - - // report the finished block ids - client - .report_shuffle_result(ReportShuffleResultRequest { - app_id: app_id.clone(), - shuffle_id: 0, - task_attempt_id: 0, - bitmap_num: 0, - partition_to_block_ids: vec![PartitionToBlockIds { - partition_id: idx, - block_ids: vec![idx as i64], - }], - }) - .await?; - } - - tokio::time::sleep(Duration::from_secs(1)).await; - - let mut accepted_block_ids = vec![]; - let mut accepted_data_bytes = BytesMut::new(); - - // firstly. read from the memory - for idx in 0..batch_size { - let block_id_result = client - .get_shuffle_result(GetShuffleResultRequest { - app_id: app_id.clone(), - shuffle_id: 0, - partition_id: idx, - }) - .await? - .into_inner(); - - assert_eq!(0, block_id_result.status); - - let block_id_bitmap = Treemap::deserialize(&*block_id_result.serialized_bitmap)?; - assert_eq!(1, block_id_bitmap.iter().count()); - assert!(block_id_bitmap.contains(idx as u64)); - - let response_data = client - .get_memory_shuffle_data(GetMemoryShuffleDataRequest { - app_id: app_id.clone(), - shuffle_id: 0, - partition_id: idx, - last_block_id: -1, - read_buffer_size: 10000000, - timestamp: 0, - serialized_expected_task_ids_bitmap: Default::default(), - }) - .await?; - let response = response_data.into_inner(); - let segments = response.shuffle_data_block_segments; - for segment in segments { - accepted_block_ids.push(segment.block_id) - } - let data = response.data; - accepted_data_bytes.extend_from_slice(&data); - } - - // secondly, read from the localfile - for idx in 0..batch_size { - let local_index_data = client - .get_local_shuffle_index(GetLocalShuffleIndexRequest { - app_id: app_id.clone(), - shuffle_id: 0, - partition_id: idx, - partition_num_per_range: 1, - partition_num: 0, - }) - .await?; - - let mut bytes = local_index_data.into_inner().index_data; - if bytes.is_empty() { - continue; - } - // index_bytes_holder.put_i64(next_offset); - // index_bytes_holder.put_i32(length); - // index_bytes_holder.put_i32(uncompress_len); - // index_bytes_holder.put_i64(crc); - // index_bytes_holder.put_i64(block_id); - // index_bytes_holder.put_i64(task_attempt_id); - bytes.get_i64(); - let len = bytes.get_i32(); - bytes.get_i32(); - bytes.get_i64(); - let id = bytes.get_i64(); - accepted_block_ids.push(id); - - let partitioned_local_data = client - .get_local_shuffle_data(GetLocalShuffleDataRequest { - app_id: app_id.clone(), - shuffle_id: 0, - partition_id: idx, - partition_num_per_range: 0, - partition_num: 0, - offset: 0, - length: len, - timestamp: 0, - }) - .await?; - accepted_data_bytes.extend_from_slice(&partitioned_local_data.into_inner().data); - } - - // check the block ids - assert_eq!(batch_size as usize, accepted_block_ids.len()); - assert_eq!(block_ids, accepted_block_ids); - - // check the shuffle data - assert_eq!(all_bytes_data.freeze(), accepted_data_bytes.freeze()); - - Ok(()) -} diff --git a/rust/experimental/server/src/main.rs b/rust/experimental/server/src/main.rs deleted file mode 100644 index f5e93c5896..0000000000 --- a/rust/experimental/server/src/main.rs +++ /dev/null @@ -1,313 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#![feature(impl_trait_in_assoc_type)] - -use crate::app::{AppManager, AppManagerRef, SHUFFLE_SERVER_ID}; -use crate::await_tree::AWAIT_TREE_REGISTRY; -use crate::config::{Config, LogConfig, RotationConfig}; -use crate::grpc::await_tree_middleware::AwaitTreeMiddlewareLayer; -use crate::grpc::metrics_middleware::MetricsMiddlewareLayer; -use crate::grpc::{DefaultShuffleServer, MAX_CONNECTION_WINDOW_SIZE, STREAM_WINDOW_SIZE}; -use crate::http::{HTTPServer, HTTP_SERVICE}; -use crate::metric::{init_metric_service, GRPC_LATENCY_TIME_SEC}; -use crate::proto::uniffle::coordinator_server_client::CoordinatorServerClient; -use crate::proto::uniffle::shuffle_server_server::ShuffleServerServer; -use crate::proto::uniffle::{ShuffleServerHeartBeatRequest, ShuffleServerId}; -use crate::runtime::manager::RuntimeManager; -use crate::signal::details::graceful_wait_for_signal; -use crate::util::{gen_worker_uid, get_local_ip}; - -use crate::mem_allocator::ALLOCATOR; -use crate::readable_size::ReadableSize; -use anyhow::Result; -use clap::{App, Arg}; -use log::{debug, error, info}; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::str::FromStr; -use std::time::Duration; -use tokio::net::TcpListener; -use tokio::sync::broadcast; -use tonic::transport::{Channel, Server}; -use tracing_appender::non_blocking::WorkerGuard; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; -use tracing_subscriber::{fmt, EnvFilter, Registry}; - -pub mod app; -mod await_tree; -mod config; -mod error; -pub mod grpc; -mod http; -mod mem_allocator; -mod metric; -pub mod proto; -mod readable_size; -pub mod runtime; -pub mod signal; -pub mod store; -mod util; - -const MAX_MEMORY_ALLOCATION_SIZE_ENV_KEY: &str = "MAX_MEMORY_ALLOCATION_SIZE"; -const DEFAULT_SHUFFLE_SERVER_TAG: &str = "ss_v4"; - -fn start_coordinator_report( - runtime_manager: RuntimeManager, - app_manager: AppManagerRef, - coordinator_quorum: Vec, - grpc_port: i32, - tags: Vec, - worker_uid: String, -) -> anyhow::Result<()> { - runtime_manager.default_runtime.spawn(async move { - let ip = get_local_ip().unwrap().to_string(); - - info!("machine ip: {}", ip.clone()); - - let shuffle_server_id = ShuffleServerId { - id: worker_uid, - ip, - port: grpc_port, - netty_port: 0, - }; - - let mut multi_coordinator_clients: Vec> = - futures::future::try_join_all( - coordinator_quorum - .iter() - .map(|quorum| CoordinatorServerClient::connect(format!("http://{}", quorum))), - ) - .await - .unwrap(); - - loop { - // todo: add interval as config var - tokio::time::sleep(Duration::from_secs(10)).await; - - let mut all_tags = vec![]; - all_tags.push(DEFAULT_SHUFFLE_SERVER_TAG.to_string()); - all_tags.extend_from_slice(&*tags); - - let healthy = app_manager.store_is_healthy().await.unwrap_or(false); - let memory_snapshot = app_manager - .store_memory_snapshot() - .await - .unwrap_or((0, 0, 0).into()); - let memory_spill_event_num = - app_manager.store_memory_spill_event_num().unwrap_or(0) as i32; - - let heartbeat_req = ShuffleServerHeartBeatRequest { - server_id: Some(shuffle_server_id.clone()), - used_memory: memory_snapshot.get_used(), - pre_allocated_memory: memory_snapshot.get_allocated(), - available_memory: memory_snapshot.get_capacity() - - memory_snapshot.get_used() - - memory_snapshot.get_allocated(), - event_num_in_flush: memory_spill_event_num, - tags: all_tags, - is_healthy: Some(healthy), - status: 0, - storage_info: Default::default(), - }; - - // It must use the 0..len to avoid borrow check in loop. - for idx in 0..multi_coordinator_clients.len() { - let client = multi_coordinator_clients.get_mut(idx).unwrap(); - let _ = client - .heartbeat(tonic::Request::new(heartbeat_req.clone())) - .await; - } - } - }); - - Ok(()) -} - -const LOG_FILE_NAME: &str = "uniffle-worker.log"; - -fn init_log(log: &LogConfig) -> WorkerGuard { - let file_appender = match log.rotation { - RotationConfig::Hourly => tracing_appender::rolling::hourly(&log.path, LOG_FILE_NAME), - RotationConfig::Daily => tracing_appender::rolling::daily(&log.path, LOG_FILE_NAME), - RotationConfig::Never => tracing_appender::rolling::never(&log.path, LOG_FILE_NAME), - }; - - let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - let formatting_layer = fmt::layer().pretty().with_writer(std::io::stderr); - - let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender); - let file_layer = fmt::layer() - .with_ansi(false) - .with_line_number(true) - .with_writer(non_blocking); - - let console_layer = console_subscriber::Builder::default() - .server_addr((Ipv4Addr::new(0, 0, 0, 0), 21002)) - .spawn(); - - Registry::default() - .with(env_filter) - .with(formatting_layer) - .with(file_layer) - .with(console_layer) - .init(); - - // Note: _guard is a WorkerGuard which is returned by tracing_appender::non_blocking to - // ensure buffered logs are flushed to their output in the case of abrupt terminations of a process. - // See WorkerGuard module for more details. - _guard -} - -fn main() -> Result<()> { - setup_max_memory_allocation(); - - let args_match = App::new("Uniffle Worker") - .version("0.9.0-SNAPSHOT") - .about("Rust based shuffle server for Apache Uniffle") - .arg( - Arg::with_name("config") - .short('c') - .long("config") - .value_name("FILE") - .default_value("./config.toml") - .help("Sets a custom config file") - .takes_value(true), - ) - .get_matches(); - - let config_path = args_match.value_of("config").unwrap_or("./config.toml"); - let config = Config::from(config_path); - - let runtime_manager = RuntimeManager::from(config.runtime_config.clone()); - - // init log - let log_config = &config.log.clone().unwrap_or(Default::default()); - let _guard = init_log(log_config); - - let rpc_port = config.grpc_port.unwrap_or(19999); - let worker_uid = gen_worker_uid(rpc_port); - // todo: remove some unnecessary worker_id transfer. - SHUFFLE_SERVER_ID.get_or_init(|| worker_uid.clone()); - - let metric_config = config.metrics.clone(); - init_metric_service(runtime_manager.clone(), &metric_config, worker_uid.clone()); - - let coordinator_quorum = config.coordinator_quorum.clone(); - let tags = config.tags.clone().unwrap_or(vec![]); - let app_manager_ref = AppManager::get_ref(runtime_manager.clone(), config.clone()); - let _ = start_coordinator_report( - runtime_manager.clone(), - app_manager_ref.clone(), - coordinator_quorum, - rpc_port, - tags, - worker_uid, - ); - - let http_port = config.http_monitor_service_port.unwrap_or(20010); - info!( - "Starting http monitor service with port:[{}] ......", - http_port - ); - HTTP_SERVICE.start(runtime_manager.clone(), http_port); - - let (tx, _) = broadcast::channel(1); - - info!("Starting GRpc server with port:[{}] ......", rpc_port); - - let available_cores = std::thread::available_parallelism()?; - debug!("GRpc service with parallelism: [{}]", &available_cores); - - for _ in 0..available_cores.into() { - let shuffle_server = DefaultShuffleServer::from(app_manager_ref.clone()); - let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), rpc_port as u16); - let service = ShuffleServerServer::new(shuffle_server) - .max_decoding_message_size(usize::MAX) - .max_encoding_message_size(usize::MAX); - let service_tx = tx.subscribe(); - runtime_manager - .grpc_runtime - .spawn(async move { grpc_serve(service, addr, service_tx).await }); - } - - graceful_wait_for_signal(tx); - - Ok(()) -} - -fn setup_max_memory_allocation() { - let _ = std::env::var(MAX_MEMORY_ALLOCATION_SIZE_ENV_KEY).map(|v| { - let readable_size = ReadableSize::from_str(v.as_str()).unwrap(); - ALLOCATOR.set_limit(readable_size.as_bytes() as usize) - }); -} - -async fn grpc_serve( - service: ShuffleServerServer, - addr: SocketAddr, - mut rx: broadcast::Receiver<()>, -) { - let sock = socket2::Socket::new( - match addr { - SocketAddr::V4(_) => socket2::Domain::IPV4, - SocketAddr::V6(_) => socket2::Domain::IPV6, - }, - socket2::Type::STREAM, - None, - ) - .unwrap(); - - sock.set_reuse_address(true).unwrap(); - sock.set_reuse_port(true).unwrap(); - sock.set_nonblocking(true).unwrap(); - sock.bind(&addr.into()).unwrap(); - sock.listen(8192).unwrap(); - - let incoming = - tokio_stream::wrappers::TcpListenerStream::new(TcpListener::from_std(sock.into()).unwrap()); - - Server::builder() - .initial_connection_window_size(MAX_CONNECTION_WINDOW_SIZE) - .initial_stream_window_size(STREAM_WINDOW_SIZE) - .tcp_nodelay(true) - .layer(MetricsMiddlewareLayer::new(GRPC_LATENCY_TIME_SEC.clone())) - .layer(AwaitTreeMiddlewareLayer::new_optional(Some( - AWAIT_TREE_REGISTRY.clone(), - ))) - .add_service(service) - .serve_with_incoming_shutdown(incoming, async { - if let Err(err) = rx.recv().await { - error!("Errors on stopping the GRPC service, err: {:?}.", err); - } else { - debug!("GRPC service has been graceful stopped."); - } - }) - .await - .unwrap(); -} - -#[cfg(test)] -mod test { - use crate::get_local_ip; - - #[test] - fn get_local_ip_test() { - let ip = get_local_ip().unwrap(); - println!("{}", ip.to_string()); - } -} diff --git a/rust/experimental/server/src/mem_allocator/default.rs b/rust/experimental/server/src/mem_allocator/default.rs deleted file mode 100644 index db7f3cb9d4..0000000000 --- a/rust/experimental/server/src/mem_allocator/default.rs +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::error::{ProfError, ProfResult}; - -pub fn is_prof_enabled() -> bool { - false -} - -pub fn dump_prof(_path: &str) -> ProfResult> { - Err(ProfError::MemProfilingNotEnabled) -} - -pub fn activate_prof() -> ProfResult<()> { - Err(ProfError::MemProfilingNotEnabled) -} - -pub fn deactivate_prof() -> ProfResult<()> { - Err(ProfError::MemProfilingNotEnabled) -} diff --git a/rust/experimental/server/src/mem_allocator/error.rs b/rust/experimental/server/src/mem_allocator/error.rs deleted file mode 100644 index e32c9c7bae..0000000000 --- a/rust/experimental/server/src/mem_allocator/error.rs +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::{error, fmt}; - -#[derive(Debug)] -pub enum ProfError { - MemProfilingNotEnabled, // when memory-prof feature is not enabled - IoError(std::io::Error), /* io error occurred, such as read/write file error, create file err - * , etc */ - JemallocError(String), // jemalloc related error - PathEncodingError(std::ffi::OsString), /* When temp files are in a non-unicode directory, - * OsString.into_string() will cause this error, */ - PathWithNulError(std::ffi::NulError), -} - -pub type ProfResult = Result; - -impl fmt::Display for ProfError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - ProfError::MemProfilingNotEnabled => { - let message = r#"memory-prof was not enabled, please make sure the feature -memory-prof is enabled when building and set corresponding environment variables such as -_RJEM_MALLOC_CONF=prof:true"#; - write!(f, "{}", message) - } - ProfError::IoError(e) => write!(f, "io error occurred {:?}", e), - ProfError::JemallocError(e) => write!(f, "jemalloc error {}", e), - ProfError::PathEncodingError(path) => { - write!(f, "Dump target path {:?} is not unicode encoding", path) - } - ProfError::PathWithNulError(path) => { - write!(f, "Dump target path {:?} contain an internal 0 byte", path) - } - } - } -} - -impl From for ProfError { - fn from(e: std::io::Error) -> Self { - ProfError::IoError(e) - } -} - -impl From for ProfError { - fn from(e: std::ffi::NulError) -> Self { - ProfError::PathWithNulError(e) - } -} - -impl error::Error for ProfError {} diff --git a/rust/experimental/server/src/mem_allocator/jemalloc.rs b/rust/experimental/server/src/mem_allocator/jemalloc.rs deleted file mode 100644 index f3fe5d7c7e..0000000000 --- a/rust/experimental/server/src/mem_allocator/jemalloc.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub type Allocator = tikv_jemallocator::Jemalloc; -pub const fn allocator() -> Allocator { - tikv_jemallocator::Jemalloc -} diff --git a/rust/experimental/server/src/mem_allocator/mod.rs b/rust/experimental/server/src/mem_allocator/mod.rs deleted file mode 100644 index d075f4ea63..0000000000 --- a/rust/experimental/server/src/mem_allocator/mod.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// Allocators -#[cfg(all(unix, feature = "jemalloc"))] -#[path = "jemalloc.rs"] -mod imp; - -#[cfg(not(all(unix, feature = "jemalloc")))] -#[path = "system_std.rs"] -mod imp; - -use cap::Cap; - -#[global_allocator] -pub static ALLOCATOR: Cap = Cap::new(imp::allocator(), usize::max_value()); - -pub mod error; -pub type AllocStats = Vec<(&'static str, usize)>; - -// when memory-prof feature is enabled, provide empty profiling functions -#[cfg(not(all(unix, feature = "memory-prof")))] -mod default; - -#[cfg(not(all(unix, feature = "memory-prof")))] -pub use default::*; - -// when memory-prof feature is enabled, provide jemalloc profiling functions -#[cfg(all(unix, feature = "memory-prof"))] -mod profiling; -#[cfg(all(unix, feature = "memory-prof"))] -pub use profiling::*; diff --git a/rust/experimental/server/src/mem_allocator/profiling.rs b/rust/experimental/server/src/mem_allocator/profiling.rs deleted file mode 100644 index 16ea21a3bb..0000000000 --- a/rust/experimental/server/src/mem_allocator/profiling.rs +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::ffi::{c_char, CString}; -use std::fs::File; -use std::io::Read; - -use super::error::{ProfError, ProfResult}; - -// C string should end with a '\0'. -const OPT_PROF: &[u8] = b"opt.prof\0"; -const PROF_ACTIVE: &[u8] = b"prof.active\0"; -const PROF_DUMP: &[u8] = b"prof.dump\0"; - -pub fn is_prof_enabled() -> bool { - unsafe { tikv_jemalloc_ctl::raw::read::(OPT_PROF).unwrap_or(false) } -} - -pub fn activate_prof() -> ProfResult<()> { - unsafe { - if let Err(e) = tikv_jemalloc_ctl::raw::update(PROF_ACTIVE, true) { - return Err(ProfError::JemallocError(format!( - "failed to activate profiling: {}", - e - ))); - } - } - Ok(()) -} - -pub fn deactivate_prof() -> ProfResult<()> { - unsafe { - if let Err(e) = tikv_jemalloc_ctl::raw::update(PROF_ACTIVE, false) { - return Err(ProfError::JemallocError(format!( - "failed to deactivate profiling: {}", - e - ))); - } - } - Ok(()) -} - -/// Dump the profile to the `path`. path usually is a temp file generated by `tempfile` crate. -pub fn dump_prof(path: &str) -> ProfResult> { - let mut bytes = CString::new(path)?.into_bytes_with_nul(); - let ptr = bytes.as_mut_ptr() as *mut c_char; - unsafe { - if let Err(e) = tikv_jemalloc_ctl::raw::write(PROF_DUMP, ptr) { - return Err(ProfError::JemallocError(format!( - "failed to dump the profile to {:?}: {}", - path, e - ))); - } - } - let mut f = File::open(path)?; - let mut buf = Vec::new(); - f.read_to_end(&mut buf)?; - Ok(buf) -} diff --git a/rust/experimental/server/src/mem_allocator/system_std.rs b/rust/experimental/server/src/mem_allocator/system_std.rs deleted file mode 100644 index cb38449f3a..0000000000 --- a/rust/experimental/server/src/mem_allocator/system_std.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub type Allocator = std::alloc::System; -pub const fn allocator() -> Allocator { - std::alloc::System -} diff --git a/rust/experimental/server/src/metric.rs b/rust/experimental/server/src/metric.rs deleted file mode 100644 index a7998a8ecb..0000000000 --- a/rust/experimental/server/src/metric.rs +++ /dev/null @@ -1,443 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::config::MetricsConfig; -use crate::runtime::manager::RuntimeManager; -use log::{error, info}; -use once_cell::sync::Lazy; -use prometheus::{ - histogram_opts, labels, register_histogram_vec_with_registry, register_int_gauge_vec, - Histogram, HistogramOpts, HistogramVec, IntCounter, IntGauge, IntGaugeVec, Registry, -}; -use std::time::Duration; - -const DEFAULT_BUCKETS: &[f64; 16] = &[ - 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 20.0, 40.0, 60.0, 80.0, 100.0, -]; - -pub static REGISTRY: Lazy = Lazy::new(Registry::new); - -pub static TOTAL_RECEIVED_DATA: Lazy = Lazy::new(|| { - IntCounter::new("total_received_data", "Incoming Requests").expect("metric should be created") -}); - -pub static TOTAL_READ_DATA: Lazy = Lazy::new(|| { - IntCounter::new("total_read_data", "Reading Data").expect("metric should be created") -}); - -pub static TOTAL_READ_DATA_FROM_MEMORY: Lazy = Lazy::new(|| { - IntCounter::new("total_read_data_from_memory", "Reading Data from memory") - .expect("metric should be created") -}); - -pub static TOTAL_READ_DATA_FROM_LOCALFILE: Lazy = Lazy::new(|| { - IntCounter::new( - "total_read_data_from_localfile", - "Reading Data from localfile", - ) - .expect("metric should be created") -}); - -pub static GRPC_GET_MEMORY_DATA_TRANSPORT_TIME: Lazy = Lazy::new(|| { - let opts = HistogramOpts::new("grpc_get_memory_data_transport_time", "none") - .buckets(Vec::from(DEFAULT_BUCKETS as &'static [f64])); - - let histogram = Histogram::with_opts(opts).unwrap(); - histogram -}); - -pub static GRPC_GET_MEMORY_DATA_PROCESS_TIME: Lazy = Lazy::new(|| { - let opts = HistogramOpts::new("grpc_get_memory_data_process_time", "none") - .buckets(Vec::from(DEFAULT_BUCKETS as &'static [f64])); - let histogram = Histogram::with_opts(opts).unwrap(); - histogram -}); - -pub static GRPC_GET_LOCALFILE_DATA_TRANSPORT_TIME: Lazy = Lazy::new(|| { - let opts = HistogramOpts::new("grpc_get_localfile_data_transport_time", "none") - .buckets(Vec::from(DEFAULT_BUCKETS as &'static [f64])); - - let histogram = Histogram::with_opts(opts).unwrap(); - histogram -}); - -pub static GRPC_GET_LOCALFILE_DATA_PROCESS_TIME: Lazy = Lazy::new(|| { - let opts = HistogramOpts::new("grpc_get_localfile_data_process_time", "none") - .buckets(Vec::from(DEFAULT_BUCKETS as &'static [f64])); - - let histogram = Histogram::with_opts(opts).unwrap(); - histogram -}); - -pub static GRPC_SEND_DATA_TRANSPORT_TIME: Lazy = Lazy::new(|| { - let opts = HistogramOpts::new("grpc_send_data_transport_time", "none") - .buckets(Vec::from(DEFAULT_BUCKETS as &'static [f64])); - - let histogram = Histogram::with_opts(opts).unwrap(); - histogram -}); - -pub static GRPC_SEND_DATA_PROCESS_TIME: Lazy = Lazy::new(|| { - let opts = HistogramOpts::new("grpc_send_data_process_time", "none") - .buckets(Vec::from(DEFAULT_BUCKETS as &'static [f64])); - - let histogram = Histogram::with_opts(opts).unwrap(); - histogram -}); - -pub static GRPC_BUFFER_REQUIRE_PROCESS_TIME: Lazy = Lazy::new(|| { - let opts = HistogramOpts::new("grpc_buffer_require_process_time", "none") - .buckets(Vec::from(DEFAULT_BUCKETS as &'static [f64])); - - let histogram = Histogram::with_opts(opts).unwrap(); - histogram -}); - -pub static GRPC_LATENCY_TIME_SEC: Lazy = Lazy::new(|| { - let opts = histogram_opts!( - "grpc_duration_seconds", - "gRPC latency", - Vec::from(DEFAULT_BUCKETS as &'static [f64]) - ); - let grpc_latency = register_histogram_vec_with_registry!(opts, &["path"], REGISTRY).unwrap(); - grpc_latency -}); - -pub static TOTAL_MEMORY_USED: Lazy = Lazy::new(|| { - IntCounter::new("total_memory_used", "Total memory used").expect("metric should be created") -}); - -pub static TOTAL_LOCALFILE_USED: Lazy = Lazy::new(|| { - IntCounter::new("total_localfile_used", "Total localfile used") - .expect("metric should be created") -}); - -pub static TOTAL_HDFS_USED: Lazy = Lazy::new(|| { - IntCounter::new("total_hdfs_used", "Total hdfs used").expect("metric should be created") -}); -pub static GAUGE_MEMORY_USED: Lazy = - Lazy::new(|| IntGauge::new("memory_used", "memory used").expect("metric should be created")); -pub static GAUGE_MEMORY_ALLOCATED: Lazy = Lazy::new(|| { - IntGauge::new("memory_allocated", "memory allocated").expect("metric should be created") -}); -pub static GAUGE_MEMORY_CAPACITY: Lazy = Lazy::new(|| { - IntGauge::new("memory_capacity", "memory capacity").expect("metric should be created") -}); -pub static TOTAL_MEMORY_SPILL_OPERATION: Lazy = Lazy::new(|| { - IntCounter::new("total_memory_spill", "memory capacity").expect("metric should be created") -}); -pub static TOTAL_MEMORY_SPILL_OPERATION_FAILED: Lazy = Lazy::new(|| { - IntCounter::new("total_memory_spill_failed", "memory capacity") - .expect("metric should be created") -}); -pub static TOTAL_MEMORY_SPILL_TO_LOCALFILE: Lazy = Lazy::new(|| { - IntCounter::new( - "total_memory_spill_to_localfile", - "memory spill to localfile", - ) - .expect("metric should be created") -}); -pub static TOTAL_MEMORY_SPILL_TO_HDFS: Lazy = Lazy::new(|| { - IntCounter::new("total_memory_spill_to_hdfs", "memory spill to hdfs") - .expect("metric should be created") -}); -pub static GAUGE_MEMORY_SPILL_OPERATION: Lazy = - Lazy::new(|| IntGauge::new("memory_spill", "memory spill").expect("metric should be created")); -pub static GAUGE_MEMORY_SPILL_TO_LOCALFILE: Lazy = Lazy::new(|| { - IntGauge::new("memory_spill_to_localfile", "memory spill to localfile") - .expect("metric should be created") -}); -pub static GAUGE_MEMORY_SPILL_TO_HDFS: Lazy = Lazy::new(|| { - IntGauge::new("memory_spill_to_hdfs", "memory spill to hdfs").expect("metric should be created") -}); -pub static TOTAL_APP_NUMBER: Lazy = Lazy::new(|| { - IntCounter::new("total_app_number", "total_app_number").expect("metrics should be created") -}); -pub static TOTAL_PARTITION_NUMBER: Lazy = Lazy::new(|| { - IntCounter::new("total_partition_number", "total_partition_number") - .expect("metrics should be created") -}); -pub static GAUGE_APP_NUMBER: Lazy = - Lazy::new(|| IntGauge::new("app_number", "app_number").expect("metrics should be created")); -pub static GAUGE_PARTITION_NUMBER: Lazy = Lazy::new(|| { - IntGauge::new("partition_number", "partition_number").expect("metrics should be created") -}); -pub static TOTAL_REQUIRE_BUFFER_FAILED: Lazy = Lazy::new(|| { - IntCounter::new("total_require_buffer_failed", "total_require_buffer_failed") - .expect("metrics should be created") -}); -pub static TOTAL_HUGE_PARTITION_REQUIRE_BUFFER_FAILED: Lazy = Lazy::new(|| { - IntCounter::new( - "total_huge_partition_require_buffer_failed", - "total_huge_partition_require_buffer_failed", - ) - .expect("metrics should be created") -}); - -pub static GAUGE_LOCAL_DISK_CAPACITY: Lazy = Lazy::new(|| { - register_int_gauge_vec!( - "local_disk_capacity", - "local disk capacity for root path", - &["root"] - ) - .unwrap() -}); - -pub static GAUGE_LOCAL_DISK_USED: Lazy = Lazy::new(|| { - register_int_gauge_vec!( - "local_disk_used", - "local disk used for root path", - &["root"] - ) - .unwrap() -}); - -pub static GAUGE_LOCAL_DISK_IS_HEALTHY: Lazy = Lazy::new(|| { - register_int_gauge_vec!( - "local_disk_is_healthy", - "local disk is_healthy for root path", - &["root"] - ) - .unwrap() -}); - -pub static GAUGE_RUNTIME_ALIVE_THREAD_NUM: Lazy = Lazy::new(|| { - register_int_gauge_vec!( - "runtime_thread_alive_gauge", - "alive thread number for runtime", - &["name"] - ) - .unwrap() -}); - -pub static GAUGE_RUNTIME_IDLE_THREAD_NUM: Lazy = Lazy::new(|| { - register_int_gauge_vec!( - "runtime_thread_idle_gauge", - "idle thread number for runtime", - &["name"] - ) - .unwrap() -}); - -pub static GAUGE_TOPN_APP_RESIDENT_DATA_SIZE: Lazy = Lazy::new(|| { - register_int_gauge_vec!( - "topN_app_resident_data_size", - "topN app resident data size", - &["app_id"] - ) - .unwrap() -}); - -pub static GAUGE_IN_SPILL_DATA_SIZE: Lazy = - Lazy::new(|| IntGauge::new("in_spill_data_size", "total data size in spill").unwrap()); - -pub static TOTAL_GRPC_REQUEST: Lazy = - Lazy::new(|| IntCounter::new("total_grpc_request_number", "total request number").expect("")); - -pub static GAUGE_GRPC_REQUEST_QUEUE_SIZE: Lazy = - Lazy::new(|| IntGauge::new("grpc_request_number", "current grpc request queue size").unwrap()); - -pub static TOTAL_SPILL_EVENTS_DROPPED: Lazy = Lazy::new(|| { - IntCounter::new( - "total_spill_events_dropped", - "total spill events dropped number", - ) - .expect("") -}); - -fn register_custom_metrics() { - REGISTRY - .register(Box::new(TOTAL_GRPC_REQUEST.clone())) - .expect(""); - - REGISTRY - .register(Box::new(GAUGE_GRPC_REQUEST_QUEUE_SIZE.clone())) - .expect(""); - - REGISTRY - .register(Box::new(TOTAL_SPILL_EVENTS_DROPPED.clone())) - .expect(""); - - REGISTRY - .register(Box::new(GAUGE_TOPN_APP_RESIDENT_DATA_SIZE.clone())) - .expect(""); - - REGISTRY - .register(Box::new(TOTAL_READ_DATA_FROM_LOCALFILE.clone())) - .expect("total_read_data must be registered"); - - REGISTRY - .register(Box::new(TOTAL_READ_DATA_FROM_MEMORY.clone())) - .expect("total_read_data must be registered"); - - REGISTRY - .register(Box::new(GAUGE_IN_SPILL_DATA_SIZE.clone())) - .expect(""); - - REGISTRY - .register(Box::new(GAUGE_LOCAL_DISK_CAPACITY.clone())) - .expect(""); - - REGISTRY - .register(Box::new(GAUGE_LOCAL_DISK_USED.clone())) - .expect(""); - - REGISTRY - .register(Box::new(GAUGE_LOCAL_DISK_IS_HEALTHY.clone())) - .expect(""); - - REGISTRY - .register(Box::new(GAUGE_RUNTIME_ALIVE_THREAD_NUM.clone())) - .expect(""); - - REGISTRY - .register(Box::new(GAUGE_RUNTIME_IDLE_THREAD_NUM.clone())) - .expect(""); - - REGISTRY - .register(Box::new(TOTAL_RECEIVED_DATA.clone())) - .expect("total_received_data must be registered"); - REGISTRY - .register(Box::new(TOTAL_READ_DATA.clone())) - .expect("total_read_data must be registered"); - REGISTRY - .register(Box::new(TOTAL_MEMORY_USED.clone())) - .expect("total_memory_used must be registered"); - REGISTRY - .register(Box::new(TOTAL_LOCALFILE_USED.clone())) - .expect("total_localfile_used must be registered"); - REGISTRY - .register(Box::new(TOTAL_HDFS_USED.clone())) - .expect("total_hdfs_used must be registered"); - REGISTRY - .register(Box::new(TOTAL_MEMORY_SPILL_OPERATION.clone())) - .expect("total_memory_spill_operation must be registered"); - REGISTRY - .register(Box::new(TOTAL_MEMORY_SPILL_OPERATION_FAILED.clone())) - .expect("total_memory_spill_operation_failed must be registered"); - REGISTRY - .register(Box::new(TOTAL_APP_NUMBER.clone())) - .expect("total_app_number must be registered"); - REGISTRY - .register(Box::new(TOTAL_PARTITION_NUMBER.clone())) - .expect("total_partition_number must be registered"); - REGISTRY - .register(Box::new(TOTAL_REQUIRE_BUFFER_FAILED.clone())) - .expect("total_require_buffer_failed must be registered"); - REGISTRY - .register(Box::new(TOTAL_HUGE_PARTITION_REQUIRE_BUFFER_FAILED.clone())) - .expect("total_huge_partition_require_buffer_failed must be registered"); - REGISTRY - .register(Box::new(TOTAL_MEMORY_SPILL_TO_LOCALFILE.clone())) - .expect("total_memory_spill_to_localfile must be registered"); - REGISTRY - .register(Box::new(TOTAL_MEMORY_SPILL_TO_HDFS.clone())) - .expect("total_memory_spill_to_hdfs must be registered"); - - REGISTRY - .register(Box::new(GAUGE_MEMORY_USED.clone())) - .expect("memory_used must be registered"); - REGISTRY - .register(Box::new(GAUGE_MEMORY_ALLOCATED.clone())) - .expect("memory_allocated must be registered"); - REGISTRY - .register(Box::new(GAUGE_MEMORY_CAPACITY.clone())) - .expect("memory_capacity must be registered"); - REGISTRY - .register(Box::new(GAUGE_APP_NUMBER.clone())) - .expect("app_number must be registered"); - REGISTRY - .register(Box::new(GAUGE_PARTITION_NUMBER.clone())) - .expect("partition_number must be registered"); - REGISTRY - .register(Box::new(GAUGE_MEMORY_SPILL_OPERATION.clone())) - .expect("memory_spill_operation must be registered"); - REGISTRY - .register(Box::new(GAUGE_MEMORY_SPILL_TO_LOCALFILE.clone())) - .expect("memory_spill_to_localfile must be registered"); - REGISTRY - .register(Box::new(GAUGE_MEMORY_SPILL_TO_HDFS.clone())) - .expect("memory_spill_to_hdfs must be registered"); - REGISTRY - .register(Box::new(GRPC_BUFFER_REQUIRE_PROCESS_TIME.clone())) - .expect("grpc_buffer_require_process_time must be registered"); - REGISTRY - .register(Box::new(GRPC_SEND_DATA_TRANSPORT_TIME.clone())) - .expect("grpc_send_data_transport_time must be registered"); - REGISTRY - .register(Box::new(GRPC_SEND_DATA_PROCESS_TIME.clone())) - .expect("grpc_send_data_process_time must be registered"); - - REGISTRY - .register(Box::new(GRPC_GET_MEMORY_DATA_PROCESS_TIME.clone())) - .expect("grpc_get_memory_data_process_time must be registered"); - REGISTRY - .register(Box::new(GRPC_GET_LOCALFILE_DATA_TRANSPORT_TIME.clone())) - .expect("grpc_get_localfile_data_transport_time must be registered"); - REGISTRY - .register(Box::new(GRPC_GET_LOCALFILE_DATA_PROCESS_TIME.clone())) - .expect("grpc_get_localfile_data_process_time must be registered"); - REGISTRY - .register(Box::new(GRPC_GET_MEMORY_DATA_TRANSPORT_TIME.clone())) - .expect("grpc_get_memory_data_transport_time must be registered"); -} - -pub fn init_metric_service( - runtime_manager: RuntimeManager, - metric_config: &Option, - worker_uid: String, -) -> bool { - if metric_config.is_none() { - return false; - } - - register_custom_metrics(); - - let job_name = "uniffle-worker"; - - let cfg = metric_config.clone().unwrap(); - - let push_gateway_endpoint = cfg.push_gateway_endpoint; - - if let Some(ref _endpoint) = push_gateway_endpoint { - let push_interval_sec = cfg.push_interval_sec.unwrap_or(60); - runtime_manager.default_runtime.spawn(async move { - info!("Starting prometheus metrics exporter..."); - loop { - tokio::time::sleep(Duration::from_secs(push_interval_sec as u64)).await; - - let general_metrics = prometheus::gather(); - let custom_metrics = REGISTRY.gather(); - let mut metrics = vec![]; - metrics.extend_from_slice(&custom_metrics); - metrics.extend_from_slice(&general_metrics); - - let pushed_result = prometheus::push_metrics( - job_name, - labels! {"worker_id".to_owned() => worker_uid.to_owned(),}, - &push_gateway_endpoint.to_owned().unwrap().to_owned(), - metrics, - None, - ); - if pushed_result.is_err() { - error!("Errors on pushing metrics. {:?}", pushed_result.err()); - } - } - }); - } - return true; -} diff --git a/rust/experimental/server/src/proto/mod.rs b/rust/experimental/server/src/proto/mod.rs deleted file mode 100644 index f9baf9bab9..0000000000 --- a/rust/experimental/server/src/proto/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#[rustfmt::skip] -pub mod uniffle; diff --git a/rust/experimental/server/src/proto/uniffle.proto b/rust/experimental/server/src/proto/uniffle.proto deleted file mode 100644 index b394c36654..0000000000 --- a/rust/experimental/server/src/proto/uniffle.proto +++ /dev/null @@ -1,523 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; -import "google/protobuf/empty.proto"; -import "google/protobuf/wrappers.proto"; -package rss.common; - -service ShuffleServer { - rpc registerShuffle (ShuffleRegisterRequest) returns (ShuffleRegisterResponse); - rpc unregisterShuffle(ShuffleUnregisterRequest) returns (ShuffleUnregisterResponse); - rpc unregisterShuffleByAppId(ShuffleUnregisterByAppIdRequest) returns (ShuffleUnregisterByAppIdResponse); - rpc sendShuffleData (SendShuffleDataRequest) returns (SendShuffleDataResponse); - rpc getLocalShuffleIndex (GetLocalShuffleIndexRequest) returns (GetLocalShuffleIndexResponse); - rpc getLocalShuffleData (GetLocalShuffleDataRequest) returns (GetLocalShuffleDataResponse); - rpc getMemoryShuffleData (GetMemoryShuffleDataRequest) returns (GetMemoryShuffleDataResponse); - rpc commitShuffleTask (ShuffleCommitRequest) returns (ShuffleCommitResponse); - rpc reportShuffleResult (ReportShuffleResultRequest) returns (ReportShuffleResultResponse); - rpc getShuffleResult (GetShuffleResultRequest) returns (GetShuffleResultResponse); - rpc getShuffleResultForMultiPart (GetShuffleResultForMultiPartRequest) returns (GetShuffleResultForMultiPartResponse); - rpc finishShuffle (FinishShuffleRequest) returns (FinishShuffleResponse); - rpc requireBuffer (RequireBufferRequest) returns (RequireBufferResponse); - rpc appHeartbeat(AppHeartBeatRequest) returns (AppHeartBeatResponse); -} - -message FinishShuffleRequest { - string appId = 1; - int32 shuffleId = 2; -} - -message FinishShuffleResponse { - StatusCode status = 1; - string retMsg = 2; -} - -message RequireBufferRequest { - int32 requireSize = 1; - string appId = 2; - int32 shuffleId = 3; - repeated int32 partitionIds = 4; -} - -message RequireBufferResponse { - int64 requireBufferId = 1; - StatusCode status = 2; - string retMsg = 3; -} - -message ShuffleDataBlockSegment { - int64 blockId = 1; - int64 offset = 2; - int32 length = 3; - int32 uncompressLength = 4; - int64 crc = 5; - int64 taskAttemptId = 6; -} - -message GetLocalShuffleDataRequest { - string appId = 1; - int32 shuffleId = 2; - int32 partitionId = 3; - int32 partitionNumPerRange = 4; - int32 partitionNum = 5; - int64 offset = 6; - int32 length = 7; - int64 timestamp = 8; -} - -message GetLocalShuffleDataResponse { - bytes data = 1; - StatusCode status = 2; - string retMsg = 3; -} - -message GetMemoryShuffleDataRequest { - string appId = 1; - int32 shuffleId = 2; - int32 partitionId = 3; - int64 lastBlockId = 4; - int32 readBufferSize = 5; - int64 timestamp = 6; - bytes serializedExpectedTaskIdsBitmap = 7; -} - -message GetMemoryShuffleDataResponse { - repeated ShuffleDataBlockSegment shuffleDataBlockSegments = 1; - bytes data = 2; - StatusCode status = 3; - string retMsg = 4; -} - -message GetLocalShuffleIndexRequest { - string appId = 1; - int32 shuffleId = 2; - int32 partitionId = 3; - int32 partitionNumPerRange = 4; - int32 partitionNum = 5; -} - -message GetLocalShuffleIndexResponse { - bytes indexData = 1; - StatusCode status = 2; - string retMsg = 3; - int64 dataFileLen = 4; -} - -message ReportShuffleResultRequest { - string appId = 1; - int32 shuffleId = 2; - int64 taskAttemptId = 3; - int32 bitmapNum = 4; - repeated PartitionToBlockIds partitionToBlockIds = 5; -} - -message PartitionToBlockIds { - int32 partitionId = 1; - repeated int64 blockIds = 2; -} - -message ReportShuffleResultResponse { - StatusCode status = 1; - string retMsg = 2; -} - -message GetShuffleResultRequest { - string appId = 1; - int32 shuffleId = 2; - int32 partitionId = 3; -} - -message GetShuffleResultResponse { - StatusCode status = 1; - string retMsg = 2; - bytes serializedBitmap = 3; -} - -message GetShuffleResultForMultiPartRequest { - string appId = 1; - int32 shuffleId = 2; - repeated int32 partitions = 3; -} - -message GetShuffleResultForMultiPartResponse { - StatusCode status = 1; - string retMsg = 2; - bytes serializedBitmap = 3; -} - -message ShufflePartitionRange { - int32 start = 1; - int32 end = 2; -} - -message ShuffleRegisterRequest { - string appId = 1; - int32 shuffleId = 2; - repeated ShufflePartitionRange partitionRanges = 3; - RemoteStorage remoteStorage = 4; - string user = 5; - DataDistribution shuffleDataDistribution = 6; - int32 maxConcurrencyPerPartitionToWrite = 7; -} - -enum DataDistribution { - NORMAL = 0; - LOCAL_ORDER = 1; -} - -message ShuffleUnregisterRequest { - string appId = 1; - int32 shuffleId = 2; -} - -message ShuffleUnregisterResponse { - StatusCode status = 1; - string retMsg = 2; -} - -message ShuffleRegisterResponse { - StatusCode status = 1; - string retMsg = 2; -} - -message ShuffleUnregisterByAppIdRequest { - string appId = 1; -} - -message ShuffleUnregisterByAppIdResponse { - StatusCode status = 1; - string retMsg = 2; -} - -message SendShuffleDataRequest { - string appId = 1; - int32 shuffleId = 2; - int64 requireBufferId = 3; - repeated ShuffleData shuffleData = 4; - int64 timestamp = 5; -} - -message SendShuffleDataResponse { - StatusCode status = 1; - string retMsg = 2; -} - -message ShuffleData { - int32 partitionId = 1; - repeated ShuffleBlock block = 2; -} - -message ShuffleBlock { - int64 blockId = 1; - int32 length = 2; - int32 uncompressLength = 3; - int64 crc = 4; - bytes data = 5; - int64 taskAttemptId = 6; -} - -message ShuffleCommitRequest { - string appId = 1; - int32 shuffleId = 2; -} - -message ShuffleCommitResponse { - int32 commitCount = 1; - StatusCode status = 2; - string retMsg = 3; -} - -enum ServerStatus { - ACTIVE = 0; - DECOMMISSIONING = 1; - DECOMMISSIONED = 2; - // todo: more status, such as UPGRADING -} - -message ShuffleServerHeartBeatRequest { - ShuffleServerId serverId = 1; - int64 usedMemory = 2; - int64 preAllocatedMemory = 3; - int64 availableMemory = 4; - int32 eventNumInFlush = 5; - repeated string tags = 6; - google.protobuf.BoolValue isHealthy = 7; - ServerStatus status = 8; - map storageInfo = 21; // mount point to storage info mapping. -} - -message ShuffleServerHeartBeatResponse { - StatusCode status = 1; - string retMsg = 2; -} - -message ShuffleServerId { - string id = 1; - string ip = 2; - int32 port = 3; - int32 netty_port = 4; -} - -message ShuffleServerResult { - StatusCode status = 1; - string retMsg = 2; -} - -/** Status code to identify the status of response */ -enum StatusCode { - SUCCESS = 0; - DOUBLE_REGISTER = 1; - NO_BUFFER = 2; - INVALID_STORAGE = 3; - NO_REGISTER = 4; - NO_PARTITION = 5; - INTERNAL_ERROR = 6; - TIMEOUT = 7; - ACCESS_DENIED = 8; - INVALID_REQUEST = 9; - NO_BUFFER_FOR_HUGE_PARTITION = 10; - // add more status -} - -message StorageInfo { - enum StorageMedia { - STORAGE_TYPE_UNKNOWN = 0; - HDD = 1; - SSD = 2; - HDFS = 3; - OBJECT_STORE = 4; - // possible other types, such as cloud-ssd. - } - - enum StorageStatus { - STORAGE_STATUS_UNKNOWN = 0; - NORMAL = 1; - UNHEALTHY = 2; - OVERUSED = 3; // indicate current disk/storage is overused. - } - - string mountPoint = 1; - StorageMedia storageMedia = 2; - int64 capacity = 3; - int64 usedBytes = 4; - int64 writingSpeed1M = 5; // writing speed of last minute - int64 writingSpeed5M = 6; // writing speed of last 5 minutes - int64 writingSpeed1H = 7; // writing speed of last hour - int64 numOfWritingFailures = 8; // number of writing failures since start up. - StorageStatus status = 9; -} - -service CoordinatorServer { - // Get Shuffle Server list - rpc getShuffleServerList(google.protobuf.Empty) returns (GetShuffleServerListResponse); - - // Count Shuffle Server number - rpc getShuffleServerNum(google.protobuf.Empty) returns (GetShuffleServerNumResponse); - - // Ask for suitable Shuffle Servers with partitions - rpc getShuffleAssignments(GetShuffleServerRequest) returns (GetShuffleAssignmentsResponse); - - // Heartbeat between Shuffle Server and Coordinator Server - rpc heartbeat(ShuffleServerHeartBeatRequest) returns (ShuffleServerHeartBeatResponse); - - // Get the global configuration of this Rss-cluster, i.e., data storage info - rpc getShuffleDataStorageInfo(google.protobuf.Empty) returns (GetShuffleDataStorageInfoResponse); - rpc checkServiceAvailable(google.protobuf.Empty) returns (CheckServiceAvailableResponse); - - // Heartbeat between Shuffle Application and Coordinator Server - rpc appHeartbeat(AppHeartBeatRequest) returns (AppHeartBeatResponse); - - // Report a client operation's result to coordinator server - rpc reportClientOperation(ReportShuffleClientOpRequest) returns (ReportShuffleClientOpResponse); - - // Report a application info to Coordinator Server - rpc registerApplicationInfo(ApplicationInfoRequest) returns (ApplicationInfoResponse); - - // Access to the remote shuffle service cluster - rpc accessCluster(AccessClusterRequest) returns (AccessClusterResponse); - - // Get basic client conf from coordinator - rpc fetchClientConf(google.protobuf.Empty) returns (FetchClientConfResponse); - - // Get remote storage from coordinator - rpc fetchRemoteStorage(FetchRemoteStorageRequest) returns (FetchRemoteStorageResponse); -} - -message AppHeartBeatRequest { - string appId = 1; -} - -message AppHeartBeatResponse { - StatusCode status = 1; - string retMsg = 2; -} - -message ApplicationInfoRequest { - string appId = 1; - string user = 2; -} - -message ApplicationInfoResponse { - StatusCode status = 1; - string retMsg = 2; -} - -message GetShuffleServerListResponse { - repeated ShuffleServerId servers = 1; -} - -message GetShuffleServerNumResponse { - int32 num = 1; -} - -message GetShuffleServerRequest { - string clientHost = 1; - string clientPort = 2; - string clientProperty = 3; - string applicationId = 4; - int32 shuffleId = 5; - int32 partitionNum = 6; - int32 partitionNumPerRange = 7; - int32 dataReplica = 8; - repeated string requireTags = 9; - int32 assignmentShuffleServerNumber = 10; - int32 estimateTaskConcurrency = 11; -} - -message PartitionRangeAssignment { - int32 startPartition = 1; - int32 endPartition = 2; - // replica - repeated ShuffleServerId server = 3; -} - -message GetShuffleAssignmentsResponse { - StatusCode status = 1; - repeated PartitionRangeAssignment assignments = 2; - string retMsg = 3; -} - -message ReportShuffleClientOpRequest { - string clientHost = 1; - int32 clientPort = 2; - ShuffleServerId server = 3; - string operation = 4; -} - -message ReportShuffleClientOpResponse { - StatusCode status = 1; - string retMsg = 2; -} - -message GetShuffleDataStorageInfoResponse { - string storage = 1; - string storagePath = 2; - string storagePattern = 3; -} - -message CheckServiceAvailableResponse { - StatusCode status = 1; - bool available = 2; -} - -message AccessClusterRequest { - string accessId = 1; - repeated string tags = 2; - map extraProperties = 3; - string user=4; -} - -message AccessClusterResponse { - StatusCode status = 1; - string retMsg = 2; - string uuid = 3; -} - -message FetchClientConfResponse { - StatusCode status = 1; - string retMsg = 2; - repeated ClientConfItem clientConf = 3; -} - -message ClientConfItem { - string key = 1; - string value = 2; -} - -message FetchRemoteStorageRequest { - string appId = 1; -} - -message RemoteStorageConfItem { - string key = 1; - string value = 2; -} - -message RemoteStorage { - string path = 1; - repeated RemoteStorageConfItem remoteStorageConf = 2; -} - -message FetchRemoteStorageResponse { - StatusCode status = 1; - RemoteStorage remoteStorage = 2; -} - -service ShuffleServerInternal { - rpc decommission(DecommissionRequest) returns (DecommissionResponse); - rpc cancelDecommission(CancelDecommissionRequest) returns (CancelDecommissionResponse); -} - -message DecommissionRequest { -} - -message DecommissionResponse { - StatusCode status = 1; - string retMsg = 2; -} - -message CancelDecommissionRequest { -} - -message CancelDecommissionResponse { - StatusCode status = 1; - string retMsg = 2; -} -// ShuffleManager service lives inside of compute-engine's application master, which handles rss shuffle specific logic -// per application. -service ShuffleManager { - rpc reportShuffleFetchFailure (ReportShuffleFetchFailureRequest) returns (ReportShuffleFetchFailureResponse); -} - -message ReportShuffleFetchFailureRequest { - // appId normally should be omitted, it's used to avoid wrongly request issued from remaining executors of another - // app which accidentally has the same shuffle manager port with this app. - string appId = 1; - int32 shuffleId = 2; - int32 stageAttemptId = 3; - int32 partitionId = 4; - string exception = 5; - // todo: report ShuffleServerId if needed - // ShuffleServerId serverId = 6; -} - -message ReportShuffleFetchFailureResponse { - StatusCode status = 1; - bool reSubmitWholeStage = 2; - string msg = 3; -} diff --git a/rust/experimental/server/src/readable_size.rs b/rust/experimental/server/src/readable_size.rs deleted file mode 100644 index 9dc52b9ccc..0000000000 --- a/rust/experimental/server/src/readable_size.rs +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright (c) 2017-present, PingCAP, Inc. Licensed under Apache-2.0. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// This file is copied from https://github.com/tikv/raft-engine/blob/8dd2a39f359ff16f5295f35343f626e0c10132fa/src/util.rs without any modification. - -use std::fmt; -use std::fmt::{Display, Write}; -use std::ops::{Div, Mul}; -use std::str::FromStr; - -use serde::de::{Unexpected, Visitor}; -use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; - -const UNIT: u64 = 1; - -const BINARY_DATA_MAGNITUDE: u64 = 1024; -pub const B: u64 = UNIT; -pub const KIB: u64 = B * BINARY_DATA_MAGNITUDE; -pub const MIB: u64 = KIB * BINARY_DATA_MAGNITUDE; -pub const GIB: u64 = MIB * BINARY_DATA_MAGNITUDE; -pub const TIB: u64 = GIB * BINARY_DATA_MAGNITUDE; -pub const PIB: u64 = TIB * BINARY_DATA_MAGNITUDE; - -#[derive(Clone, Debug, Copy, PartialEq, Eq, PartialOrd)] -pub struct ReadableSize(pub u64); - -impl ReadableSize { - pub const fn kb(count: u64) -> ReadableSize { - ReadableSize(count * KIB) - } - - pub const fn mb(count: u64) -> ReadableSize { - ReadableSize(count * MIB) - } - - pub const fn gb(count: u64) -> ReadableSize { - ReadableSize(count * GIB) - } - - pub const fn as_mb(self) -> u64 { - self.0 / MIB - } - - pub const fn as_bytes(self) -> u64 { - self.0 - } -} - -impl Div for ReadableSize { - type Output = ReadableSize; - - fn div(self, rhs: u64) -> ReadableSize { - ReadableSize(self.0 / rhs) - } -} - -impl Div for ReadableSize { - type Output = u64; - - fn div(self, rhs: ReadableSize) -> u64 { - self.0 / rhs.0 - } -} - -impl Mul for ReadableSize { - type Output = ReadableSize; - - fn mul(self, rhs: u64) -> ReadableSize { - ReadableSize(self.0 * rhs) - } -} - -impl Serialize for ReadableSize { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let size = self.0; - let mut buffer = String::new(); - if size == 0 { - write!(buffer, "{}KiB", size).unwrap(); - } else if size % PIB == 0 { - write!(buffer, "{}PiB", size / PIB).unwrap(); - } else if size % TIB == 0 { - write!(buffer, "{}TiB", size / TIB).unwrap(); - } else if size % GIB as u64 == 0 { - write!(buffer, "{}GiB", size / GIB).unwrap(); - } else if size % MIB as u64 == 0 { - write!(buffer, "{}MiB", size / MIB).unwrap(); - } else if size % KIB as u64 == 0 { - write!(buffer, "{}KiB", size / KIB).unwrap(); - } else { - return serializer.serialize_u64(size); - } - serializer.serialize_str(&buffer) - } -} - -impl FromStr for ReadableSize { - type Err = String; - - // This method parses value in binary unit. - fn from_str(s: &str) -> Result { - let size_str = s.trim(); - if size_str.is_empty() { - return Err(format!("{:?} is not a valid size.", s)); - } - - if !size_str.is_ascii() { - return Err(format!("ASCII string is expected, but got {:?}", s)); - } - - // size: digits and '.' as decimal separator - let size_len = size_str - .to_string() - .chars() - .take_while(|c| char::is_ascii_digit(c) || ['.', 'e', 'E', '-', '+'].contains(c)) - .count(); - - // unit: alphabetic characters - let (size, unit) = size_str.split_at(size_len); - - let unit = match unit.trim() { - "K" | "KB" | "KiB" => KIB, - "M" | "MB" | "MiB" => MIB, - "G" | "GB" | "GiB" => GIB, - "T" | "TB" | "TiB" => TIB, - "P" | "PB" | "PiB" => PIB, - "B" | "" => B, - _ => { - return Err(format!( - "only B, KB, KiB, MB, MiB, GB, GiB, TB, TiB, PB, and PiB are supported: {:?}", - s - )); - } - }; - - match size.parse::() { - Ok(n) => Ok(ReadableSize((n * unit as f64) as u64)), - Err(_) => Err(format!("invalid size string: {:?}", s)), - } - } -} - -impl Display for ReadableSize { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.0 >= PIB { - write!(f, "{:.1}PiB", self.0 as f64 / PIB as f64) - } else if self.0 >= TIB { - write!(f, "{:.1}TiB", self.0 as f64 / TIB as f64) - } else if self.0 >= GIB { - write!(f, "{:.1}GiB", self.0 as f64 / GIB as f64) - } else if self.0 >= MIB { - write!(f, "{:.1}MiB", self.0 as f64 / MIB as f64) - } else if self.0 >= KIB { - write!(f, "{:.1}KiB", self.0 as f64 / KIB as f64) - } else { - write!(f, "{}B", self.0) - } - } -} - -impl<'de> Deserialize<'de> for ReadableSize { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct SizeVisitor; - - impl<'de> Visitor<'de> for SizeVisitor { - type Value = ReadableSize; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("valid size") - } - - fn visit_i64(self, size: i64) -> Result - where - E: de::Error, - { - if size >= 0 { - self.visit_u64(size as u64) - } else { - Err(E::invalid_value(Unexpected::Signed(size), &self)) - } - } - - fn visit_u64(self, size: u64) -> Result - where - E: de::Error, - { - Ok(ReadableSize(size)) - } - - fn visit_str(self, size_str: &str) -> Result - where - E: de::Error, - { - size_str.parse().map_err(E::custom) - } - } - - deserializer.deserialize_any(SizeVisitor) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_readable_size() { - let s = ReadableSize::kb(2); - assert_eq!(s.0, 2048); - assert_eq!(s.as_mb(), 0); - let s = ReadableSize::mb(2); - assert_eq!(s.0, 2 * 1024 * 1024); - assert_eq!(s.as_mb(), 2); - let s = ReadableSize::gb(2); - assert_eq!(s.0, 2 * 1024 * 1024 * 1024); - assert_eq!(s.as_mb(), 2048); - - assert_eq!((ReadableSize::mb(2) / 2).0, MIB); - assert_eq!((ReadableSize::mb(1) / 2).0, 512 * KIB); - assert_eq!(ReadableSize::mb(2) / ReadableSize::kb(1), 2048); - } - - #[test] - fn test_parse_readable_size() { - #[derive(Serialize, Deserialize)] - struct SizeHolder { - s: ReadableSize, - } - - let legal_cases = vec![ - (0, "0KiB"), - (2 * KIB, "2KiB"), - (4 * MIB, "4MiB"), - (5 * GIB, "5GiB"), - (7 * TIB, "7TiB"), - (11 * PIB, "11PiB"), - ]; - for (size, exp) in legal_cases { - let c = SizeHolder { - s: ReadableSize(size), - }; - let res_str = toml::to_string(&c).unwrap(); - let exp_str = format!("s = {:?}\n", exp); - assert_eq!(res_str, exp_str); - let res_size: SizeHolder = toml::from_str(&exp_str).unwrap(); - assert_eq!(res_size.s.0, size); - } - - let c = SizeHolder { - s: ReadableSize(512), - }; - let res_str = toml::to_string(&c).unwrap(); - assert_eq!(res_str, "s = 512\n"); - let res_size: SizeHolder = toml::from_str(&res_str).unwrap(); - assert_eq!(res_size.s.0, c.s.0); - - let decode_cases = vec![ - (" 0.5 PB", PIB / 2), - ("0.5 TB", TIB / 2), - ("0.5GB ", GIB / 2), - ("0.5MB", MIB / 2), - ("0.5KB", KIB / 2), - ("0.5P", PIB / 2), - ("0.5T", TIB / 2), - ("0.5G", GIB / 2), - ("0.5M", MIB / 2), - ("0.5K", KIB / 2), - ("23", 23), - ("1", 1), - ("1024B", KIB), - // units with binary prefixes - (" 0.5 PiB", PIB / 2), - ("1PiB", PIB), - ("0.5 TiB", TIB / 2), - ("2 TiB", TIB * 2), - ("0.5GiB ", GIB / 2), - ("787GiB ", GIB * 787), - ("0.5MiB", MIB / 2), - ("3MiB", MIB * 3), - ("0.5KiB", KIB / 2), - ("1 KiB", KIB), - // scientific notation - ("0.5e6 B", B * 500000), - ("0.5E6 B", B * 500000), - ("1e6B", B * 1000000), - ("8E6B", B * 8000000), - ("8e7", B * 80000000), - ("1e-1MB", MIB / 10), - ("1e+1MB", MIB * 10), - ("0e+10MB", 0), - ]; - for (src, exp) in decode_cases { - let src = format!("s = {:?}", src); - let res: SizeHolder = toml::from_str(&src).unwrap(); - assert_eq!(res.s.0, exp); - } - - let illegal_cases = vec![ - "0.5kb", "0.5kB", "0.5Kb", "0.5k", "0.5g", "b", "gb", "1b", "B", "1K24B", " 5_KB", - "4B7", "5M_", - ]; - for src in illegal_cases { - let src_str = format!("s = {:?}", src); - assert!(toml::from_str::(&src_str).is_err(), "{}", src); - } - } -} diff --git a/rust/experimental/server/src/runtime/manager.rs b/rust/experimental/server/src/runtime/manager.rs deleted file mode 100644 index 0dbee3caa0..0000000000 --- a/rust/experimental/server/src/runtime/manager.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::config::RuntimeConfig; -use crate::runtime::{Builder, RuntimeRef}; -use std::future::Future; -use std::sync::Arc; - -#[derive(Clone, Debug)] -pub struct RuntimeManager { - // for reading data - pub read_runtime: RuntimeRef, - // for writing data - pub write_runtime: RuntimeRef, - // for grpc service - pub grpc_runtime: RuntimeRef, - // for http monitor service - pub http_runtime: RuntimeRef, - // the default runtime for not important tasks. - // like the data purging/ heartbeat / metric push - pub default_runtime: RuntimeRef, -} - -impl Default for RuntimeManager { - fn default() -> Self { - RuntimeManager::from(Default::default()) - } -} - -impl RuntimeManager { - pub fn from(config: RuntimeConfig) -> Self { - fn create_runtime(pool_size: usize, name: &str) -> RuntimeRef { - Arc::new( - Builder::default() - .worker_threads(pool_size as usize) - .thread_name(name) - .enable_all() - .build() - .unwrap(), - ) - } - - Self { - read_runtime: create_runtime(config.read_thread_num, "read_thread_pool"), - write_runtime: create_runtime(config.write_thread_num, "write_thread_pool"), - grpc_runtime: create_runtime(config.grpc_thread_num, "grpc_thread_pool"), - http_runtime: create_runtime(config.http_thread_num, "http_thread_pool"), - default_runtime: create_runtime(config.default_thread_num, "default_thread_pool"), - } - } - - // for test cases to wait the future - pub fn wait(&self, future: F) -> F::Output { - self.default_runtime.block_on(future) - } -} diff --git a/rust/experimental/server/src/runtime/metrics.rs b/rust/experimental/server/src/runtime/metrics.rs deleted file mode 100644 index 9a40edd250..0000000000 --- a/rust/experimental/server/src/runtime/metrics.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metric::{GAUGE_RUNTIME_ALIVE_THREAD_NUM, GAUGE_RUNTIME_IDLE_THREAD_NUM}; -use prometheus::IntGauge; - -#[derive(Debug)] -pub struct Metrics { - pub thread_alive_gauge: IntGauge, - pub thread_idle_gauge: IntGauge, -} - -impl Metrics { - pub fn new(name: &str) -> Self { - Self { - thread_alive_gauge: GAUGE_RUNTIME_ALIVE_THREAD_NUM.with_label_values(&[name]), - thread_idle_gauge: GAUGE_RUNTIME_IDLE_THREAD_NUM.with_label_values(&[name]), - } - } - - #[inline] - pub fn on_thread_start(&self) { - self.thread_alive_gauge.inc(); - } - - #[inline] - pub fn on_thread_stop(&self) { - self.thread_alive_gauge.dec(); - } - - #[inline] - pub fn on_thread_park(&self) { - self.thread_idle_gauge.inc(); - } - - #[inline] - pub fn on_thread_unpark(&self) { - self.thread_idle_gauge.dec(); - } -} diff --git a/rust/experimental/server/src/runtime/mod.rs b/rust/experimental/server/src/runtime/mod.rs deleted file mode 100644 index 257f29ffa5..0000000000 --- a/rust/experimental/server/src/runtime/mod.rs +++ /dev/null @@ -1,264 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod manager; -mod metrics; - -use crate::runtime::metrics::Metrics; -use anyhow::anyhow; -use pin_project_lite::pin_project; -use std::fmt::Debug; -use std::{ - future::Future, - pin::Pin, - sync::Arc, - task::{Context, Poll}, -}; -use tokio::runtime::Builder as TokioRuntimeBuilder; -use tokio::runtime::Runtime as TokioRuntime; -use tokio::task::JoinHandle as TokioJoinHandle; - -pub type RuntimeRef = Arc; - -#[derive(Debug)] -pub struct Runtime { - rt: TokioRuntime, - metrics: Arc, -} - -impl Runtime { - pub fn spawn(&self, future: F) -> JoinHandle - where - F: Future + Send + 'static, - F::Output: Send + 'static, - { - JoinHandle { - inner: self.rt.spawn(future), - } - } - - pub fn spawn_blocking(&self, func: F) -> JoinHandle - where - F: FnOnce() -> R + Send + 'static, - R: Send + 'static, - { - JoinHandle { - inner: self.rt.spawn_blocking(func), - } - } - - pub fn block_on(&self, future: F) -> F::Output { - self.rt.block_on(future) - } - - pub fn stats(&self) -> RuntimeStats { - RuntimeStats { - alive_thread_num: self.metrics.thread_alive_gauge.get(), - idle_thread_num: self.metrics.thread_idle_gauge.get(), - } - } -} - -#[derive(Debug)] -pub struct RuntimeStats { - pub alive_thread_num: i64, - pub idle_thread_num: i64, -} - -pin_project! { - #[derive(Debug)] - pub struct JoinHandle { - #[pin] - inner: TokioJoinHandle, - } -} - -impl JoinHandle { - pub fn abort(&self) { - self.inner.abort(); - } -} - -impl Future for JoinHandle { - type Output = Result; - - fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { - let this = self.project(); - this.inner - .poll(ctx) - .map_err(|_source| anyhow!("errors on polling for future.")) - } -} - -pub struct Builder { - thread_name: String, - builder: TokioRuntimeBuilder, -} - -impl Default for Builder { - fn default() -> Self { - Self { - thread_name: "runtime-worker".to_string(), - builder: TokioRuntimeBuilder::new_multi_thread(), - } - } -} - -fn with_metrics(metrics: &Arc, f: F) -> impl Fn() -where - F: Fn(&Arc) + 'static, -{ - let m = metrics.clone(); - move || { - f(&m); - } -} - -impl Builder { - /// Sets the number of worker threads the Runtime will use. - /// - /// This can be any number above 0 - pub fn worker_threads(&mut self, val: usize) -> &mut Self { - self.builder.worker_threads(val); - self - } - - /// Sets name of threads spawned by the Runtime thread pool - pub fn thread_name(&mut self, val: impl Into) -> &mut Self { - self.thread_name = val.into(); - self - } - - /// Enable all feature of the underlying runtime - pub fn enable_all(&mut self) -> &mut Self { - self.builder.enable_all(); - self - } - - pub fn build(&mut self) -> anyhow::Result { - let metrics = Arc::new(Metrics::new(&self.thread_name)); - - let rt = self - .builder - .thread_name(self.thread_name.clone()) - .on_thread_start(with_metrics(&metrics, |m| { - m.on_thread_start(); - })) - .on_thread_stop(with_metrics(&metrics, |m| { - m.on_thread_stop(); - })) - .on_thread_park(with_metrics(&metrics, |m| { - m.on_thread_park(); - })) - .on_thread_unpark(with_metrics(&metrics, |m| { - m.on_thread_unpark(); - })) - .build()?; - - Ok(Runtime { rt, metrics }) - } -} - -#[cfg(test)] -mod tests { - use crate::runtime::{Builder, Runtime}; - use std::sync::Arc; - use std::thread; - use std::time::Duration; - - fn create_runtime(pool_size: usize, name: &str) -> Arc { - let runtime = Builder::default() - .worker_threads(pool_size as usize) - .thread_name(name) - .enable_all() - .build(); - assert!(runtime.is_ok()); - Arc::new(runtime.unwrap()) - } - - #[test] - fn test_metrics() { - let runtime = create_runtime(2usize, "test_metrics"); - thread::sleep(Duration::from_millis(60)); - - let stats = runtime.stats(); - assert_eq!(2, stats.idle_thread_num); - assert_eq!(2, stats.alive_thread_num); - - runtime.spawn(async { - thread::sleep(Duration::from_millis(1000)); - }); - - // waiting the task is invoked. - thread::sleep(Duration::from_millis(50)); - - let stats = runtime.stats(); - assert_eq!(2, stats.alive_thread_num); - assert_eq!(1, stats.idle_thread_num); - } - - #[test] - fn test_nested_spawn() { - let runtime = create_runtime(4usize, "test_nested_spawn"); - let cloned_rt = runtime.clone(); - - let handle = runtime.spawn(async move { - let mut counter = 0; - for _ in 0..3 { - counter += cloned_rt - .spawn(async move { - thread::sleep(Duration::from_millis(50)); - 1 - }) - .await - .unwrap() - } - counter - }); - - assert_eq!(3, runtime.block_on(handle).unwrap()) - } - - #[test] - fn test_spawn_block() { - let runtime = create_runtime(2usize, "test_spawn_block"); - - let handle = runtime.spawn(async { - thread::sleep(Duration::from_millis(50)); - 1 - }); - - assert_eq!(1, runtime.block_on(handle).unwrap()); - } - - #[test] - fn test_spawn() { - let runtime = create_runtime(2usize, "test_spawn"); - - let rt_cloned = runtime.clone(); - - let res = runtime.block_on(async { - rt_cloned - .spawn_blocking(move || { - thread::sleep(Duration::from_millis(100)); - 2 - }) - .await - }); - assert_eq!(res.unwrap(), 2); - } -} diff --git a/rust/experimental/server/src/signal.rs b/rust/experimental/server/src/signal.rs deleted file mode 100644 index da8b2faf6b..0000000000 --- a/rust/experimental/server/src/signal.rs +++ /dev/null @@ -1,58 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#[cfg(unix)] -pub mod details { - use log::info; - use signal_hook::{consts::TERM_SIGNALS, iterator::Signals}; - - pub fn wait_for_signal() { - let mut sigs = Signals::new(TERM_SIGNALS).expect("Failed to register signal handlers"); - - for signal in &mut sigs { - if TERM_SIGNALS.contains(&signal) { - info!("Received signal {}, stopping server...", signal); - break; - } - } - } - - pub fn graceful_wait_for_signal(tx: tokio::sync::broadcast::Sender<()>) { - let mut sigs = Signals::new(TERM_SIGNALS).expect("Failed to register signal handlers"); - - for signal in &mut sigs { - if TERM_SIGNALS.contains(&signal) { - info!("Received signal {}, stopping server...", signal); - let _ = tx.send(()); - break; - } - } - } -} - -#[cfg(not(unix))] -pub mod details { - use std::thread; - use std::time::Duration; - - // todo: this should be handled elegantly. - pub fn wait_for_signal() { - while 1 { - thread::sleep(Duration::from_millis(20)); - } - } -} diff --git a/rust/experimental/server/src/store/hdfs.rs b/rust/experimental/server/src/store/hdfs.rs deleted file mode 100644 index a3e602a630..0000000000 --- a/rust/experimental/server/src/store/hdfs.rs +++ /dev/null @@ -1,410 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::app::{ - PartitionedUId, PurgeDataContext, ReadingIndexViewContext, ReadingViewContext, - RegisterAppContext, ReleaseBufferContext, RequireBufferContext, WritingViewContext, -}; -use crate::config::{HdfsStoreConfig, StorageType}; -use crate::error::WorkerError; -use std::collections::HashMap; - -use crate::metric::TOTAL_HDFS_USED; -use crate::store::{Persistent, RequireBufferResponse, ResponseData, ResponseDataIndex, Store}; -use anyhow::{anyhow, Result}; - -use async_trait::async_trait; -use await_tree::InstrumentAwait; -use bytes::{BufMut, Bytes, BytesMut}; -use dashmap::DashMap; - -use log::info; - -use std::path::Path; - -use hdfs_native::{Client, WriteOptions}; -use std::sync::Arc; -use tokio::sync::{Mutex, Semaphore}; - -use tracing::debug; -use url::Url; - -struct PartitionCachedMeta { - is_file_created: bool, - data_len: i64, -} - -impl PartitionCachedMeta { - pub fn reset(&mut self, len: i64) { - self.data_len = len; - } -} - -impl Default for PartitionCachedMeta { - fn default() -> Self { - Self { - is_file_created: true, - data_len: 0, - } - } -} - -pub struct HdfsStore { - concurrency_access_limiter: Semaphore, - - // key: app_id, value: hdfs_native_client - app_remote_clients: DashMap, - - partition_file_locks: DashMap>>, - partition_cached_meta: DashMap, -} - -unsafe impl Send for HdfsStore {} -unsafe impl Sync for HdfsStore {} -impl Persistent for HdfsStore {} - -impl HdfsStore { - pub fn from(conf: HdfsStoreConfig) -> Self { - HdfsStore { - partition_file_locks: DashMap::new(), - concurrency_access_limiter: Semaphore::new(conf.max_concurrency.unwrap_or(1) as usize), - partition_cached_meta: Default::default(), - app_remote_clients: Default::default(), - } - } - - fn get_app_dir(&self, app_id: &str) -> String { - format!("{}/", app_id) - } - - /// the dir created with app_id/shuffle_id - fn get_shuffle_dir(&self, app_id: &str, shuffle_id: i32) -> String { - format!("{}/{}/", app_id, shuffle_id) - } - - fn get_file_path_by_uid(&self, uid: &PartitionedUId) -> (String, String) { - let app_id = &uid.app_id; - let shuffle_id = &uid.shuffle_id; - let p_id = &uid.partition_id; - - let worker_id = crate::app::SHUFFLE_SERVER_ID.get().unwrap(); - ( - format!( - "{}/{}/{}-{}/{}.data", - app_id, shuffle_id, p_id, p_id, worker_id - ), - format!( - "{}/{}/{}-{}/{}.index", - app_id, shuffle_id, p_id, p_id, worker_id - ), - ) - } -} - -#[async_trait] -impl Store for HdfsStore { - fn start(self: Arc) { - info!("There is nothing to do in hdfs store"); - } - - async fn insert(&self, ctx: WritingViewContext) -> Result<(), WorkerError> { - let uid = ctx.uid; - let data_blocks = ctx.data_blocks; - - let (data_file_path, index_file_path) = self.get_file_path_by_uid(&uid); - - let concurrency_guarder = self - .concurrency_access_limiter - .acquire() - .instrument_await(format!( - "hdfs concurrency limiter. path: {}", - data_file_path - )) - .await - .map_err(|e| WorkerError::from(e))?; - - let lock_cloned = self - .partition_file_locks - .entry(data_file_path.clone()) - .or_insert_with(|| Arc::new(Mutex::new(()))) - .clone(); - let _lock_guard = lock_cloned - .lock() - .instrument_await(format!( - "hdfs partition file lock. path: {}", - data_file_path - )) - .await; - - let filesystem = self.app_remote_clients.get(&uid.app_id).ok_or( - WorkerError::HDFS_NATIVE_CLIENT_NOT_FOUND(uid.app_id.to_string()), - )?; - - let mut next_offset = match self.partition_cached_meta.get(&data_file_path) { - None => { - // setup the parent folder - let parent_dir = Path::new(data_file_path.as_str()).parent().unwrap(); - let parent_path_str = format!("{}/", parent_dir.to_str().unwrap()); - debug!("creating dir: {}", parent_path_str.as_str()); - - filesystem.create_dir(parent_path_str.as_str()).await?; - - // setup the file - filesystem.touch(&data_file_path).await?; - filesystem.touch(&index_file_path).await?; - - self.partition_cached_meta - .insert(data_file_path.to_string(), Default::default()); - 0 - } - Some(meta) => meta.data_len, - }; - - let mut index_bytes_holder = BytesMut::new(); - let mut data_bytes_holder = BytesMut::new(); - - let mut total_flushed = 0; - for data_block in data_blocks { - let block_id = data_block.block_id; - let crc = data_block.crc; - let length = data_block.length; - let task_attempt_id = data_block.task_attempt_id; - let uncompress_len = data_block.uncompress_length; - - index_bytes_holder.put_i64(next_offset); - index_bytes_holder.put_i32(length); - index_bytes_holder.put_i32(uncompress_len); - index_bytes_holder.put_i64(crc); - index_bytes_holder.put_i64(block_id); - index_bytes_holder.put_i64(task_attempt_id); - - let data = data_block.data; - data_bytes_holder.extend_from_slice(&data); - - next_offset += length as i64; - - total_flushed += length; - } - - filesystem - .append(&data_file_path, data_bytes_holder.freeze()) - .instrument_await(format!("hdfs writing data. path: {}", data_file_path)) - .await?; - filesystem - .append(&index_file_path, index_bytes_holder.freeze()) - .instrument_await(format!("hdfs writing index. path: {}", data_file_path)) - .await?; - - let mut partition_cached_meta = - self.partition_cached_meta.get_mut(&data_file_path).unwrap(); - partition_cached_meta.reset(next_offset); - - TOTAL_HDFS_USED.inc_by(total_flushed as u64); - - drop(concurrency_guarder); - - Ok(()) - } - - async fn get(&self, _ctx: ReadingViewContext) -> Result { - Err(WorkerError::NOT_READ_HDFS_DATA_FROM_SERVER) - } - - async fn get_index( - &self, - _ctx: ReadingIndexViewContext, - ) -> Result { - Err(WorkerError::NOT_READ_HDFS_DATA_FROM_SERVER) - } - - async fn require_buffer( - &self, - _ctx: RequireBufferContext, - ) -> Result { - todo!() - } - - async fn release_buffer(&self, _ctx: ReleaseBufferContext) -> Result { - todo!() - } - - async fn purge(&self, ctx: PurgeDataContext) -> Result { - let app_id = ctx.app_id; - let filesystem = self.app_remote_clients.get(&app_id).ok_or( - WorkerError::HDFS_NATIVE_CLIENT_NOT_FOUND(app_id.to_string()), - )?; - - let dir = match ctx.shuffle_id { - Some(shuffle_id) => self.get_shuffle_dir(app_id.as_str(), shuffle_id), - _ => self.get_app_dir(app_id.as_str()), - }; - - let keys_to_delete: Vec<_> = self - .partition_file_locks - .iter() - .filter(|entry| entry.key().starts_with(dir.as_str())) - .map(|entry| entry.key().to_string()) - .collect(); - - let mut removed_size = 0i64; - for deleted_key in keys_to_delete { - self.partition_file_locks.remove(&deleted_key); - if let Some(meta) = self.partition_cached_meta.remove(&deleted_key) { - removed_size += meta.1.data_len; - } - } - - info!("The hdfs data for {} has been deleted", &dir); - filesystem.delete_dir(dir.as_str()).await?; - drop(filesystem); - - if ctx.shuffle_id.is_none() { - self.app_remote_clients.remove(&app_id); - } - - Ok(removed_size) - } - - async fn is_healthy(&self) -> Result { - Ok(true) - } - - async fn register_app(&self, ctx: RegisterAppContext) -> Result<()> { - let remote_storage_conf_option = ctx.app_config_options.remote_storage_config_option; - if remote_storage_conf_option.is_none() { - return Err(anyhow!( - "The remote config must be populated by app registry action!" - )); - } - - let remote_storage_conf = remote_storage_conf_option.unwrap(); - let client = HdfsNativeClient::new(remote_storage_conf.root, remote_storage_conf.configs)?; - - let app_id = ctx.app_id.clone(); - self.app_remote_clients - .entry(app_id) - .or_insert_with(|| client); - Ok(()) - } - - async fn name(&self) -> StorageType { - StorageType::HDFS - } -} - -#[async_trait] -trait HdfsDelegator { - async fn touch(&self, file_path: &str) -> Result<()>; - async fn append(&self, file_path: &str, data: Bytes) -> Result<()>; - async fn len(&self, file_path: &str) -> Result; - - async fn create_dir(&self, dir: &str) -> Result<()>; - async fn delete_dir(&self, dir: &str) -> Result<()>; -} - -struct HdfsNativeClient { - client: Client, - root: String, -} - -impl HdfsNativeClient { - fn new(root: String, configs: HashMap) -> Result { - // todo: do more optimizations! - let url = Url::parse(root.as_str())?; - let url_header = format!("{}://{}", url.scheme(), url.host().unwrap()); - - let root_path = url.path(); - - info!( - "Created hdfs client, header: {}, path: {}", - &url_header, root_path - ); - - let client = Client::new_with_config(url_header.as_str(), configs)?; - Ok(Self { - client, - root: root_path.to_string(), - }) - } - - fn wrap_root(&self, path: &str) -> String { - format!("{}/{}", &self.root, path) - } -} - -#[async_trait] -impl HdfsDelegator for HdfsNativeClient { - async fn touch(&self, file_path: &str) -> Result<()> { - let file_path = &self.wrap_root(file_path); - self.client - .create(file_path, WriteOptions::default()) - .await? - .close() - .await?; - Ok(()) - } - - async fn append(&self, file_path: &str, data: Bytes) -> Result<()> { - let file_path = &self.wrap_root(file_path); - let mut file_writer = self.client.append(file_path).await?; - file_writer.write(data).await?; - file_writer.close().await?; - Ok(()) - } - - async fn len(&self, file_path: &str) -> Result { - let file_path = &self.wrap_root(file_path); - let file_info = self.client.get_file_info(file_path).await?; - Ok(file_info.length as u64) - } - - async fn create_dir(&self, dir: &str) -> Result<()> { - let dir = &self.wrap_root(dir); - let _ = self.client.mkdirs(dir, 777, true).await?; - Ok(()) - } - - async fn delete_dir(&self, dir: &str) -> Result<()> { - let dir = &self.wrap_root(dir); - self.client.delete(dir, true).await?; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use std::path::Path; - use url::Url; - - #[test] - fn url_test() { - let url = Url::parse("hdfs://rbf-1:19999/a/b").unwrap(); - assert_eq!("hdfs", url.scheme()); - assert_eq!("rbf-1", url.host().unwrap().to_string()); - assert_eq!(19999, url.port().unwrap()); - assert_eq!("/a/b", url.path()); - } - - #[test] - fn dir_test() -> anyhow::Result<()> { - let file_path = "app/0/1.data"; - let parent_path = Path::new(file_path).parent().unwrap(); - println!("{}", parent_path.to_str().unwrap()); - - Ok(()) - } -} diff --git a/rust/experimental/server/src/store/hybrid.rs b/rust/experimental/server/src/store/hybrid.rs deleted file mode 100644 index 76bb3e83d2..0000000000 --- a/rust/experimental/server/src/store/hybrid.rs +++ /dev/null @@ -1,922 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::app::{ - PartitionedUId, PurgeDataContext, ReadingIndexViewContext, ReadingOptions, ReadingViewContext, - RegisterAppContext, ReleaseBufferContext, RequireBufferContext, WritingViewContext, -}; -use crate::await_tree::AWAIT_TREE_REGISTRY; - -use crate::config::{Config, HybridStoreConfig, StorageType}; -use crate::error::WorkerError; -use crate::metric::{ - GAUGE_IN_SPILL_DATA_SIZE, GAUGE_MEMORY_SPILL_OPERATION, GAUGE_MEMORY_SPILL_TO_HDFS, - GAUGE_MEMORY_SPILL_TO_LOCALFILE, TOTAL_MEMORY_SPILL_OPERATION, - TOTAL_MEMORY_SPILL_OPERATION_FAILED, TOTAL_MEMORY_SPILL_TO_HDFS, - TOTAL_MEMORY_SPILL_TO_LOCALFILE, TOTAL_SPILL_EVENTS_DROPPED, -}; -use crate::readable_size::ReadableSize; -#[cfg(feature = "hdfs")] -use crate::store::hdfs::HdfsStore; -use crate::store::localfile::LocalFileStore; -use crate::store::memory::{MemorySnapshot, MemoryStore}; - -use crate::store::{ - PartitionedDataBlock, Persistent, RequireBufferResponse, ResponseData, ResponseDataIndex, Store, -}; -use anyhow::{anyhow, Result}; - -use async_trait::async_trait; -use log::{debug, error, info, warn}; -use prometheus::core::{Atomic, AtomicU64}; -use std::any::Any; - -use std::collections::VecDeque; - -use await_tree::InstrumentAwait; -use spin::mutex::Mutex; -use std::str::FromStr; -use std::sync::Arc; - -use crate::runtime::manager::RuntimeManager; -use tokio::sync::Semaphore; - -trait PersistentStore: Store + Persistent + Send + Sync {} -impl PersistentStore for LocalFileStore {} - -#[cfg(feature = "hdfs")] -impl PersistentStore for HdfsStore {} - -const DEFAULT_MEMORY_SPILL_MAX_CONCURRENCY: i32 = 20; - -pub struct HybridStore { - // Box will build fail - hot_store: Box, - - warm_store: Option>, - cold_store: Option>, - - config: HybridStoreConfig, - memory_spill_lock: Mutex<()>, - memory_spill_recv: async_channel::Receiver, - memory_spill_send: async_channel::Sender, - memory_spill_event_num: AtomicU64, - - memory_spill_to_cold_threshold_size: Option, - - memory_spill_max_concurrency: i32, - - memory_watermark_flush_trigger_sender: async_channel::Sender<()>, - memory_watermark_flush_trigger_recv: async_channel::Receiver<()>, - - runtime_manager: RuntimeManager, -} - -#[derive(Clone)] -struct SpillMessage { - ctx: WritingViewContext, - id: i64, - retry_cnt: i32, - previous_spilled_storage: Option>>, -} - -unsafe impl Send for HybridStore {} -unsafe impl Sync for HybridStore {} - -impl HybridStore { - pub fn from(config: Config, runtime_manager: RuntimeManager) -> Self { - let store_type = &config.store_type.unwrap_or(StorageType::MEMORY); - if !StorageType::contains_memory(&store_type) { - panic!("Storage type must contains memory."); - } - - let mut persistent_stores: VecDeque> = VecDeque::with_capacity(2); - if StorageType::contains_localfile(&store_type) { - let localfile_store = - LocalFileStore::from(config.localfile_store.unwrap(), runtime_manager.clone()); - persistent_stores.push_back(Box::new(localfile_store)); - } - - if StorageType::contains_hdfs(&store_type) { - #[cfg(not(feature = "hdfs"))] - panic!("The binary is not compiled with feature of hdfs! So the storage type can't involve hdfs."); - - #[cfg(feature = "hdfs")] - let hdfs_store = HdfsStore::from(config.hdfs_store.unwrap()); - #[cfg(feature = "hdfs")] - persistent_stores.push_back(Box::new(hdfs_store)); - } - - let (send, recv) = async_channel::unbounded(); - let hybrid_conf = config.hybrid_store.unwrap(); - let memory_spill_to_cold_threshold_size = - match &hybrid_conf.memory_spill_to_cold_threshold_size { - Some(v) => Some(ReadableSize::from_str(&v.clone()).unwrap().as_bytes()), - _ => None, - }; - let memory_spill_max_concurrency = hybrid_conf - .memory_spill_max_concurrency - .unwrap_or(DEFAULT_MEMORY_SPILL_MAX_CONCURRENCY); - - let (watermark_flush_send, watermark_flush_recv) = async_channel::unbounded(); - - let store = HybridStore { - hot_store: Box::new(MemoryStore::from( - config.memory_store.unwrap(), - runtime_manager.clone(), - )), - warm_store: persistent_stores.pop_front(), - cold_store: persistent_stores.pop_front(), - config: hybrid_conf, - memory_spill_lock: Mutex::new(()), - memory_spill_recv: recv, - memory_spill_send: send, - memory_spill_event_num: AtomicU64::new(0), - memory_spill_to_cold_threshold_size, - memory_spill_max_concurrency, - memory_watermark_flush_trigger_sender: watermark_flush_send, - memory_watermark_flush_trigger_recv: watermark_flush_recv, - runtime_manager, - }; - store - } - - fn is_memory_only(&self) -> bool { - self.cold_store.is_none() && self.warm_store.is_none() - } - - fn is_localfile(&self, store: &dyn Any) -> bool { - store.is::() - } - - #[allow(unused)] - fn is_hdfs(&self, store: &dyn Any) -> bool { - #[cfg(feature = "hdfs")] - return store.is::(); - - #[cfg(not(feature = "hdfs"))] - false - } - - async fn memory_spill_to_persistent_store( - &self, - spill_message: SpillMessage, - ) -> Result { - let mut ctx: WritingViewContext = spill_message.ctx; - let retry_cnt = spill_message.retry_cnt; - - if retry_cnt > 3 { - let app_id = ctx.uid.app_id; - return Err(WorkerError::SPILL_EVENT_EXCEED_RETRY_MAX_LIMIT(app_id)); - } - - let blocks = &ctx.data_blocks; - let mut spill_size = 0i64; - for block in blocks { - spill_size += block.length as i64; - } - - let warm = self - .warm_store - .as_ref() - .ok_or(anyhow!("empty warm store. It should not happen"))?; - let cold = self.cold_store.as_ref().unwrap_or(warm); - - // we should cover the following cases - // 1. local store is unhealthy. spill to hdfs - // 2. event flushed to localfile failed. and exceed retry max cnt, fallback to hdfs - // 3. huge partition directly flush to hdfs - - // normal assignment - let mut candidate_store = if warm.is_healthy().await? { - let cold_spilled_size = self.memory_spill_to_cold_threshold_size.unwrap_or(u64::MAX); - if cold_spilled_size < spill_size as u64 || ctx.owned_by_huge_partition { - cold - } else { - warm - } - } else { - cold - }; - - // fallback assignment. propose hdfs always is active and stable - if retry_cnt >= 1 { - candidate_store = cold; - } - - match candidate_store.name().await { - StorageType::LOCALFILE => { - TOTAL_MEMORY_SPILL_TO_LOCALFILE.inc(); - GAUGE_MEMORY_SPILL_TO_LOCALFILE.inc(); - } - StorageType::HDFS => { - TOTAL_MEMORY_SPILL_TO_HDFS.inc(); - GAUGE_MEMORY_SPILL_TO_HDFS.inc(); - } - _ => {} - } - - let message = format!( - "partition uid: {:?}, memory spilled size: {}", - &ctx.uid, spill_size - ); - - // Resort the blocks by task_attempt_id to support LOCAL ORDER by default. - // This is for spark AQE. - ctx.data_blocks.sort_by_key(|block| block.task_attempt_id); - - // when throwing the data lost error, it should fast fail for this partition data. - let inserted = candidate_store - .insert(ctx) - .instrument_await("inserting into the persistent store, invoking [write]") - .await; - if let Err(err) = inserted { - match err { - WorkerError::PARTIAL_DATA_LOST(msg) => { - let err_msg = format!( - "Partial data has been lost. Let's abort this partition data. error: {}", - &msg - ); - error!("{}", err_msg) - } - others => return Err(others), - } - } - - match candidate_store.name().await { - StorageType::LOCALFILE => { - GAUGE_MEMORY_SPILL_TO_LOCALFILE.dec(); - } - StorageType::HDFS => { - GAUGE_MEMORY_SPILL_TO_HDFS.dec(); - } - _ => {} - } - - Ok(message) - } - - pub async fn free_hot_store_allocated_memory_size(&self, size: i64) -> Result { - self.hot_store.free_allocated(size).await - } - - pub async fn get_hot_store_memory_snapshot(&self) -> Result { - self.hot_store.memory_snapshot().await - } - - pub async fn get_hot_store_memory_partitioned_buffer_size( - &self, - uid: &PartitionedUId, - ) -> Result { - self.hot_store.get_partitioned_buffer_size(uid).await - } - - pub fn memory_spill_event_num(&self) -> Result { - Ok(self.memory_spill_event_num.get()) - } - - async fn make_memory_buffer_flush( - &self, - in_flight_uid: i64, - blocks: Vec, - uid: PartitionedUId, - ) -> Result<()> { - let writing_ctx = WritingViewContext::from(uid, blocks); - - if self - .memory_spill_send - .send(SpillMessage { - ctx: writing_ctx, - id: in_flight_uid, - retry_cnt: 0, - previous_spilled_storage: None, - }) - .await - .is_err() - { - error!("Errors on sending spill message to queue. This should not happen."); - } else { - self.memory_spill_event_num.inc_by(1); - } - - Ok(()) - } - - async fn release_data_in_memory(&self, data_size: u64, message: &SpillMessage) -> Result<()> { - let uid = &message.ctx.uid; - let in_flight_id = message.id; - self.hot_store - .release_in_flight_blocks_in_underlying_staging_buffer(uid.clone(), in_flight_id) - .await?; - self.hot_store.free_used(data_size as i64).await?; - self.hot_store.desc_to_in_flight_buffer_size(data_size); - Ok(()) - } -} - -#[async_trait] -impl Store for HybridStore { - /// Using the async_channel to keep the immutable self to - /// the self as the Arc rather than mpsc::channel, which - /// uses the recv(&mut self). I don't hope so. - fn start(self: Arc) { - if self.is_memory_only() { - return; - } - - // the handler to accept watermark flush trigger - let hybrid_store = self.clone(); - self.runtime_manager.default_runtime.spawn(async move { - let store = hybrid_store.clone(); - while let Ok(_) = &store.memory_watermark_flush_trigger_recv.recv().await { - if let Err(e) = watermark_flush(store.clone()).await { - error!("Errors on handling watermark flush events. err: {:?}", e) - } - } - }); - - let await_tree_registry = AWAIT_TREE_REGISTRY.clone(); - let store = self.clone(); - let concurrency_limiter = - Arc::new(Semaphore::new(store.memory_spill_max_concurrency as usize)); - self.runtime_manager.write_runtime.spawn(async move { - while let Ok(mut message) = store.memory_spill_recv.recv().await { - let await_root = await_tree_registry - .register(format!("hot->warm flush. uid: {:#?}", &message.ctx.uid)) - .await; - - // using acquire_owned(), refer to https://github.com/tokio-rs/tokio/issues/1998 - let concurrency_guarder = concurrency_limiter - .clone() - .acquire_owned() - .instrument_await("waiting for the spill concurrent lock.") - .await - .unwrap(); - - TOTAL_MEMORY_SPILL_OPERATION.inc(); - GAUGE_MEMORY_SPILL_OPERATION.inc(); - let store_ref = store.clone(); - store - .runtime_manager - .write_runtime - .spawn(await_root.instrument(async move { - let mut size = 0u64; - for block in &message.ctx.data_blocks { - size += block.length as u64; - } - - GAUGE_IN_SPILL_DATA_SIZE.add(size as i64); - match store_ref - .memory_spill_to_persistent_store(message.clone()) - .instrument_await("memory_spill_to_persistent_store.") - .await - { - Ok(msg) => { - debug!("{}", msg); - if let Err(err) = store_ref.release_data_in_memory(size, &message).await { - error!("Errors on releasing memory data, that should not happen. err: {:#?}", err); - } - } - Err(WorkerError::SPILL_EVENT_EXCEED_RETRY_MAX_LIMIT(_)) | Err(WorkerError::PARTIAL_DATA_LOST(_)) => { - warn!("Dropping the spill event for app: {:?}. Attention: this will make data lost!", message.ctx.uid.app_id); - if let Err(err) = store_ref.release_data_in_memory(size, &message).await { - error!("Errors on releasing memory data when dropping the spill event, that should not happen. err: {:#?}", err); - } - TOTAL_SPILL_EVENTS_DROPPED.inc(); - } - Err(error) => { - TOTAL_MEMORY_SPILL_OPERATION_FAILED.inc(); - error!( - "Errors on spill memory data to persistent storage for event_id:{:?}. The error: {:#?}", - message.id, - error); - - message.retry_cnt = message.retry_cnt + 1; - // re-push to the queue to execute - let _ = store_ref.memory_spill_send.send(message).await; - } - } - store_ref.memory_spill_event_num.dec_by(1); - GAUGE_IN_SPILL_DATA_SIZE.sub(size as i64); - GAUGE_MEMORY_SPILL_OPERATION.dec(); - drop(concurrency_guarder); - })); - } - }); - } - - async fn insert(&self, ctx: WritingViewContext) -> Result<(), WorkerError> { - let uid = ctx.uid.clone(); - - let insert_result = self - .hot_store - .insert(ctx) - .instrument_await("inserting data into memory") - .await; - if self.is_memory_only() { - return insert_result; - } - - if let Some(max_spill_size) = &self.config.memory_single_buffer_max_spill_size { - // single buffer flush - if let Ok(size) = ReadableSize::from_str(max_spill_size.as_str()) { - let buffer = self - .hot_store - .get_or_create_underlying_staging_buffer(uid.clone()); - let mut buffer_inner = buffer.lock(); - if size.as_bytes() < buffer_inner.get_staging_size()? as u64 { - let (in_flight_uid, blocks) = buffer_inner.migrate_staging_to_in_flight()?; - self.make_memory_buffer_flush(in_flight_uid, blocks, uid.clone()) - .await?; - } - } - } - - // if the used size exceed the ratio of high watermark, - // then send watermark flush trigger - if let Some(_lock) = self.memory_spill_lock.try_lock() { - let used_ratio = self.hot_store.memory_used_ratio().await; - if used_ratio > self.config.memory_spill_high_watermark { - if let Err(e) = self.memory_watermark_flush_trigger_sender.send(()).await { - error!( - "Errors on send watermark flush event to handler. err: {:?}", - e - ); - } - } - } - - insert_result - } - - async fn get(&self, ctx: ReadingViewContext) -> Result { - match ctx.reading_options { - ReadingOptions::MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE(_, _) => { - self.hot_store.get(ctx).await - } - _ => self.warm_store.as_ref().unwrap().get(ctx).await, - } - } - - async fn get_index( - &self, - ctx: ReadingIndexViewContext, - ) -> Result { - self.warm_store.as_ref().unwrap().get_index(ctx).await - } - - async fn require_buffer( - &self, - ctx: RequireBufferContext, - ) -> Result { - let uid = &ctx.uid.clone(); - self.hot_store - .require_buffer(ctx) - .instrument_await(format!("requiring buffers. uid: {:?}", uid)) - .await - } - - async fn release_buffer(&self, ctx: ReleaseBufferContext) -> Result { - self.hot_store.release_buffer(ctx).await - } - - async fn purge(&self, ctx: PurgeDataContext) -> Result { - let app_id = &ctx.app_id; - let mut removed_size = 0i64; - - removed_size += self.hot_store.purge(ctx.clone()).await?; - info!("Removed data of app:[{}] in hot store", app_id); - if self.warm_store.is_some() { - removed_size += self.warm_store.as_ref().unwrap().purge(ctx.clone()).await?; - info!("Removed data of app:[{}] in warm store", app_id); - } - if self.cold_store.is_some() { - removed_size += self.cold_store.as_ref().unwrap().purge(ctx.clone()).await?; - info!("Removed data of app:[{}] in cold store", app_id); - } - Ok(removed_size) - } - - async fn is_healthy(&self) -> Result { - async fn check_healthy(store: Option<&Box>) -> Result { - match store { - Some(store) => store.is_healthy().await, - _ => Ok(true), - } - } - let warm = check_healthy(self.warm_store.as_ref()) - .await - .unwrap_or(false); - let cold = check_healthy(self.cold_store.as_ref()) - .await - .unwrap_or(false); - Ok(self.hot_store.is_healthy().await? && (warm || cold)) - } - - async fn register_app(&self, ctx: RegisterAppContext) -> Result<()> { - self.hot_store.register_app(ctx.clone()).await?; - if self.warm_store.is_some() { - self.warm_store - .as_ref() - .unwrap() - .register_app(ctx.clone()) - .await?; - } - if self.cold_store.is_some() { - self.cold_store - .as_ref() - .unwrap() - .register_app(ctx.clone()) - .await?; - } - Ok(()) - } - - async fn name(&self) -> StorageType { - unimplemented!() - } -} - -pub async fn watermark_flush(store: Arc) -> Result<()> { - let used_ratio = store.hot_store.memory_used_ratio().await; - if used_ratio <= store.config.memory_spill_high_watermark { - return Ok(()); - } - let target_size = - (store.hot_store.get_capacity()? as f32 * store.config.memory_spill_low_watermark) as i64; - let buffers = store - .hot_store - .get_required_spill_buffer(target_size) - .instrument_await(format!("getting spill buffers.")) - .await; - - let mut flushed_size = 0u64; - for (partition_id, buffer) in buffers { - let mut buffer_inner = buffer.lock(); - let (in_flight_uid, blocks) = buffer_inner.migrate_staging_to_in_flight()?; - drop(buffer_inner); - for block in &blocks { - flushed_size += block.length as u64; - } - store - .make_memory_buffer_flush(in_flight_uid, blocks, partition_id) - .await?; - } - store.hot_store.add_to_in_flight_buffer_size(flushed_size); - debug!("Trigger spilling in background...."); - Ok(()) -} - -#[cfg(test)] -mod tests { - use crate::app::ReadingOptions::MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE; - use crate::app::{ - PartitionedUId, ReadingIndexViewContext, ReadingOptions, ReadingViewContext, - WritingViewContext, - }; - use crate::config::{ - Config, HybridStoreConfig, LocalfileStoreConfig, MemoryStoreConfig, StorageType, - }; - - use crate::store::hybrid::HybridStore; - use crate::store::ResponseData::Mem; - use crate::store::{PartitionedDataBlock, ResponseData, ResponseDataIndex, Store}; - use bytes::{Buf, Bytes}; - - use std::any::Any; - use std::collections::VecDeque; - - use std::sync::Arc; - use std::thread; - - use std::time::Duration; - - #[test] - fn type_downcast_check() { - trait Fruit {} - - struct Banana {} - impl Fruit for Banana {} - - struct Apple {} - impl Fruit for Apple {} - - fn is_apple(store: &dyn Any) -> bool { - store.is::() - } - - assert_eq!(true, is_apple(&Apple {})); - assert_eq!(false, is_apple(&Banana {})); - - let boxed_apple = Box::new(Apple {}); - assert_eq!(true, is_apple(&*boxed_apple)); - assert_eq!(false, is_apple(&boxed_apple)); - } - - #[test] - fn test_only_memory() { - let mut config = Config::default(); - config.memory_store = Some(MemoryStoreConfig::new("20M".to_string())); - config.hybrid_store = Some(HybridStoreConfig::new(0.8, 0.2, None)); - config.store_type = Some(StorageType::MEMORY); - let store = HybridStore::from(config, Default::default()); - - let runtime = store.runtime_manager.clone(); - assert_eq!(true, runtime.wait(store.is_healthy()).unwrap()); - } - - #[test] - fn test_vec_pop() { - let mut stores = VecDeque::with_capacity(2); - stores.push_back(1); - stores.push_back(2); - assert_eq!(1, stores.pop_front().unwrap()); - assert_eq!(2, stores.pop_front().unwrap()); - assert_eq!(None, stores.pop_front()); - } - - fn start_store( - memory_single_buffer_max_spill_size: Option, - memory_capacity: String, - ) -> Arc { - let data = b"hello world!"; - let _data_len = data.len(); - - let temp_dir = tempdir::TempDir::new("test_local_store").unwrap(); - let temp_path = temp_dir.path().to_str().unwrap().to_string(); - println!("init local file path: {}", temp_path); - - let mut config = Config::default(); - config.memory_store = Some(MemoryStoreConfig::new(memory_capacity)); - config.localfile_store = Some(LocalfileStoreConfig::new(vec![temp_path])); - config.hybrid_store = Some(HybridStoreConfig::new( - 0.8, - 0.2, - memory_single_buffer_max_spill_size, - )); - config.store_type = Some(StorageType::MEMORY_LOCALFILE); - - // The hybrid store will flush the memory data to file when - // the data reaches the number of 4 - let store = Arc::new(HybridStore::from(config, Default::default())); - store - } - - async fn write_some_data( - store: Arc, - uid: PartitionedUId, - data_len: i32, - data: &[u8; 12], - batch_size: i64, - ) -> Vec { - let mut block_ids = vec![]; - for i in 0..batch_size { - block_ids.push(i); - let writing_ctx = WritingViewContext::from( - uid.clone(), - vec![PartitionedDataBlock { - block_id: i, - length: data_len as i32, - uncompress_length: 100, - crc: 0, - data: Bytes::copy_from_slice(data), - task_attempt_id: 0, - }], - ); - let _ = store.insert(writing_ctx).await; - } - - block_ids - } - - #[test] - fn single_buffer_spill_test() -> anyhow::Result<()> { - let data = b"hello world!"; - let data_len = data.len(); - - let store = start_store( - Some("1".to_string()), - ((data_len * 10000) as i64).to_string(), - ); - store.clone().start(); - - let runtime = store.runtime_manager.clone(); - - let uid = PartitionedUId { - app_id: "1000".to_string(), - shuffle_id: 0, - partition_id: 0, - }; - let expected_block_ids = runtime.wait(write_some_data( - store.clone(), - uid.clone(), - data_len as i32, - data, - 100, - )); - - thread::sleep(Duration::from_secs(1)); - - // read from memory and then from localfile - let response_data = runtime.wait(store.get(ReadingViewContext { - uid: uid.clone(), - reading_options: MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE(-1, 1024 * 1024 * 1024), - serialized_expected_task_ids_bitmap: Default::default(), - }))?; - - let mut accepted_block_ids = vec![]; - for segment in response_data.from_memory().shuffle_data_block_segments { - accepted_block_ids.push(segment.block_id); - } - - let local_index_data = runtime.wait(store.get_index(ReadingIndexViewContext { - partition_id: uid.clone(), - }))?; - - match local_index_data { - ResponseDataIndex::Local(index) => { - let mut index_bytes = index.index_data; - while index_bytes.has_remaining() { - // index_bytes_holder.put_i64(next_offset); - // index_bytes_holder.put_i32(length); - // index_bytes_holder.put_i32(uncompress_len); - // index_bytes_holder.put_i64(crc); - // index_bytes_holder.put_i64(block_id); - // index_bytes_holder.put_i64(task_attempt_id); - index_bytes.get_i64(); - index_bytes.get_i32(); - index_bytes.get_i32(); - index_bytes.get_i64(); - let id = index_bytes.get_i64(); - index_bytes.get_i64(); - - accepted_block_ids.push(id); - } - } - } - - accepted_block_ids.sort(); - assert_eq!(accepted_block_ids, expected_block_ids); - - Ok(()) - } - - #[tokio::test] - async fn get_data_from_localfile() { - let data = b"hello world!"; - let data_len = data.len(); - - let store = start_store(None, ((data_len * 1) as i64).to_string()); - store.clone().start(); - - let uid = PartitionedUId { - app_id: "1000".to_string(), - shuffle_id: 0, - partition_id: 0, - }; - write_some_data(store.clone(), uid.clone(), data_len as i32, data, 4).await; - tokio::time::sleep(Duration::from_secs(1)).await; - - // case1: all data has been flushed to localfile. the data in memory should be empty - let last_block_id = -1; - let reading_view_ctx = ReadingViewContext { - uid: uid.clone(), - reading_options: ReadingOptions::MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE( - last_block_id, - data_len as i64, - ), - serialized_expected_task_ids_bitmap: Default::default(), - }; - - let read_data = store.get(reading_view_ctx).await; - if read_data.is_err() { - panic!(); - } - let read_data = read_data.unwrap(); - match read_data { - Mem(mem_data) => { - assert_eq!(0, mem_data.shuffle_data_block_segments.len()); - } - _ => panic!(), - } - - // case2: read data from localfile - // 1. read index file - // 2. read data - let index_view_ctx = ReadingIndexViewContext { - partition_id: uid.clone(), - }; - match store.get_index(index_view_ctx).await.unwrap() { - ResponseDataIndex::Local(index) => { - let mut index_data = index.index_data; - while index_data.has_remaining() { - let offset = index_data.get_i64(); - let length = index_data.get_i32(); - let _uncompress = index_data.get_i32(); - let _crc = index_data.get_i64(); - let _block_id = index_data.get_i64(); - let _task_id = index_data.get_i64(); - - let reading_view_ctx = ReadingViewContext { - uid: uid.clone(), - reading_options: ReadingOptions::FILE_OFFSET_AND_LEN(offset, length as i64), - serialized_expected_task_ids_bitmap: None, - }; - println!("reading. offset: {:?}. len: {:?}", offset, length); - let read_data = store.get(reading_view_ctx).await.unwrap(); - match read_data { - ResponseData::Local(local_data) => { - assert_eq!(Bytes::copy_from_slice(data), local_data.data); - } - _ => panic!(), - } - } - } - } - } - - #[tokio::test] - async fn test_localfile_disk_corrupted() { - // when the local disk is corrupted, the data will be aborted. - // Anyway, this partition's data should be not reserved on the memory to effect other - // apps - } - - #[tokio::test] - async fn test_localfile_disk_unhealthy() { - // when the local disk is unhealthy, the data should be flushed - // to the cold store(like hdfs). If not having cold, it will retry again - // then again. - } - - #[test] - fn test_insert_and_get_from_memory() { - let data = b"hello world!"; - let data_len = data.len(); - - let store = start_store(None, ((data_len * 1) as i64).to_string()); - let runtime = store.runtime_manager.clone(); - - let uid = PartitionedUId { - app_id: "1000".to_string(), - shuffle_id: 0, - partition_id: 0, - }; - runtime.wait(write_some_data( - store.clone(), - uid.clone(), - data_len as i32, - data, - 4, - )); - let mut last_block_id = -1; - // read data one by one - for idx in 0..=10 { - let reading_view_ctx = ReadingViewContext { - uid: uid.clone(), - reading_options: ReadingOptions::MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE( - last_block_id, - data_len as i64, - ), - serialized_expected_task_ids_bitmap: Default::default(), - }; - - let read_data = runtime.wait(store.get(reading_view_ctx)); - if read_data.is_err() { - panic!(); - } - - match read_data.unwrap() { - Mem(mem_data) => { - if idx >= 4 { - println!( - "idx: {}, len: {}", - idx, - mem_data.shuffle_data_block_segments.len() - ); - continue; - } - assert_eq!(Bytes::copy_from_slice(data), mem_data.data); - let segments = mem_data.shuffle_data_block_segments; - assert_eq!(1, segments.len()); - last_block_id = segments.get(0).unwrap().block_id; - } - _ => panic!(), - } - } - } -} diff --git a/rust/experimental/server/src/store/local/disk.rs b/rust/experimental/server/src/store/local/disk.rs deleted file mode 100644 index 7302903db9..0000000000 --- a/rust/experimental/server/src/store/local/disk.rs +++ /dev/null @@ -1,409 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metric::{ - GAUGE_LOCAL_DISK_CAPACITY, GAUGE_LOCAL_DISK_IS_HEALTHY, GAUGE_LOCAL_DISK_USED, -}; -use crate::runtime::manager::RuntimeManager; -use anyhow::{anyhow, Result}; -use await_tree::InstrumentAwait; -use bytes::{Bytes, BytesMut}; -use futures::AsyncWriteExt; -use log::{error, info, warn}; -use opendal::services::Fs; -use opendal::Operator; -use std::io::SeekFrom; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::time::Duration; -use tokio::io::{AsyncReadExt, AsyncSeekExt}; -use tokio::sync::Semaphore; - -pub struct LocalDiskConfig { - pub(crate) high_watermark: f32, - pub(crate) low_watermark: f32, - pub(crate) max_concurrency: i32, -} - -impl LocalDiskConfig { - pub fn create_mocked_config() -> Self { - LocalDiskConfig { - high_watermark: 1.0, - low_watermark: 0.6, - max_concurrency: 20, - } - } -} - -impl Default for LocalDiskConfig { - fn default() -> Self { - LocalDiskConfig { - high_watermark: 0.8, - low_watermark: 0.6, - max_concurrency: 40, - } - } -} - -pub struct LocalDisk { - pub(crate) root: String, - operator: Operator, - concurrency_limiter: Semaphore, - is_corrupted: AtomicBool, - is_healthy: AtomicBool, - config: LocalDiskConfig, - - capacity: u64, -} - -impl LocalDisk { - pub fn new( - root: String, - config: LocalDiskConfig, - runtime_manager: RuntimeManager, - ) -> Arc { - let mut builder = Fs::default(); - builder.root(&root); - let operator: Operator = Operator::new(builder).unwrap().finish(); - - let disk_capacity = - Self::get_disk_capacity(&root).expect("Errors on getting disk capacity"); - - let instance = LocalDisk { - root: root.to_string(), - operator, - concurrency_limiter: Semaphore::new(config.max_concurrency as usize), - is_corrupted: AtomicBool::new(false), - is_healthy: AtomicBool::new(true), - config, - capacity: disk_capacity, - }; - let instance = Arc::new(instance); - - let runtime = runtime_manager.default_runtime.clone(); - let cloned = instance.clone(); - runtime.spawn(async { - info!("Starting the disk healthy check, root: {}", &cloned.root); - LocalDisk::loop_check_disk(cloned).await; - }); - - GAUGE_LOCAL_DISK_CAPACITY - .with_label_values(&[&root]) - .set(disk_capacity as i64); - - instance - } - - async fn write_read_check(local_disk: Arc) -> Result<()> { - let temp_path = "corruption_check.file"; - // cleanup remaining files before checking. - local_disk.delete(temp_path).await?; - - let written_data = Bytes::copy_from_slice(b"file corruption check"); - local_disk.write(written_data.clone(), temp_path).await?; - let read_data = local_disk.read(temp_path, 0, None).await?; - local_disk.delete(temp_path).await?; - - if written_data != read_data { - let msg = format!( - "The local disk has been corrupted. path: {}. expected: {:?}, actual: {:?}", - &local_disk.root, &written_data, &read_data - ); - Err(anyhow!(msg)) - } else { - Ok(()) - } - } - - async fn loop_check_disk(local_disk: Arc) { - loop { - tokio::time::sleep(Duration::from_secs(10)).await; - - if local_disk.is_corrupted().unwrap() { - return; - } - - let root_ref = &local_disk.root; - - let check_succeed: Result<()> = LocalDisk::write_read_check(local_disk.clone()).await; - if check_succeed.is_err() { - local_disk.mark_corrupted(); - GAUGE_LOCAL_DISK_IS_HEALTHY - .with_label_values(&[root_ref]) - .set(1i64); - error!( - "Errors on checking local disk corruption. err: {:#?}", - check_succeed.err() - ); - } - - // get the disk used ratio. - let disk_capacity = local_disk.capacity; - let disk_available = Self::get_disk_available(root_ref); - if disk_available.is_err() { - error!( - "Errors on getting the available of the local disk. err: {:?}", - disk_available.err() - ); - continue; - } - let disk_available = disk_available.unwrap(); - let used_ratio = 1.0 - (disk_available as f64 / disk_capacity as f64); - - GAUGE_LOCAL_DISK_USED - .with_label_values(&[root_ref]) - .set((disk_capacity - disk_available) as i64); - - if local_disk.is_healthy().unwrap() - && used_ratio > local_disk.config.high_watermark as f64 - { - warn!("Disk={} has been unhealthy.", &local_disk.root); - local_disk.mark_unhealthy(); - GAUGE_LOCAL_DISK_IS_HEALTHY - .with_label_values(&[root_ref]) - .set(1i64); - continue; - } - - if !local_disk.is_healthy().unwrap() - && used_ratio < local_disk.config.low_watermark as f64 - { - warn!("Disk={} has been healthy.", &local_disk.root); - local_disk.mark_healthy(); - GAUGE_LOCAL_DISK_IS_HEALTHY - .with_label_values(&[root_ref]) - .set(0i64); - continue; - } - } - } - - pub async fn create_dir(&self, dir: &str) -> Result<()> { - self.operator.create_dir(dir).await?; - Ok(()) - } - - // this will ensure the data flushed into the file - async fn write(&self, data: Bytes, path: &str) -> Result<()> { - self.operator.write(path, data).await?; - Ok(()) - } - - pub async fn append(&self, data: Bytes, path: &str) -> Result<()> { - let _concurrency_guarder = self - .concurrency_limiter - .acquire() - .instrument_await("meet the concurrency limiter") - .await?; - - let mut writer = self - .operator - .writer_with(path) - .append(true) - .instrument_await("creating the writer...") - .await?; - // we must use the write_all to ensure the buffer consumed by the OS. - // Please see the detail: https://doc.rust-lang.org/std/io/trait.Write.html#method.write_all - writer - .write_all(&*data) - .instrument_await("writing the data into buffer...") - .await?; - writer - .flush() - .instrument_await("committing the data into file...") - .await?; - - Ok(()) - } - - pub async fn get_file_len(&self, path: &str) -> Result { - match self.operator.stat(path).await { - Ok(meta) => Ok(meta.content_length() as i64), - Err(_) => Ok(0), - } - } - - pub async fn read(&self, path: &str, offset: i64, length: Option) -> Result { - if length.is_none() { - return Ok(Bytes::from(self.operator.read(path).await?)); - } - - let mut reader = self.operator.reader(path).await?; - reader.seek(SeekFrom::Start(offset as u64)).await?; - - let mut buffer = vec![0; length.unwrap() as usize]; - reader.read_exact(buffer.as_mut()).await?; - - let mut bytes_buffer = BytesMut::new(); - bytes_buffer.extend_from_slice(&*buffer); - Ok(bytes_buffer.freeze()) - } - - pub async fn delete(&self, path: &str) -> Result<()> { - self.operator.remove_all(path).await?; - Ok(()) - } - - pub fn mark_corrupted(&self) { - self.is_corrupted.store(true, Ordering::SeqCst); - } - - pub fn mark_unhealthy(&self) { - self.is_healthy.store(false, Ordering::SeqCst); - } - - pub fn mark_healthy(&self) { - self.is_healthy.store(true, Ordering::SeqCst); - } - - pub fn is_corrupted(&self) -> Result { - Ok(self.is_corrupted.load(Ordering::SeqCst)) - } - - pub fn is_healthy(&self) -> Result { - Ok(self.is_healthy.load(Ordering::SeqCst)) - } - - fn get_disk_used_ratio(root: &str, capacity: u64) -> Result { - // Get the total and available space in bytes - let available_space = fs2::available_space(root)?; - Ok(1.0 - (available_space as f64 / capacity as f64)) - } - - fn get_disk_capacity(root: &str) -> Result { - Ok(fs2::total_space(root)?) - } - - fn get_disk_available(root: &str) -> Result { - Ok(fs2::available_space(root)?) - } -} - -#[cfg(test)] -mod tests { - use crate::runtime::manager::RuntimeManager; - use crate::store::local::disk::{LocalDisk, LocalDiskConfig}; - use bytes::Bytes; - use std::time::Duration; - - #[test] - fn test_local_disk_delete_operation() { - let temp_dir = tempdir::TempDir::new("test_local_disk_delete_operation-dir").unwrap(); - let temp_path = temp_dir.path().to_str().unwrap().to_string(); - - println!("init the path: {}", &temp_path); - - let runtime: RuntimeManager = Default::default(); - let local_disk = LocalDisk::new( - temp_path.clone(), - LocalDiskConfig::default(), - runtime.clone(), - ); - - let data = b"hello!"; - runtime.wait(local_disk.create_dir("a/")).unwrap(); - runtime - .wait(local_disk.append(Bytes::copy_from_slice(data), "a/b")) - .unwrap(); - - assert_eq!( - true, - runtime - .wait(tokio::fs::try_exists(format!( - "{}/{}", - &temp_path, - "a/b".to_string() - ))) - .unwrap() - ); - - runtime - .wait(local_disk.delete("a/")) - .expect("TODO: panic message"); - assert_eq!( - false, - runtime - .wait(tokio::fs::try_exists(format!( - "{}/{}", - &temp_path, - "a/b".to_string() - ))) - .unwrap() - ); - } - - #[test] - fn local_disk_corruption_healthy_check() { - let temp_dir = tempdir::TempDir::new("test_directory").unwrap(); - let temp_path = temp_dir.path().to_str().unwrap().to_string(); - - let local_disk = LocalDisk::new( - temp_path.clone(), - LocalDiskConfig::create_mocked_config(), - Default::default(), - ); - - awaitility::at_most(Duration::from_secs(10)).until(|| local_disk.is_healthy().unwrap()); - assert_eq!(false, local_disk.is_corrupted().unwrap()); - } - - #[test] - fn local_disk_test() { - let temp_dir = tempdir::TempDir::new("test_directory").unwrap(); - let temp_path = temp_dir.path().to_str().unwrap().to_string(); - - let runtime: RuntimeManager = Default::default(); - let local_disk = LocalDisk::new( - temp_path.clone(), - LocalDiskConfig::default(), - runtime.clone(), - ); - - let data = b"Hello, World!"; - - let relative_path = "app-id/test_file.txt"; - - runtime.wait(local_disk.create_dir("app-id/")).unwrap(); - - for _ in 0..2 { - let write_result = - runtime.wait(local_disk.append(Bytes::copy_from_slice(data), relative_path)); - assert!(write_result.is_ok()); - } - - let read_result = runtime.wait(local_disk.read(relative_path, 0, Some(data.len() as i64))); - assert!(read_result.is_ok()); - let read_data = read_result.unwrap(); - let expected = b"Hello, World!"; - assert_eq!(read_data.as_ref(), expected); - - // read the middle word - let read_result = runtime.wait(local_disk.read( - relative_path, - data.len() as i64, - Some(data.len() as i64), - )); - assert_eq!(read_result.unwrap().as_ref(), expected); - - // read all words - let read_result = runtime.wait(local_disk.read(relative_path, 0, None)); - let expected = b"Hello, World!Hello, World!"; - assert_eq!(read_result.unwrap().as_ref(), expected); - - temp_dir.close().unwrap(); - } -} diff --git a/rust/experimental/server/src/store/local/mod.rs b/rust/experimental/server/src/store/local/mod.rs deleted file mode 100644 index 63ef41a5bb..0000000000 --- a/rust/experimental/server/src/store/local/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod disk; diff --git a/rust/experimental/server/src/store/localfile.rs b/rust/experimental/server/src/store/localfile.rs deleted file mode 100644 index 27852e20a2..0000000000 --- a/rust/experimental/server/src/store/localfile.rs +++ /dev/null @@ -1,720 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::app::ReadingOptions::FILE_OFFSET_AND_LEN; -use crate::app::{ - PartitionedUId, PurgeDataContext, ReadingIndexViewContext, ReadingViewContext, - RegisterAppContext, ReleaseBufferContext, RequireBufferContext, WritingViewContext, -}; -use crate::config::{LocalfileStoreConfig, StorageType}; -use crate::error::WorkerError; -use crate::metric::TOTAL_LOCALFILE_USED; -use crate::store::ResponseDataIndex::Local; -use crate::store::{ - LocalDataIndex, PartitionedLocalData, Persistent, RequireBufferResponse, ResponseData, - ResponseDataIndex, Store, -}; -use std::ops::Deref; -use std::path::Path; - -use anyhow::Result; -use async_trait::async_trait; -use await_tree::InstrumentAwait; -use bytes::{BufMut, BytesMut}; -use dashmap::DashMap; - -use log::{debug, error, warn}; - -use crate::runtime::manager::RuntimeManager; -use dashmap::mapref::entry::Entry; -use std::sync::atomic::{AtomicI64, Ordering}; -use std::sync::Arc; - -use crate::store::local::disk::{LocalDisk, LocalDiskConfig}; - -struct LockedObj { - disk: Arc, - pointer: AtomicI64, -} - -impl From> for LockedObj { - fn from(value: Arc) -> Self { - Self { - disk: value.clone(), - pointer: Default::default(), - } - } -} - -pub struct LocalFileStore { - local_disks: Vec>, - healthy_check_min_disks: i32, - runtime_manager: RuntimeManager, - partition_locks: DashMap>, -} - -impl Persistent for LocalFileStore {} - -unsafe impl Send for LocalFileStore {} -unsafe impl Sync for LocalFileStore {} - -impl LocalFileStore { - // only for test cases - pub fn new(local_disks: Vec) -> Self { - let mut local_disk_instances = vec![]; - let runtime_manager: RuntimeManager = Default::default(); - for path in local_disks { - local_disk_instances.push(LocalDisk::new( - path, - LocalDiskConfig::default(), - runtime_manager.clone(), - )); - } - LocalFileStore { - local_disks: local_disk_instances, - healthy_check_min_disks: 1, - runtime_manager, - partition_locks: Default::default(), - } - } - - pub fn from(localfile_config: LocalfileStoreConfig, runtime_manager: RuntimeManager) -> Self { - let mut local_disk_instances = vec![]; - for path in localfile_config.data_paths { - let config = LocalDiskConfig { - high_watermark: localfile_config.disk_high_watermark.unwrap_or(0.8), - low_watermark: localfile_config.disk_low_watermark.unwrap_or(0.6), - max_concurrency: localfile_config.disk_max_concurrency.unwrap_or(40), - }; - - local_disk_instances.push(LocalDisk::new(path, config, runtime_manager.clone())); - } - LocalFileStore { - local_disks: local_disk_instances, - healthy_check_min_disks: localfile_config.healthy_check_min_disks.unwrap_or(1), - runtime_manager, - partition_locks: Default::default(), - } - } - - fn gen_relative_path_for_app(app_id: &str) -> String { - format!("{}", app_id) - } - - fn gen_relative_path_for_shuffle(app_id: &str, shuffle_id: i32) -> String { - format!("{}/{}", app_id, shuffle_id) - } - - fn gen_relative_path_for_partition(uid: &PartitionedUId) -> (String, String) { - ( - format!( - "{}/{}/partition-{}.data", - uid.app_id, uid.shuffle_id, uid.partition_id - ), - format!( - "{}/{}/partition-{}.index", - uid.app_id, uid.shuffle_id, uid.partition_id - ), - ) - } - - fn healthy_check(&self) -> Result { - let mut available = 0; - for local_disk in &self.local_disks { - if local_disk.is_healthy()? && !local_disk.is_corrupted()? { - available += 1; - } - } - - debug!( - "disk: available={}, healthy_check_min={}", - available, self.healthy_check_min_disks - ); - Ok(available > self.healthy_check_min_disks) - } - - fn select_disk(&self, uid: &PartitionedUId) -> Result, WorkerError> { - let hash_value = PartitionedUId::get_hash(uid); - - let mut candidates = vec![]; - for local_disk in &self.local_disks { - if !local_disk.is_corrupted().unwrap() && local_disk.is_healthy().unwrap() { - candidates.push(local_disk); - } - } - - let len = candidates.len(); - if len == 0 { - error!("There is no available local disk!"); - return Err(WorkerError::NO_AVAILABLE_LOCAL_DISK); - } - - let index = (hash_value % len as u64) as usize; - if let Some(&disk) = candidates.get(index) { - Ok(disk.clone()) - } else { - Err(WorkerError::INTERNAL_ERROR) - } - } -} - -#[async_trait] -impl Store for LocalFileStore { - fn start(self: Arc) { - todo!() - } - - async fn insert(&self, ctx: WritingViewContext) -> Result<(), WorkerError> { - if ctx.data_blocks.len() <= 0 { - return Ok(()); - } - - let uid = ctx.uid; - let (data_file_path, index_file_path) = - LocalFileStore::gen_relative_path_for_partition(&uid); - - let mut parent_dir_is_created = false; - let locked_obj = match self.partition_locks.entry(data_file_path.clone()) { - Entry::Vacant(e) => { - parent_dir_is_created = true; - let disk = self.select_disk(&uid)?; - let locked_obj = Arc::new(LockedObj::from(disk)); - let obj = e.insert_entry(locked_obj.clone()); - obj.get().clone() - } - Entry::Occupied(v) => v.get().clone(), - }; - - let local_disk = &locked_obj.disk; - let mut next_offset = locked_obj.pointer.load(Ordering::SeqCst); - - if local_disk.is_corrupted()? { - return Err(WorkerError::PARTIAL_DATA_LOST(local_disk.root.to_string())); - } - - if !local_disk.is_healthy()? { - return Err(WorkerError::LOCAL_DISK_UNHEALTHY( - local_disk.root.to_string(), - )); - } - - if !parent_dir_is_created { - if let Some(path) = Path::new(&data_file_path).parent() { - local_disk - .create_dir(format!("{:?}/", path.to_str().unwrap()).as_str()) - .await?; - } - } - - let mut index_bytes_holder = BytesMut::new(); - let mut data_bytes_holder = BytesMut::new(); - - let mut total_size = 0; - for block in ctx.data_blocks { - let block_id = block.block_id; - let length = block.length; - let uncompress_len = block.uncompress_length; - let task_attempt_id = block.task_attempt_id; - let crc = block.crc; - - total_size += length; - - index_bytes_holder.put_i64(next_offset); - index_bytes_holder.put_i32(length); - index_bytes_holder.put_i32(uncompress_len); - index_bytes_holder.put_i64(crc); - index_bytes_holder.put_i64(block_id); - index_bytes_holder.put_i64(task_attempt_id); - - let data = block.data; - - data_bytes_holder.extend_from_slice(&data); - next_offset += length as i64; - } - - local_disk - .append(data_bytes_holder.freeze(), &data_file_path) - .instrument_await(format!("localfile writing data. path: {}", &data_file_path)) - .await?; - local_disk - .append(index_bytes_holder.freeze(), &index_file_path) - .instrument_await(format!( - "localfile writing index. path: {}", - &index_file_path - )) - .await?; - - TOTAL_LOCALFILE_USED.inc_by(total_size as u64); - - locked_obj - .deref() - .pointer - .store(next_offset, Ordering::SeqCst); - - Ok(()) - } - - async fn get(&self, ctx: ReadingViewContext) -> Result { - let uid = ctx.uid; - let (offset, len) = match ctx.reading_options { - FILE_OFFSET_AND_LEN(offset, len) => (offset, len), - _ => (0, 0), - }; - - if len == 0 { - warn!("There is no data in localfile for [{:?}]", &uid); - return Ok(ResponseData::Local(PartitionedLocalData { - data: Default::default(), - })); - } - - let (data_file_path, _) = LocalFileStore::gen_relative_path_for_partition(&uid); - - if !self.partition_locks.contains_key(&data_file_path) { - warn!( - "There is no cached data in localfile store for [{:?}]", - &uid - ); - return Ok(ResponseData::Local(PartitionedLocalData { - data: Default::default(), - })); - } - - let locked_object = self - .partition_locks - .entry(data_file_path.clone()) - .or_insert_with(|| Arc::new(LockedObj::from(self.select_disk(&uid).unwrap()))) - .clone(); - - let local_disk = &locked_object.disk; - - if local_disk.is_corrupted()? { - return Err(WorkerError::LOCAL_DISK_OWNED_BY_PARTITION_CORRUPTED( - local_disk.root.to_string(), - )); - } - - let data = local_disk - .read(&data_file_path, offset, Some(len)) - .instrument_await(format!( - "getting data from localfile: {:?}", - &data_file_path - )) - .await?; - Ok(ResponseData::Local(PartitionedLocalData { data })) - } - - async fn get_index( - &self, - ctx: ReadingIndexViewContext, - ) -> Result { - let uid = ctx.partition_id; - let (data_file_path, index_file_path) = - LocalFileStore::gen_relative_path_for_partition(&uid); - - if !self.partition_locks.contains_key(&data_file_path) { - warn!( - "There is no cached data in localfile store for [{:?}]", - &uid - ); - return Ok(Local(LocalDataIndex { - index_data: Default::default(), - data_file_len: 0, - })); - } - - let locked_object = self - .partition_locks - .entry(data_file_path.clone()) - .or_insert_with(|| Arc::new(LockedObj::from(self.select_disk(&uid).unwrap()))) - .clone(); - - let local_disk = &locked_object.disk; - if local_disk.is_corrupted()? { - return Err(WorkerError::LOCAL_DISK_OWNED_BY_PARTITION_CORRUPTED( - local_disk.root.to_string(), - )); - } - - let index_data_result = local_disk - .read(&index_file_path, 0, None) - .instrument_await(format!( - "reading index data from file: {:?}", - &index_file_path - )) - .await?; - let len = local_disk - .get_file_len(&data_file_path) - .instrument_await(format!("getting file len from file: {:?}", &data_file_path)) - .await?; - Ok(Local(LocalDataIndex { - index_data: index_data_result, - data_file_len: len, - })) - } - - async fn purge(&self, ctx: PurgeDataContext) -> Result { - let app_id = ctx.app_id; - let shuffle_id_option = ctx.shuffle_id; - - let data_relative_dir_path = match shuffle_id_option { - Some(shuffle_id) => LocalFileStore::gen_relative_path_for_shuffle(&app_id, shuffle_id), - _ => LocalFileStore::gen_relative_path_for_app(&app_id), - }; - - for local_disk_ref in &self.local_disks { - let disk = local_disk_ref.clone(); - disk.delete(&data_relative_dir_path).await?; - } - - let keys_to_delete: Vec<_> = self - .partition_locks - .iter() - .filter(|entry| entry.key().starts_with(&data_relative_dir_path)) - .map(|entry| entry.key().to_string()) - .collect(); - - let mut removed_data_size = 0i64; - for key in keys_to_delete { - let meta = self.partition_locks.remove(&key); - if let Some(x) = meta { - let size = x.1.pointer.load(Ordering::SeqCst); - removed_data_size += size; - } - } - - Ok(removed_data_size) - } - - async fn is_healthy(&self) -> Result { - self.healthy_check() - } - - async fn require_buffer( - &self, - _ctx: RequireBufferContext, - ) -> Result { - todo!() - } - - async fn release_buffer(&self, _ctx: ReleaseBufferContext) -> Result { - todo!() - } - - async fn register_app(&self, _ctx: RegisterAppContext) -> Result<()> { - Ok(()) - } - - async fn name(&self) -> StorageType { - StorageType::LOCALFILE - } -} - -#[cfg(test)] -mod test { - use crate::app::{ - PartitionedUId, PurgeDataContext, ReadingIndexViewContext, ReadingOptions, - ReadingViewContext, WritingViewContext, - }; - use crate::store::localfile::LocalFileStore; - - use crate::error::WorkerError; - use crate::store::{PartitionedDataBlock, ResponseData, ResponseDataIndex, Store}; - use bytes::{Buf, Bytes, BytesMut}; - use log::{error, info}; - - fn create_writing_ctx() -> WritingViewContext { - let uid = PartitionedUId { - app_id: "100".to_string(), - shuffle_id: 0, - partition_id: 0, - }; - - let data = b"hello world!hello china!"; - let size = data.len(); - let writing_ctx = WritingViewContext::from( - uid.clone(), - vec![ - PartitionedDataBlock { - block_id: 0, - length: size as i32, - uncompress_length: 200, - crc: 0, - data: Bytes::copy_from_slice(data), - task_attempt_id: 0, - }, - PartitionedDataBlock { - block_id: 1, - length: size as i32, - uncompress_length: 200, - crc: 0, - data: Bytes::copy_from_slice(data), - task_attempt_id: 0, - }, - ], - ); - - writing_ctx - } - - #[test] - fn local_disk_under_exception_test() -> anyhow::Result<()> { - let temp_dir = tempdir::TempDir::new("local_disk_under_exception_test").unwrap(); - let temp_path = temp_dir.path().to_str().unwrap().to_string(); - println!("init local file path: {}", &temp_path); - let local_store = LocalFileStore::new(vec![temp_path.to_string()]); - - let runtime = local_store.runtime_manager.clone(); - - let writing_view_ctx = create_writing_ctx(); - let insert_result = runtime.wait(local_store.insert(writing_view_ctx)); - - if insert_result.is_err() { - println!("{:?}", insert_result.err()); - panic!() - } - - // case1: mark the local disk unhealthy, that will the following flush throw exception directly. - let local_disk = local_store.local_disks[0].clone(); - local_disk.mark_unhealthy(); - - let writing_view_ctx = create_writing_ctx(); - let insert_result = runtime.wait(local_store.insert(writing_view_ctx)); - match insert_result { - Err(WorkerError::LOCAL_DISK_UNHEALTHY(_)) => {} - _ => panic!(), - } - - // case2: mark the local disk healthy, all things work! - local_disk.mark_healthy(); - let writing_view_ctx = create_writing_ctx(); - let insert_result = runtime.wait(local_store.insert(writing_view_ctx)); - match insert_result { - Err(WorkerError::LOCAL_DISK_UNHEALTHY(_)) => panic!(), - _ => {} - } - - // case3: mark the local disk corrupted, fail directly. - local_disk.mark_corrupted(); - let writing_view_ctx = create_writing_ctx(); - let insert_result = runtime.wait(local_store.insert(writing_view_ctx)); - match insert_result { - Err(WorkerError::PARTIAL_DATA_LOST(_)) => {} - _ => panic!(), - } - - Ok(()) - } - - #[test] - fn purge_test() -> anyhow::Result<()> { - let temp_dir = tempdir::TempDir::new("test_local_store").unwrap(); - let temp_path = temp_dir.path().to_str().unwrap().to_string(); - println!("init local file path: {}", &temp_path); - let local_store = LocalFileStore::new(vec![temp_path.clone()]); - - let runtime = local_store.runtime_manager.clone(); - - let app_id = "purge_test-app-id".to_string(); - let uid = PartitionedUId { - app_id: app_id.clone(), - shuffle_id: 0, - partition_id: 0, - }; - - let data = b"hello world!hello china!"; - let size = data.len(); - let writing_ctx = WritingViewContext::from( - uid.clone(), - vec![ - PartitionedDataBlock { - block_id: 0, - length: size as i32, - uncompress_length: 200, - crc: 0, - data: Bytes::copy_from_slice(data), - task_attempt_id: 0, - }, - PartitionedDataBlock { - block_id: 1, - length: size as i32, - uncompress_length: 200, - crc: 0, - data: Bytes::copy_from_slice(data), - task_attempt_id: 0, - }, - ], - ); - - let insert_result = runtime.wait(local_store.insert(writing_ctx)); - if insert_result.is_err() { - println!("{:?}", insert_result.err()); - panic!() - } - assert_eq!( - true, - runtime.wait(tokio::fs::try_exists(format!( - "{}/{}/{}/partition-{}.data", - &temp_path, &app_id, "0", "0" - )))? - ); - - // shuffle level purge - runtime - .wait(local_store.purge(PurgeDataContext::new(app_id.to_string(), Some(0)))) - .expect(""); - assert_eq!( - false, - runtime.wait(tokio::fs::try_exists(format!( - "{}/{}/{}", - &temp_path, &app_id, 0 - )))? - ); - - // app level purge - runtime.wait(local_store.purge((&*app_id).into()))?; - assert_eq!( - false, - runtime.wait(tokio::fs::try_exists(format!("{}/{}", &temp_path, &app_id)))? - ); - - Ok(()) - } - - #[test] - #[ignore] - fn local_store_test() { - let temp_dir = tempdir::TempDir::new("test_local_store").unwrap(); - let temp_path = temp_dir.path().to_str().unwrap().to_string(); - info!("init local file path: {}", temp_path); - let mut local_store = LocalFileStore::new(vec![temp_path]); - - let runtime = local_store.runtime_manager.clone(); - - let uid = PartitionedUId { - app_id: "100".to_string(), - shuffle_id: 0, - partition_id: 0, - }; - - let data = b"hello world!hello china!"; - let size = data.len(); - let writing_ctx = WritingViewContext::from( - uid.clone(), - vec![ - PartitionedDataBlock { - block_id: 0, - length: size as i32, - uncompress_length: 200, - crc: 0, - data: Bytes::copy_from_slice(data), - task_attempt_id: 0, - }, - PartitionedDataBlock { - block_id: 1, - length: size as i32, - uncompress_length: 200, - crc: 0, - data: Bytes::copy_from_slice(data), - task_attempt_id: 0, - }, - ], - ); - - let insert_result = runtime.wait(local_store.insert(writing_ctx)); - if insert_result.is_err() { - println!("{:?}", insert_result.err()); - panic!() - } - - async fn get_and_check_partitial_data( - local_store: &mut LocalFileStore, - uid: PartitionedUId, - size: i64, - expected: &[u8], - ) { - let reading_ctx = ReadingViewContext { - uid, - reading_options: ReadingOptions::FILE_OFFSET_AND_LEN(0, size as i64), - serialized_expected_task_ids_bitmap: Default::default(), - }; - - let read_result = local_store.get(reading_ctx).await; - if read_result.is_err() { - error!("failed to get the localfile data: {:?}", read_result.err()); - panic!() - } - - match read_result.unwrap() { - ResponseData::Local(partitioned_data) => { - assert_eq!(expected, partitioned_data.data.as_ref()); - } - _ => panic!(), - } - } - - // case1: read the one partition block data - runtime.wait(get_and_check_partitial_data( - &mut local_store, - uid.clone(), - size as i64, - data, - )); - - // case2: read the complete block data - let mut expected = BytesMut::with_capacity(size * 2); - expected.extend_from_slice(data); - expected.extend_from_slice(data); - runtime.wait(get_and_check_partitial_data( - &mut local_store, - uid.clone(), - size as i64 * 2, - expected.freeze().as_ref(), - )); - - // case3: get the index data - let reading_index_view_ctx = ReadingIndexViewContext { - partition_id: uid.clone(), - }; - let result = runtime.wait(local_store.get_index(reading_index_view_ctx)); - if result.is_err() { - panic!() - } - - match result.unwrap() { - ResponseDataIndex::Local(data) => { - let mut index = data.index_data; - let offset_1 = index.get_i64(); - assert_eq!(0, offset_1); - let length_1 = index.get_i32(); - assert_eq!(size as i32, length_1); - index.get_i32(); - index.get_i64(); - let block_id_1 = index.get_i64(); - assert_eq!(0, block_id_1); - let task_id = index.get_i64(); - assert_eq!(0, task_id); - - let offset_2 = index.get_i64(); - assert_eq!(size as i64, offset_2); - assert_eq!(size as i32, index.get_i32()); - } - } - - temp_dir.close().unwrap(); - } -} diff --git a/rust/experimental/server/src/store/mem/mod.rs b/rust/experimental/server/src/store/mem/mod.rs deleted file mode 100644 index 84061067d9..0000000000 --- a/rust/experimental/server/src/store/mem/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod ticket; - -pub use await_tree::InstrumentAwait; diff --git a/rust/experimental/server/src/store/mem/ticket.rs b/rust/experimental/server/src/store/mem/ticket.rs deleted file mode 100644 index 2879dd59d9..0000000000 --- a/rust/experimental/server/src/store/mem/ticket.rs +++ /dev/null @@ -1,232 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::error::WorkerError; -use crate::runtime::manager::RuntimeManager; -use anyhow::Result; -use dashmap::DashMap; -use log::debug; -use std::sync::Arc; -use std::thread; -use std::time::Duration; - -#[derive(Clone)] -pub struct Ticket { - id: i64, - created_time: u64, - size: i64, - owned_by_app_id: String, -} - -impl Ticket { - pub fn new(ticket_id: i64, created_time: u64, size: i64, app_id: &str) -> Self { - Self { - id: ticket_id, - created_time, - size, - owned_by_app_id: app_id.into(), - } - } - - pub fn get_size(&self) -> i64 { - self.size - } - - pub fn is_timeout(&self, timeout_sec: i64) -> bool { - (crate::util::current_timestamp_sec() - self.created_time) as i64 > timeout_sec - } - - pub fn get_id(&self) -> i64 { - self.id - } -} - -#[derive(Clone)] -pub struct TicketManager { - // key: ticket_id - ticket_store: Arc>, - - ticket_timeout_sec: i64, - ticket_timeout_check_interval_sec: i64, -} - -impl TicketManager { - pub fn new bool + Send + 'static>( - ticket_timeout_sec: i64, - ticket_timeout_check_interval_sec: i64, - free_allocated_size_func: F, - runtime_manager: RuntimeManager, - ) -> Self { - let manager = Self { - ticket_store: Default::default(), - ticket_timeout_sec, - ticket_timeout_check_interval_sec, - }; - Self::schedule_ticket_check(manager.clone(), free_allocated_size_func, runtime_manager); - manager - } - - /// check the ticket existence - pub fn exist(&self, ticket_id: i64) -> bool { - self.ticket_store.contains_key(&ticket_id) - } - - /// Delete one ticket by its id, and it will return the allocated size for this ticket - pub fn delete(&self, ticket_id: i64) -> Result { - if let Some(entry) = self.ticket_store.remove(&ticket_id) { - Ok(entry.1.size) - } else { - Err(WorkerError::TICKET_ID_NOT_EXIST(ticket_id)) - } - } - - /// Delete all the ticket owned by the app id. And - /// it will return all the allocated size of ticket ids that owned by this app_id - pub fn delete_by_app_id(&self, app_id: &str) -> i64 { - let read_view = self.ticket_store.clone(); - let mut deleted_ids = vec![]; - for ticket in read_view.iter() { - if ticket.owned_by_app_id == *app_id { - deleted_ids.push(ticket.id); - } - } - - let mut size = 0i64; - for deleted_id in deleted_ids { - size += self - .ticket_store - .remove(&deleted_id) - .map_or(0, |val| val.1.size); - } - size - } - - /// insert one ticket managed by this ticket manager - pub fn insert(&self, ticket_id: i64, size: i64, created_timestamp: u64, app_id: &str) -> bool { - let ticket = Ticket { - id: ticket_id, - created_time: created_timestamp, - size, - owned_by_app_id: app_id.into(), - }; - - self.ticket_store - .insert(ticket_id, ticket) - .map_or(false, |_| true) - } - - fn schedule_ticket_check bool + Send + 'static>( - ticket_manager: TicketManager, - mut free_allocated_fn: F, - _runtime_manager: RuntimeManager, - ) { - thread::spawn(move || { - let ticket_store = ticket_manager.ticket_store; - loop { - let read_view = ticket_store.clone(); - let mut timeout_tickets = vec![]; - for ticket in read_view.iter() { - if ticket.is_timeout(ticket_manager.ticket_timeout_sec) { - timeout_tickets.push(ticket.id); - } - } - - let mut total_removed_size = 0i64; - for timeout_ticket_id in timeout_tickets.iter() { - total_removed_size += ticket_store - .remove(timeout_ticket_id) - .map_or(0, |val| val.1.size); - } - if total_removed_size != 0 { - free_allocated_fn(total_removed_size); - debug!("remove {:#?} memory allocated tickets, release pre-allocated memory size: {:?}", timeout_tickets, total_removed_size); - } - thread::sleep(Duration::from_secs( - ticket_manager.ticket_timeout_check_interval_sec as u64, - )); - } - }); - } -} - -#[cfg(test)] -mod test { - use crate::runtime::manager::RuntimeManager; - use crate::store::mem::ticket::TicketManager; - use dashmap::DashMap; - use std::sync::{Arc, Mutex}; - use std::thread; - use std::thread::JoinHandle; - use std::time::Duration; - - #[test] - fn test_closure() { - let state = Arc::new(DashMap::new()); - state.insert(1, 1); - - fn schedule(mut callback: impl FnMut(i64) -> i64 + Send + 'static) -> JoinHandle { - thread::spawn(move || callback(2)) - } - - let state_cloned = state.clone(); - let callback = move |a: i64| { - state_cloned.insert(a, a); - a + 1 - }; - schedule(callback).join().expect(""); - - assert!(state.contains_key(&2)); - } - - #[test] - #[ignore] - fn test_ticket_manager() { - let released_size = Arc::new(Mutex::new(0)); - - let release_size_cloned = released_size.clone(); - let free_allocated_size_func = move |size: i64| { - *(release_size_cloned.lock().unwrap()) += size; - true - }; - let ticket_manager = - TicketManager::new(1, 1, free_allocated_size_func, RuntimeManager::default()); - let app_id = "test_ticket_manager_app_id"; - - assert!(ticket_manager.delete(1000).is_err()); - - // case1 - ticket_manager.insert(1, 10, crate::util::current_timestamp_sec() + 1, app_id); - ticket_manager.insert(2, 10, crate::util::current_timestamp_sec() + 1, app_id); - assert!(ticket_manager.exist(1)); - assert!(ticket_manager.exist(2)); - - // case2 - ticket_manager.delete(1).expect(""); - assert!(!ticket_manager.exist(1)); - assert!(ticket_manager.exist(2)); - - // case3 - ticket_manager.delete_by_app_id(app_id); - assert!(!ticket_manager.exist(2)); - - // case4 - ticket_manager.insert(3, 10, crate::util::current_timestamp_sec() + 1, app_id); - assert!(ticket_manager.exist(3)); - awaitility::at_most(Duration::from_secs(5)).until(|| !ticket_manager.exist(3)); - assert_eq!(10, *released_size.lock().unwrap()); - } -} diff --git a/rust/experimental/server/src/store/memory.rs b/rust/experimental/server/src/store/memory.rs deleted file mode 100644 index e93e4ae7f4..0000000000 --- a/rust/experimental/server/src/store/memory.rs +++ /dev/null @@ -1,1105 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::app::ReadingOptions::MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE; -use crate::app::{ - PartitionedUId, PurgeDataContext, ReadingIndexViewContext, ReadingViewContext, - RegisterAppContext, ReleaseBufferContext, RequireBufferContext, WritingViewContext, -}; -use crate::config::{MemoryStoreConfig, StorageType}; -use crate::error::WorkerError; -use crate::metric::{ - GAUGE_MEMORY_ALLOCATED, GAUGE_MEMORY_CAPACITY, GAUGE_MEMORY_USED, TOTAL_MEMORY_USED, -}; -use crate::readable_size::ReadableSize; -use crate::store::{ - DataSegment, PartitionedDataBlock, PartitionedMemoryData, RequireBufferResponse, ResponseData, - ResponseDataIndex, Store, -}; -use crate::*; -use async_trait::async_trait; -use bytes::BytesMut; -use dashmap::DashMap; - -use std::collections::{BTreeMap, HashMap}; - -use std::str::FromStr; - -use crate::store::mem::ticket::TicketManager; -use croaring::Treemap; -use spin::mutex::Mutex; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; - -pub struct MemoryStore { - // todo: change to RW lock - state: DashMap>>, - budget: MemoryBudget, - // key: app_id, value: allocated memory size - memory_capacity: i64, - in_flush_buffer_size: AtomicU64, - runtime_manager: RuntimeManager, - ticket_manager: TicketManager, -} - -unsafe impl Send for MemoryStore {} -unsafe impl Sync for MemoryStore {} - -impl MemoryStore { - // only for test cases - pub fn new(max_memory_size: i64) -> Self { - let budget = MemoryBudget::new(max_memory_size); - let runtime_manager: RuntimeManager = Default::default(); - - let budget_clone = budget.clone(); - let free_allocated_size_func = - move |size: i64| budget_clone.free_allocated(size).map_or(false, |v| v); - let ticket_manager = TicketManager::new( - 5 * 60, - 10, - free_allocated_size_func, - runtime_manager.clone(), - ); - MemoryStore { - budget, - state: DashMap::new(), - memory_capacity: max_memory_size, - ticket_manager, - in_flush_buffer_size: Default::default(), - runtime_manager, - } - } - - pub fn from(conf: MemoryStoreConfig, runtime_manager: RuntimeManager) -> Self { - let capacity = ReadableSize::from_str(&conf.capacity).unwrap(); - let budget = MemoryBudget::new(capacity.as_bytes() as i64); - - let budget_clone = budget.clone(); - let free_allocated_size_func = - move |size: i64| budget_clone.free_allocated(size).map_or(false, |v| v); - let ticket_manager = TicketManager::new( - 5 * 60, - 10, - free_allocated_size_func, - runtime_manager.clone(), - ); - MemoryStore { - state: DashMap::new(), - budget: MemoryBudget::new(capacity.as_bytes() as i64), - memory_capacity: capacity.as_bytes() as i64, - ticket_manager, - in_flush_buffer_size: Default::default(), - runtime_manager, - } - } - - // todo: make this used size as a var - pub async fn memory_usage_ratio(&self) -> f32 { - let snapshot = self.budget.snapshot(); - snapshot.get_used_percent() - } - - pub async fn memory_snapshot(&self) -> Result { - Ok(self.budget.snapshot()) - } - - pub fn get_capacity(&self) -> Result { - Ok(self.memory_capacity) - } - - pub async fn memory_used_ratio(&self) -> f32 { - let snapshot = self.budget.snapshot(); - (snapshot.used + snapshot.allocated - - self.in_flush_buffer_size.load(Ordering::SeqCst) as i64) as f32 - / snapshot.capacity as f32 - } - - pub fn add_to_in_flight_buffer_size(&self, size: u64) { - self.in_flush_buffer_size.fetch_add(size, Ordering::SeqCst); - } - - pub fn desc_to_in_flight_buffer_size(&self, size: u64) { - self.in_flush_buffer_size.fetch_sub(size, Ordering::SeqCst); - } - - pub async fn free_used(&self, size: i64) -> Result { - self.budget.free_used(size) - } - - pub async fn free_allocated(&self, size: i64) -> Result { - self.budget.free_allocated(size) - } - - pub async fn get_required_spill_buffer( - &self, - target_len: i64, - ) -> HashMap>> { - // sort - // get the spill buffers - - let snapshot = self.budget.snapshot(); - let removed_size = snapshot.used - target_len; - if removed_size <= 0 { - return HashMap::new(); - } - - let mut sorted_tree_map = BTreeMap::new(); - - let buffers = self.state.clone().into_read_only(); - for buffer in buffers.iter() { - let staging_size = buffer.1.lock().staging_size; - let valset = sorted_tree_map - .entry(staging_size) - .or_insert_with(|| vec![]); - let key = buffer.0; - valset.push(key); - } - - let mut current_removed = 0; - - let mut required_spill_buffers = HashMap::new(); - - let iter = sorted_tree_map.iter().rev(); - 'outer: for (size, vals) in iter { - for pid in vals { - if current_removed >= removed_size || size.to_be() == 0 { - break 'outer; - } - current_removed += size.to_be() as i64; - let partition_uid = (*pid).clone(); - - let buffer = self.get_underlying_partition_buffer(&partition_uid); - required_spill_buffers.insert(partition_uid, buffer); - } - } - - required_spill_buffers - } - - pub async fn get_partitioned_buffer_size(&self, uid: &PartitionedUId) -> Result { - let buffer = self.get_underlying_partition_buffer(uid); - let buffer = buffer.lock(); - Ok(buffer.total_size as u64) - } - - fn get_underlying_partition_buffer(&self, pid: &PartitionedUId) -> Arc> { - self.state.get(pid).unwrap().clone() - } - - pub async fn release_in_flight_blocks_in_underlying_staging_buffer( - &self, - uid: PartitionedUId, - in_flight_blocks_id: i64, - ) -> Result<()> { - let buffer = self.get_or_create_underlying_staging_buffer(uid); - let mut buffer_ref = buffer.lock(); - buffer_ref.flight_finished(&in_flight_blocks_id)?; - Ok(()) - } - - pub fn get_or_create_underlying_staging_buffer( - &self, - uid: PartitionedUId, - ) -> Arc> { - let buffer = self - .state - .entry(uid) - .or_insert_with(|| Arc::new(Mutex::new(StagingBuffer::new()))); - buffer.clone() - } - - pub(crate) fn read_partial_data_with_max_size_limit_and_filter<'a>( - &'a self, - blocks: Vec<&'a PartitionedDataBlock>, - fetched_size_limit: i64, - serialized_expected_task_ids_bitmap: Option, - ) -> (Vec<&PartitionedDataBlock>, i64) { - let mut fetched = vec![]; - let mut fetched_size = 0; - - for block in blocks { - if let Some(ref filter) = serialized_expected_task_ids_bitmap { - if !filter.contains(block.task_attempt_id as u64) { - continue; - } - } - if fetched_size >= fetched_size_limit { - break; - } - fetched_size += block.length as i64; - fetched.push(block); - } - - (fetched, fetched_size) - } -} - -#[async_trait] -impl Store for MemoryStore { - fn start(self: Arc) { - // ignore - } - - async fn insert(&self, ctx: WritingViewContext) -> Result<(), WorkerError> { - let uid = ctx.uid; - let buffer = self.get_or_create_underlying_staging_buffer(uid.clone()); - let mut buffer_guarded = buffer.lock(); - - let blocks = ctx.data_blocks; - let inserted_size = buffer_guarded.add(blocks)?; - drop(buffer_guarded); - - self.budget.allocated_to_used(inserted_size)?; - - TOTAL_MEMORY_USED.inc_by(inserted_size as u64); - - Ok(()) - } - - async fn get(&self, ctx: ReadingViewContext) -> Result { - let uid = ctx.uid; - let buffer = self.get_or_create_underlying_staging_buffer(uid); - let buffer = buffer.lock(); - - let options = ctx.reading_options; - let (fetched_blocks, length) = match options { - MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE(last_block_id, max_size) => { - let mut last_block_id = last_block_id.clone(); - let mut in_flight_flatten_blocks = vec![]; - for (_, blocks) in buffer.in_flight.iter() { - for in_flight_block in blocks { - in_flight_flatten_blocks.push(in_flight_block); - } - } - let in_flight_flatten_blocks = Arc::new(in_flight_flatten_blocks); - - let mut staging_blocks = vec![]; - for block in &buffer.staging { - staging_blocks.push(block); - } - let staging_blocks = Arc::new(staging_blocks); - - let mut candidate_blocks = vec![]; - // todo: optimize to the way of recursion - let mut exited = false; - while !exited { - exited = true; - if last_block_id == -1 { - // Anyway, it will always read from in_flight to staging - let mut extends: Vec<&PartitionedDataBlock> = vec![]; - extends.extend_from_slice(&*in_flight_flatten_blocks); - extends.extend_from_slice(&*staging_blocks); - candidate_blocks = extends; - } else { - // check whether the in_fight_blocks exist the last_block_id - let mut in_flight_exist = false; - let mut unread_in_flight_blocks = vec![]; - for block in in_flight_flatten_blocks.clone().iter() { - if !in_flight_exist { - if block.block_id == last_block_id { - in_flight_exist = true; - } - continue; - } - unread_in_flight_blocks.push(*block); - } - - if in_flight_exist { - let mut extends: Vec<&PartitionedDataBlock> = vec![]; - extends.extend_from_slice(&*unread_in_flight_blocks); - extends.extend_from_slice(&*staging_blocks); - candidate_blocks = extends; - } else { - let mut staging_exist = false; - let mut unread_staging_buffers = vec![]; - for block in staging_blocks.clone().iter() { - if !staging_exist { - if block.block_id == last_block_id { - staging_exist = true; - } - continue; - } - unread_staging_buffers.push(*block); - } - - if staging_exist { - candidate_blocks = unread_staging_buffers; - } else { - // when the last_block_id is not found, maybe the partial has been flush - // to file. if having rest data, let's read it from the head - exited = false; - last_block_id = -1; - } - } - } - } - - self.read_partial_data_with_max_size_limit_and_filter( - candidate_blocks, - max_size, - ctx.serialized_expected_task_ids_bitmap, - ) - } - _ => (vec![], 0), - }; - - let mut bytes_holder = BytesMut::with_capacity(length as usize); - let mut segments = vec![]; - let mut offset = 0; - for block in fetched_blocks { - let data = &block.data; - bytes_holder.extend_from_slice(data); - segments.push(DataSegment { - block_id: block.block_id, - offset, - length: block.length, - uncompress_length: block.uncompress_length, - crc: block.crc, - task_attempt_id: block.task_attempt_id, - }); - offset += block.length as i64; - } - - Ok(ResponseData::Mem(PartitionedMemoryData { - shuffle_data_block_segments: segments, - data: bytes_holder.freeze(), - })) - } - - async fn get_index( - &self, - _ctx: ReadingIndexViewContext, - ) -> Result { - panic!("It should not be invoked.") - } - - async fn purge(&self, ctx: PurgeDataContext) -> Result { - let app_id = ctx.app_id; - let shuffle_id_option = ctx.shuffle_id; - - // remove the corresponding app's data - let read_only_state_view = self.state.clone().into_read_only(); - let mut _removed_list = vec![]; - for entry in read_only_state_view.iter() { - let pid = entry.0; - if pid.app_id == app_id { - if ctx.shuffle_id.is_some() { - if pid.shuffle_id == shuffle_id_option.unwrap() { - _removed_list.push(pid); - } else { - continue; - } - } else { - _removed_list.push(pid); - } - } - } - - let mut used = 0; - for removed_pid in _removed_list { - if let Some(entry) = self.state.remove(removed_pid) { - used += entry.1.lock().total_size; - } - } - - // free used - self.budget.free_used(used)?; - - info!( - "removed used buffer size:[{}] for [{:?}], [{:?}]", - used, app_id, shuffle_id_option - ); - - Ok(used) - } - - async fn is_healthy(&self) -> Result { - Ok(true) - } - - async fn require_buffer( - &self, - ctx: RequireBufferContext, - ) -> Result { - let (succeed, ticket_id) = self.budget.pre_allocate(ctx.size)?; - match succeed { - true => { - let require_buffer_resp = RequireBufferResponse::new(ticket_id); - self.ticket_manager.insert( - ticket_id, - ctx.size, - require_buffer_resp.allocated_timestamp, - &ctx.uid.app_id, - ); - Ok(require_buffer_resp) - } - _ => Err(WorkerError::NO_ENOUGH_MEMORY_TO_BE_ALLOCATED), - } - } - - async fn release_buffer(&self, ctx: ReleaseBufferContext) -> Result { - let ticket_id = ctx.ticket_id; - self.ticket_manager.delete(ticket_id) - } - - async fn register_app(&self, _ctx: RegisterAppContext) -> Result<()> { - Ok(()) - } - - async fn name(&self) -> StorageType { - StorageType::MEMORY - } -} - -/// thread safe, this will be guarded by the lock -#[derive(Debug, Clone)] -pub struct StagingBuffer { - // optimize this size. - total_size: i64, - staging_size: i64, - in_flight_size: i64, - staging: Vec, - in_flight: BTreeMap>, - id_generator: i64, -} - -impl StagingBuffer { - pub fn new() -> StagingBuffer { - StagingBuffer { - total_size: 0, - staging_size: 0, - in_flight_size: 0, - staging: vec![], - in_flight: BTreeMap::new(), - id_generator: 0, - } - } - - /// add the blocks to the staging - pub fn add(&mut self, blocks: Vec) -> Result { - let mut added = 0i64; - for block in blocks { - added += block.length as i64; - self.staging.push(block); - } - self.staging_size += added; - self.total_size += added; - Ok(added) - } - - /// make the blocks sent to persistent storage - pub fn migrate_staging_to_in_flight(&mut self) -> Result<(i64, Vec)> { - self.in_flight_size += self.staging_size; - self.staging_size = 0; - - let blocks = self.staging.to_owned(); - self.staging.clear(); - - let id = self.id_generator; - self.id_generator += 1; - self.in_flight.insert(id.clone(), blocks.clone()); - - Ok((id, blocks)) - } - - /// clear the blocks which are flushed to persistent storage - pub fn flight_finished(&mut self, flight_id: &i64) -> Result { - let done = self.in_flight.remove(flight_id); - - let mut removed_size = 0i64; - if let Some(removed_blocks) = done { - for removed_block in removed_blocks { - removed_size += removed_block.length as i64; - } - } - - self.total_size -= removed_size; - self.in_flight_size -= removed_size; - - Ok(removed_size) - } - - pub fn get_staging_size(&self) -> Result { - Ok(self.staging_size) - } - - pub fn get_in_flight_size(&self) -> Result { - Ok(self.in_flight_size) - } - - pub fn get_total_size(&self) -> Result { - Ok(self.total_size) - } -} - -pub struct MemorySnapshot { - capacity: i64, - allocated: i64, - used: i64, -} - -impl From<(i64, i64, i64)> for MemorySnapshot { - fn from(value: (i64, i64, i64)) -> Self { - MemorySnapshot { - capacity: value.0, - allocated: value.1, - used: value.2, - } - } -} - -impl MemorySnapshot { - pub fn get_capacity(&self) -> i64 { - self.capacity - } - pub fn get_allocated(&self) -> i64 { - self.allocated - } - pub fn get_used(&self) -> i64 { - self.used - } - fn get_used_percent(&self) -> f32 { - (self.allocated + self.used) as f32 / self.capacity as f32 - } -} - -#[derive(Clone)] -pub struct MemoryBudget { - inner: Arc>, -} - -struct MemoryBudgetInner { - capacity: i64, - allocated: i64, - used: i64, - allocation_incr_id: i64, -} - -impl MemoryBudget { - fn new(capacity: i64) -> MemoryBudget { - GAUGE_MEMORY_CAPACITY.set(capacity); - MemoryBudget { - inner: Arc::new(std::sync::Mutex::new(MemoryBudgetInner { - capacity, - allocated: 0, - used: 0, - allocation_incr_id: 0, - })), - } - } - - pub fn snapshot(&self) -> MemorySnapshot { - let inner = self.inner.lock().unwrap(); - (inner.capacity, inner.allocated, inner.used).into() - } - - fn pre_allocate(&self, size: i64) -> Result<(bool, i64)> { - let mut inner = self.inner.lock().unwrap(); - let free_space = inner.capacity - inner.allocated - inner.used; - if free_space < size { - Ok((false, -1)) - } else { - inner.allocated += size; - let now = inner.allocation_incr_id; - inner.allocation_incr_id += 1; - GAUGE_MEMORY_ALLOCATED.set(inner.allocated); - Ok((true, now)) - } - } - - fn allocated_to_used(&self, size: i64) -> Result { - let mut inner = self.inner.lock().unwrap(); - if inner.allocated < size { - inner.allocated = 0; - } else { - inner.allocated -= size; - } - inner.used += size; - GAUGE_MEMORY_ALLOCATED.set(inner.allocated); - GAUGE_MEMORY_USED.set(inner.used); - Ok(true) - } - - fn free_used(&self, size: i64) -> Result { - let mut inner = self.inner.lock().unwrap(); - if inner.used < size { - inner.used = 0; - // todo: metric - } else { - inner.used -= size; - } - GAUGE_MEMORY_USED.set(inner.used); - Ok(true) - } - - fn free_allocated(&self, size: i64) -> Result { - let mut inner = self.inner.lock().unwrap(); - if inner.allocated < size { - inner.allocated = 0; - } else { - inner.allocated -= size; - } - GAUGE_MEMORY_ALLOCATED.set(inner.allocated); - Ok(true) - } -} - -#[cfg(test)] -mod test { - use crate::app::{ - PartitionedUId, PurgeDataContext, ReadingOptions, ReadingViewContext, RequireBufferContext, - WritingViewContext, - }; - - use crate::store::memory::MemoryStore; - use crate::store::ResponseData::Mem; - - use crate::store::{PartitionedDataBlock, PartitionedMemoryData, ResponseData, Store}; - - use bytes::BytesMut; - use core::panic; - use std::sync::Arc; - - use anyhow::Result; - use croaring::Treemap; - - #[test] - fn test_read_buffer_in_flight() { - let store = MemoryStore::new(1024); - let runtime = store.runtime_manager.clone(); - - let uid = PartitionedUId { - app_id: "100".to_string(), - shuffle_id: 0, - partition_id: 0, - }; - let writing_view_ctx = create_writing_ctx_with_blocks(10, 10, uid.clone()); - let _ = runtime.wait(store.insert(writing_view_ctx)); - - let default_single_read_size = 20; - - // case1: read from -1 - let mem_data = runtime.wait(get_data_with_last_block_id( - default_single_read_size, - -1, - &store, - uid.clone(), - )); - assert_eq!(2, mem_data.shuffle_data_block_segments.len()); - assert_eq!( - 0, - mem_data - .shuffle_data_block_segments - .get(0) - .unwrap() - .block_id - ); - assert_eq!( - 1, - mem_data - .shuffle_data_block_segments - .get(1) - .unwrap() - .block_id - ); - - // case2: when the last_block_id doesn't exist, it should return the data like when last_block_id=-1 - let mem_data = runtime.wait(get_data_with_last_block_id( - default_single_read_size, - 100, - &store, - uid.clone(), - )); - assert_eq!(2, mem_data.shuffle_data_block_segments.len()); - assert_eq!( - 0, - mem_data - .shuffle_data_block_segments - .get(0) - .unwrap() - .block_id - ); - assert_eq!( - 1, - mem_data - .shuffle_data_block_segments - .get(1) - .unwrap() - .block_id - ); - - // case3: read from 3 - let mem_data = runtime.wait(get_data_with_last_block_id( - default_single_read_size, - 3, - &store, - uid.clone(), - )); - assert_eq!(2, mem_data.shuffle_data_block_segments.len()); - assert_eq!( - 4, - mem_data - .shuffle_data_block_segments - .get(0) - .unwrap() - .block_id - ); - assert_eq!( - 5, - mem_data - .shuffle_data_block_segments - .get(1) - .unwrap() - .block_id - ); - - // case4: some data are in inflight blocks - let buffer = store.get_or_create_underlying_staging_buffer(uid.clone()); - let mut buffer = buffer.lock(); - let owned = buffer.staging.to_owned(); - buffer.staging.clear(); - let mut idx = 0; - for block in owned { - buffer.in_flight.insert(idx, vec![block]); - idx += 1; - } - drop(buffer); - - // all data will be fetched from in_flight data - let mem_data = runtime.wait(get_data_with_last_block_id( - default_single_read_size, - 3, - &store, - uid.clone(), - )); - assert_eq!(2, mem_data.shuffle_data_block_segments.len()); - assert_eq!( - 4, - mem_data - .shuffle_data_block_segments - .get(0) - .unwrap() - .block_id - ); - assert_eq!( - 5, - mem_data - .shuffle_data_block_segments - .get(1) - .unwrap() - .block_id - ); - - // case5: old data in in_flight and latest data in staging. - // read it from the block id 9, and read size of 30 - let buffer = store.get_or_create_underlying_staging_buffer(uid.clone()); - let mut buffer = buffer.lock(); - buffer.staging.push(PartitionedDataBlock { - block_id: 20, - length: 10, - uncompress_length: 0, - crc: 0, - data: BytesMut::with_capacity(10).freeze(), - task_attempt_id: 0, - }); - drop(buffer); - - let mem_data = runtime.wait(get_data_with_last_block_id(30, 7, &store, uid.clone())); - assert_eq!(3, mem_data.shuffle_data_block_segments.len()); - assert_eq!( - 8, - mem_data - .shuffle_data_block_segments - .get(0) - .unwrap() - .block_id - ); - assert_eq!( - 9, - mem_data - .shuffle_data_block_segments - .get(1) - .unwrap() - .block_id - ); - assert_eq!( - 20, - mem_data - .shuffle_data_block_segments - .get(2) - .unwrap() - .block_id - ); - - // case6: read the end to return empty result - let mem_data = runtime.wait(get_data_with_last_block_id(30, 20, &store, uid.clone())); - assert_eq!(0, mem_data.shuffle_data_block_segments.len()); - } - - async fn get_data_with_last_block_id( - default_single_read_size: i64, - last_block_id: i64, - store: &MemoryStore, - uid: PartitionedUId, - ) -> PartitionedMemoryData { - let ctx = ReadingViewContext { - uid: uid.clone(), - reading_options: ReadingOptions::MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE( - last_block_id, - default_single_read_size, - ), - serialized_expected_task_ids_bitmap: Default::default(), - }; - if let Ok(data) = store.get(ctx).await { - match data { - Mem(mem_data) => mem_data, - _ => panic!(), - } - } else { - panic!(); - } - } - - fn create_writing_ctx_with_blocks( - _block_number: i32, - single_block_size: i32, - uid: PartitionedUId, - ) -> WritingViewContext { - let mut data_blocks = vec![]; - for idx in 0..=9 { - data_blocks.push(PartitionedDataBlock { - block_id: idx, - length: single_block_size.clone(), - uncompress_length: 0, - crc: 0, - data: BytesMut::with_capacity(single_block_size as usize).freeze(), - task_attempt_id: 0, - }); - } - WritingViewContext::from(uid, data_blocks) - } - - #[test] - fn test_allocated_and_purge_for_memory() { - let store = MemoryStore::new(1024 * 1024 * 1024); - let runtime = store.runtime_manager.clone(); - - let ctx = RequireBufferContext { - uid: PartitionedUId { - app_id: "100".to_string(), - shuffle_id: 0, - partition_id: 0, - }, - size: 10000, - }; - match runtime.default_runtime.block_on(store.require_buffer(ctx)) { - Ok(_) => { - let _ = runtime.default_runtime.block_on(store.purge("100".into())); - } - _ => panic!(), - } - - let budget = store.budget.inner.lock().unwrap(); - assert_eq!(0, budget.used); - assert_eq!(1024 * 1024 * 1024, budget.capacity); - } - - #[test] - fn test_purge() -> Result<()> { - let store = MemoryStore::new(1024); - let runtime = store.runtime_manager.clone(); - - let app_id = "purge_app"; - let shuffle_id = 1; - let partition = 1; - - let uid = PartitionedUId::from(app_id.to_string(), shuffle_id, partition); - - // the buffer requested - - let _buffer = runtime - .wait(store.require_buffer(RequireBufferContext::new(uid.clone(), 40))) - .expect(""); - - let writing_ctx = WritingViewContext::from( - uid.clone(), - vec![PartitionedDataBlock { - block_id: 0, - length: 10, - uncompress_length: 100, - crc: 99, - data: Default::default(), - task_attempt_id: 0, - }], - ); - runtime.wait(store.insert(writing_ctx)).expect(""); - - let reading_ctx = ReadingViewContext { - uid: uid.clone(), - reading_options: ReadingOptions::MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE(-1, 1000000), - serialized_expected_task_ids_bitmap: Default::default(), - }; - let data = runtime.wait(store.get(reading_ctx.clone())).expect(""); - assert_eq!(1, data.from_memory().shuffle_data_block_segments.len()); - - // get weak reference to ensure purge can successfully free memory - let weak_ref_before = store - .state - .get(&uid) - .map(|entry| Arc::downgrade(&entry.value())); - assert!( - weak_ref_before.is_some(), - "Failed to obtain weak reference before purge" - ); - - // partial purge for app's one shuffle data - runtime - .wait(store.purge(PurgeDataContext::new(app_id.to_string(), Some(shuffle_id)))) - .expect(""); - assert!(!store.state.contains_key(&PartitionedUId::from( - app_id.to_string(), - shuffle_id, - partition - ))); - - // purge - runtime.wait(store.purge(app_id.into())).expect(""); - assert!( - weak_ref_before.clone().unwrap().upgrade().is_none(), - "Arc should not exist after purge" - ); - let snapshot = store.budget.snapshot(); - assert_eq!(snapshot.used, 0); - assert_eq!(snapshot.capacity, 1024); - let data = runtime.wait(store.get(reading_ctx.clone())).expect(""); - assert_eq!(0, data.from_memory().shuffle_data_block_segments.len()); - - Ok(()) - } - - #[test] - fn test_put_and_get_for_memory() { - let store = MemoryStore::new(1024 * 1024 * 1024); - let runtime = store.runtime_manager.clone(); - - let writing_ctx = WritingViewContext::from( - Default::default(), - vec![ - PartitionedDataBlock { - block_id: 0, - length: 10, - uncompress_length: 100, - crc: 99, - data: Default::default(), - task_attempt_id: 0, - }, - PartitionedDataBlock { - block_id: 1, - length: 20, - uncompress_length: 200, - crc: 99, - data: Default::default(), - task_attempt_id: 1, - }, - ], - ); - runtime.wait(store.insert(writing_ctx)).unwrap(); - - let reading_ctx = ReadingViewContext { - uid: Default::default(), - reading_options: ReadingOptions::MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE(-1, 1000000), - serialized_expected_task_ids_bitmap: Default::default(), - }; - - match runtime.wait(store.get(reading_ctx)).unwrap() { - ResponseData::Mem(data) => { - assert_eq!(data.shuffle_data_block_segments.len(), 2); - assert_eq!(data.shuffle_data_block_segments.get(0).unwrap().offset, 0); - assert_eq!(data.shuffle_data_block_segments.get(1).unwrap().offset, 10); - } - _ => panic!("should not"), - } - } - - #[test] - fn test_block_id_filter_for_memory() { - let store = MemoryStore::new(1024 * 1024 * 1024); - let runtime = store.runtime_manager.clone(); - - // 1. insert 2 block - let writing_ctx = WritingViewContext::from( - Default::default(), - vec![ - PartitionedDataBlock { - block_id: 0, - length: 10, - uncompress_length: 100, - crc: 99, - data: Default::default(), - task_attempt_id: 0, - }, - PartitionedDataBlock { - block_id: 1, - length: 20, - uncompress_length: 200, - crc: 99, - data: Default::default(), - task_attempt_id: 1, - }, - ], - ); - runtime.wait(store.insert(writing_ctx)).unwrap(); - - // 2. block_ids_filter is empty, should return 2 blocks - let mut reading_ctx = ReadingViewContext { - uid: Default::default(), - reading_options: ReadingOptions::MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE(-1, 1000000), - serialized_expected_task_ids_bitmap: Default::default(), - }; - - match runtime.wait(store.get(reading_ctx)).unwrap() { - Mem(data) => { - assert_eq!(data.shuffle_data_block_segments.len(), 2); - } - _ => panic!("should not"), - } - - // 3. set serialized_expected_task_ids_bitmap, and set last_block_id equals 1, should return 1 block - let mut bitmap = Treemap::default(); - bitmap.add(1); - reading_ctx = ReadingViewContext { - uid: Default::default(), - reading_options: ReadingOptions::MEMORY_LAST_BLOCK_ID_AND_MAX_SIZE(0, 1000000), - serialized_expected_task_ids_bitmap: Option::from(bitmap.clone()), - }; - - match runtime.wait(store.get(reading_ctx)).unwrap() { - Mem(data) => { - assert_eq!(data.shuffle_data_block_segments.len(), 1); - assert_eq!(data.shuffle_data_block_segments.get(0).unwrap().offset, 0); - assert_eq!( - data.shuffle_data_block_segments - .get(0) - .unwrap() - .uncompress_length, - 200 - ); - } - _ => panic!("should not"), - } - } -} diff --git a/rust/experimental/server/src/store/mod.rs b/rust/experimental/server/src/store/mod.rs deleted file mode 100644 index 99bd36e6f6..0000000000 --- a/rust/experimental/server/src/store/mod.rs +++ /dev/null @@ -1,192 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#[cfg(feature = "hdfs")] -pub mod hdfs; -pub mod hybrid; -pub mod local; -pub mod localfile; -pub mod mem; -pub mod memory; - -use crate::app::{ - PurgeDataContext, ReadingIndexViewContext, ReadingViewContext, RegisterAppContext, - ReleaseBufferContext, RequireBufferContext, WritingViewContext, -}; -use crate::config::{Config, StorageType}; -use crate::error::WorkerError; -use crate::proto::uniffle::{ShuffleData, ShuffleDataBlockSegment}; -use crate::store::hybrid::HybridStore; - -use crate::util::current_timestamp_sec; -use anyhow::Result; -use async_trait::async_trait; -use bytes::Bytes; - -use crate::runtime::manager::RuntimeManager; -use std::sync::Arc; - -#[derive(Debug)] -pub struct PartitionedData { - pub partition_id: i32, - pub blocks: Vec, -} - -#[derive(Debug, Clone)] -pub struct PartitionedDataBlock { - pub block_id: i64, - pub length: i32, - pub uncompress_length: i32, - pub crc: i64, - pub data: Bytes, - pub task_attempt_id: i64, -} - -impl From for PartitionedData { - fn from(shuffle_data: ShuffleData) -> PartitionedData { - let mut blocks = vec![]; - for data in shuffle_data.block { - let block = PartitionedDataBlock { - block_id: data.block_id, - length: data.length, - uncompress_length: data.uncompress_length, - crc: data.crc, - data: data.data, - task_attempt_id: data.task_attempt_id, - }; - blocks.push(block); - } - PartitionedData { - partition_id: shuffle_data.partition_id, - blocks, - } - } -} - -pub enum ResponseDataIndex { - Local(LocalDataIndex), -} - -pub struct LocalDataIndex { - pub index_data: Bytes, - pub data_file_len: i64, -} - -pub enum ResponseData { - Local(PartitionedLocalData), - Mem(PartitionedMemoryData), -} - -impl ResponseData { - pub fn from_local(&self) -> Bytes { - match self { - ResponseData::Local(data) => data.data.clone(), - _ => Default::default(), - } - } - - pub fn from_memory(&self) -> PartitionedMemoryData { - match self { - ResponseData::Mem(data) => data.clone(), - _ => Default::default(), - } - } -} - -pub struct PartitionedLocalData { - pub data: Bytes, -} - -#[derive(Clone, Default)] -pub struct PartitionedMemoryData { - pub shuffle_data_block_segments: Vec, - pub data: Bytes, -} - -#[derive(Clone)] -pub struct DataSegment { - pub block_id: i64, - pub offset: i64, - pub length: i32, - pub uncompress_length: i32, - pub crc: i64, - pub task_attempt_id: i64, -} - -impl Into for DataSegment { - fn into(self) -> ShuffleDataBlockSegment { - ShuffleDataBlockSegment { - block_id: self.block_id, - offset: self.offset, - length: self.length, - uncompress_length: self.uncompress_length, - crc: self.crc, - task_attempt_id: self.task_attempt_id, - } - } -} - -// ===================================================== - -#[derive(Clone, Debug)] -pub struct RequireBufferResponse { - pub ticket_id: i64, - pub allocated_timestamp: u64, -} - -impl RequireBufferResponse { - fn new(ticket_id: i64) -> Self { - Self { - ticket_id, - allocated_timestamp: current_timestamp_sec(), - } - } -} - -// ===================================================== - -#[async_trait] -pub trait Store { - fn start(self: Arc); - async fn insert(&self, ctx: WritingViewContext) -> Result<(), WorkerError>; - async fn get(&self, ctx: ReadingViewContext) -> Result; - async fn get_index( - &self, - ctx: ReadingIndexViewContext, - ) -> Result; - async fn purge(&self, ctx: PurgeDataContext) -> Result; - async fn is_healthy(&self) -> Result; - - async fn require_buffer( - &self, - ctx: RequireBufferContext, - ) -> Result; - async fn release_buffer(&self, ctx: ReleaseBufferContext) -> Result; - async fn register_app(&self, ctx: RegisterAppContext) -> Result<()>; - - async fn name(&self) -> StorageType; -} - -pub trait Persistent {} - -pub struct StoreProvider {} - -impl StoreProvider { - pub fn get(runtime_manager: RuntimeManager, config: Config) -> HybridStore { - HybridStore::from(config, runtime_manager) - } -} diff --git a/rust/experimental/server/src/util.rs b/rust/experimental/server/src/util.rs deleted file mode 100644 index 96e49de04a..0000000000 --- a/rust/experimental/server/src/util.rs +++ /dev/null @@ -1,145 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use bytes::Bytes; -use crc32fast::Hasher; - -use std::net::IpAddr; -use std::sync::Mutex; -use std::time::{SystemTime, UNIX_EPOCH}; - -const WORKER_IP: &str = "WORKER_IP"; - -pub fn get_local_ip() -> Result { - let ip = std::env::var(WORKER_IP); - if ip.is_ok() { - Ok(ip.unwrap().parse().unwrap()) - } else { - let socket = std::net::UdpSocket::bind("0.0.0.0:0")?; - socket.connect("8.8.8.8:80")?; - let local_addr = socket.local_addr()?; - Ok(local_addr.ip()) - } -} - -pub fn gen_worker_uid(grpc_port: i32) -> String { - let ip = get_local_ip().unwrap().to_string(); - format!("{}-{}", ip.clone(), grpc_port) -} - -const LENGTH_PER_CRC: usize = 4 * 1024; -pub fn get_crc(bytes: &Bytes) -> i64 { - let mut crc32 = Hasher::new(); - let offset = 0; - let length = bytes.len(); - let crc_buffer = &bytes[offset..(offset + length)]; - - for i in (0..length).step_by(LENGTH_PER_CRC) { - let len = std::cmp::min(LENGTH_PER_CRC, length - i); - let crc_slice = &crc_buffer[i..(i + len)]; - - crc32.update(crc_slice); - } - - crc32.finalize() as i64 -} - -pub fn current_timestamp_ms() -> u128 { - let current_time = SystemTime::now(); - let timestamp = current_time.duration_since(UNIX_EPOCH).unwrap().as_millis(); - timestamp -} - -pub fn current_timestamp_sec() -> u64 { - let current_time = SystemTime::now(); - let timestamp = current_time.duration_since(UNIX_EPOCH).unwrap().as_secs(); - timestamp -} - -pub struct ConcurrencyLimiter { - remaining: Mutex, -} - -pub struct Ticket<'a> { - limiter: &'a ConcurrencyLimiter, - id: u32, -} - -impl ConcurrencyLimiter { - fn new(size: u32) -> Self { - ConcurrencyLimiter { - remaining: Mutex::new(size), - } - } - - pub fn try_acquire(&self) -> Option> { - let mut remaining = self.remaining.lock().unwrap(); - if *remaining <= 0 { - None - } else { - let id = *remaining; - *remaining -= 1; - Some(Ticket { limiter: self, id }) - } - } -} - -impl Drop for Ticket<'_> { - fn drop(&mut self) { - let mut lock = self.limiter.remaining.lock().unwrap(); - *lock += 1; - } -} - -#[cfg(test)] -mod test { - use crate::util::{current_timestamp_sec, get_crc, ConcurrencyLimiter}; - use bytes::Bytes; - - #[test] - fn ticket_test() { - let limiter = ConcurrencyLimiter::new(2); - { - let ticket = limiter.try_acquire().unwrap(); - assert_eq!(2, ticket.id); - - let ticket = limiter.try_acquire().unwrap(); - assert_eq!(1, ticket.id); - - assert_eq!(true, limiter.try_acquire().is_none()); - } - - let ticket = limiter.try_acquire(); - assert_eq!(true, ticket.is_some()); - } - - #[test] - fn time_test() { - println!("{}", current_timestamp_sec()); - } - - #[test] - fn crc_test() { - let data = Bytes::from("hello world! hello china!"); - let crc_value = get_crc(&data); - // This value is the same with java's implementation - assert_eq!(3871485936, crc_value); - } - - #[test] - fn drop_test() {} -} diff --git a/rust/experimental/server/tests/graceful_shutdown.rs b/rust/experimental/server/tests/graceful_shutdown.rs deleted file mode 100644 index 54f09046b1..0000000000 --- a/rust/experimental/server/tests/graceful_shutdown.rs +++ /dev/null @@ -1,104 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#[cfg(test)] -mod test { - use std::time::Duration; - - use signal_hook::consts::SIGTERM; - use signal_hook::low_level::raise; - use tonic::transport::Channel; - - use uniffle_worker::config::{ - Config, HybridStoreConfig, LocalfileStoreConfig, MemoryStoreConfig, MetricsConfig, - StorageType, - }; - use uniffle_worker::proto::uniffle::shuffle_server_client::ShuffleServerClient; - use uniffle_worker::{start_uniffle_worker, write_read_for_one_time}; - - fn create_mocked_config(grpc_port: i32, capacity: String, local_data_path: String) -> Config { - Config { - memory_store: Some(MemoryStoreConfig::new(capacity)), - localfile_store: Some(LocalfileStoreConfig { - data_paths: vec![local_data_path], - healthy_check_min_disks: Some(0), - disk_high_watermark: None, - disk_low_watermark: None, - disk_max_concurrency: None, - }), - hybrid_store: Some(HybridStoreConfig::new(0.9, 0.5, None)), - hdfs_store: None, - store_type: Some(StorageType::MEMORY_LOCALFILE), - runtime_config: Default::default(), - metrics: Some(MetricsConfig { - push_gateway_endpoint: None, - push_interval_sec: None, - }), - grpc_port: Some(grpc_port), - coordinator_quorum: vec![], - tags: None, - log: None, - app_heartbeat_timeout_min: None, - huge_partition_marked_threshold: None, - huge_partition_memory_max_used_percent: None, - http_monitor_service_port: None, - } - } - - async fn get_data_from_remote( - _client: &ShuffleServerClient, - _app_id: &str, - _shuffle_id: i32, - _partitions: Vec, - ) { - } - - async fn start_embedded_worker(path: String, port: i32) { - let config = create_mocked_config(port, "1G".to_string(), path); - let _ = start_uniffle_worker(config).await; - tokio::time::sleep(Duration::from_secs(1)).await; - } - - #[tokio::test] - async fn graceful_shutdown_test_with_embedded_worker_successfully_shutdown() { - let temp_dir = tempdir::TempDir::new("test_write_read").unwrap(); - let temp_path = temp_dir.path().to_str().unwrap().to_string(); - println!("created the temp file path: {}", &temp_path); - - let port = 21101; - let _ = start_embedded_worker(temp_path, port).await; - - let client = - match ShuffleServerClient::connect(format!("http://{}:{}", "0.0.0.0", port)).await { - Ok(client) => client, - Err(e) => { - // Handle the error, e.g., by panicking or logging it. - panic!("Failed to connect: {}", e); - } - }; - - let jh = tokio::spawn(async move { write_read_for_one_time(client).await }); - - // raise shutdown signal - tokio::spawn(async { - raise(SIGTERM).expect("failed to raise shutdown signal"); - eprintln!("successfully raised shutdown signal"); - }); - - let _ = jh.await.expect("Task panicked or failed."); - } -} diff --git a/rust/experimental/server/tests/lock_benchmark.rs b/rust/experimental/server/tests/lock_benchmark.rs deleted file mode 100644 index 3b0fc3ee92..0000000000 --- a/rust/experimental/server/tests/lock_benchmark.rs +++ /dev/null @@ -1,138 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#[cfg(test)] -mod test { - use dashmap::DashMap; - use std::sync::Arc; - use std::time::Duration; - use tokio::sync::Mutex; - use tokio::time::Instant; - - struct Buffer { - idx: i64, - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 8)] - #[ignore] - async fn lock_test() -> anyhow::Result<()> { - // 100 requests(4000 insert in single batch) - - let concurrency = 1000; - let loop_size = 4000; - - let buffer_map: Arc>>> = Arc::new(DashMap::new()); - - let mut handlers = vec![]; - let timer = Instant::now(); - for _ in 0..concurrency { - let buffer = buffer_map.clone(); - handlers.push(tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(10)).await; - for k in 0..loop_size { - let buf = buffer - .entry(k.to_string()) - .or_insert_with(|| Arc::new(Mutex::new(Buffer { idx: 0 }))) - .clone(); - let mut buf = buf.lock().await; - buf.idx += 1; - } - })); - } - - for handler in handlers { - let _ = handler.await; - } - - println!( - "concurrent tokio lock time cost: {} ms", - timer.elapsed().as_millis() - 10 - ); - - let buffer_map: Arc>>> = Arc::new(DashMap::new()); - - let timer = Instant::now(); - for _ in 0..concurrency { - let buffer = buffer_map.clone(); - for k in 0..loop_size { - let buf = buffer - .entry(k.to_string()) - .or_insert_with(|| Arc::new(Mutex::new(Buffer { idx: 0 }))) - .clone(); - let mut buf = buf.lock().await; - buf.idx += 1; - } - } - - println!( - "sequence tokio lock time cost: {} ms", - timer.elapsed().as_millis() - 10 - ); - - let buffer_map: Arc>>> = - Arc::new(DashMap::new()); - - let timer = Instant::now(); - for _ in 0..concurrency { - let buffer = buffer_map.clone(); - for k in 0..loop_size { - let buf = buffer - .entry(k.to_string()) - .or_insert_with(|| Arc::new(std::sync::Mutex::new(Buffer { idx: 0 }))) - .clone(); - let mut buf = buf.lock().unwrap(); - buf.idx += 1; - } - } - - println!( - "sequence std lock time cost: {} ms", - timer.elapsed().as_millis() - 10 - ); - - let buffer_map: Arc>>> = - Arc::new(DashMap::new()); - - let mut handlers = vec![]; - let timer = Instant::now(); - for _ in 0..concurrency { - let buffer = buffer_map.clone(); - handlers.push(tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(10)).await; - for k in 0..loop_size { - let buf = buffer - .entry(k.to_string()) - .or_insert_with(|| Arc::new(std::sync::Mutex::new(Buffer { idx: 0 }))) - .clone(); - let mut buf = buf.lock().unwrap(); - buf.idx += 1; - } - })); - } - - for handler in handlers { - let _ = handler.await; - } - - println!( - "concurrent std lock time cost: {} ms", - timer.elapsed().as_millis() - 10 - ); - - Ok(()) - } -} diff --git a/rust/experimental/server/tests/write_read.rs b/rust/experimental/server/tests/write_read.rs deleted file mode 100644 index 09f29a682a..0000000000 --- a/rust/experimental/server/tests/write_read.rs +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#[cfg(test)] -mod tests { - use anyhow::Result; - use uniffle_worker::config::{ - Config, HybridStoreConfig, LocalfileStoreConfig, MemoryStoreConfig, MetricsConfig, - StorageType, - }; - use uniffle_worker::proto::uniffle::shuffle_server_client::ShuffleServerClient; - use uniffle_worker::{start_uniffle_worker, write_read_for_one_time}; - - use std::time::Duration; - use tonic::transport::Channel; - use uniffle_worker::metric::GAUGE_MEMORY_ALLOCATED; - - fn create_mocked_config(grpc_port: i32, capacity: String, local_data_path: String) -> Config { - Config { - memory_store: Some(MemoryStoreConfig::new(capacity)), - localfile_store: Some(LocalfileStoreConfig { - data_paths: vec![local_data_path], - healthy_check_min_disks: Some(0), - disk_high_watermark: None, - disk_low_watermark: None, - disk_max_concurrency: None, - }), - hybrid_store: Some(HybridStoreConfig::new(0.9, 0.5, None)), - hdfs_store: None, - store_type: Some(StorageType::MEMORY_LOCALFILE), - runtime_config: Default::default(), - metrics: Some(MetricsConfig { - push_gateway_endpoint: None, - push_interval_sec: None, - }), - grpc_port: Some(grpc_port), - coordinator_quorum: vec![], - tags: None, - log: None, - app_heartbeat_timeout_min: None, - huge_partition_marked_threshold: None, - huge_partition_memory_max_used_percent: None, - http_monitor_service_port: None, - } - } - - async fn get_data_from_remote( - _client: &ShuffleServerClient, - _app_id: &str, - _shuffle_id: i32, - _partitions: Vec, - ) { - } - - async fn start_embedded_worker(path: String, port: i32) { - let config = create_mocked_config(port, "1G".to_string(), path); - let _ = start_uniffle_worker(config).await; - tokio::time::sleep(Duration::from_secs(1)).await; - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn write_read_test_with_embedded_worker() -> Result<()> { - let temp_dir = tempdir::TempDir::new("test_write_read").unwrap(); - let temp_path = temp_dir.path().to_str().unwrap().to_string(); - println!("created the temp file path: {}", &temp_path); - - let port = 21101; - let _ = start_embedded_worker(temp_path, port).await; - - let client = ShuffleServerClient::connect(format!("http://{}:{}", "0.0.0.0", port)).await?; - - // after one batch write/read process, the allocated memory size should be 0 - assert_eq!(0, GAUGE_MEMORY_ALLOCATED.get()); - - write_read_for_one_time(client).await - } -}