From 1aae8ed3a7e72e32bb41e24f934b41268e5eff95 Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Fri, 18 Aug 2023 02:03:55 +0000 Subject: [PATCH 1/2] refactor: re-implement the slight shim with libcontainer instance Signed-off-by: jiaxiao zhou --- containerd-shim-slight-v1/Cargo.lock | 544 ++++++++++++++++------ containerd-shim-slight-v1/Cargo.toml | 11 +- containerd-shim-slight-v1/src/executor.rs | 80 ++++ containerd-shim-slight-v1/src/main.rs | 267 ++++------- images/slight/Dockerfile | 5 +- tests/workloads/workload.yaml | 1 - 6 files changed, 592 insertions(+), 316 deletions(-) create mode 100644 containerd-shim-slight-v1/src/executor.rs diff --git a/containerd-shim-slight-v1/Cargo.lock b/containerd-shim-slight-v1/Cargo.lock index e476132c..cdb39873 100644 --- a/containerd-shim-slight-v1/Cargo.lock +++ b/containerd-shim-slight-v1/Cargo.lock @@ -210,7 +210,7 @@ dependencies = [ "async-lock", "async-task", "concurrent-queue", - "fastrand", + "fastrand 1.9.0", "futures-lite", "slab", ] @@ -429,7 +429,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes 1.4.0", - "fastrand", + "fastrand 1.9.0", "http", "regex", "tokio-stream", @@ -460,7 +460,7 @@ dependencies = [ "aws-types", "bytes 1.4.0", "bytes-utils", - "fastrand", + "fastrand 1.9.0", "http", "http-body", "once_cell", @@ -603,7 +603,7 @@ dependencies = [ "aws-smithy-protocol-test", "aws-smithy-types", "bytes 1.4.0", - "fastrand", + "fastrand 1.9.0", "http", "http-body", "hyper", @@ -746,7 +746,7 @@ checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" dependencies = [ "async-trait", "axum-core", - "bitflags", + "bitflags 1.3.2", "bytes 1.4.0", "futures-util", "http", @@ -826,7 +826,7 @@ dependencies = [ "serde_json", "time 0.3.20", "url", - "uuid 1.3.2", + "uuid", ] [[package]] @@ -853,7 +853,7 @@ dependencies = [ "serde_json", "time 0.3.20", "url", - "uuid 1.3.2", + "uuid", ] [[package]] @@ -895,7 +895,7 @@ dependencies = [ "sha2 0.10.6", "time 0.3.20", "url", - "uuid 1.3.2", + "uuid", ] [[package]] @@ -918,7 +918,7 @@ dependencies = [ "sha2 0.10.6", "time 0.3.20", "url", - "uuid 1.3.2", + "uuid", ] [[package]] @@ -941,7 +941,7 @@ dependencies = [ "serde_json", "time 0.3.20", "url", - "uuid 1.3.2", + "uuid", ] [[package]] @@ -962,7 +962,7 @@ dependencies = [ "serde_json", "time 0.3.20", "url", - "uuid 1.3.2", + "uuid", ] [[package]] @@ -1029,6 +1029,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + [[package]] name = "block-buffer" version = "0.9.0" @@ -1057,7 +1063,7 @@ dependencies = [ "async-lock", "async-task", "atomic-waker", - "fastrand", + "fastrand 1.9.0", "futures-lite", "log", ] @@ -1228,6 +1234,31 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cgroups-rs" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3845d8ddaca63e9975f07b7a32262afe284561c2f0f620aa968913a65f671fd2" +dependencies = [ + "libc", + "log", + "nix 0.24.3", + "regex", +] + +[[package]] +name = "cgroups-rs" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fb3af90c8d48ad5f432d8afb521b5b40c2a2fce46dd60e05912de51c47fba64" +dependencies = [ + "libc", + "log", + "nix 0.25.1", + "regex", + "thiserror", +] + [[package]] name = "chrono" version = "0.4.24" @@ -1238,6 +1269,7 @@ dependencies = [ "js-sys", "num-integer", "num-traits", + "serde", "time 0.1.45", "wasm-bindgen", "winapi", @@ -1271,7 +1303,7 @@ checksum = "914c8c79fb560f238ef6429439a30023c862f7a28e688c58f7203f12b29970bd" dependencies = [ "anstream", "anstyle", - "bitflags", + "bitflags 1.3.2", "clap_lex", "strsim", ] @@ -1300,7 +1332,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ee4e061ea30800291ca09663878f3953840a69b08ce244b3e8b26e894d9f60f" dependencies = [ - "bitflags", + "bitflags 1.3.2", "uapi", ] @@ -1372,51 +1404,64 @@ checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" [[package]] name = "containerd-shim" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b7dc0083bae091806fe57ef508eada5a96244b41c6b8a555d1ef316f433b6de" +checksum = "34b838538bbc58599a085b1d9e525eb15eae53c48c756f3339dc61524f77891a" dependencies = [ + "cgroups-rs 0.2.11", "command-fds", "containerd-shim-protos", "go-flag", "lazy_static", "libc", "log", - "nix 0.23.2", - "oci-spec 0.5.8", + "mio", + "nix 0.26.2", + "oci-spec", + "os_pipe", + "page_size", "prctl", + "regex", "serde", "serde_derive", "serde_json", "signal-hook", "thiserror", "time 0.3.20", - "uuid 0.8.2", + "windows-sys 0.48.0", ] [[package]] name = "containerd-shim-protos" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "077ec778a0835d9d85502e8535362130187759b69eddabe2bdb3a68ffb575bd0" +checksum = "4dbbd1b58f3aa972bb5218a0a79c9e96939113f5db5cacbd15be5022e02ba2d5" dependencies = [ - "protobuf", + "protobuf 3.2.0", "ttrpc", + "ttrpc-codegen", ] [[package]] name = "containerd-shim-slight-v1" version = "0.8.0" dependencies = [ + "anyhow", "chrono", "clap", "containerd-shim", "containerd-shim-wasm", "curl", "isahc 1.7.2", + "libc", + "libcontainer", "log", + "nix 0.26.2", + "oci-spec", "openssl", "openssl-sys", + "serde", + "serde_json", "slight", "tokio", "tokio-util", @@ -1428,22 +1473,23 @@ dependencies = [ [[package]] name = "containerd-shim-wasm" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2671b0d99b6656f32f8ff4e6d228ad547f199fb943f545fd7bfb36e666a57e72" +version = "0.2.1" +source = "git+https://github.com/containerd/runwasi?rev=6287dff637b0ac96a43a6bbe2c7919f8b2f2cf27#6287dff637b0ac96a43a6bbe2c7919f8b2f2cf27" dependencies = [ "anyhow", "caps", + "cgroups-rs 0.3.3", "chrono", "clone3", "command-fds", "containerd-shim", "libc", + "libcontainer", "log", "nix 0.26.2", - "oci-spec 0.6.0", + "oci-spec", "proc-mounts", - "protobuf", + "protobuf 3.2.0", "serde", "serde_json", "thiserror", @@ -1859,12 +1905,14 @@ dependencies = [ ] [[package]] -name = "derive_builder" -version = "0.11.2" +name = "derive-new" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3" +checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" dependencies = [ - "derive_builder_macro 0.11.2", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -1873,19 +1921,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" dependencies = [ - "derive_builder_macro 0.12.0", -] - -[[package]] -name = "derive_builder_core" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 1.0.109", + "derive_builder_macro", ] [[package]] @@ -1900,23 +1936,13 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "derive_builder_macro" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" -dependencies = [ - "derive_builder_core 0.11.2", - "syn 1.0.109", -] - [[package]] name = "derive_builder_macro" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" dependencies = [ - "derive_builder_core 0.12.0", + "derive_builder_core", "syn 1.0.109", ] @@ -2087,7 +2113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4319dc0fb739a6e84cb8678b8cf50c9bcfa4712ae826b33ecf00cc0850550a58" dependencies = [ "http", - "prost", + "prost 0.11.9", "tokio", "tokio-stream", "tonic", @@ -2129,6 +2155,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + [[package]] name = "fd-lock" version = "3.0.12" @@ -2162,6 +2194,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "fixedbitset" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" + [[package]] name = "fixedbitset" version = "0.4.2" @@ -2292,6 +2330,7 @@ dependencies = [ "futures-core", "futures-task", "futures-util", + "num_cpus", ] [[package]] @@ -2306,7 +2345,7 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ - "fastrand", + "fastrand 1.9.0", "futures-core", "futures-io", "memchr", @@ -2951,9 +2990,52 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.142" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "libcgroups" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f6fef16f505466473eeeee906244e03a437beaf41ccd85c39355b4077890c9" +dependencies = [ + "fixedbitset 0.4.2", + "nix 0.26.2", + "oci-spec", + "procfs", + "serde", + "thiserror", + "tracing", +] + +[[package]] +name = "libcontainer" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac48a05819bd5bd31390bd1874f5a94f711c248677fc908801de4789bdd1fbad" +dependencies = [ + "bitflags 2.4.0", + "caps", + "chrono", + "clone3", + "fastrand 2.0.0", + "futures", + "libc", + "libcgroups", + "nix 0.26.2", + "oci-spec", + "once_cell", + "prctl", + "procfs", + "regex", + "rust-criu", + "safe-path", + "serde", + "serde_json", + "thiserror", + "tracing", +] [[package]] name = "libmosquitto-sys" @@ -3051,7 +3133,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] @@ -3207,7 +3289,7 @@ dependencies = [ "base64-url", "blocking", "crossbeam-channel", - "fastrand", + "fastrand 1.9.0", "itoa", "json", "lazy_static", @@ -3239,7 +3321,7 @@ version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4916f159ed8e5de0082076562152a76b7a1f64a01fd9d1e0fea002c37624faf" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cc", "cfg-if 1.0.0", "libc", @@ -3248,15 +3330,25 @@ dependencies = [ [[package]] name = "nix" -version = "0.23.2" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ - "bitflags", - "cc", + "bitflags 1.3.2", + "cfg-if 1.0.0", + "libc", +] + +[[package]] +name = "nix" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" +dependencies = [ + "autocfg", + "bitflags 1.3.2", "cfg-if 1.0.0", "libc", - "memoffset 0.6.5", ] [[package]] @@ -3265,7 +3357,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if 1.0.0", "libc", "memoffset 0.7.1", @@ -3360,24 +3452,11 @@ dependencies = [ [[package]] name = "oci-spec" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98135224dd4faeb24c05a2fac911ed53ea6b09ecb09d7cada1cb79963ab2ee34" -dependencies = [ - "derive_builder 0.11.2", - "getset", - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "oci-spec" -version = "0.6.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214b837f7dde5026f2028ead5ae720073277c19f82ff85623b142c39d4b843e7" +checksum = "9421b067205c68dc80af7c68599a9c1eb113f975aafeb874cea7f4d5d41ce3fb" dependencies = [ - "derive_builder 0.12.0", + "derive_builder", "getset", "serde", "serde_json", @@ -3386,9 +3465,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "opaque-debug" @@ -3402,7 +3481,7 @@ version = "0.10.52" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if 1.0.0", "foreign-types", "libc", @@ -3450,6 +3529,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "os_pipe" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae859aa07428ca9a929b936690f8b12dc5f11dd8c6992a18ca93919f28bc177" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "output_vt100" version = "0.1.3" @@ -3471,6 +3560,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "page_size" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b7663cbd190cfd818d08efa8497f6cd383076688c49a391ef7c0d03cd12b561" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking" version = "2.1.0" @@ -3530,13 +3629,23 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +[[package]] +name = "petgraph" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" +dependencies = [ + "fixedbitset 0.2.0", + "indexmap", +] + [[package]] name = "petgraph" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ - "fixedbitset", + "fixedbitset 0.4.2", "indexmap", ] @@ -3615,7 +3724,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" dependencies = [ "autocfg", - "bitflags", + "bitflags 1.3.2", "cfg-if 1.0.0", "concurrent-queue", "libc", @@ -3764,6 +3873,31 @@ dependencies = [ "partition-identity", ] +[[package]] +name = "procfs" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943ca7f9f29bab5844ecd8fdb3992c5969b6622bb9609b9502fef9b4310e3f1f" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "chrono", + "flate2", + "hex", + "lazy_static", + "rustix 0.36.13", +] + +[[package]] +name = "prost" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020" +dependencies = [ + "bytes 1.4.0", + "prost-derive 0.8.0", +] + [[package]] name = "prost" version = "0.11.9" @@ -3771,7 +3905,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ "bytes 1.4.0", - "prost-derive", + "prost-derive 0.11.9", +] + +[[package]] +name = "prost-build" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603" +dependencies = [ + "bytes 1.4.0", + "heck 0.3.3", + "itertools", + "log", + "multimap", + "petgraph 0.5.1", + "prost 0.8.0", + "prost-types 0.8.0", + "tempfile", + "which", ] [[package]] @@ -3786,16 +3938,29 @@ dependencies = [ "lazy_static", "log", "multimap", - "petgraph", + "petgraph 0.6.3", "prettyplease", - "prost", - "prost-types", + "prost 0.11.9", + "prost-types 0.11.9", "regex", "syn 1.0.109", "tempfile", "which", ] +[[package]] +name = "prost-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "600d2f334aa05acb02a755e217ef1ab6dea4d51b58b7846588b747edec04efba" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "prost-derive" version = "0.11.9" @@ -3809,13 +3974,23 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-types" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" +dependencies = [ + "bytes 1.4.0", + "prost 0.8.0", +] + [[package]] name = "prost-types" version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" dependencies = [ - "prost", + "prost 0.11.9", ] [[package]] @@ -3824,23 +3999,64 @@ version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +[[package]] +name = "protobuf" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55bad9126f378a853655831eb7363b7b01b81d19f8cb1218861086ca4a1a61e" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror", +] + [[package]] name = "protobuf-codegen" version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "033460afb75cf755fcfc16dfaed20b86468082a2ea24e05ac35ab4a099a017d6" dependencies = [ - "protobuf", + "protobuf 2.28.0", ] [[package]] -name = "protobuf-codegen-pure" -version = "2.28.0" +name = "protobuf-codegen" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd418ac3c91caa4032d37cb80ff0d44e2ebe637b2fb243b6234bf89cdac4901" +dependencies = [ + "anyhow", + "once_cell", + "protobuf 3.2.0", + "protobuf-parse", + "regex", + "tempfile", + "thiserror", +] + +[[package]] +name = "protobuf-parse" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d39b14605eaa1f6a340aec7f320b34064feb26c93aec35d6a9a2272a8ddfa49" +dependencies = [ + "anyhow", + "indexmap", + "log", + "protobuf 3.2.0", + "protobuf-support", + "tempfile", + "thiserror", + "which", +] + +[[package]] +name = "protobuf-support" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a29399fc94bcd3eeaa951c715f7bea69409b2445356b00519740bcd6ddd865" +checksum = "a5d4d7b8601c814cfb36bcebb79f0e61e45e1e93640cf778837833bbed05c372" dependencies = [ - "protobuf", - "protobuf-codegen", + "thiserror", ] [[package]] @@ -3858,7 +4074,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffade02495f22453cd593159ea2f59827aae7f53fa8323f756799b670881dcf8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "memchr", "unicase", ] @@ -3995,7 +4211,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -4004,7 +4220,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -4032,13 +4248,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.1", + "regex-automata 0.3.6", + "regex-syntax 0.7.4", ] [[package]] @@ -4050,6 +4267,17 @@ dependencies = [ "regex-syntax 0.6.29", ] +[[package]] +name = "regex-automata" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.4", +] + [[package]] name = "regex-syntax" version = "0.6.29" @@ -4058,9 +4286,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.1" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "reqwest" @@ -4148,6 +4376,18 @@ dependencies = [ "xmlparser", ] +[[package]] +name = "rust-criu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4737b28406b3395359f485127073117a11cedc8942738b69ba6ab9a79432acbc" +dependencies = [ + "anyhow", + "libc", + "protobuf 3.2.0", + "protobuf-codegen 3.2.0", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -4178,7 +4418,7 @@ version = "0.33.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "938a344304321a9da4973b9ff4f9f8db9caf4597dfd9dda6a60b523340a0fff0" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno 0.2.8", "io-lifetimes 0.5.3", "itoa", @@ -4194,7 +4434,7 @@ version = "0.36.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a38f9520be93aba504e8ca974197f46158de5dcaa9fa04b57c57cd6a679d658" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno 0.3.1", "io-lifetimes 1.0.10", "libc", @@ -4208,7 +4448,7 @@ version = "0.37.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bbfc1d1c7c40c01715f47d71444744a81669ca84e8b63e25a55e169b1f86433" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno 0.3.1", "io-lifetimes 1.0.10", "itoa", @@ -4297,6 +4537,15 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "safe-path" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "980abdd3220aa19b67ca3ea07b173ca36383f18ae48cde696d90c8af39447ffb" +dependencies = [ + "libc", +] + [[package]] name = "schannel" version = "0.1.21" @@ -4344,7 +4593,7 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -4816,7 +5065,7 @@ dependencies = [ "tokio", "tracing", "url", - "uuid 1.3.2", + "uuid", "wit-bindgen-wasmtime 0.2.0 (git+https://github.com/fermyon/wit-bindgen-backport)", "wit-error-rs", ] @@ -5087,7 +5336,7 @@ version = "0.25.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "928ebd55ab758962e230f51ca63735c5b283f26292297c81404289cda5d78631" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cap-fs-ext", "cap-std 1.0.14", "fd-lock", @@ -5121,7 +5370,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if 1.0.0", - "fastrand", + "fastrand 1.9.0", "redox_syscall 0.3.5", "rustix 0.37.18", "windows-sys 0.45.0", @@ -5432,8 +5681,8 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", - "prost-derive", + "prost 0.11.9", + "prost-derive 0.11.9", "tokio", "tokio-stream", "tokio-util", @@ -5452,7 +5701,7 @@ checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" dependencies = [ "prettyplease", "proc-macro2", - "prost-build", + "prost-build 0.11.9", "quote", "syn 1.0.109", ] @@ -5570,17 +5819,45 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "ttrpc" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ecfff459a859c6ba6668ff72b34c2f1d94d9d58f7088414c2674ad0f31cc7d8" +checksum = "adb03d0f5219ec54d870cb3d58719a2dc0b8849405b75a2e0968b3590392a5b0" dependencies = [ "byteorder", "libc", "log", - "nix 0.23.2", - "protobuf", - "protobuf-codegen-pure", + "nix 0.26.2", + "protobuf 3.2.0", + "protobuf-codegen 3.2.0", "thiserror", + "windows-sys 0.48.0", +] + +[[package]] +name = "ttrpc-codegen" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d7f7631d7a9ebed715a47cd4cb6072cbc7ae1d4ec01598971bbec0024340c2" +dependencies = [ + "protobuf 2.28.0", + "protobuf-codegen 3.2.0", + "protobuf-support", + "ttrpc-compiler", +] + +[[package]] +name = "ttrpc-compiler" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3cb5dbf1f0865a34fe3f722290fe776cacb16f50428610b779467b76ddf647" +dependencies = [ + "derive-new", + "prost 0.8.0", + "prost-build 0.8.0", + "prost-types 0.8.0", + "protobuf 2.28.0", + "protobuf-codegen 2.28.0", + "tempfile", ] [[package]] @@ -5703,15 +5980,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" -[[package]] -name = "uuid" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" -dependencies = [ - "getrandom 0.2.9", -] - [[package]] name = "uuid" version = "1.3.2" @@ -5820,7 +6088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "008136464e438c5049a614b6ea1bae9f6c4d354ce9ee2b4d9a1ac6e73f31aafc" dependencies = [ "anyhow", - "bitflags", + "bitflags 1.3.2", "cap-rand", "cap-std 1.0.14", "io-extras 0.17.4", @@ -6289,7 +6557,7 @@ checksum = "6b16a7462893c46c6d3dd2a1f99925953bdbb921080606e1a4c9344864492fa4" dependencies = [ "anyhow", "async-trait", - "bitflags", + "bitflags 1.3.2", "thiserror", "tracing", "wasmtime", @@ -6534,7 +6802,7 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d5973cb8cd94a77d03ad7e23bbe14889cb29805da1cec0e4aff75e21aebded" dependencies = [ - "bitflags", + "bitflags 1.3.2", "io-lifetimes 0.5.3", "winapi", ] @@ -6545,7 +6813,7 @@ version = "0.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c52a121f0fbf9320d5f2a9a5d82f6cb7557eda5e8b47fc3e7f359ec866ae960" dependencies = [ - "bitflags", + "bitflags 1.3.2", "io-lifetimes 1.0.10", "windows-sys 0.48.0", ] @@ -6613,7 +6881,7 @@ source = "git+https://github.com/danbugs/wit-bindgen?branch=backport-http-server dependencies = [ "anyhow", "async-trait", - "bitflags", + "bitflags 1.3.2", "thiserror", "wasmtime", "wit-bindgen-wasmtime-impl 0.2.0 (git+https://github.com/danbugs/wit-bindgen?branch=backport-http-server)", @@ -6626,7 +6894,7 @@ source = "git+https://github.com/fermyon/wit-bindgen-backport#ba1636af0338623b54 dependencies = [ "anyhow", "async-trait", - "bitflags", + "bitflags 1.3.2", "thiserror", "wasmtime", "wit-bindgen-wasmtime-impl 0.2.0 (git+https://github.com/fermyon/wit-bindgen-backport)", diff --git a/containerd-shim-slight-v1/Cargo.toml b/containerd-shim-slight-v1/Cargo.toml index 27a4ded7..4041d8bb 100644 --- a/containerd-shim-slight-v1/Cargo.toml +++ b/containerd-shim-slight-v1/Cargo.toml @@ -13,8 +13,8 @@ Containerd shim for running Slight workloads. [dependencies] chrono = "0.4" clap = { version = "4.1", features = ["derive", "env"] } -containerd-shim = "0.3" -containerd-shim-wasm = "0.1" +containerd-shim = "0.4.0" +containerd-shim-wasm = { git = "https://github.com/containerd/runwasi", rev = "6287dff637b0ac96a43a6bbe2c7919f8b2f2cf27", features = ["cgroupsv2"]} log = "0.4" tokio = { version = "1", features = [ "full" ] } tokio-util = { version = "0.7", features = [ "codec" ]} @@ -24,6 +24,13 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing = { version = "0.1", features = ["log"] } slight = { git = "https://github.com/deislabs/spiderlightning", version = "0.5.0" } openssl = { version = "0.10", features = ["vendored"] } +libcontainer = { version = "0.1", features = ["v2"], default-features = false } +oci-spec = "0.6.2" +libc = "0.2.147" +nix = "0.26.2" +anyhow = "1.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" [dependencies.openssl-sys] features = ["vendored"] diff --git a/containerd-shim-slight-v1/src/executor.rs b/containerd-shim-slight-v1/src/executor.rs new file mode 100644 index 00000000..51d4e1c2 --- /dev/null +++ b/containerd-shim-slight-v1/src/executor.rs @@ -0,0 +1,80 @@ +use anyhow::Result; +use log::info; +use nix::unistd::{dup, dup2}; +use std::{os::fd::RawFd, path::PathBuf}; +use tokio::runtime::Runtime; + +use containerd_shim_wasm::sandbox::oci; +use libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; +use libcontainer::workload::{Executor, ExecutorError}; +use oci_spec::runtime::Spec; +use slight_lib::commands::run::{handle_run, RunArgs}; + +const EXECUTOR_NAME: &str = "slight"; + +pub struct SlightExecutor { + pub stdin: Option, + pub stdout: Option, + pub stderr: Option, +} + +impl SlightExecutor {} + +impl Executor for SlightExecutor { + fn exec(&self, spec: &Spec) -> Result<(), ExecutorError> { + let args = oci::get_args(spec); + if args.is_empty() { + return Err(ExecutorError::InvalidArg); + } + + let mod_path = PathBuf::from("/slightfile.toml"); + let wasm_path = PathBuf::from("/app.wasm"); + + prepare_stdio(self.stdin, self.stdout, self.stderr).map_err(|err| { + ExecutorError::Other(format!("failed to prepare stdio for container: {}", err)) + })?; + + let rt = Runtime::new().unwrap(); + let args = RunArgs { + module: wasm_path, + slightfile: PathBuf::from(&mod_path), + io_redirects: None, + link_all_capabilities: true, + }; + rt.block_on(async { + let f = handle_run(args); + info!(" >>> notifying main thread we are about to start"); + tokio::select! { + _ = f => { + log::info!(" >>> server shut down: exiting"); + std::process::exit(0); + }, + }; + }); + std::process::exit(137); + } + + fn can_handle(&self, _spec: &Spec) -> bool { + true + } + + fn name(&self) -> &'static str { + EXECUTOR_NAME + } +} + +fn prepare_stdio(stdin: Option, stdout: Option, stderr: Option) -> Result<()> { + if let Some(stdin) = stdin { + dup(STDIN_FILENO)?; + dup2(stdin, STDIN_FILENO)?; + } + if let Some(stdout) = stdout { + dup(STDOUT_FILENO)?; + dup2(stdout, STDOUT_FILENO)?; + } + if let Some(stderr) = stderr { + dup(STDERR_FILENO)?; + dup2(stderr, STDERR_FILENO)?; + } + Ok(()) +} diff --git a/containerd-shim-slight-v1/src/main.rs b/containerd-shim-slight-v1/src/main.rs index 1b4bf347..b45fb791 100644 --- a/containerd-shim-slight-v1/src/main.rs +++ b/containerd-shim-slight-v1/src/main.rs @@ -1,24 +1,29 @@ +use std::fs::File; +use std::io::ErrorKind; +use std::io::Read; +use std::option::Option; use std::path::Path; use std::path::PathBuf; -use std::sync::mpsc::channel; -use std::sync::mpsc::Sender; -use std::sync::Arc; -use std::sync::{Condvar, Mutex}; -use std::thread; -use chrono::{DateTime, Utc}; +use anyhow::{Context, Result}; use containerd_shim as shim; -use containerd_shim_wasm::sandbox::error::Error; -use containerd_shim_wasm::sandbox::instance::EngineGetter; -use containerd_shim_wasm::sandbox::oci; -use containerd_shim_wasm::sandbox::Instance; -use containerd_shim_wasm::sandbox::{instance::InstanceConfig, ShimCli}; -use log::info; +use containerd_shim_wasm::libcontainer_instance::LibcontainerInstance; +use containerd_shim_wasm::libcontainer_instance::LinuxContainerExecutor; +use containerd_shim_wasm::sandbox::instance::ExitCode; +use containerd_shim_wasm::sandbox::instance_utils::maybe_open_stdio; +use containerd_shim_wasm::sandbox::{error::Error, InstanceConfig, ShimCli}; +use executor::SlightExecutor; +use libcontainer::container::builder::ContainerBuilder; +use libcontainer::container::Container; +use libcontainer::syscall::syscall::create_syscall; +use serde::Deserialize; +use serde::Serialize; +use std::os::fd::IntoRawFd; + +mod executor; + +static DEFAULT_CONTAINER_ROOT_DIR: &str = "/run/containerd/slight"; -use slight_lib::commands::run::{handle_run, IORedirects, RunArgs}; -use tokio::runtime::Runtime; - -type ExitCode = Arc<(Mutex)>>, Condvar)>; pub struct Wasi { exit_code: ExitCode, id: String, @@ -26,186 +31,102 @@ pub struct Wasi { stdout: String, stderr: String, bundle: String, - shutdown_signal: Arc<(Mutex, Condvar)>, + rootdir: PathBuf, } -pub fn prepare_module(bundle: String) -> Result<(PathBuf, PathBuf), Error> { - let mut spec = oci::load(Path::new(&bundle).join("config.json").to_str().unwrap()) - .expect("unable to load OCI bundle"); - - spec.canonicalize_rootfs(&bundle) - .map_err(|err| Error::Others(format!("could not canonicalize rootfs: {err}")))?; - - let working_dir = oci::get_root(&spec); - - // TODO(DJ): use something like https://github.com/containerd/runwasi/pull/78 rather than chroot. - // - // change the working directory to the rootfs - // std::os::unix::fs::chroot(working_dir).unwrap(); - std::env::set_current_dir(working_dir).unwrap(); - - // add env to current proc - let env = spec.process().as_ref().unwrap().env().as_ref().unwrap(); - for v in env { - match v.split_once('=') { - None => {} - Some(t) => std::env::set_var(t.0, t.1), - }; - } +#[derive(Serialize, Deserialize)] +struct Options { + root: Option, +} - let mod_path = PathBuf::from("slightfile.toml"); - let wasm_path = PathBuf::from("app.wasm"); - Ok((wasm_path, mod_path)) +fn determine_rootdir>(bundle: P, namespace: String) -> Result { + log::info!( + "determining rootdir for bundle: {}", + bundle.as_ref().display() + ); + let mut file = match File::open(bundle.as_ref().join("options.json")) { + Ok(f) => f, + Err(err) => match err.kind() { + ErrorKind::NotFound => { + return Ok(<&str as Into>::into(DEFAULT_CONTAINER_ROOT_DIR).join(namespace)) + } + _ => return Err(err.into()), + }, + }; + let mut data = String::new(); + file.read_to_string(&mut data)?; + let options: Options = serde_json::from_str(&data)?; + let path = options + .root + .unwrap_or(PathBuf::from(DEFAULT_CONTAINER_ROOT_DIR)) + .join(namespace); + log::info!("youki root path is: {}", path.display()); + Ok(path) } -impl Instance for Wasi { - type E = (); - fn new(id: String, cfg: Option<&InstanceConfig>) -> Self { - info!(">>> new instance"); +impl LibcontainerInstance for Wasi { + type Engine = (); + fn new_libcontainer(id: String, cfg: Option<&InstanceConfig>) -> Self { + log::info!(">>> new instance"); let cfg = cfg.unwrap(); + let bundle = cfg.get_bundle().unwrap_or_default(); + let rootdir = determine_rootdir(bundle.as_str(), cfg.get_namespace()).unwrap(); Wasi { - exit_code: Arc::new((Mutex::new(None), Condvar::new())), + exit_code: Default::default(), id, stdin: cfg.get_stdin().unwrap(), stdout: cfg.get_stdout().unwrap(), stderr: cfg.get_stderr().unwrap(), - bundle: cfg.get_bundle().unwrap_or_default(), - shutdown_signal: Arc::new((Mutex::new(false), Condvar::new())), + bundle, + rootdir, } } - fn start(&self) -> Result { - info!(">>> shim starts"); - let exit_code = self.exit_code.clone(); - let shutdown_signal = self.shutdown_signal.clone(); - let (tx, rx) = channel::>(); - let bundle = self.bundle.clone(); - let pod_stdin = self.stdin.clone(); - let pod_stdout = self.stdout.clone(); - let pod_stderr = self.stderr.clone(); - - thread::Builder::new() - .name(self.id.clone()) - .spawn(move || { - let (wasm_path, mod_path) = match prepare_module(bundle) { - Ok(f) => f, - Err(err) => { - tx.send(Err(err)).unwrap(); - return; - } - }; - - info!(" >>> loading module: {}", mod_path.display()); - info!(" >>> wasm path: {}", wasm_path.display()); - info!(" >>> starting slight"); - - let rt = Runtime::new().unwrap(); - rt.block_on(async { - let rx_future = tokio::task::spawn_blocking(move || { - let (lock, cvar) = &*shutdown_signal; - let mut shutdown = lock.lock().unwrap(); - while !*shutdown { - shutdown = cvar.wait(shutdown).unwrap(); - } - }); - - let args = RunArgs { - module: wasm_path, - slightfile: PathBuf::from(&mod_path), - io_redirects: Some(IORedirects { - stdin_path: Some(PathBuf::from(pod_stdin)), - stdout_path: Some(PathBuf::from(pod_stdout)), - stderr_path: Some(PathBuf::from(pod_stderr)), - }), - link_all_capabilities: true, - }; - let f = handle_run(args); - - info!(" >>> notifying main thread we are about to start"); - tx.send(Ok(())).unwrap(); - tokio::select! { - res = f => { - log::info!(" >>> server shut down: exiting"); - if res.is_err() { - log::error!(" >>> error: {:?}", res); - } - let (lock, cvar) = &*exit_code; - let mut ec = lock.lock().unwrap(); - *ec = Some((137, Utc::now())); - cvar.notify_all(); - }, - _ = rx_future => { - log::info!(" >>> user requested shutdown: exiting"); - let (lock, cvar) = &*exit_code; - let mut ec = lock.lock().unwrap(); - *ec = Some((0, Utc::now())); - cvar.notify_all(); - }, - }; - }) - })?; - - info!(" >>> waiting for start notification"); - match rx.recv().unwrap() { - Ok(_) => { - info!(" >>> started the instance"); - } - Err(err) => { - info!(" >>> error starting instance: {}", err); - let code = self.exit_code.clone(); - let (lock, cvar) = &*code; - let mut ec = lock.lock().unwrap(); - *ec = Some((139, Utc::now())); - cvar.notify_all(); - return Err(err); - } - } - - Ok(1) // TODO: PID: I wanted to use a thread ID here, but threads use a u64, the API wants a u32 + fn get_exit_code(&self) -> ExitCode { + self.exit_code.clone() } - fn kill(&self, signal: u32) -> Result<(), Error> { - if signal != 9 && signal != 2 { - return Err(Error::InvalidArgument( - "only SIGKILL and SIGINT are supported".to_string(), - )); - } - - let (lock, cvar) = &*self.shutdown_signal; - let mut shutdown = lock.lock().unwrap(); - *shutdown = true; - cvar.notify_all(); - - Ok(()) + fn get_id(&self) -> String { + self.id.clone() } - fn delete(&self) -> Result<(), Error> { - Ok(()) + fn get_root_dir(&self) -> std::result::Result { + Ok(self.rootdir.clone()) } - fn wait(&self, channel: Sender<(u32, DateTime)>) -> Result<(), Error> { - let code = self.exit_code.clone(); - thread::spawn(move || { - let (lock, cvar) = &*code; - let mut exit = lock.lock().unwrap(); - while (*exit).is_none() { - exit = cvar.wait(exit).unwrap(); - } - let ec = (*exit).unwrap(); - channel.send(ec).unwrap(); + fn build_container(&self) -> std::result::Result { + let syscall = create_syscall(); + let stdin = maybe_open_stdio(&self.stdin) + .context("could not open stdin")? + .map(|f| f.into_raw_fd()); + let stdout = maybe_open_stdio(&self.stdout) + .context("could not open stdout")? + .map(|f| f.into_raw_fd()); + let stderr = maybe_open_stdio(&self.stderr) + .context("could not open stderr")? + .map(|f| f.into_raw_fd()); + let err_others = |err| Error::Others(format!("failed to create container: {}", err)); + let spin_executor = Box::new(SlightExecutor { + stdin, + stdout, + stderr, }); - - Ok(()) - } -} - -impl EngineGetter for Wasi { - type E = (); - fn new_engine() -> Result { - Ok(()) + let default_executor = Box::::default(); + + let container = ContainerBuilder::new(self.id.clone(), syscall.as_ref()) + .with_executor(vec![default_executor, spin_executor]) + .map_err(err_others)? + .with_root_path(self.rootdir.clone()) + .map_err(err_others)? + .as_init(&self.bundle) + .with_systemd(false) + .with_detach(true) + .build() + .map_err(err_others)?; + Ok(container) } } fn main() { - shim::run::>("io.containerd.slight.v1", None); + shim::run::>("io.containerd.slight.v1", None); } diff --git a/images/slight/Dockerfile b/images/slight/Dockerfile index a44bdde1..b66bea3e 100644 --- a/images/slight/Dockerfile +++ b/images/slight/Dockerfile @@ -5,6 +5,7 @@ RUN rustup target add wasm32-wasi && cargo build --target wasm32-wasi --release RUN apt-get update && apt-get install ca-certificates -y FROM scratch -COPY --from=build /opt/build/target/wasm32-wasi/release/http_server_lib.wasm ./app.wasm +COPY --from=build --chmod=0755 /opt/build/target/wasm32-wasi/release/http_server_lib.wasm ./app.wasm COPY --from=build /opt/build/slightfile.toml . -COPY --from=build /etc/ssl /etc/ssl \ No newline at end of file +COPY --from=build /etc/ssl /etc/ssl +ENTRYPOINT ["/app.wasm"] \ No newline at end of file diff --git a/tests/workloads/workload.yaml b/tests/workloads/workload.yaml index 9df958ac..611cae26 100644 --- a/tests/workloads/workload.yaml +++ b/tests/workloads/workload.yaml @@ -17,7 +17,6 @@ spec: - name: testwasm image: docker.io/library/slight-hello-world:latest imagePullPolicy: Never # prevent k8s from pulling the image from a registry - command: ["/"] resources: # limit the resources to 128Mi of memory and 100m of CPU limits: cpu: 100m From 6105fb535907a5fceae14d01fec8e8825e51ff3b Mon Sep 17 00:00:00 2001 From: jiaxiao zhou Date: Sun, 20 Aug 2023 22:57:57 +0000 Subject: [PATCH 2/2] split python setup and tearmdown scripts to bash scripts this allows the CI and makefile to have more configurations Signed-off-by: jiaxiao zhou --- .github/workflows/build.yaml | 3 + .github/workflows/ci.yaml | 19 +++---- Makefile | 27 ++++++++- scripts/check-bins.sh | 41 ++++++++++++++ scripts/down.sh | 15 +++++ scripts/move-bins.sh | 46 ++++++++++++++++ scripts/pod-status-check.sh | 21 +++++++ scripts/up.sh | 29 ++++++++++ scripts/workloads.sh | 26 +++++++++ tests/setup.py | 103 ----------------------------------- tests/teardown.py | 12 ---- 11 files changed, 212 insertions(+), 130 deletions(-) create mode 100755 scripts/check-bins.sh create mode 100755 scripts/down.sh create mode 100755 scripts/move-bins.sh create mode 100755 scripts/pod-status-check.sh create mode 100755 scripts/up.sh create mode 100755 scripts/workloads.sh delete mode 100644 tests/setup.py delete mode 100644 tests/teardown.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c0445470..d7458aca 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -35,6 +35,9 @@ jobs: - name: build run: | VERBOSE=1 make build + - name: unit tests + run: | + VERBOSE=1 make unit-tests - name: lowercase the runner OS name shell: bash run: | diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f31d327b..88ae9e58 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,27 +26,22 @@ jobs: build: uses: ./.github/workflows/build.yaml test: + needs: build runs-on: ubuntu-latest env: ARCH: x86_64 steps: - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - with: - workspaces: | - "containerd-shim-*-v1 -> target" - - name: "Install Rust Wasm targets" + - uses: actions/download-artifact@v3 + - name: Extract containerd-wasm-shims-v1-linux-${{ env.ARCH }} run: | - make install-rust-targets - - name: "Install dependencies" - run: | - sudo apt-get update - sudo apt-get install protobuf-compiler -y + mkdir -p ./bin + tar -xzf containerd-wasm-shims-v1-linux-${{ env.ARCH }}/containerd-wasm-shims-v1-linux-${{ env.ARCH }}.tar.gz -C ./bin - name: install k3d run: make install-k3d working-directory: ./deployments/k3d - name: run integration tests - run: make test + run: BIN_DIR="./bin" make integration-tests - name: clean up k3d if: always() - run: make test/clean \ No newline at end of file + run: make test/clean diff --git a/Makefile b/Makefile index 8b31649a..e2b2dfd9 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,8 @@ else VERBOSE_FLAG := -vvv endif +BIN_DIR ?= + .PHONY: test test: unit-tests integration-tests @@ -23,14 +25,33 @@ test: unit-tests integration-tests unit-tests: build $(foreach shim,$(SHIMS),cross test --release --manifest-path=containerd-shim-$(shim)-v1/Cargo.toml --target $(TARGET);) +.PHONY: check-bins +check-bins: + ./scripts/check-bins.sh + +./PHONY: move-bins +move-bins: + ./scripts/move-bins.sh $(BIN_DIR) + +./PHONY: up +up: + ./scripts/up.sh + +./PHONY: pod-status-check +pod-status-check: + ./scripts/pod-status-check.sh + +./PHONY: workloads +workloads: + ./scripts/workloads.sh + .PHONY: integration-tests -integration-tests: build - $(PYTHON) tests/setup.py $(TARGET) +integration-tests: install-cross check-bins move-bins up pod-status-check workloads cargo test -- --nocapture .PHONY: tests/clean test/clean: - $(PYTHON) tests/teardown.py + ./scripts/down.sh .PHONY: fmt fmt: diff --git a/scripts/check-bins.sh b/scripts/check-bins.sh new file mode 100755 index 00000000..1d72fcd8 --- /dev/null +++ b/scripts/check-bins.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Description: +# This script checks for the existence of specific binaries on the system. +# It uses a function called which_binary to accomplish this. +# The script first verifies the existence of the binaries and then prints their paths. + +# Usage: +# ./check-bins.sh + +# Dependencies: +# The script expects the following binaries to be present in the system's PATH: +# k3d, cross, docker, kubectl + +set -euo pipefail + +# Function: which_binary +# Description: +# Finds and prints the path of the specified binary if it exists in the system's PATH. +# If the binary is not found, it prints an error message. +# Parameters: +# $1 - The name of the binary to locate. +which_binary() { + local binary_name="$1" + local binary_path + binary_path=$(command -v "$binary_name") + if [[ -n "$binary_path" ]]; then + echo "$binary_path" + else + echo "Could not find $binary_name" >&2 + exit 1 + fi +} + + +# List of binary names +binaries=("k3d" "cross" "docker" "kubectl") + +for binary in "${binaries[@]}"; do + which_binary "$binary" +done diff --git a/scripts/down.sh b/scripts/down.sh new file mode 100755 index 00000000..21e3d3b8 --- /dev/null +++ b/scripts/down.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euo pipefail + +cluster_name="test-cluster" + +teardown_test() { + # delete k3d cluster + k3d cluster delete "$cluster_name" + + # delete docker image + docker rmi k3d-shim-test +} + +teardown_test \ No newline at end of file diff --git a/scripts/move-bins.sh b/scripts/move-bins.sh new file mode 100755 index 00000000..2e08a624 --- /dev/null +++ b/scripts/move-bins.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Containerd Shim Installer Script +# +# This script automates the installation of specific containerd shim versions (slight, spin, wws) +# by checking their existence and copying them to a desired location if not found. +# +# Usage: +# ./move-bins.sh [release_pattern] [target] +# +# Arguments: +# 1. release_pattern (Optional): The pattern used to locate the shim binaries. +# 2. target (Optional): The target architecture used in the release path. +# Default value is `x86_64-unknown-linux-musl`. +# +# Example: +# ./move-bins.sh +# + +set -euo pipefail + +target="${2:-x86_64-unknown-linux-musl}" +release_pattern="${1:-containerd-shim-%s-v1/target/$target/release}" + +dockerfile_path="deployments/k3d" +bin_path="${dockerfile_path}/.tmp/" +cluster_name="test-cluster" +default_shim_path="${bin_path}containerd-shim-" + +declare -A shims=( + [slight]="${default_shim_path}slight-v1" + [spin]="${default_shim_path}spin-v1" + [wws]="${default_shim_path}wws-v1" +) + +mkdir -p "$bin_path" + +for shim_key in "${!shims[@]}"; do + shim_path=${shims[$shim_key]} + release_path=$(printf "$release_pattern" "$shim_key") + + if [ ! -f "$shim_path" ]; then + echo ">>> install containerd-shim-${shim_key}-v1 from $release_path" + cp "$(eval echo $release_path)/containerd-shim-${shim_key}-v1" "${bin_path}containerd-shim-${shim_key}-v1" + fi +done diff --git a/scripts/pod-status-check.sh b/scripts/pod-status-check.sh new file mode 100755 index 00000000..a78a90e4 --- /dev/null +++ b/scripts/pod-status-check.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -euo pipefail + +# Get the status of all pods +pod_statuses=$(kubectl get pods --no-headers -o custom-columns=":status.phase") + +# Check if all pods are running fine +all_running=true +for status in $pod_statuses; do + if [ "$status" != "Running" ]; then + all_running=false + break + fi +done + +if $all_running; then + echo "All pods are running fine." +else + echo "Not all pods are running fine. Please check the status." +fi \ No newline at end of file diff --git a/scripts/up.sh b/scripts/up.sh new file mode 100755 index 00000000..a6acbca8 --- /dev/null +++ b/scripts/up.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -euo pipefail + +cluster_name="test-cluster" # name of the k3d cluster +dockerfile_path="deployments/k3d" # path to the Dockerfile + +DOCKER_IMAGES=("slight" "spin" "wws") +OUT_DIRS=("test/out_slight" "test/out_spin" "test/out_wws") +IMAGES=("slight-hello-world" "spin-hello-world" "wws-hello-world") + +# build the Docker image for the k3d cluster +docker build -t k3d-shim-test "$dockerfile_path" + +k3d cluster create "$cluster_name" --image k3d-shim-test --api-port 6551 -p '8082:80@loadbalancer' --agents 2 + +kubectl wait --for=condition=ready node --all --timeout=120s + +# Iterate through the Docker images and build them +for i in "${!DOCKER_IMAGES[@]}"; do + docker buildx build -t "${IMAGES[$i]}:latest" "./images/${DOCKER_IMAGES[$i]}" --load + mkdir -p "${OUT_DIRS[$i]}" + docker save -o "${OUT_DIRS[$i]}/img.tar" "${IMAGES[$i]}:latest" + k3d image import "${OUT_DIRS[$i]}/img.tar" -c "$cluster_name" +done + +sleep 5 + +echo ">>> cluster is ready" diff --git a/scripts/workloads.sh b/scripts/workloads.sh new file mode 100755 index 00000000..210b9a0e --- /dev/null +++ b/scripts/workloads.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -euo pipefail + +# apply the workloads +echo ">>> apply workloads" +kubectl apply -f tests/workloads + + +# wait for all the pods to be ready +kubectl wait --for=condition=ready --timeout=50s pod --all + +# get and describe all the pods +echo ">>> Pods:" +kubectl get pods -o wide +kubectl describe pods + +# get and describe all the deployments +echo ">>> Deployments:" +kubectl get deployments -o wide +kubectl describe deployments + +# get and describe all the services +echo ">>> Services:" +kubectl get services -o wide +kubectl describe services \ No newline at end of file diff --git a/tests/setup.py b/tests/setup.py deleted file mode 100644 index 0f97d0d9..00000000 --- a/tests/setup.py +++ /dev/null @@ -1,103 +0,0 @@ -import time -import os -import sys - -def which(binary_name): - """Return the path to a binary, or None if it is not found.""" - for path in os.environ["PATH"].split(os.pathsep): - binary_path = os.path.join(path, binary_name) - if os.path.exists(binary_path): - return binary_path - # panic - raise RuntimeError("Could not find %s" % binary_name) - - -def setup_test(target): - # run this as root - which("k3d") - which("cross") - which("docker") - which("kubectl") - - dockerfile_path = "deployments/k3d" - bin_path = "deployments/k3d/.tmp/" - slight_shim_path = "deployments/k3d/.tmp/containerd-shim-slight-v1" - spin_shim_path = "deployments/k3d/.tmp/containerd-shim-spin-v1" - wws_shim_path = "deployments/k3d/.tmp/containerd-shim-wws-v1" - cluster_name = "test-cluster" - - # create bin_path if not exists - if not os.path.exists(bin_path): - os.makedirs(bin_path) - - try: - which(slight_shim_path) - except RuntimeError: - print(">>> install containerd-shim-slight-v1") - os.system(f"cp containerd-shim-slight-v1/target/{target}/release/containerd-shim-slight-v1 {bin_path}/containerd-shim-slight-v1") - - try: - which(spin_shim_path) - except RuntimeError: - print(">>> install containerd-shim-spin-v1") - os.system(f"cp containerd-shim-spin-v1/target/{target}/release/containerd-shim-spin-v1 {bin_path}/containerd-shim-spin-v1") - - try: - which(wws_shim_path) - except RuntimeError: - print(">>> install containerd-shim-wws-v1") - os.system(f"cp containerd-shim-wws-v1/target/{target}/release/containerd-shim-wws-v1 {bin_path}/containerd-shim-wws-v1") - - # build the docker image - os.system(f"docker build -t k3d-shim-test {dockerfile_path}") - - # create the cluster - os.system(f"k3d cluster create {cluster_name} --image k3d-shim-test --api-port 6551 -p '8082:80@loadbalancer' --agents 2") - - # wait for the cluster to be ready - os.system("kubectl wait --for=condition=ready node --all --timeout=120s") - - # build slight and spin images locally - os.system("docker buildx build -t slight-hello-world:latest ./images/slight --load") - os.system("docker buildx build -t spin-hello-world:latest ./images/spin --load") - os.system("docker buildx build -t wws-hello-world:latest ./images/wws --load") - - # create dir if not exists - if not os.path.exists("test/out_slight"): - os.makedirs("test/out_slight") - if not os.path.exists("test/out_spin"): - os.makedirs("test/out_spin") - if not os.path.exists("test/out_wws"): - os.makedirs("test/out_wws") - - # save docker images to tar ball - os.system("docker save -o test/out_slight/img.tar slight-hello-world:latest") - os.system("docker save -o test/out_spin/img.tar spin-hello-world:latest") - os.system("docker save -o test/out_wws/img.tar wws-hello-world:latest") - - # load tar ball to k3d cluster - os.system(f"k3d image import test/out_slight/img.tar -c {cluster_name}") - os.system(f"k3d image import test/out_spin/img.tar -c {cluster_name}") - os.system(f"k3d image import test/out_wws/img.tar -c {cluster_name}") - - # wait for 5 seconds - time.sleep(5) - - print(">>> apply workloads") - os.system("kubectl apply -f tests/workloads") - - # wait for 25 seconds - time.sleep(25) - - os.system("kubectl describe pods") - os.system("kubectl describe deployments") - os.system("kubectl describe services") - - print(">>> cluster is ready") - -if __name__ == '__main__': - if len(sys.argv) < 2: - target = "x86_64-unknown-linux-musl" - else: - target = sys.argv[1] - setup_test(target = target) \ No newline at end of file diff --git a/tests/teardown.py b/tests/teardown.py deleted file mode 100644 index 563c49a7..00000000 --- a/tests/teardown.py +++ /dev/null @@ -1,12 +0,0 @@ -import os - -def teardown_test(): - # delete k3d cluster - os.system("k3d cluster delete test-cluster") - - # delete docker image - os.system("docker rmi k3d-shim-test") - - -if __name__ == '__main__': - teardown_test() \ No newline at end of file