From 536f19b58b6018bcb9a5435efe2eb555a216d463 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Tue, 17 May 2022 14:18:38 -0700 Subject: [PATCH] DLPX-80453 DOSE-Azure: zpool import (#428) Signed-off-by: Paul Dagnelie --- cmd/zdb/zdb.c | 2 +- cmd/zfs_object_agent/Cargo.lock | 231 +++++++++++------- cmd/zfs_object_agent/client/src/main.rs | 3 + cmd/zfs_object_agent/util/src/logging.rs | 8 + cmd/zfs_object_agent/zettaobject/src/debug.rs | 27 +- .../zettaobject/src/public_connection.rs | 56 ++--- cmd/zpool/zpool_vdev.c | 5 + etc/zfs/zoa_log4rs.yml | 8 + lib/libzutil/zutil_import.c | 26 +- 9 files changed, 196 insertions(+), 170 deletions(-) diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index d3c43d3a0271..1bbd2aa6eba8 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -8665,7 +8665,7 @@ main(int argc, char **argv) }; while ((c = getopt_long(argc, argv, - "a:AB:bcCdDeEf:Fg:GhiI:klLmMNo:Op:pPqrRsSt:TuU:vVx:XYyZz:", + "a:AB:bcCdDeEf:Fg:GhiI:klLmMNo:Op:pPqrRsSt:T:uU:vVx:XYyZz:", long_options, NULL)) != -1) { switch (c) { case 'b': diff --git a/cmd/zfs_object_agent/Cargo.lock b/cmd/zfs_object_agent/Cargo.lock index d5150ada2122..122c9af247b7 100644 --- a/cmd/zfs_object_agent/Cargo.lock +++ b/cmd/zfs_object_agent/Cargo.lock @@ -360,6 +360,15 @@ dependencies = [ "generic-array 0.14.5", ] +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array 0.14.5", +] + [[package]] name = "block-padding" version = "0.1.5" @@ -446,9 +455,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.12" +version = "3.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c167e37342afc5f33fd87bbc870cedd020d2a6dffa05d45ccd9241fbdd146db" +checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b" dependencies = [ "atty", "bitflags", @@ -463,9 +472,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.1.7" +version = "3.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1" +checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c" dependencies = [ "heck 0.4.0", "proc-macro-error", @@ -476,9 +485,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669" +checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213" dependencies = [ "os_str_bytes", ] @@ -590,6 +599,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crypto-common" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +dependencies = [ + "generic-array 0.14.5", + "typenum", +] + [[package]] name = "crypto-mac" version = "0.11.1" @@ -618,13 +637,13 @@ checksum = "ef8ae57c4978a2acd8b869ce6b9ca1dfe817bff704c220209fdef2c0b75a01b9" [[package]] name = "dashmap" -version = "5.2.0" +version = "5.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8858831f7781322e539ea39e72449c46b059638250c14344fec8d0aa6e539c" +checksum = "391b56fbd302e585b7a9494fb70e40949567b1cf9003a8e4a6041a1687c26573" dependencies = [ "cfg-if", - "num_cpus", - "parking_lot 0.12.0", + "hashbrown 0.12.1", + "lock_api", ] [[package]] @@ -667,6 +686,16 @@ dependencies = [ "generic-array 0.14.5", ] +[[package]] +name = "digest" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +dependencies = [ + "block-buffer 0.10.2", + "crypto-common", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -1039,9 +1068,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c21d40587b92fa6a6c6e3c1bdbf87d75511db5672f9c93175574b3a00df1758" +checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" dependencies = [ "ahash", ] @@ -1113,9 +1142,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" dependencies = [ "bytes", "fnv", @@ -1277,9 +1306,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.124" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50" +checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" [[package]] name = "linked-hash-map" @@ -1299,9 +1328,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", "serde", @@ -1405,9 +1434,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" @@ -1485,25 +1514,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" dependencies = [ "libc", "log", - "miow", - "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", - "winapi 0.3.9", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi 0.3.9", + "windows-sys", ] [[package]] @@ -1572,9 +1590,9 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", @@ -1582,9 +1600,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] @@ -1607,8 +1625,8 @@ checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" [[package]] name = "nvpair" -version = "0.5.1-delphix0" -source = "git+https://github.com/delphix/rust-libzfs#77bca6ab28e563ae97512108ea92e869798db026" +version = "0.5.2-delphix0" +source = "git+https://github.com/delphix/rust-libzfs#215ad1544495128ccba63010d1825cb70fa14123" dependencies = [ "cstr-argument", "foreign-types 0.5.0", @@ -1619,13 +1637,13 @@ dependencies = [ [[package]] name = "nvpair-sys" version = "0.4.1-delphix0" -source = "git+https://github.com/delphix/rust-libzfs#77bca6ab28e563ae97512108ea92e869798db026" +source = "git+https://github.com/delphix/rust-libzfs#215ad1544495128ccba63010d1825cb70fa14123" [[package]] name = "oauth2" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e47cfc4c0a1a519d9a025ebfbac3a2439d1b5cdf397d72dcb79b11d9920dab" +checksum = "c3bd7d544f02ae0fa9e06137962703d043870d7ad6e6d44786d6a5f20679b2c9" dependencies = [ "base64", "chrono", @@ -1636,16 +1654,16 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", - "sha2", + "sha2 0.10.2", "thiserror", "url", ] [[package]] name = "object" -version = "0.28.3" +version = "0.28.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40bec70ba014595f99f7aa110b84331ffe1ee9aece7fe6f387cc7e3ecda4d456" +checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424" dependencies = [ "memchr", ] @@ -1670,18 +1688,30 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.38" +version = "0.10.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e" dependencies = [ "bitflags", "cfg-if", "foreign-types 0.3.2", "libc", "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -1690,9 +1720,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.72" +version = "0.9.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0" dependencies = [ "autocfg", "cc", @@ -1717,7 +1747,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" dependencies = [ "dlv-list", - "hashbrown 0.12.0", + "hashbrown 0.12.1", ] [[package]] @@ -1744,7 +1774,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ "lock_api", - "parking_lot_core 0.9.2", + "parking_lot_core 0.9.3", ] [[package]] @@ -1763,9 +1793,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if", "libc", @@ -1911,9 +1941,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" +checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa" dependencies = [ "unicode-xid", ] @@ -2171,7 +2201,7 @@ dependencies = [ "rusoto_credential", "rustc_version", "serde", - "sha2", + "sha2 0.9.9", "tokio", ] @@ -2299,15 +2329,15 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" [[package]] name = "serde" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] @@ -2336,18 +2366,18 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16ae07dd2f88a366f15bd0632ba725227018c69a1c8550a927324f8eb8368bb9" +checksum = "212e73464ebcde48d723aa02eb270ba62eff38a9b732df31f33f1b4e145f3a54" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" dependencies = [ "proc-macro2", "quote", @@ -2356,9 +2386,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "itoa", "ryu", @@ -2388,9 +2418,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a521f2940385c165a24ee286aa8599633d162077a54bdcae2a6fd5a7bfa7a0" +checksum = "707d15895415db6628332b737c838b88c598522e4dc70647e59b72312924aebc" dependencies = [ "indexmap", "ryu", @@ -2447,6 +2477,17 @@ dependencies = [ "opaque-debug 0.3.0", ] +[[package]] +name = "sha2" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.3", +] + [[package]] name = "shlex" version = "1.1.0" @@ -2539,9 +2580,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.91" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" +checksum = "04066589568b72ec65f42d65a1a52436e954b168773148893c020269563decf2" dependencies = [ "proc-macro2", "quote", @@ -2561,9 +2602,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.23.10" +version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eea2ed6847da2e0c7289f72cb4f285f0bd704694ca067d32be811b2a45ea858" +checksum = "56b1e20ee77901236c389ff74618a899ff5fd34719a7ff0fd1d64f0acca5179a" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2630,18 +2671,18 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", @@ -2687,9 +2728,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.18.0" +version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f48b6d60512a392e34dbf7fd456249fd2de3c83669ab642e021903f4015185b" +checksum = "4903bf0427cf68dddd5aa6a93220756f8be0c34fcfa9f5e6191e103e15a31395" dependencies = [ "bytes", "libc", @@ -2868,9 +2909,9 @@ checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" [[package]] name = "unicode-xid" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "unsafe-any" @@ -3112,9 +3153,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -3125,33 +3166,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.34.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "winreg" @@ -3215,9 +3256,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb5728b8afd3f280a869ce1d4c554ffaed35f45c231fc41bfbd0381bef50317" +checksum = "94693807d016b2f2d2e14420eb3bfcca689311ff775dcf113d74ea624b7cdf07" [[package]] name = "zettacache" diff --git a/cmd/zfs_object_agent/client/src/main.rs b/cmd/zfs_object_agent/client/src/main.rs index cf79b85c68af..671f6b794c6c 100644 --- a/cmd/zfs_object_agent/client/src/main.rs +++ b/cmd/zfs_object_agent/client/src/main.rs @@ -40,6 +40,7 @@ use zettaobject::object_access::BlobCredentials; use zettaobject::object_access::BucketAccess; use zettaobject::Pool; mod client; +use ::util::setup_logging; use itertools::Itertools; use zettaobject::object_access::ObjectAccess; use zettaobject::object_access::ObjectAccessProtocol; @@ -597,6 +598,8 @@ async fn main() -> Result<(), Box> { ) .await; + setup_logging(cli.verbose as u64, None, None, false); + match cli.command { Commands::S3Rusoto => do_s3_rusoto().await?, Commands::Blob => do_blob(cli.bucket, cli.profile).await?, diff --git a/cmd/zfs_object_agent/util/src/logging.rs b/cmd/zfs_object_agent/util/src/logging.rs index 8c6b93265b7c..644bda9ca87c 100644 --- a/cmd/zfs_object_agent/util/src/logging.rs +++ b/cmd/zfs_object_agent/util/src/logging.rs @@ -231,6 +231,10 @@ fn setup_console_logging(verbosity: u64) { // These are too chatty by default, so increase their minimum log level. .logger(Logger::builder().build("rusoto_core::request", LevelFilter::Info)) .logger(Logger::builder().build("want", LevelFilter::Debug)) + .logger(Logger::builder().build("azure_storage::core", LevelFilter::Info)) + .logger(Logger::builder().build("azure_storage_blobs::blob", LevelFilter::Info)) + .logger(Logger::builder().build("serde_xml_rs::de", LevelFilter::Info)) + .logger(Logger::builder().build("reqwest::async_impl::client", LevelFilter::Info)) .build( Root::builder() .appender("memory") @@ -265,6 +269,10 @@ fn setup_logfile(verbosity: u64, logfile: &Path) { // These are too chatty by default, so increase their minimum log level. .logger(Logger::builder().build("rusoto_core::request", LevelFilter::Info)) .logger(Logger::builder().build("want", LevelFilter::Debug)) + .logger(Logger::builder().build("azure_storage::core", LevelFilter::Info)) + .logger(Logger::builder().build("azure_storage_blobs::blob", LevelFilter::Info)) + .logger(Logger::builder().build("serde_xml_rs::de", LevelFilter::Info)) + .logger(Logger::builder().build("reqwest::async_impl::client", LevelFilter::Info)) .build( Root::builder() .appender("memory") diff --git a/cmd/zfs_object_agent/zettaobject/src/debug.rs b/cmd/zfs_object_agent/zettaobject/src/debug.rs index 059312090a6a..3b2f3213dedf 100644 --- a/cmd/zfs_object_agent/zettaobject/src/debug.rs +++ b/cmd/zfs_object_agent/zettaobject/src/debug.rs @@ -13,7 +13,6 @@ use zettacache::base_types::PoolGuid; use crate::base_types::Txg; use crate::object_access::ObjectAccess; use crate::object_access::ObjectAccessProtocol; -use crate::object_access::S3Credentials; use crate::pool; use crate::pool::Pool; use crate::pool::PoolPhys; @@ -26,30 +25,12 @@ pub struct DebugHandle { } async fn get_object_access(nvl: &NvListRef) -> Arc { + let protocol: ObjectAccessProtocol = nvpair::from_nvlist(nvl).unwrap(); let bucket_name = nvl.lookup_string("bucket").unwrap(); - let region_str = nvl.lookup_string("region").unwrap(); - let endpoint = nvl.lookup_string("endpoint").unwrap(); - let credentials_profile: Option = nvl - .lookup_string("credentials_profile") - .ok() - .map(|s| s.to_string_lossy().to_string()); - let credentials = match credentials_profile { - Some(profile) => S3Credentials::Profile(profile), - None => S3Credentials::Automatic, - }; - - ObjectAccess::new( - ObjectAccessProtocol::S3 { - endpoint: endpoint.to_str().unwrap().to_string(), - region: region_str.to_str().unwrap().to_string(), - credentials, - }, - bucket_name.to_str().unwrap().to_string(), - true, - ) - .await - .unwrap() + ObjectAccess::new(protocol, bucket_name.to_str().unwrap().to_string(), true) + .await + .unwrap() } impl DebugHandle { diff --git a/cmd/zfs_object_agent/zettaobject/src/public_connection.rs b/cmd/zfs_object_agent/zettaobject/src/public_connection.rs index 4b7ff4d64f48..42318ede4be9 100644 --- a/cmd/zfs_object_agent/zettaobject/src/public_connection.rs +++ b/cmd/zfs_object_agent/zettaobject/src/public_connection.rs @@ -7,6 +7,7 @@ use futures::stream::StreamExt; use log::*; use nvpair::NvList; use semver::Version; +use serde::Deserialize; use serde::Serialize; use util::maybe_die_with; use util::message::*; @@ -18,7 +19,6 @@ use zettacache::ZettaCache; use crate::object_access::BucketAccess; use crate::object_access::ObjectAccess; use crate::object_access::ObjectAccessProtocol; -use crate::object_access::S3Credentials; use crate::pool::*; use crate::pool_destroy; use crate::server::return_result; @@ -95,33 +95,22 @@ impl PublicConnectionState { } async fn get_pools_impl(nvl: NvList) -> Result> { - // XXX convert to use serde nvlist request and response - let region_cstr = nvl.lookup_string("region")?; - let endpoint_cstr = nvl.lookup_string("endpoint")?; - let region = region_cstr.to_str()?.to_string(); - let endpoint = endpoint_cstr.to_str()?.to_string(); - let readonly = nvl.exists("readonly"); - let credentials_profile: Option = nvl - .lookup_string("credentials_profile") - .ok() - .map(|s| s.to_string_lossy().to_string()); - - let credentials = match credentials_profile { - Some(profile) => S3Credentials::Profile(profile), - None => S3Credentials::Automatic, - }; - - let bucket_access = BucketAccess::new(ObjectAccessProtocol::S3 { - endpoint: endpoint.clone(), - region: region.clone(), - credentials: credentials.clone(), - }) - .await?; + #[derive(Debug, Deserialize)] + struct GetPoolsRequest { + #[serde(flatten)] + protocol: ObjectAccessProtocol, + #[serde(default)] + readonly: bool, + bucket: Option, + guid: Option, + } + let request: GetPoolsRequest = nvpair::from_nvlist(&nvl)?; + + let bucket_access = BucketAccess::new(request.protocol.clone()).await?; let mut buckets = vec![]; - let bucket_result = nvl.lookup_string("bucket"); - if let Ok(bucket) = bucket_result { - buckets.push(bucket.into_string()?); + if let Some(bucket) = request.bucket { + buckets.push(bucket); } else { buckets.append(&mut bucket_access.list_buckets().await); } @@ -129,18 +118,9 @@ impl PublicConnectionState { maybe_die_with(|| "in get_pools_impl"); let response = Arc::new(Mutex::new(NvList::new_unique_names())); for buck in buckets { - let object_access = ObjectAccess::new( - ObjectAccessProtocol::S3 { - endpoint: endpoint.clone(), - region: region.clone(), - credentials: credentials.clone(), - }, - buck, - readonly, - ) - .await?; - let guid_result = nvl.lookup_uint64("guid"); - if let Ok(guid) = guid_result { + let object_access = + ObjectAccess::new(request.protocol.clone(), buck, request.readonly).await?; + if let Some(guid) = request.guid { if !Pool::exists(&object_access, PoolGuid(guid)).await { continue; } diff --git a/cmd/zpool/zpool_vdev.c b/cmd/zpool/zpool_vdev.c index 28cd756b9796..70069a87c10f 100644 --- a/cmd/zpool/zpool_vdev.c +++ b/cmd/zpool/zpool_vdev.c @@ -307,6 +307,11 @@ make_objstore_vdev(nvlist_t *props, const char *protocol, const char *arg) if ((nvlist_lookup_string(props, zpool_prop_to_name(ZPOOL_PROP_OBJ_CRED_PROFILE), &profile)) == 0) { fnvlist_add_string(vdev, ZPOOL_CONFIG_CRED_PROFILE, profile); + } else { + fprintf(stderr, gettext("Credentials profile required for " + "objstore vdev %s"), arg); + fnvlist_free(vdev); + return (NULL); } return (vdev); diff --git a/etc/zfs/zoa_log4rs.yml b/etc/zfs/zoa_log4rs.yml index 2a804ff51d48..9377b8abd384 100644 --- a/etc/zfs/zoa_log4rs.yml +++ b/etc/zfs/zoa_log4rs.yml @@ -113,3 +113,11 @@ loggers: level: info want: level: debug + azure_storage::core: + level: info + azure_storage_blobs::blob: + level: info + serde_xml_rs::de: + level: info + reqwest::async_impl::client: + level: info diff --git a/lib/libzutil/zutil_import.c b/lib/libzutil/zutil_import.c index 4cdb94e44dc1..ebb1ac2a55d8 100644 --- a/lib/libzutil/zutil_import.c +++ b/lib/libzutil/zutil_import.c @@ -1868,21 +1868,17 @@ static void zpool_find_import_agent(libpc_handle_t *hdl, importargs_t *iarg, pthread_mutex_t *lock, avl_tree_t *cache) { - char *profile = NULL, *bucket = NULL, *endpoint, *region; + char *profile = NULL, *bucket = NULL, *endpoint = NULL; + char *region = NULL, *protocol = NULL; // TODO: We don't handle multiple search paths yet nvlist_lookup_string(iarg->props, "path", &bucket); if (bucket == NULL && iarg->path != NULL) { bucket = iarg->path[0]; } - if ((nvlist_lookup_string(iarg->props, "object-endpoint", - &endpoint)) != 0) { - return; - } - if ((nvlist_lookup_string(iarg->props, "object-region", - ®ion)) != 0) { - return; - } + nvlist_lookup_string(iarg->props, "object-protocol", &protocol); + nvlist_lookup_string(iarg->props, "object-endpoint", &endpoint); + nvlist_lookup_string(iarg->props, "object-region", ®ion); nvlist_lookup_string(iarg->props, "object-credentials-profile", &profile); @@ -1890,11 +1886,15 @@ zpool_find_import_agent(libpc_handle_t *hdl, importargs_t *iarg, fnvlist_add_string(msg, AGENT_REQUEST_TYPE, AGENT_TYPE_GET_POOLS); if (bucket != NULL) fnvlist_add_string(msg, AGENT_BUCKET, bucket); - fnvlist_add_string(msg, AGENT_REGION, region); - fnvlist_add_string(msg, AGENT_ENDPOINT, endpoint); - if (profile != NULL) { + if (region != NULL) + fnvlist_add_string(msg, AGENT_REGION, region); + if (endpoint != NULL) + fnvlist_add_string(msg, AGENT_ENDPOINT, endpoint); + if (protocol != NULL) + fnvlist_add_string(msg, AGENT_PROTOCOL, protocol); + if (profile != NULL) fnvlist_add_string(msg, AGENT_CRED_PROFILE, profile); - } + if (iarg->guid != 0) fnvlist_add_uint64(msg, AGENT_GUID, iarg->guid);