From c833176b0d34e8f20405353d6aa2347539d34dac Mon Sep 17 00:00:00 2001 From: James MacMahon Date: Fri, 18 Mar 2022 15:24:12 -0400 Subject: [PATCH] Supply real volume data to propolis Instance creation now can specify that existing disks be attached to the instance (disk creation during instance creation is not yet implemented). There's a few important changes here. Nexus will instruct the sled agent to send volume construction requests when it ensures the instance, and therefore has to build these requests. This is done in sdc_regions_ensure as regions are allocated. sic_create_instance_record no longer returns the runtime of an instance, but the instance's name. Importantly, the instance creation saga now calls Nexus' instance_set_runtime instead of duplicating the logic to call instance_put. Nexus' instance_set_runtime will populate the InstanceHardware with NICs and disks. Nexus' disk_set_runtime now *optionally* calls the sled agent's disk_put if the instance is running already, otherwise the disk will be attached as part of the instance ensure by specifying volume requests. request_body_max_bytes had to be increased to 1M because the volume requests could be arbitrarily large (and eventually, the cloud init bytes will add to the body size too). --- Cargo.lock | 232 +++++++++++++++- common/src/api/external/error.rs | 6 + common/src/api/external/mod.rs | 4 + nexus/Cargo.toml | 3 +- nexus/src/db/model.rs | 31 ++- nexus/src/external_api/params.rs | 21 ++ nexus/src/nexus.rs | 140 ++++++++-- nexus/src/sagas.rs | 257 ++++++++++++++---- nexus/test-utils/src/resource_helpers.rs | 1 + nexus/tests/integration_tests/endpoints.rs | 1 + nexus/tests/integration_tests/instances.rs | 95 +++++++ .../integration_tests/subnet_allocation.rs | 1 + openapi/nexus.json | 74 +++++ openapi/sled-agent.json | 177 ++++++++++++ sled-agent/Cargo.toml | 4 +- sled-agent/src/bin/sled-agent-sim.rs | 2 +- sled-agent/src/bin/sled-agent.rs | 6 +- sled-agent/src/instance.rs | 13 +- sled-agent/src/instance_manager.rs | 1 + sled-agent/src/params.rs | 1 + sled-agent/src/sim/storage.rs | 1 - tools/oxapi_demo | 19 +- 22 files changed, 1001 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b3e92df11d9..b7cd2d86a85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,53 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aead" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877" +dependencies = [ + "generic-array 0.14.4", +] + +[[package]] +name = "aes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", + "opaque-debug 0.3.0", +] + +[[package]] +name = "aes-gcm-siv" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589c637f0e68c877bbd59a4599bbe849cac8e5f3e4b5a3ebae8f528cd218dcdc" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "polyval", + "subtle", + "zeroize", +] + +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -305,6 +352,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "cipher" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +dependencies = [ + "generic-array 0.14.4", +] + [[package]] name = "clap" version = "2.34.0" @@ -518,10 +574,42 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crucible" +version = "0.0.1" +source = "git+https://github.com/oxidecomputer/crucible?rev=4090a023b77dcab7a5000057cf2c96cdbb0469b6#4090a023b77dcab7a5000057cf2c96cdbb0469b6" +dependencies = [ + "aes-gcm-siv", + "anyhow", + "base64", + "bytes", + "crucible-common", + "crucible-protocol", + "crucible-scope", + "dropshot", + "futures", + "futures-core", + "rand", + "rand_chacha", + "reqwest", + "ringbuffer", + "schemars", + "serde", + "serde_json", + "structopt", + "tokio", + "tokio-rustls", + "tokio-util 0.7.0", + "toml", + "tracing", + "usdt 0.3.1", + "uuid", +] + [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=3e7e49eeb88fa8ad74375b0642aabd4224b1f2cb#3e7e49eeb88fa8ad74375b0642aabd4224b1f2cb" +source = "git+https://github.com/oxidecomputer/crucible?rev=32e603bdc3b6e5eb6b880a2ddde7e05f043b5357#32e603bdc3b6e5eb6b880a2ddde7e05f043b5357" dependencies = [ "anyhow", "chrono", @@ -533,6 +621,53 @@ dependencies = [ "serde_json", ] +[[package]] +name = "crucible-common" +version = "0.0.0" +source = "git+https://github.com/oxidecomputer/crucible?rev=4090a023b77dcab7a5000057cf2c96cdbb0469b6#4090a023b77dcab7a5000057cf2c96cdbb0469b6" +dependencies = [ + "anyhow", + "rusqlite", + "rustls-pemfile 0.3.0", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tokio-rustls", + "toml", + "twox-hash", + "uuid", +] + +[[package]] +name = "crucible-protocol" +version = "0.0.0" +source = "git+https://github.com/oxidecomputer/crucible?rev=4090a023b77dcab7a5000057cf2c96cdbb0469b6#4090a023b77dcab7a5000057cf2c96cdbb0469b6" +dependencies = [ + "anyhow", + "bincode", + "bytes", + "crucible-common", + "serde", + "tokio-util 0.7.0", + "uuid", +] + +[[package]] +name = "crucible-scope" +version = "0.0.0" +source = "git+https://github.com/oxidecomputer/crucible?rev=4090a023b77dcab7a5000057cf2c96cdbb0469b6#4090a023b77dcab7a5000057cf2c96cdbb0469b6" +dependencies = [ + "anyhow", + "futures", + "futures-core", + "serde", + "serde_json", + "tokio", + "tokio-util 0.7.0", + "toml", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -592,6 +727,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +dependencies = [ + "cipher", +] + [[package]] name = "curve25519-dalek" version = "4.0.0-pre.1" @@ -929,6 +1073,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "1.6.0" @@ -1251,6 +1401,18 @@ name = "hashbrown" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" +dependencies = [ + "hashbrown", +] [[package]] name = "headers" @@ -1603,6 +1765,16 @@ version = "0.2.119" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" +[[package]] +name = "libsqlite3-sys" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb644c388dfaefa18035c12614156d285364769e818893da0dda9030c80ad2ba" +dependencies = [ + "pkg-config", + "vcpkg", +] + [[package]] name = "lock_api" version = "0.4.6" @@ -1973,6 +2145,7 @@ dependencies = [ "api_identity", "async-bb8-diesel", "async-trait", + "base64", "bb8", "chrono", "cookie", @@ -2697,6 +2870,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "polyval" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug 0.3.0", + "universal-hash", +] + [[package]] name = "postgres-protocol" version = "0.6.3" @@ -2881,8 +3066,9 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=0e3798510ae190131f63b9df767ec01b2beacf91#0e3798510ae190131f63b9df767ec01b2beacf91" +source = "git+https://github.com/oxidecomputer/propolis?rev=832a86afce308d3210685af987ace1ba74c2ecd6#832a86afce308d3210685af987ace1ba74c2ecd6" dependencies = [ + "crucible", "reqwest", "ring", "schemars", @@ -3112,6 +3298,21 @@ dependencies = [ "array-init", ] +[[package]] +name = "rusqlite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "memchr", + "smallvec", +] + [[package]] name = "rustc_version" version = "0.1.7" @@ -3742,6 +3943,12 @@ dependencies = [ "der", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "steno" version = "0.1.0" @@ -4299,6 +4506,17 @@ dependencies = [ "toml", ] +[[package]] +name = "twox-hash" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee73e6e4924fe940354b8d4d98cad5231175d615cd855b758adc658c0aac6a0" +dependencies = [ + "cfg-if", + "rand", + "static_assertions", +] + [[package]] name = "typenum" version = "1.14.0" @@ -4400,6 +4618,16 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" +[[package]] +name = "universal-hash" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" +dependencies = [ + "generic-array 0.14.4", + "subtle", +] + [[package]] name = "untrusted" version = "0.7.1" diff --git a/common/src/api/external/error.rs b/common/src/api/external/error.rs index f66de9cd301..7efeae48156 100644 --- a/common/src/api/external/error.rs +++ b/common/src/api/external/error.rs @@ -296,6 +296,12 @@ impl From> for Error { } } +impl From for Error { + fn from(e: serde_json::Error) -> Self { + Error::internal_error(&e.to_string()) + } +} + /// Like [`assert!`], except that instead of panicking, this function returns an /// `Err(Error::InternalError)` with an appropriate message if the given /// condition is not true. diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index f79e7a4b69c..68a57161ded 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -467,6 +467,10 @@ impl Generation { assert!(next_gen <= u64::try_from(i64::MAX).unwrap()); Generation(next_gen) } + + pub fn get(&self) -> u64 { + self.0 + } } impl Display for Generation { diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 87ad1391026..7bdf38cb2d6 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -11,9 +11,10 @@ path = "../rpaths" anyhow = "1.0" async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "c849b717be" } async-trait = "0.1.51" +base64 = "0.13.0" bb8 = "0.7.1" cookie = "0.16" -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "3e7e49eeb88fa8ad74375b0642aabd4224b1f2cb" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "32e603bdc3b6e5eb6b880a2ddde7e05f043b5357" } # Tracking pending 2.0 version. diesel = { git = "https://github.com/diesel-rs/diesel", rev = "ce77c382", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } futures = "0.3.21" diff --git a/nexus/src/db/model.rs b/nexus/src/db/model.rs index ebc837db711..410380e1570 100644 --- a/nexus/src/db/model.rs +++ b/nexus/src/db/model.rs @@ -646,7 +646,11 @@ impl Sled { pub fn address(&self) -> SocketAddr { // TODO: avoid this unwrap - SocketAddr::new(self.ip.ip(), u16::try_from(self.port).unwrap()) + self.address_with_port(u16::try_from(self.port).unwrap()) + } + + pub fn address_with_port(&self, port: u16) -> SocketAddr { + SocketAddr::new(self.ip.ip(), port) } } @@ -787,7 +791,11 @@ impl Dataset { pub fn address(&self) -> SocketAddr { // TODO: avoid this unwrap - SocketAddr::new(self.ip.ip(), u16::try_from(self.port).unwrap()) + self.address_with_port(u16::try_from(self.port).unwrap()) + } + + pub fn address_with_port(&self, port: u16) -> SocketAddr { + SocketAddr::new(self.ip.ip(), port) } } @@ -905,6 +913,10 @@ impl Volume { data, } } + + pub fn data(&self) -> &String { + &self.data + } } /// Describes an organization within the database. @@ -1266,6 +1278,10 @@ impl Disk { pub fn runtime(&self) -> DiskRuntimeState { self.runtime_state.clone() } + + pub fn id(&self) -> Uuid { + self.identity.id + } } /// Conversion to the external API type. @@ -1319,6 +1335,17 @@ impl DiskRuntimeState { } } + pub fn attach(self, instance_id: Uuid) -> Self { + Self { + disk_state: external::DiskState::Attached(instance_id) + .label() + .to_string(), + attach_instance_id: Some(instance_id), + gen: self.gen.next().into(), + time_updated: Utc::now(), + } + } + pub fn detach(self) -> Self { Self { disk_state: external::DiskState::Detached.label().to_string(), diff --git a/nexus/src/external_api/params.rs b/nexus/src/external_api/params.rs index 2775df390a4..16734a47586 100644 --- a/nexus/src/external_api/params.rs +++ b/nexus/src/external_api/params.rs @@ -105,6 +105,23 @@ impl Default for InstanceNetworkInterfaceAttachment { } } +/// Describe the instance's disks at creation time +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum InstanceDiskAttachment { + /// During instance creation, create and attach disks + Create(DiskCreate), + + /// During instance creation, attach this disk + Attach(InstanceDiskAttach), +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct InstanceDiskAttach { + /// A disk name to attach + pub disk: Name, +} + /// Create-time parameters for an [`Instance`](omicron_common::api::external::Instance) #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct InstanceCreate { @@ -117,6 +134,10 @@ pub struct InstanceCreate { /// The network interfaces to be created for this instance. #[serde(default)] pub network_interfaces: InstanceNetworkInterfaceAttachment, + + /// The disks to be created or attached for this instance. + #[serde(default)] + pub disks: Vec, } /// Migration parameters for an [`Instance`](omicron_common::api::external::Instance) diff --git a/nexus/src/nexus.rs b/nexus/src/nexus.rs index 96131ff6e9e..383f6898ff0 100644 --- a/nexus/src/nexus.rs +++ b/nexus/src/nexus.rs @@ -858,6 +858,8 @@ impl Nexus { let saga_params = Arc::new(sagas::ParamsInstanceCreate { serialized_authn: authn::saga::Serialized::for_opctx(opctx), + organization_name: organization_name.clone().into(), + project_name: project_name.clone().into(), project_id: authz_project.id(), create_params: params.clone(), }); @@ -1209,7 +1211,7 @@ impl Nexus { /// Modifies the runtime state of the Instance as requested. This generally /// means booting or halting the Instance. - async fn instance_set_runtime( + pub(crate) async fn instance_set_runtime( &self, opctx: &OpContext, authz_instance: &authz::Instance, @@ -1223,23 +1225,71 @@ impl Nexus { &requested, )?; - let sa = self.instance_sled(&db_instance).await?; + // Gather disk information and turn that into DiskRequests + let disks = self + .db_datastore + .instance_list_disks( + &opctx, + &authz_instance, + &DataPageParams { + marker: None, + direction: dropshot::PaginationOrder::Ascending, + // TODO: is there a limit to the number of disks an instance + // can have attached? + limit: std::num::NonZeroU32::new(8).unwrap(), + }, + ) + .await?; + + let mut disk_reqs = vec![]; + for (i, disk) in disks.iter().enumerate() { + let volume = self.db_datastore.volume_get(disk.volume_id).await?; + disk_reqs.push(sled_agent_client::types::DiskRequest { + name: disk.name().to_string(), + slot: sled_agent_client::types::Slot(i as u8), + // TODO offer ability to attach read-only? + read_only: false, + device: "nvme".to_string(), + gen: disk.runtime_state.gen.0.get(), + volume_construction_request: serde_json::from_str( + &volume.data(), + )?, + }); + } + + let nics: Vec = self + .db_datastore + .instance_list_network_interfaces( + &opctx, + &authz_instance, + &DataPageParams { + marker: None, + direction: dropshot::PaginationOrder::Ascending, + // TODO: is there a limit to the number of NICs an instance + // can have attached? + limit: std::num::NonZeroU32::new(8).unwrap(), + }, + ) + .await? + .iter() + .map(|x| x.clone().into()) + .collect(); // Ask the sled agent to begin the state change. Then update the // database to reflect the new intermediate state. If this update is // not the newest one, that's fine. That might just mean the sled agent // beat us to it. - // TODO: Populate this with an appropriate NIC. - // See also: sic_create_instance_record in sagas.rs for a similar - // construction. let instance_hardware = sled_agent_client::types::InstanceHardware { runtime: sled_agent_client::types::InstanceRuntimeState::from( db_instance.runtime().clone(), ), - nics: vec![], + nics: nics.iter().map(|nic| nic.clone().into()).collect(), + disks: disk_reqs, }; + let sa = self.instance_sled(&db_instance).await?; + let new_runtime = sa .instance_put( &db_instance.id(), @@ -1381,6 +1431,7 @@ impl Nexus { opctx, &authz_disk, &db_disk, + &db_instance, self.instance_sled(&db_instance).await?, sled_agent_client::types::DiskStateRequested::Attached( *instance_id, @@ -1455,6 +1506,7 @@ impl Nexus { opctx, &authz_disk, &db_disk, + &db_instance, self.instance_sled(&db_instance).await?, sled_agent_client::types::DiskStateRequested::Detached, ) @@ -1469,30 +1521,74 @@ impl Nexus { opctx: &OpContext, authz_disk: &authz::Disk, db_disk: &db::model::Disk, + db_instance: &db::model::Instance, sa: Arc, requested: sled_agent_client::types::DiskStateRequested, ) -> Result<(), Error> { + use sled_agent_client::types::DiskStateRequested; + let runtime: DiskRuntimeState = db_disk.runtime().into(); opctx.authorize(authz::Action::Modify, authz_disk).await?; - // Ask the Sled Agent to begin the state change. Then update the - // database to reflect the new intermediate state. - let new_runtime = sa - .disk_put( - &authz_disk.id(), - &sled_agent_client::types::DiskEnsureBody { - initial_runtime: - sled_agent_client::types::DiskRuntimeState::from( - runtime, - ), - target: requested, - }, - ) - .await - .map_err(Error::from)?; + let new_runtime: DiskRuntimeState = match &db_instance + .runtime_state + .state + .state() + { + InstanceState::Running | InstanceState::Starting => { + /* + * If there's a propolis zone for this instnace, ask the Sled + * Agent to hot-plug or hot-remove disk. Then update the + * database to reflect the new intermediate state. + * + * TODO this will probably involve volume construction requests + * as well! + */ + let new_runtime = sa + .disk_put( + &authz_disk.id(), + &sled_agent_client::types::DiskEnsureBody { + initial_runtime: + sled_agent_client::types::DiskRuntimeState::from( + runtime, + ), + target: requested, + }, + ) + .await + .map_err(Error::from)?; + + new_runtime.into_inner().into() + } - let new_runtime: DiskRuntimeState = new_runtime.into_inner().into(); + InstanceState::Creating => { + // If we're still creating this instance, then the disks will be + // attached as part of the instance ensure by specifying volume + // construction requests. + match requested { + DiskStateRequested::Detached => { + db_disk.runtime().detach().into() + } + DiskStateRequested::Attached(id) => { + db_disk.runtime().attach(id).into() + } + DiskStateRequested::Destroyed => { + todo!() + } + DiskStateRequested::Faulted => { + todo!() + } + } + } + + _ => { + todo!( + "implement state {:?}", + db_instance.runtime_state.state.state() + ); + } + }; self.db_datastore .disk_update_runtime(opctx, authz_disk, &new_runtime.into()) diff --git a/nexus/src/sagas.rs b/nexus/src/sagas.rs index 53197ecb607..1191b6e4669 100644 --- a/nexus/src/sagas.rs +++ b/nexus/src/sagas.rs @@ -30,6 +30,7 @@ use omicron_common::api::external::Name; use omicron_common::api::external::NetworkInterface; use omicron_common::api::internal::nexus::InstanceRuntimeState; use omicron_common::backoff::{self, BackoffError}; +use rand::{rngs::StdRng, RngCore, SeedableRng}; use serde::Deserialize; use serde::Serialize; use sled_agent_client::types::InstanceEnsureBody; @@ -113,6 +114,8 @@ async fn saga_generate_uuid( #[derive(Debug, Deserialize, Serialize)] pub struct ParamsInstanceCreate { pub serialized_authn: authn::saga::Serialized, + pub organization_name: Name, + pub project_name: Name, pub project_id: Uuid, pub create_params: params::InstanceCreate, } @@ -149,7 +152,7 @@ pub fn saga_instance_create() -> SagaTemplate { ); template_builder.append( - "initial_runtime", + "instance_name", "CreateInstanceRecord", new_action_noop_undo(sic_create_instance_record), ); @@ -184,12 +187,22 @@ pub fn saga_instance_create() -> SagaTemplate { sic_create_network_interfaces_undo, ), ); + template_builder.append( "network_interfaces", "CreateNetworkInterfaces", new_action_noop_undo(sic_create_network_interfaces), ); + template_builder.append( + "attach_disks", + "AttachDisksToInstance", + ActionFunc::new_action( + sic_attach_disks_to_instance, + sic_attach_disks_to_instance_undo, + ), + ); + template_builder.append( "instance_ensure", "InstanceEnsure", @@ -475,9 +488,77 @@ async fn sic_create_network_interfaces_undo( Ok(()) } +async fn sic_attach_disks_to_instance( + sagactx: ActionContext, +) -> Result<(), ActionError> { + ensure_instance_disk_attach_state(sagactx, true).await +} + +async fn sic_attach_disks_to_instance_undo( + sagactx: ActionContext, +) -> Result<(), anyhow::Error> { + Ok(ensure_instance_disk_attach_state(sagactx, false).await?) +} + +async fn ensure_instance_disk_attach_state( + sagactx: ActionContext, + attached: bool, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let saga_params = sagactx.saga_params(); + let opctx = + OpContext::for_saga_action(&sagactx, &saga_params.serialized_authn); + + let saga_disks = &saga_params.create_params.disks; + let instance_name = sagactx.lookup::("instance_name")?; + + let organization_name: db::model::Name = + saga_params.organization_name.clone().into(); + let project_name: db::model::Name = saga_params.project_name.clone().into(); + + for disk in saga_disks { + match disk { + params::InstanceDiskAttachment::Create(_) => { + todo!(); + } + params::InstanceDiskAttachment::Attach(instance_disk_attach) => { + let disk_name: db::model::Name = + instance_disk_attach.disk.clone().into(); + + if attached { + osagactx + .nexus() + .instance_attach_disk( + &opctx, + &organization_name, + &project_name, + &instance_name, + &disk_name, + ) + .await + } else { + osagactx + .nexus() + .instance_detach_disk( + &opctx, + &organization_name, + &project_name, + &instance_name, + &disk_name, + ) + .await + } + .map_err(ActionError::action_failed)?; + } + } + } + + Ok(()) +} + async fn sic_create_instance_record( sagactx: ActionContext, -) -> Result { +) -> Result { let osagactx = sagactx.user_data(); let params = sagactx.saga_params(); let sled_uuid = sagactx.lookup::("server_id"); @@ -511,7 +592,7 @@ async fn sic_create_instance_record( .await .map_err(ActionError::action_failed)?; - Ok(instance.runtime().clone().into()) + Ok(instance.name().clone()) } async fn sic_instance_ensure( @@ -519,50 +600,39 @@ async fn sic_instance_ensure( ) -> Result<(), ActionError> { // TODO-correctness is this idempotent? let osagactx = sagactx.user_data(); + let params = sagactx.saga_params(); let runtime_params = InstanceRuntimeStateRequested { run_state: InstanceStateRequested::Running, migration_params: None, }; - let instance_id = sagactx.lookup::("instance_id")?; - let sled_uuid = sagactx.lookup::("server_id")?; - let nics = sagactx - .lookup::>>("network_interfaces")? - .unwrap_or_default(); - let runtime = sagactx.lookup::("initial_runtime")?; - let initial_hardware = InstanceHardware { - runtime: runtime.into(), - nics: nics.into_iter().map(|nic| nic.into()).collect(), - }; - let sa = osagactx - .sled_client(&sled_uuid) + + let instance_name = sagactx.lookup::("instance_name")?; + let opctx = OpContext::for_saga_action(&sagactx, ¶ms.serialized_authn); + + let authz_project = osagactx + .datastore() + .project_lookup_by_id(params.project_id) .await .map_err(ActionError::action_failed)?; - // Ask the sled agent to begin the state change. Then update the database - // to reflect the new intermediate state. If this update is not the newest - // one, that's fine. That might just mean the sled agent beat us to it. - let new_runtime_state = sa - .instance_put( - &instance_id, - &InstanceEnsureBody { - initial: initial_hardware, - target: runtime_params, - migrate: None, - }, - ) + let (authz_instance, instance) = osagactx + .datastore() + .instance_fetch(&opctx, &authz_project, &instance_name) .await - .map_err(omicron_common::api::external::Error::from) .map_err(ActionError::action_failed)?; - let new_runtime_state: InstanceRuntimeState = - new_runtime_state.into_inner().into(); - osagactx - .datastore() - .instance_update_runtime(&instance_id, &new_runtime_state.into()) + .nexus() + .instance_set_runtime( + &opctx, + &authz_instance, + &instance, + runtime_params, + ) .await - .map(|_| ()) - .map_err(ActionError::action_failed) + .map_err(ActionError::action_failed)?; + + Ok(()) } // "Migrate Instance" saga template @@ -670,6 +740,8 @@ async fn sim_instance_migrate( runtime: runtime.into(), // TODO: populate NICs nics: vec![], + // TODO: populate disks + disks: vec![], }; let target = InstanceRuntimeStateRequested { run_state: InstanceStateRequested::Migrating, @@ -879,7 +951,6 @@ async fn ensure_region_in_dataset( // TODO: Can we avoid casting from UUID to string? // NOTE: This'll require updating the crucible agent client. id: RegionId(region.id().to_string()), - volume_id: region.volume_id().to_string(), encrypted: region.encrypted(), cert_pem: None, key_pem: None, @@ -927,30 +998,120 @@ const MAX_CONCURRENT_REGION_REQUESTS: usize = 3; async fn sdc_regions_ensure( sagactx: ActionContext, -) -> Result<(), ActionError> { +) -> Result { let log = sagactx.user_data().log(); let datasets_and_regions = sagactx .lookup::>( "datasets_and_regions", )?; + let request_count = datasets_and_regions.len(); - futures::stream::iter(datasets_and_regions) + + // Allocate regions, and additionally return the dataset that the region was + // allocated in. + let datasets_and_regions: Vec<( + db::model::Dataset, + crucible_agent_client::types::Region, + )> = futures::stream::iter(datasets_and_regions) .map(|(dataset, region)| async move { - ensure_region_in_dataset(log, &dataset, ®ion).await + match ensure_region_in_dataset(log, &dataset, ®ion).await { + Ok(result) => Ok((dataset, result)), + Err(e) => Err(e), + } }) // Execute the allocation requests concurrently. .buffer_unordered(std::cmp::min( request_count, MAX_CONCURRENT_REGION_REQUESTS, )) - .collect::>>() + .collect::, + >>() .await .into_iter() - .collect::, _>>() + .collect::, + Error, + >>() .map_err(ActionError::action_failed)?; - // TODO: Region has a port value, we could store this in the DB? - Ok(()) + // Assert each region has the same block size, otherwise Volume creation + // will fail. + let all_region_have_same_block_size = datasets_and_regions + .windows(2) + .all(|w| w[0].1.block_size == w[1].1.block_size); + + if !all_region_have_same_block_size { + return Err(ActionError::new_subsaga( + // XXX wrong error type? + anyhow::anyhow!( + "volume creation will fail due to block size mismatch" + ), + )); + } + + let block_size = datasets_and_regions[0].1.block_size; + + // Store volume details in db + let mut rng = StdRng::from_entropy(); + let volume_construction_request = + sled_agent_client::types::VolumeConstructionRequest::Volume { + block_size, + sub_volumes: vec![ + // XXX allocation algorithm only supports one sub vol? + sled_agent_client::types::VolumeConstructionRequest::Region { + block_size, + // gen of 0 is here, these regions were just allocated. + gen: 0, + opts: sled_agent_client::types::CrucibleOpts { + target: datasets_and_regions + .iter() + .map(|(dataset, region)| { + dataset + .address_with_port(region.port_number) + .to_string() + }) + .collect(), + + lossy: false, + + // all downstairs will expect encrypted blocks + key: Some(base64::encode({ + // XXX the current encryption key + // requirement is 32 bytes, what if that + // changes? + let mut random_bytes: [u8; 32] = [0; 32]; + rng.fill_bytes(&mut random_bytes); + random_bytes + })), + + // TODO TLS, which requires sending X509 stuff during + // downstairs region allocation too. + cert_pem: None, + key_pem: None, + root_cert_pem: None, + + // TODO open a control socket for the whole volume, not + // in the sub volumes + control: None, + }, + }, + ], + read_only_parent: None, + }; + + let volume_data = serde_json::to_string(&volume_construction_request) + .map_err(|_| { + ActionError::new_subsaga( + // XXX wrong error type? + anyhow::anyhow!("serde_json::to_string"), + ) + })?; + + Ok(volume_data) } async fn delete_regions( @@ -1008,16 +1169,16 @@ async fn sdc_create_volume_record( let osagactx = sagactx.user_data(); let volume_id = sagactx.lookup::("volume_id")?; - let volume = db::model::Volume::new( - volume_id, - // TODO: Patch this up with serialized contents that Crucible can use. - "Some Data".to_string(), - ); + let volume_data = sagactx.lookup::("regions_ensure")?; + + let volume = db::model::Volume::new(volume_id, volume_data); + let volume_created = osagactx .datastore() .volume_create(volume) .await .map_err(ActionError::action_failed)?; + Ok(volume_created) } diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index 3f7068e77d3..2896244be63 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -143,6 +143,7 @@ pub async fn create_instance( hostname: String::from("the_host"), network_interfaces: params::InstanceNetworkInterfaceAttachment::Default, + disks: vec![], }, ) .await diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index 6af67d4f043..492ec4c4b8d 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -166,6 +166,7 @@ lazy_static! { hostname: String::from("demo-instance"), network_interfaces: params::InstanceNetworkInterfaceAttachment::Default, + disks: vec![], }; } diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index e166ad3fde9..3e9ad1f32c3 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -9,8 +9,12 @@ use http::StatusCode; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; +use nexus_test_utils::resource_helpers::create_disk; use nexus_test_utils::resource_helpers::objects_list_page_authz; +use nexus_test_utils::resource_helpers::DiskTest; use omicron_common::api::external::ByteCount; +use omicron_common::api::external::Disk; +use omicron_common::api::external::DiskState; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::Instance; use omicron_common::api::external::InstanceCpuCount; @@ -144,6 +148,7 @@ async fn test_instances_create_reboot_halt( hostname: instance.hostname.clone(), network_interfaces: params::InstanceNetworkInterfaceAttachment::Default, + disks: vec![], })) .expect_status(Some(StatusCode::BAD_REQUEST)), ) @@ -551,6 +556,7 @@ async fn test_instance_with_single_explicit_ip_address( memory: ByteCount::from_mebibytes_u32(4), hostname: String::from("nic-test"), network_interfaces: interface_params, + disks: vec![], }; let response = NexusRequest::objects_post(client, &url_instances, &instance_params) @@ -666,6 +672,7 @@ async fn test_instance_with_new_custom_network_interfaces( memory: ByteCount::from_mebibytes_u32(4), hostname: String::from("nic-test"), network_interfaces: interface_params, + disks: vec![], }; let response = NexusRequest::objects_post(client, &url_instances, &instance_params) @@ -758,6 +765,7 @@ async fn test_instance_create_delete_network_interface( memory: ByteCount::from_mebibytes_u32(4), hostname: String::from("nic-test"), network_interfaces: params::InstanceNetworkInterfaceAttachment::None, + disks: vec![], }; let response = NexusRequest::objects_post(client, &url_instances, &instance_params) @@ -940,6 +948,7 @@ async fn test_instance_with_multiple_nics_unwinds_completely( memory: ByteCount::from_mebibytes_u32(4), hostname: String::from("nic-test"), network_interfaces: interface_params, + disks: vec![], }; let builder = RequestBuilder::new(client, http::Method::POST, &url_instances) @@ -969,6 +978,92 @@ async fn test_instance_with_multiple_nics_unwinds_completely( ); } +/// Create a disk and attach during instance creation +#[nexus_test] +async fn test_attach_one_disk_to_instance(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + + const ORGANIZATION_NAME: &str = "bobs-barrel-of-bytes"; + const PROJECT_NAME: &str = "bit-barrel"; + + // Test pre-reqs + DiskTest::new(&cptestctx).await; + create_organization(&client, ORGANIZATION_NAME).await; + create_project(client, ORGANIZATION_NAME, PROJECT_NAME).await; + + // Create the "probablydata" disk + create_disk(&client, ORGANIZATION_NAME, PROJECT_NAME, "probablydata").await; + + // Verify disk is there and currently detached + let url_project_disks = format!( + "/organizations/{}/projects/{}/disks", + ORGANIZATION_NAME, PROJECT_NAME, + ); + let disks: Vec = NexusRequest::iter_collection_authn( + client, + &url_project_disks, + "", + None, + ) + .await + .expect("failed to list disks") + .all_items; + assert_eq!(disks.len(), 1); + assert_eq!(disks[0].state, DiskState::Detached); + + // Create the instance + let instance_params = params::InstanceCreate { + identity: IdentityMetadataCreateParams { + name: Name::try_from(String::from("nfs")).unwrap(), + description: String::from("probably serving data"), + }, + ncpus: InstanceCpuCount::try_from(2).unwrap(), + memory: ByteCount::from_mebibytes_u32(4), + hostname: String::from("nfs"), + network_interfaces: params::InstanceNetworkInterfaceAttachment::Default, + disks: vec![params::InstanceDiskAttachment::Attach( + params::InstanceDiskAttach { + disk: Name::try_from(String::from("probablydata")).unwrap(), + }, + )], + }; + + let url_instances = format!( + "/organizations/{}/projects/{}/instances", + ORGANIZATION_NAME, PROJECT_NAME + ); + let builder = + RequestBuilder::new(client, http::Method::POST, &url_instances) + .body(Some(&instance_params)) + .expect_status(Some(http::StatusCode::CREATED)); + let response = NexusRequest::new(builder) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("Expected instance creation to work!"); + + let instance = response.parsed_body::().unwrap(); + + // Verify disk is attached to the instance + let url_instance_disks = format!( + "/organizations/{}/projects/{}/instances/{}/disks", + ORGANIZATION_NAME, + PROJECT_NAME, + instance.identity.name.as_str(), + ); + let disks: Vec = NexusRequest::iter_collection_authn( + client, + &url_instance_disks, + "", + None, + ) + .await + .expect("failed to list disks") + .all_items; + assert_eq!(disks.len(), 1); + assert_eq!(disks[0].state, DiskState::Attached(instance.identity.id)); +} + async fn instance_get( client: &ClientTestContext, instance_url: &str, diff --git a/nexus/tests/integration_tests/subnet_allocation.rs b/nexus/tests/integration_tests/subnet_allocation.rs index 47a0e5ef723..9c8279d8520 100644 --- a/nexus/tests/integration_tests/subnet_allocation.rs +++ b/nexus/tests/integration_tests/subnet_allocation.rs @@ -40,6 +40,7 @@ async fn create_instance_expect_failure( memory: ByteCount::from_mebibytes_u32(256), hostname: name.to_string(), network_interfaces: params::InstanceNetworkInterfaceAttachment::Default, + disks: vec![], }; NexusRequest::new( diff --git a/openapi/nexus.json b/openapi/nexus.json index e14c22ff713..97aea629146 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -4597,6 +4597,13 @@ "description": { "type": "string" }, + "disks": { + "description": "The disks to be created or attached for this instance.", + "type": "array", + "items": { + "$ref": "#/components/schemas/InstanceDiskAttachment" + } + }, "hostname": { "type": "string" }, @@ -4626,6 +4633,73 @@ "ncpus" ] }, + "InstanceDiskAttachment": { + "description": "Describe the instance's disks at creation time", + "oneOf": [ + { + "description": "During instance creation, create and attach disks", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + }, + "size": { + "description": "size of the Disk", + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + }, + "snapshot_id": { + "nullable": true, + "description": "id for snapshot from which the Disk should be created, if any", + "type": "string", + "format": "uuid" + }, + "type": { + "type": "string", + "enum": [ + "create" + ] + } + }, + "required": [ + "description", + "name", + "size", + "type" + ] + }, + { + "description": "During instance creation, attach this disk", + "type": "object", + "properties": { + "disk": { + "description": "A disk name to attach", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "attach" + ] + } + }, + "required": [ + "disk", + "type" + ] + } + ] + }, "InstanceMigrate": { "description": "Migration parameters for an [`Instance`](omicron_common::api::external::Instance)", "type": "object", diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 2b5a3bc1b95..5d3fe21f20a 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -199,6 +199,44 @@ "format": "uint64", "minimum": 0 }, + "CrucibleOpts": { + "type": "object", + "properties": { + "cert_pem": { + "nullable": true, + "type": "string" + }, + "control": { + "nullable": true, + "type": "string" + }, + "key": { + "nullable": true, + "type": "string" + }, + "key_pem": { + "nullable": true, + "type": "string" + }, + "lossy": { + "type": "boolean" + }, + "root_cert_pem": { + "nullable": true, + "type": "string" + }, + "target": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "lossy", + "target" + ] + }, "DatasetEnsureBody": { "description": "Used to request a new partition kind exists within a zpool.\n\nMany partition types are associated with services that will be instantiated when the partition is detected.", "type": "object", @@ -301,6 +339,39 @@ "target" ] }, + "DiskRequest": { + "type": "object", + "properties": { + "device": { + "type": "string" + }, + "gen": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "name": { + "type": "string" + }, + "read_only": { + "type": "boolean" + }, + "slot": { + "$ref": "#/components/schemas/Slot" + }, + "volume_construction_request": { + "$ref": "#/components/schemas/VolumeConstructionRequest" + } + }, + "required": [ + "device", + "gen", + "name", + "read_only", + "slot", + "volume_construction_request" + ] + }, "DiskRuntimeState": { "description": "Runtime state of the Disk, which includes its attach state and some minimal metadata", "type": "object", @@ -594,6 +665,12 @@ "description": "Describes the instance hardware.", "type": "object", "properties": { + "disks": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DiskRequest" + } + }, "nics": { "type": "array", "items": { @@ -605,6 +682,7 @@ } }, "required": [ + "disks", "nics", "runtime" ] @@ -895,6 +973,12 @@ "name" ] }, + "Slot": { + "description": "A stable index which is translated by Propolis into a PCI BDF, visible to the guest.", + "type": "integer", + "format": "uint8", + "minimum": 0 + }, "UpdateArtifact": { "description": "Description of a single update artifact.", "type": "object", @@ -922,6 +1006,99 @@ "enum": [ "zone" ] + }, + "VolumeConstructionRequest": { + "oneOf": [ + { + "type": "object", + "properties": { + "block_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "read_only_parent": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/VolumeConstructionRequest" + } + ] + }, + "sub_volumes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VolumeConstructionRequest" + } + }, + "type": { + "type": "string", + "enum": [ + "Volume" + ] + } + }, + "required": [ + "block_size", + "sub_volumes", + "type" + ] + }, + { + "type": "object", + "properties": { + "block_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "Url" + ] + }, + "url": { + "type": "string" + } + }, + "required": [ + "block_size", + "type", + "url" + ] + }, + { + "type": "object", + "properties": { + "block_size": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "gen": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "opts": { + "$ref": "#/components/schemas/CrucibleOpts" + }, + "type": { + "type": "string", + "enum": [ + "Region" + ] + } + }, + "required": [ + "block_size", + "gen", + "opts", + "type" + ] + } + ] } } } diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 6e155cf282b..ef1ce6abe44 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -12,7 +12,7 @@ bytes = "1.1" cfg-if = "1.0" chrono = { version = "0.4", features = [ "serde" ] } # Only used by the simulated sled agent. -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "3e7e49eeb88fa8ad74375b0642aabd4224b1f2cb" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "32e603bdc3b6e5eb6b880a2ddde7e05f043b5357" } dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } futures = "0.3.21" ipnetwork = "0.18" @@ -21,7 +21,7 @@ omicron-common = { path = "../common" } p256 = "0.9.0" percent-encoding = "2.1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "0e3798510ae190131f63b9df767ec01b2beacf91" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "832a86afce308d3210685af987ace1ba74c2ecd6" } rand = { version = "0.8.5", features = ["getrandom"] } reqwest = { version = "0.11.8", default-features = false, features = ["rustls-tls", "stream"] } schemars = { version = "0.8", features = [ "chrono", "uuid" ] } diff --git a/sled-agent/src/bin/sled-agent-sim.rs b/sled-agent/src/bin/sled-agent-sim.rs index cc7b3ddf7f3..61adec0ce58 100644 --- a/sled-agent/src/bin/sled-agent-sim.rs +++ b/sled-agent/src/bin/sled-agent-sim.rs @@ -68,7 +68,7 @@ async fn do_run() -> Result<(), CmdError> { nexus_address: args.nexus_addr, dropshot: ConfigDropshot { bind_address: args.sled_agent_addr, - request_body_max_bytes: 2048, + request_body_max_bytes: 1024 * 1024, ..Default::default() }, log: ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Info }, diff --git a/sled-agent/src/bin/sled-agent.rs b/sled-agent/src/bin/sled-agent.rs index 4edfe39f853..6c24ed541a4 100644 --- a/sled-agent/src/bin/sled-agent.rs +++ b/sled-agent/src/bin/sled-agent.rs @@ -81,8 +81,10 @@ async fn do_run() -> Result<(), CmdError> { } }, Args::Run { config_path } => { - let config = SledConfig::from_file(&config_path) + let mut config = SledConfig::from_file(&config_path) .map_err(|e| CmdError::Failure(e.to_string()))?; + config.dropshot.request_body_max_bytes = 1024 * 1024; + let config = config; // - Sled agent starts with the normal config file - typically // called "config.toml". @@ -116,7 +118,7 @@ async fn do_run() -> Result<(), CmdError> { id: config.id, dropshot: ConfigDropshot { bind_address: config.bootstrap_address, - request_body_max_bytes: 2048, + request_body_max_bytes: 1024 * 1024, ..Default::default() }, log: ConfigLogging::StderrTerminal { diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index fe202861cf3..904b7c80b87 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -22,6 +22,7 @@ use futures::lock::{Mutex, MutexGuard}; use omicron_common::api::external::NetworkInterface; use omicron_common::api::internal::nexus::InstanceRuntimeState; use omicron_common::backoff; +use propolis_client::api::DiskRequest; use propolis_client::Client as PropolisClient; use slog::Logger; use std::net::SocketAddr; @@ -62,6 +63,9 @@ pub enum Error { #[error(transparent)] RunningZone(#[from] crate::illumos::running_zone::Error), + + #[error("serde_json failure: {0}")] + SerdeJsonError(#[from] serde_json::Error), } // Issues read-only, idempotent HTTP requests at propolis until it responds with @@ -181,6 +185,9 @@ struct InstanceInner { requested_nics: Vec, vlan: Option, + // Disk related properties + requested_disks: Vec, + // Internal State management state: InstanceStates, running_state: Option, @@ -284,9 +291,9 @@ impl InstanceInner { let request = propolis_client::api::InstanceEnsureRequest { properties: self.properties.clone(), nics, - // TODO: Actual disks need to be wired up here. - disks: vec![], + disks: self.requested_disks.clone(), migrate, + cloud_init_bytes: None, }; info!(self.log, "Sending ensure request to propolis: {:?}", request); @@ -416,6 +423,7 @@ impl Instance { }, vnic_allocator, requested_nics: initial.nics, + requested_disks: initial.disks, vlan, state: InstanceStates::new(initial.runtime), running_state: None, @@ -678,6 +686,7 @@ mod test { time_updated: Utc::now(), }, nics: vec![], + disks: vec![], } } diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index d8b7e946c84..af250d2f935 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -236,6 +236,7 @@ mod test { time_updated: Utc::now(), }, nics: vec![], + disks: vec![], } } diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index c691d90eac0..23c8f739a17 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -50,6 +50,7 @@ pub struct DiskEnsureBody { pub struct InstanceHardware { pub runtime: InstanceRuntimeState, pub nics: Vec, + pub disks: Vec, } /// Sent to a sled agent to establish the runtime state of an Instance diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 7d9770a2494..61cd59ec6a0 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -52,7 +52,6 @@ impl CrucibleDataInner { let region = Region { id: params.id, - volume_id: params.volume_id, block_size: params.block_size, extent_size: params.extent_size, extent_count: params.extent_count, diff --git a/tools/oxapi_demo b/tools/oxapi_demo index e22ca9f221c..1b8fce3e7c0 100755 --- a/tools/oxapi_demo +++ b/tools/oxapi_demo @@ -272,12 +272,18 @@ function cmd_project_list_vpcs function cmd_instance_create_demo { - # memory is 1024 * 1024 * 256 [[ $# != 3 ]] && usage "expected ORGANIZATION_NAME PROJECT_NAME INSTANCE_NAME" - mkjson name="$3" description="an instance called $3" ncpus=1 \ - memory=268435456 boot_disk_size=1 hostname="$3" | - do_curl_authn "/organizations/$1/projects/$2/instances" \ - -X POST -T - + do_curl_authn "/organizations/$1/projects/$2/instances" \ + -X POST -T - <