diff --git a/Cargo.lock b/Cargo.lock index f0f12429b..0d2aa618f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5926,16 +5926,13 @@ version = "0.33.0" dependencies = [ "anyhow", "chrono", - "dunce", "futures-core", - "home", "prost 0.12.2", "prost-types", "serde_json", "shuttle-common", "tokio", "tonic 0.10.2", - "tower", "tracing", ] @@ -6028,12 +6025,15 @@ dependencies = [ "anyhow", "async-trait", "cargo_metadata 0.18.1", + "dunce", "serde", "shuttle-common", + "shuttle-proto", "strfmt", "thiserror", "tokio", "toml 0.8.8", + "tower", "tracing", ] diff --git a/builder/Cargo.toml b/builder/Cargo.toml index 57380702a..2bb30e436 100644 --- a/builder/Cargo.toml +++ b/builder/Cargo.toml @@ -7,7 +7,7 @@ repository.workspace = true [dependencies] shuttle-common = { workspace = true, features = ["backend", "tonic"] } -shuttle-proto = { workspace = true } +shuttle-proto = { workspace = true, features = ["builder"] } async-trait = { workspace = true } clap = { workspace = true } diff --git a/cargo-shuttle/Cargo.toml b/cargo-shuttle/Cargo.toml index 9e3d8b328..e026ed35e 100644 --- a/cargo-shuttle/Cargo.toml +++ b/cargo-shuttle/Cargo.toml @@ -10,8 +10,8 @@ rust-version = "1.70" [dependencies] shuttle-common = { workspace = true, features = ["models"] } -shuttle-proto = { workspace = true } -shuttle-service = { workspace = true, features = ["builder"] } +shuttle-proto = { workspace = true, features = ["provisioner", "runtime"] } +shuttle-service = { workspace = true, features = ["builder", "runner"] } anyhow = { workspace = true } async-trait = { workspace = true } diff --git a/cargo-shuttle/src/args.rs b/cargo-shuttle/src/args.rs index 21830c44f..11803565a 100644 --- a/cargo-shuttle/src/args.rs +++ b/cargo-shuttle/src/args.rs @@ -12,7 +12,8 @@ use clap::{ Parser, ValueEnum, }; use clap_complete::Shell; -use shuttle_common::{models::project::DEFAULT_IDLE_MINUTES, resource}; +use shuttle_common::constants::DEFAULT_IDLE_MINUTES; +use shuttle_common::resource; use uuid::Uuid; #[derive(Parser)] diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index c94d5d7cc..671eaada2 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -17,8 +17,9 @@ use std::str::FromStr; use shuttle_common::{ claims::{ClaimService, InjectPropagation}, constants::{ - API_URL_DEFAULT, EXECUTABLE_DIRNAME, SHUTTLE_CLI_DOCS_URL, SHUTTLE_GH_ISSUE_URL, - SHUTTLE_IDLE_DOCS_URL, SHUTTLE_INSTALL_DOCS_URL, SHUTTLE_LOGIN_URL, STORAGE_DIRNAME, + API_URL_DEFAULT, DEFAULT_IDLE_MINUTES, EXECUTABLE_DIRNAME, SHUTTLE_CLI_DOCS_URL, + SHUTTLE_GH_ISSUE_URL, SHUTTLE_IDLE_DOCS_URL, SHUTTLE_INSTALL_DOCS_URL, SHUTTLE_LOGIN_URL, + STORAGE_DIRNAME, }, deployment::{DEPLOYER_END_MESSAGES_BAD, DEPLOYER_END_MESSAGES_GOOD}, models::{ @@ -27,14 +28,15 @@ use shuttle_common::{ GIT_STRINGS_MAX_LENGTH, }, error::ApiError, - project::{self, DEFAULT_IDLE_MINUTES}, + project, resource::get_resource_tables, }, resource, semvers_are_compatible, ApiKey, LogItem, VersionInfo, }; use shuttle_proto::runtime::{ - self, runtime_client::RuntimeClient, LoadRequest, StartRequest, StopRequest, + runtime_client::RuntimeClient, LoadRequest, StartRequest, StopRequest, }; +use shuttle_service::runner; use shuttle_service::{ builder::{build_workspace, BuiltService}, Environment, @@ -922,7 +924,7 @@ impl Shuttle { }; // Child process and gRPC client for sending requests to it - let (mut runtime, mut runtime_client) = runtime::start( + let (mut runtime, mut runtime_client) = runner::start( service.is_wasm, Environment::Local, &format!("http://localhost:{provisioner_port}"), diff --git a/common-tests/Cargo.toml b/common-tests/Cargo.toml index 8f1c6400e..de1f74858 100644 --- a/common-tests/Cargo.toml +++ b/common-tests/Cargo.toml @@ -8,7 +8,7 @@ publish = false [dependencies] cargo-shuttle = { path = "../cargo-shuttle" } -shuttle-proto = { workspace = true } +shuttle-proto = { workspace = true, features = ["builder", "logger"] } hyper = { workspace = true } portpicker = { workspace = true } diff --git a/common/Cargo.toml b/common/Cargo.toml index 512f0b637..0678142e2 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -67,9 +67,6 @@ backend = [ "tracing-subscriber/env-filter", "tracing-subscriber/fmt", "ttl_cache", - "sqlx", - "sqlx/postgres", # Fix for derive macro error when different backends enable sqlite & postgres (mainly for grouped compilation in clippy CI and Docker build) - "sqlx/sqlite", ] claims = [ "bytes", @@ -87,16 +84,9 @@ claims = [ ] display = ["chrono/clock", "comfy-table", "crossterm"] openapi = ["utoipa/chrono", "utoipa/uuid"] -models = [ - "async-trait", - "display", - "http", - "reqwest", - "service", - "thiserror", -] -persist = ["sqlx/sqlite", "rand"] -service = ["chrono/serde", "tracing", "tracing-subscriber", "uuid"] +models = ["async-trait", "http", "reqwest", "service", "thiserror"] +persist = ["sqlx", "rand"] +service = ["chrono/serde", "display", "tracing", "tracing-subscriber", "uuid"] tracing = ["dep:tracing"] wasm = [ "chrono/clock", diff --git a/common/src/claims.rs b/common/src/claims.rs index 4bca392cb..e0a5409ee 100644 --- a/common/src/claims.rs +++ b/common/src/claims.rs @@ -171,10 +171,10 @@ impl Default for ScopeBuilder { #[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, Eq, PartialEq)] #[serde(rename_all = "lowercase")] -#[cfg_attr(feature = "backend", derive(strum::Display))] -#[cfg_attr(feature = "backend", derive(sqlx::Type))] -#[cfg_attr(feature = "backend", sqlx(rename_all = "lowercase"))] -#[cfg_attr(feature = "backend", strum(serialize_all = "lowercase"))] +#[cfg_attr(feature = "display", derive(strum::Display))] +#[cfg_attr(feature = "display", strum(serialize_all = "lowercase"))] +#[cfg_attr(feature = "persist", derive(sqlx::Type))] +#[cfg_attr(feature = "persist", sqlx(rename_all = "lowercase"))] pub enum AccountTier { #[default] Basic, diff --git a/common/src/constants.rs b/common/src/constants.rs index af8ad1699..0bdf6276a 100644 --- a/common/src/constants.rs +++ b/common/src/constants.rs @@ -1,6 +1,5 @@ -// -// Constants regarding the deployer environment and conventions -// +//! Shared constants used across Shuttle crates + /// Where executables are moved to in order to persist across deploys, relative to workspace root pub const EXECUTABLE_DIRNAME: &str = ".shuttle-executables"; /// Where general files will persist across deploys, relative to workspace root. Used by plugins. @@ -27,6 +26,14 @@ pub const SHUTTLE_EXAMPLES_README: &str = pub const NEXT_NAME: &str = "shuttle-next"; pub const RUNTIME_NAME: &str = "shuttle-runtime"; +/// Timeframe before a project is considered idle +pub const DEFAULT_IDLE_MINUTES: u64 = 30; + +/// Function to set [DEFAULT_IDLE_MINUTES] as a serde default +pub const fn default_idle_minutes() -> u64 { + DEFAULT_IDLE_MINUTES +} + pub mod limits { pub const MAX_PROJECTS_DEFAULT: u32 = 3; pub const MAX_PROJECTS_EXTRA: u32 = 15; diff --git a/common/src/models/mod.rs b/common/src/models/mod.rs index 3fbc6c1de..1d687f638 100644 --- a/common/src/models/mod.rs +++ b/common/src/models/mod.rs @@ -1,7 +1,6 @@ pub mod admin; pub mod deployment; pub mod error; -#[cfg(feature = "backend")] pub mod project; pub mod resource; pub mod service; diff --git a/common/src/models/project.rs b/common/src/models/project.rs index d4c000a77..688f8d817 100644 --- a/common/src/models/project.rs +++ b/common/src/models/project.rs @@ -1,8 +1,6 @@ -use std::collections::HashSet; use std::fmt::Display; use std::fmt::Formatter; use std::str::FromStr; -use std::sync::OnceLock; use comfy_table::{ modifiers::UTF8_ROUND_CORNERS, @@ -10,9 +8,7 @@ use comfy_table::{ Attribute, Cell, CellAlignment, Color, ContentArrangement, Table, }; use crossterm::style::Stylize; -use rustrict::{Censor, Type}; -use serde::de::Error as DeError; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Serialize}; use strum::EnumString; #[cfg(feature = "openapi")] @@ -20,16 +16,6 @@ use crate::ulid_type; #[cfg(feature = "openapi")] use utoipa::ToSchema; -use super::error::InvalidProjectName; - -/// Timeframe before a project is considered idle -pub const DEFAULT_IDLE_MINUTES: u64 = 30; - -/// Function to set [DEFAULT_IDLE_MINUTES] as a serde default -pub const fn default_idle_minutes() -> u64 { - DEFAULT_IDLE_MINUTES -} - #[derive(Deserialize, Serialize, Clone)] #[cfg_attr(feature = "openapi", derive(ToSchema))] #[cfg_attr(feature = "openapi", schema(as = shuttle_common::models::project::Response))] @@ -259,142 +245,159 @@ pub fn get_projects_table( } } -/// Project names must conform to valid Host segments (or labels) -/// as per [IETF RFC 1123](https://datatracker.ietf.org/doc/html/rfc1123). -/// Initially we'll implement a strict subset of the IETF RFC 1123. -/// Additionaly, while host segments are technically case-insensitive, the filesystem isn't, -/// so we restrict project names to be lower case. We also restrict the use of profanity, -/// as well as a list of reserved words. -#[derive(Clone, Serialize, Debug, Eq, Hash, PartialEq, sqlx::Type)] -#[sqlx(transparent)] -pub struct ProjectName(String); - -impl ProjectName { - pub fn new(name: &str) -> Result { - if Self::is_valid(name) { - Ok(Self(name.to_owned())) - } else { - Err(InvalidProjectName) +#[cfg(feature = "backend")] +pub use name::ProjectName; +#[cfg(feature = "backend")] +pub mod name { + use std::collections::HashSet; + use std::fmt::Formatter; + use std::str::FromStr; + use std::sync::OnceLock; + + use rustrict::{Censor, Type}; + use serde::de::Error as DeError; + use serde::{Deserialize, Deserializer, Serialize}; + + use crate::models::error::InvalidProjectName; + + /// Project names must conform to valid Host segments (or labels) + /// as per [IETF RFC 1123](https://datatracker.ietf.org/doc/html/rfc1123). + /// Initially we'll implement a strict subset of the IETF RFC 1123. + /// Additionaly, while host segments are technically case-insensitive, the filesystem isn't, + /// so we restrict project names to be lower case. We also restrict the use of profanity, + /// as well as a list of reserved words. + #[derive(Clone, Serialize, Debug, Eq, Hash, PartialEq)] + #[cfg_attr(feature = "persist", derive(sqlx::Type))] + #[cfg_attr(feature = "persist", sqlx(transparent))] + pub struct ProjectName(String); + + impl ProjectName { + pub fn new(name: &str) -> Result { + if Self::is_valid(name) { + Ok(Self(name.to_owned())) + } else { + Err(InvalidProjectName) + } } - } - pub fn is_valid(name: &str) -> bool { - fn is_valid_char(byte: u8) -> bool { - matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'-') - } + pub fn is_valid(name: &str) -> bool { + fn is_valid_char(byte: u8) -> bool { + matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'-') + } - fn is_profanity_free(name: &str) -> bool { - let (_censored, analysis) = Censor::from_str(name).censor_and_analyze(); - !analysis.is(Type::MODERATE_OR_HIGHER) - } + fn is_profanity_free(name: &str) -> bool { + let (_censored, analysis) = Censor::from_str(name).censor_and_analyze(); + !analysis.is(Type::MODERATE_OR_HIGHER) + } - fn is_reserved(name: &str) -> bool { - static INSTANCE: OnceLock> = OnceLock::new(); - INSTANCE.get_or_init(|| { - HashSet::from(["shuttleapp", "shuttle", "console", "unstable", "staging"]) - }); + fn is_reserved(name: &str) -> bool { + static INSTANCE: OnceLock> = OnceLock::new(); + INSTANCE.get_or_init(|| { + HashSet::from(["shuttleapp", "shuttle", "console", "unstable", "staging"]) + }); - INSTANCE - .get() - .expect("Reserved words not set") - .contains(name) - } + INSTANCE + .get() + .expect("Reserved words not set") + .contains(name) + } - !name.is_empty() - && name.len() < 64 - && !name.starts_with('-') - && !name.ends_with('-') - && !is_reserved(name) - && name.bytes().all(is_valid_char) - && is_profanity_free(name) + !name.is_empty() + && name.len() < 64 + && !name.starts_with('-') + && !name.ends_with('-') + && !is_reserved(name) + && name.bytes().all(is_valid_char) + && is_profanity_free(name) + } } -} -impl std::ops::Deref for ProjectName { - type Target = String; + impl std::ops::Deref for ProjectName { + type Target = String; - fn deref(&self) -> &Self::Target { - &self.0 + fn deref(&self) -> &Self::Target { + &self.0 + } } -} -impl std::fmt::Display for ProjectName { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - self.0.fmt(f) + impl std::fmt::Display for ProjectName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } } -} -impl<'de> Deserialize<'de> for ProjectName { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s: String = String::deserialize(deserializer)?; - s.parse().map_err(DeError::custom) + impl<'de> Deserialize<'de> for ProjectName { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s: String = String::deserialize(deserializer)?; + s.parse().map_err(DeError::custom) + } } -} -impl FromStr for ProjectName { - type Err = InvalidProjectName; + impl FromStr for ProjectName { + type Err = InvalidProjectName; - fn from_str(s: &str) -> Result { - ProjectName::new(s) + fn from_str(s: &str) -> Result { + ProjectName::new(s) + } } -} -/// Test examples taken from a [Pop-OS project](https://github.com/pop-os/hostname-validator/blob/master/src/lib.rs) -/// and modified to our use case -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn valid_labels() { - for name in [ - "50-name", - "235235", - "123", - "kebab-case", - "lowercase", - "myassets", - "dachterrasse", - "another-valid-project-name", - "x", - ] { - assert!(ProjectName::is_valid(name)); + /// Test examples taken from a [Pop-OS project](https://github.com/pop-os/hostname-validator/blob/master/src/lib.rs) + /// and modified to our use case + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn valid_labels() { + for name in [ + "50-name", + "235235", + "123", + "kebab-case", + "lowercase", + "myassets", + "dachterrasse", + "another-valid-project-name", + "x", + ] { + assert!(ProjectName::is_valid(name)); + } } - } - #[test] - fn invalid_labels() { - for name in [ - "UPPERCASE", - "CamelCase", - "pascalCase", - "InVaLid", - "-invalid-name", - "also-invalid-", - "asdf@fasd", - "@asdfl", - "asd f@", - ".invalid", - "invalid.name", - "invalid.name.", - "__dunder_like__", - "__invalid", - "invalid__", - "test-condom-condom", - "s________e", - "snake_case", - "exactly-16-chars\ + #[test] + fn invalid_labels() { + for name in [ + "UPPERCASE", + "CamelCase", + "pascalCase", + "InVaLid", + "-invalid-name", + "also-invalid-", + "asdf@fasd", + "@asdfl", + "asd f@", + ".invalid", + "invalid.name", + "invalid.name.", + "__dunder_like__", + "__invalid", + "invalid__", + "test-condom-condom", + "s________e", + "snake_case", + "exactly-16-chars\ exactly-16-chars\ exactly-16-chars\ exactly-16-chars", - "shuttle", - "shuttleapp", - "", - ] { - assert!(!ProjectName::is_valid(name)); + "shuttle", + "shuttleapp", + "", + ] { + assert!(!ProjectName::is_valid(name)); + } } } } diff --git a/deployer/Cargo.toml b/deployer/Cargo.toml index 40ae6ba08..94b7766d6 100644 --- a/deployer/Cargo.toml +++ b/deployer/Cargo.toml @@ -7,8 +7,8 @@ description = "Service with instances created per project for handling the compi [dependencies] shuttle-common = { workspace = true, features = ["backend", "models", "openapi"] } -shuttle-proto = { workspace = true } -shuttle-service = { workspace = true, features = ["builder"] } +shuttle-proto = { workspace = true, features = ["resource-recorder"] } +shuttle-service = { workspace = true, features = ["builder", "runner"] } anyhow = { workspace = true } async-trait = { workspace = true } diff --git a/deployer/src/runtime_manager.rs b/deployer/src/runtime_manager.rs index 23fdc11a0..8dcd5bb63 100644 --- a/deployer/src/runtime_manager.rs +++ b/deployer/src/runtime_manager.rs @@ -13,9 +13,9 @@ use shuttle_common::{ }; use shuttle_proto::{ logger::{logger_client::LoggerClient, Batcher, LogItem, LogLine}, - runtime::{self, runtime_client::RuntimeClient, StopRequest}, + runtime::{runtime_client::RuntimeClient, StopRequest}, }; -use shuttle_service::Environment; +use shuttle_service::{runner, Environment}; use tokio::{io::AsyncBufReadExt, io::BufReader, process, sync::Mutex}; use tonic::transport::Channel; use tracing::{debug, error, info, trace, warn}; @@ -125,7 +125,7 @@ impl RuntimeManager { .join("bin/shuttle-next") }; - let (mut process, runtime_client) = runtime::start( + let (mut process, runtime_client) = runner::start( is_next, Environment::Deployment, &self.provisioner_address, diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index d463233f7..b21fa22a8 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -10,8 +10,9 @@ shuttle-common = { workspace = true, features = [ "backend", "models", "openapi", + "persist", ] } -shuttle-proto = { workspace = true } +shuttle-proto = { workspace = true, features = ["provisioner"] } shuttle-orchestrator = { workspace = true } async-trait = { workspace = true } diff --git a/gateway/src/project.rs b/gateway/src/project.rs index 1f29d1071..2f7f563e0 100644 --- a/gateway/src/project.rs +++ b/gateway/src/project.rs @@ -24,7 +24,8 @@ use once_cell::sync::Lazy; use rand::distributions::{Alphanumeric, DistString}; use serde::{Deserialize, Serialize}; use shuttle_common::backends::headers::{X_SHUTTLE_ACCOUNT_NAME, X_SHUTTLE_ADMIN_SECRET}; -use shuttle_common::models::project::{default_idle_minutes, ProjectName, DEFAULT_IDLE_MINUTES}; +use shuttle_common::constants::{default_idle_minutes, DEFAULT_IDLE_MINUTES}; +use shuttle_common::models::project::ProjectName; use shuttle_common::models::service; use tokio::time::{sleep, timeout}; use tracing::{debug, error, info, instrument, trace, warn}; diff --git a/logger/Cargo.toml b/logger/Cargo.toml index 106e67fdc..050c5ef93 100644 --- a/logger/Cargo.toml +++ b/logger/Cargo.toml @@ -7,7 +7,7 @@ repository.workspace = true [dependencies] shuttle-common = { workspace = true, features = ["backend", "tonic"] } -shuttle-proto = { workspace = true } +shuttle-proto = { workspace = true, features = ["logger"] } async-trait = { workspace = true } chrono = { workspace = true } diff --git a/proto/Cargo.toml b/proto/Cargo.toml index 478a88110..c0ac48575 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -6,23 +6,29 @@ license.workspace = true description = "Library for all the gRPC definitions used by shuttle" [dependencies] -shuttle-common = { workspace = true, features = [ - "claims", - "service", - "wasm", - "models", - "backend", -] } +shuttle-common = { workspace = true } -anyhow = { workspace = true } -chrono = { workspace = true } -dunce = { workspace = true } +anyhow = { workspace = true, optional = true } +chrono = { workspace = true, optional = true } futures-core = "0.3.28" -home = { workspace = true } prost = { workspace = true } prost-types = { workspace = true } -tokio = { workspace = true, features = ["process"] } +tokio = { workspace = true, optional = true } tonic = { workspace = true } -tower = { workspace = true } -tracing = { workspace = true } -serde_json = { workspace = true } +tracing = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } + +[features] +default = [] + +builder = [] +logger = [ + "shuttle-common/service", + "chrono", + "tracing", + "tokio/macros", + "tokio/time", +] +provisioner = [] +resource-recorder = ["anyhow", "serde_json"] +runtime = [] diff --git a/proto/src/lib.rs b/proto/src/lib.rs index e2626d651..11f2eb9e3 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -1,5 +1,11 @@ mod generated; +// useful re-exports if types are needed in other crates +pub use prost; +pub use prost_types; +pub use tonic; + +#[cfg(feature = "provisioner")] pub mod provisioner { use std::fmt::Display; @@ -92,106 +98,12 @@ pub mod provisioner { } } +#[cfg(feature = "runtime")] pub mod runtime { - use std::{ - path::{Path, PathBuf}, - process::Stdio, - time::Duration, - }; - - use anyhow::Context; - use shuttle_common::{ - claims::{ClaimLayer, ClaimService, InjectPropagation, InjectPropagationLayer}, - deployment::Environment, - }; - use tokio::process; - use tonic::transport::{Channel, Endpoint}; - use tower::ServiceBuilder; - use tracing::{info, trace}; - pub use super::generated::runtime::*; - - pub async fn start( - wasm: bool, - environment: Environment, - provisioner_address: &str, - auth_uri: Option<&String>, - port: u16, - runtime_executable: PathBuf, - project_path: &Path, - ) -> anyhow::Result<( - process::Child, - runtime_client::RuntimeClient>>, - )> { - let port = &port.to_string(); - let environment = &environment.to_string(); - - let args = if wasm { - vec!["--port", port] - } else { - let mut args = vec![ - "--port", - port, - "--provisioner-address", - provisioner_address, - "--env", - environment, - ]; - - if let Some(auth_uri) = auth_uri { - args.append(&mut vec!["--auth-uri", auth_uri]); - } - - args - }; - - info!( - "Spawning runtime process: {} {}", - runtime_executable.display(), - args.join(" ") - ); - let runtime = process::Command::new( - dunce::canonicalize(runtime_executable).context("canonicalize path of executable")?, - ) - .current_dir(project_path) - .args(&args) - .stdout(Stdio::piped()) - .kill_on_drop(true) - .spawn() - .context("spawning runtime process")?; - - info!("connecting runtime client"); - let conn = Endpoint::new(format!("http://127.0.0.1:{port}")) - .context("creating runtime client endpoint")? - .connect_timeout(Duration::from_secs(5)); - - // Wait for the spawned process to open the control port. - // Connecting instantly does not give it enough time. - let channel = tokio::time::timeout(Duration::from_millis(7000), async move { - let mut ms = 5; - loop { - if let Ok(channel) = conn.connect().await { - break channel; - } - trace!("waiting for runtime control port to open"); - // exponential backoff - tokio::time::sleep(Duration::from_millis(ms)).await; - ms *= 2; - } - }) - .await - .context("runtime control port did not open in time")?; - - let channel = ServiceBuilder::new() - .layer(ClaimLayer) - .layer(InjectPropagationLayer) - .service(channel); - let runtime_client = runtime_client::RuntimeClient::new(channel); - - Ok((runtime, runtime_client)) - } } +#[cfg(feature = "resource-recorder")] pub mod resource_recorder { use anyhow::Context; use std::str::FromStr; @@ -238,10 +150,12 @@ pub mod resource_recorder { } } +#[cfg(feature = "builder")] pub mod builder { pub use super::generated::builder::*; } +#[cfg(feature = "logger")] pub mod logger { use std::str::FromStr; use std::time::Duration; @@ -261,8 +175,6 @@ pub mod logger { DeploymentId, }; - use self::logger_client::LoggerClient; - pub use super::generated::logger::*; impl From for LogItem { @@ -333,7 +245,7 @@ pub mod logger { } #[async_trait] - impl VecReceiver for LoggerClient + impl VecReceiver for logger_client::LoggerClient where T: tonic::client::GrpcService + Send + Sync + Clone, T::Error: Into, diff --git a/provisioner/Cargo.toml b/provisioner/Cargo.toml index 34f2f41c8..aaec177c1 100644 --- a/provisioner/Cargo.toml +++ b/provisioner/Cargo.toml @@ -7,8 +7,8 @@ description = "Service responsible for provisioning and managing resources for s publish = false [dependencies] -shuttle-common = { workspace = true, features = ["backend", "tonic"] } -shuttle-proto = { workspace = true } +shuttle-common = { workspace = true, features = ["backend", "models", "service", "tonic"] } +shuttle-proto = { workspace = true, features = ["provisioner"] } aws-config = "0.56.1" aws-sdk-rds = "0.33.1" diff --git a/resource-recorder/Cargo.toml b/resource-recorder/Cargo.toml index 6d30ce415..f66c651f3 100644 --- a/resource-recorder/Cargo.toml +++ b/resource-recorder/Cargo.toml @@ -7,7 +7,7 @@ repository.workspace = true [dependencies] shuttle-common = { workspace = true, features = ["backend", "tonic"] } -shuttle-proto = { workspace = true } +shuttle-proto = { workspace = true, features = ["resource-recorder"] } async-trait = { workspace = true } chrono = { workspace = true } diff --git a/resources/turso/Cargo.toml b/resources/turso/Cargo.toml index 39895d801..fbe18aa6c 100644 --- a/resources/turso/Cargo.toml +++ b/resources/turso/Cargo.toml @@ -9,12 +9,11 @@ keywords = ["shuttle-service", "turso"] [dependencies] async-trait = "0.1.56" dunce = "1.0.4" -libsql-client = { version = "0.31.0" } +libsql-client = "0.31.0" # Stays on 0.31 until https://github.com/libsql/libsql-client-rs/issues/52 serde = { version = "1.0.148", features = ["derive"] } shuttle-service = { path = "../../service", version = "0.33.0", default-features = false } url = { version = "2.3.1", features = ["serde"] } - [dev-dependencies] tempfile = "3.3.0" tokio = "1.28.2" diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index fccca82f4..4ac724f36 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -14,8 +14,8 @@ doctest = false [dependencies] shuttle-codegen = { workspace = true, features = ["frameworks"] } -shuttle-common = { workspace = true, features = ["claims"] } -shuttle-proto = { workspace = true } +shuttle-common = { workspace = true, features = ["backend", "claims"] } +shuttle-proto = { workspace = true, features = ["provisioner", "runtime"] } shuttle-service = { workspace = true } anyhow = { workspace = true } @@ -43,11 +43,12 @@ wasmtime-wasi = { version = "13.0.0", optional = true } [dev-dependencies] portpicker = "0.1.1" futures = { workspace = true } -shuttle-service = { workspace = true, features = ["builder"] } +shuttle-service = { workspace = true, features = ["builder", "runner"] } uuid = { workspace = true } [features] default = ["setup-tracing"] + next = [ "cap-std", "futures", diff --git a/runtime/tests/integration/helpers.rs b/runtime/tests/integration/helpers.rs index 47552e675..25b7005b3 100644 --- a/runtime/tests/integration/helpers.rs +++ b/runtime/tests/integration/helpers.rs @@ -12,9 +12,9 @@ use shuttle_proto::{ provisioner_server::{Provisioner, ProvisionerServer}, DatabaseDeletionResponse, DatabaseRequest, DatabaseResponse, Ping, Pong, }, - runtime::{self, runtime_client::RuntimeClient}, + runtime::runtime_client::RuntimeClient, }; -use shuttle_service::{builder::build_workspace, Environment}; +use shuttle_service::{builder::build_workspace, runner, Environment}; use tokio::process::Child; use tonic::{ transport::{Channel, Server}, @@ -49,7 +49,7 @@ pub async fn spawn_runtime(project_path: String, service_name: &str) -> Result, + port: u16, + runtime_executable: PathBuf, + project_path: &Path, +) -> anyhow::Result<( + process::Child, + runtime_client::RuntimeClient>>, +)> { + let port = &port.to_string(); + let environment = &environment.to_string(); + + let args = if wasm { + vec!["--port", port] + } else { + let mut args = vec![ + "--port", + port, + "--provisioner-address", + provisioner_address, + "--env", + environment, + ]; + + if let Some(auth_uri) = auth_uri { + args.append(&mut vec!["--auth-uri", auth_uri]); + } + + args + }; + + info!( + "Spawning runtime process: {} {}", + runtime_executable.display(), + args.join(" ") + ); + let runtime = process::Command::new( + dunce::canonicalize(runtime_executable).context("canonicalize path of executable")?, + ) + .current_dir(project_path) + .args(&args) + .stdout(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .context("spawning runtime process")?; + + info!("connecting runtime client"); + let conn = Endpoint::new(format!("http://127.0.0.1:{port}")) + .context("creating runtime client endpoint")? + .connect_timeout(Duration::from_secs(5)); + + // Wait for the spawned process to open the control port. + // Connecting instantly does not give it enough time. + let channel = tokio::time::timeout(Duration::from_millis(7000), async move { + let mut ms = 5; + loop { + if let Ok(channel) = conn.connect().await { + break channel; + } + trace!("waiting for runtime control port to open"); + // exponential backoff + tokio::time::sleep(Duration::from_millis(ms)).await; + ms *= 2; + } + }) + .await + .context("runtime control port did not open in time")?; + + let channel = ServiceBuilder::new() + .layer(ClaimLayer) + .layer(InjectPropagationLayer) + .service(channel); + let runtime_client = runtime_client::RuntimeClient::new(channel); + + Ok((runtime, runtime_client)) +}