From 6d2bf78804218aa351387fdba154a076bd6f0f26 Mon Sep 17 00:00:00 2001 From: priks Date: Mon, 24 Apr 2023 12:56:01 +0530 Subject: [PATCH] Removed cargo dependency from deployer --- Cargo.lock | 1 - deployer/Cargo.toml | 2 +- deployer/src/deployment/queue.rs | 71 +++++++++++++++++--------------- deployer/src/error.rs | 37 ++++++++++++++++- 4 files changed, 74 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 486439f99..4d7ea7466 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5156,7 +5156,6 @@ dependencies = [ "async-trait", "axum", "bytes", - "cargo", "cargo_metadata", "chrono", "clap", diff --git a/deployer/Cargo.toml b/deployer/Cargo.toml index deba2cb8b..971a307ab 100644 --- a/deployer/Cargo.toml +++ b/deployer/Cargo.toml @@ -10,7 +10,7 @@ anyhow = { workspace = true } async-trait = { workspace = true } axum = { workspace = true, features = ["headers", "json", "query", "ws"] } bytes = { workspace = true } -cargo = { workspace = true } +#cargo = { workspace = true } cargo_metadata = { workspace = true } chrono = { workspace = true } clap = { workspace = true } diff --git a/deployer/src/deployment/queue.rs b/deployer/src/deployment/queue.rs index bdc251739..c86839cd0 100644 --- a/deployer/src/deployment/queue.rs +++ b/deployer/src/deployment/queue.rs @@ -1,18 +1,19 @@ use super::deploy_layer::{Log, LogRecorder, LogType}; use super::gateway_client::BuildQueueClient; use super::{Built, QueueReceiver, RunSender, State}; -use crate::error::{Error, Result, TestError}; +use crate::error::{CliError, Error, Result, TestError}; use crate::persistence::{DeploymentUpdater, LogLevel, SecretRecorder}; use shuttle_common::storage_manager::{ArtifactsStorageManager, StorageManager}; -use cargo::util::interning::InternedString; +//use cargo::util::interning::InternedString; use cargo_metadata::Message; use chrono::Utc; use crossbeam_channel::Sender; use opentelemetry::global; use serde_json::json; use shuttle_common::claims::Claim; -use shuttle_service::builder::{build_workspace, get_config, BuiltService}; +use shuttle_service::builder::{build_workspace, BuiltService}; +use tokio::process::Command; use tokio::time::{sleep, timeout}; use tracing::{debug, debug_span, error, info, instrument, trace, warn, Instrument, Span}; use tracing_opentelemetry::OpenTelemetrySpanExt; @@ -25,9 +26,9 @@ use std::io::Read; use std::path::{Path, PathBuf}; use std::time::Duration; -use cargo::core::compiler::{CompileMode, MessageFormat}; -use cargo::core::Workspace; -use cargo::ops::{self, CompileOptions, TestOptions}; +//use cargo::core::compiler::{CompileMode, MessageFormat}; +//use cargo::core::Workspace; +//use cargo::ops::{self, CompileOptions, TestOptions}; use flate2::read::GzDecoder; use tar::Archive; use tokio::fs; @@ -351,15 +352,16 @@ async fn run_pre_deploy_tests( project_path: &Path, tx: Sender, ) -> std::result::Result<(), TestError> { - let (read, write) = pipe::pipe(); + let (read, _) = pipe::pipe(); let project_path = project_path.to_owned(); // This needs to be on a separate thread, else deployer will block (reason currently unknown :D) + let tx_clone = tx.clone(); tokio::task::spawn_blocking(move || { for message in Message::parse_stream(read) { match message { Ok(message) => { - if let Err(error) = tx.send(message) { + if let Err(error) = tx_clone.send(message) { error!("failed to send cargo message on channel: {error}"); } } @@ -370,36 +372,37 @@ async fn run_pre_deploy_tests( } }); - let config = get_config(write)?; let manifest_path = project_path.join("Cargo.toml"); - let ws = Workspace::new(&manifest_path, &config)?; - - let mut compile_opts = CompileOptions::new(&config, CompileMode::Test)?; - - compile_opts.build_config.message_format = MessageFormat::Json { - render_diagnostics: false, - short: false, - ansi: false, - }; - - // We set the tests to build with the release profile since deployments compile - // with the release profile by default. This means crates don't need to be - // recompiled in debug mode for the tests, reducing memory usage during deployment. - compile_opts.build_config.requested_profile = InternedString::new("release"); - - // Build tests with a maximum of 4 workers. - compile_opts.build_config.jobs = 4; - - compile_opts.spec = ops::Packages::All; + let mut cargo = Command::new("cargo"); + cargo + .arg("test") + .arg("--manifest-path") + .arg(&manifest_path) + .arg("--message-format=json") + .arg("--release") + .arg("--jobs=4"); + + let output = cargo.output().await.unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + + for line in stdout.lines() { + let message: Message = serde_json::from_str(line).unwrap(); + if let Err(error) = tx.send(message) { + error!("failed to send cargo message on channel: {error}"); + } + } - let opts = TestOptions { - compile_opts, - no_run: false, - no_fail_fast: false, - }; + if !output.status.success() { + return Err(TestError::Failed(CliError { + error: Some(anyhow::Error::msg( + String::from_utf8_lossy(&output.stderr).to_string(), + )), + exit_code: output.status.code().unwrap_or(-1), + })); + } - ops::run_tests(&ws, &opts, &[]).map_err(TestError::Failed) + Ok(()) } /// This will store the path to the executable for each runtime, which will be the users project with diff --git a/deployer/src/error.rs b/deployer/src/error.rs index bd3bd1056..bfa8d9f7e 100644 --- a/deployer/src/error.rs +++ b/deployer/src/error.rs @@ -1,8 +1,9 @@ use std::error::Error as StdError; +use std::fmt::{Display, Formatter}; use std::io; use thiserror::Error; -use cargo::util::errors::CliError; +//use cargo::util::errors::CliError; use crate::deployment::gateway_client; @@ -36,6 +37,40 @@ pub enum Error { Start(String), } +#[derive(Error, Debug)] +pub struct CliError { + pub error: Option, + pub exit_code: i32, +} + +impl From for CliError { + fn from(err: anyhow::Error) -> CliError { + CliError::new(err, 101) + } +} + +impl From for CliError { + fn from(err: clap::Error) -> CliError { + let code = if err.use_stderr() { 1 } else { 0 }; + CliError::new(err.into(), code) + } +} + +impl Display for CliError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "error: {}", self.error.as_ref().unwrap()) + } +} + +impl CliError { + pub fn new(error: anyhow::Error, code: i32) -> CliError { + CliError { + error: Some(error), + exit_code: code, + } + } +} + #[derive(Error, Debug)] pub enum TestError { #[error("The deployment's tests failed.")]