fix(deployer): handle gracefully builder connection failure (#1264)

* fix(deployer): handle gracefully builder connection failure * style(deployer): fmt fix * misc(deployer): rename var and adjust event
shuttle-hq · Sep 22, 2023 · 7991c58 · 7991c58
1 parent 2af0076
commit 7991c58
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 42 deletions.
diff --git a/deployer/src/deployment/mod.rs b/deployer/src/deployment/mod.rs
@@ -86,13 +86,15 @@ where
 
     pub fn builder_client(
         mut self,
-        builder_client: BuilderClient<
-            shuttle_common::claims::ClaimService<
-                shuttle_common::claims::InjectPropagation<tonic::transport::Channel>,
+        builder_client: Option<
+            BuilderClient<
+                shuttle_common::claims::ClaimService<
+                    shuttle_common::claims::InjectPropagation<tonic::transport::Channel>,
+                >,
             >,
         >,
     ) -> Self {
-        self.builder_client = Some(builder_client);
+        self.builder_client = builder_client;
 
         self
     }
@@ -166,7 +168,6 @@ where
         let secret_getter = self.secret_getter.expect("a secret getter to be set");
         let resource_manager = self.resource_manager.expect("a resource manager to be set");
         let logs_fetcher = self.logs_fetcher.expect("a logs fetcher to be set");
-        let builder_client = self.builder_client.expect("a builder client to be set");
 
         let (queue_send, queue_recv) = mpsc::channel(QUEUE_BUFFER_SIZE);
         let (run_send, run_recv) = mpsc::channel(RUN_BUFFER_SIZE);
@@ -184,7 +185,7 @@ where
             build_log_recorder,
             secret_recorder,
             queue_client,
-            builder_client,
+            self.builder_client,
             builds_path.clone(),
         ));
         // Run queue. Waits for built deployments and runs them.

diff --git a/deployer/src/deployment/queue.rs b/deployer/src/deployment/queue.rs
@@ -44,9 +44,11 @@ pub async fn task(
     log_recorder: impl LogRecorder,
     secret_recorder: impl SecretRecorder,
     queue_client: impl BuildQueueClient,
-    builder_client: BuilderClient<
-        shuttle_common::claims::ClaimService<
-            shuttle_common::claims::InjectPropagation<tonic::transport::Channel>,
+    builder_client: Option<
+        BuilderClient<
+            shuttle_common::claims::ClaimService<
+                shuttle_common::claims::InjectPropagation<tonic::transport::Channel>,
+            >,
         >,
     >,
     builds_path: PathBuf,
@@ -67,7 +69,7 @@ pub async fn task(
                 let secret_recorder = secret_recorder.clone();
                 let queue_client = queue_client.clone();
                 let builds_path = builds_path.clone();
-                let mut builder_client = builder_client.clone();
+                let builder_client = builder_client.clone();
 
                 tasks.spawn(async move {
                     let parent_cx = global::get_text_map_propagator(|propagator| {
@@ -89,24 +91,26 @@ pub async fn task(
                             Err(err) => return build_failed(&id, err),
                         }
 
-                        let deployment_id = queued.id.to_string();
-                        let archive = queued.data.clone();
-                        let claim = queued.claim.clone();
-                        tokio::spawn(async move {
-                            let mut req = Request::new(BuildRequest {
-                                deployment_id,
-                                archive,
+                        if let Some(mut inner) = builder_client {
+                            let deployment_id = queued.id.to_string();
+                            let archive = queued.data.clone();
+                            let claim = queued.claim.clone();
+                            tokio::spawn(async move {
+                                let mut req = Request::new(BuildRequest {
+                                    deployment_id,
+                                    archive,
+                                });
+                                req.extensions_mut().insert(claim);
+
+                                match inner.build(req).await {
+                                    Ok(inner) =>  {
+                                        let response = inner.into_inner();
+                                        info!(id = %queued.id, "shuttle-builder finished building the deployment: image length is {} bytes, is_wasm flag is {} and there are {} secrets", response.image.len(), response.is_wasm, response.secrets.len());
+                                    },
+                                    Err(err) => error!(id = %queued.id, "shuttle-builder errored while building: {}", err)
+                                };
                             });
-                            req.extensions_mut().insert(claim);
-
-                            match builder_client.build(req).await {
-                                Ok(inner) =>  {
-                                    let response = inner.into_inner();
-                                    info!(id = %queued.id, "shuttle-builder finished building the deployment: image length is {} bytes, is_wasm flag is {} and there are {} secrets", response.image.len(), response.is_wasm, response.secrets.len());
-                                },
-                                Err(err) => error!(id = %queued.id, "shuttle-builder errored while building: {}", err)
-                            };
-                        });
+                        }
 
                         match queued
                             .handle(

diff --git a/deployer/src/deployment/state_change_layer.rs b/deployer/src/deployment/state_change_layer.rs
@@ -800,7 +800,7 @@ mod tests {
             .secret_getter(StubSecretGetter)
             .resource_manager(StubResourceManager)
             .log_fetcher(logger_client.clone())
-            .builder_client(builder_client.clone())
+            .builder_client(Some(builder_client))
             .runtime(get_runtime_manager(Batcher::wrap(logger_client)).await)
             .deployment_updater(StubDeploymentUpdater)
             .queue_client(StubBuildQueueClient)

diff --git a/deployer/src/lib.rs b/deployer/src/lib.rs
@@ -37,9 +37,11 @@ pub async fn start(
             shuttle_common::claims::InjectPropagation<tonic::transport::Channel>,
         >,
     >,
-    builder_client: BuilderClient<
-        shuttle_common::claims::ClaimService<
-            shuttle_common::claims::InjectPropagation<tonic::transport::Channel>,
+    builder_client: Option<
+        BuilderClient<
+            shuttle_common::claims::ClaimService<
+                shuttle_common::claims::InjectPropagation<tonic::transport::Channel>,
+            >,
         >,
     >,
     args: Args,

diff --git a/deployer/src/main.rs b/deployer/src/main.rs
@@ -45,17 +45,18 @@ async fn main() {
     let logger_client = LoggerClient::new(channel);
     let logger_batcher = Batcher::wrap(logger_client.clone());
 
-    let channel = ServiceBuilder::new()
-        .layer(ClaimLayer)
-        .layer(InjectPropagationLayer)
-        .service(
-            args.builder_uri
-                .connect()
-                .await
-                .expect("failed to connect to builder"),
-        );
-
-    let builder_client = BuilderClient::new(channel);
+    let builder_client = match args.builder_uri.connect().await {
+        Ok(channel) => Some(BuilderClient::new(
+            ServiceBuilder::new()
+                .layer(ClaimLayer)
+                .layer(InjectPropagationLayer)
+                .service(channel),
+        )),
+        Err(err) => {
+            error!("Couldn't connect to the shuttle-builder: {err}");
+            None
+        }
+    };
 
     setup_tracing(
         tracing_subscriber::registry()