From 61fdc8c43e49d4bff9bee8b57b78305850e17d4d Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:04:42 +0100 Subject: [PATCH 1/6] feat: use reqwest client --- common/Cargo.toml | 3 +- common/src/backends/client/gateway.rs | 94 ++++++++--------------- common/src/backends/client/mod.rs | 86 +++++++++++---------- deployer/src/deployment/gateway_client.rs | 23 ++---- deployer/src/lib.rs | 8 +- provisioner/src/lib.rs | 6 +- resource-recorder/src/lib.rs | 6 +- resource-recorder/src/main.rs | 4 +- resource-recorder/tests/integration.rs | 4 +- 9 files changed, 97 insertions(+), 137 deletions(-) diff --git a/common/Cargo.toml b/common/Cargo.toml index f57c7ecab..c08e8fac7 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -57,11 +57,12 @@ backend = [ "axum/matched-path", "axum/json", "claims", - "hyper/client", + "hyper", "opentelemetry_sdk", "opentelemetry-appender-tracing", "opentelemetry-otlp", "models", + "reqwest", "rustrict", # only ProjectName model uses it "thiserror", "tokio", diff --git a/common/src/backends/client/gateway.rs b/common/src/backends/client/gateway.rs index 0c76e3245..a73ce759d 100644 --- a/common/src/backends/client/gateway.rs +++ b/common/src/backends/client/gateway.rs @@ -1,37 +1,9 @@ -use headers::Authorization; -use http::{Method, Uri}; +use http::Method; use tracing::instrument; use crate::models; -use super::{Error, ServicesApiClient}; - -/// Wrapper struct to make API calls to gateway easier -#[derive(Clone)] -pub struct Client { - public_client: ServicesApiClient, - private_client: ServicesApiClient, -} - -impl Client { - /// Make a gateway client that is able to call the public and private APIs on gateway - pub fn new(public_uri: Uri, private_uri: Uri) -> Self { - Self { - public_client: ServicesApiClient::new(public_uri), - private_client: ServicesApiClient::new(private_uri), - } - } - - /// Get the client of public API calls - pub fn public_client(&self) -> &ServicesApiClient { - &self.public_client - } - - /// Get the client of private API calls - pub fn private_client(&self) -> &ServicesApiClient { - &self.private_client - } -} +use super::{header_map_with_bearer, Error, ServicesApiClient}; /// Interact with all the data relating to projects #[allow(async_fn_in_trait)] @@ -65,33 +37,31 @@ pub trait ProjectsDal { } } -impl ProjectsDal for Client { +impl ProjectsDal for ServicesApiClient { #[instrument(skip_all)] async fn get_user_project( &self, user_token: &str, project_name: &str, ) -> Result { - self.public_client - .request( - Method::GET, - format!("projects/{}", project_name).as_str(), - None::<()>, - Some(Authorization::bearer(user_token).expect("to build an authorization bearer")), - ) - .await + self.request( + Method::GET, + format!("projects/{}", project_name).as_str(), + None::<()>, + Some(header_map_with_bearer(user_token)), + ) + .await } #[instrument(skip_all)] async fn head_user_project(&self, user_token: &str, project_name: &str) -> Result { - self.public_client - .request_raw( - Method::HEAD, - format!("projects/{}", project_name).as_str(), - None::<()>, - Some(Authorization::bearer(user_token).expect("to build an authorization bearer")), - ) - .await?; + self.request_raw( + Method::HEAD, + format!("projects/{}", project_name).as_str(), + None::<()>, + Some(header_map_with_bearer(user_token)), + ) + .await?; Ok(true) } @@ -101,14 +71,13 @@ impl ProjectsDal for Client { &self, user_token: &str, ) -> Result, Error> { - self.public_client - .request( - Method::GET, - "projects", - None::<()>, - Some(Authorization::bearer(user_token).expect("to build an authorization bearer")), - ) - .await + self.request( + Method::GET, + "projects", + None::<()>, + Some(header_map_with_bearer(user_token)), + ) + .await } } @@ -116,24 +85,25 @@ impl ProjectsDal for Client { mod tests { use test_context::{test_context, AsyncTestContext}; + use crate::backends::client::ServicesApiClient; use crate::models::project::{Response, State}; use crate::test_utils::get_mocked_gateway_server; - use super::{Client, ProjectsDal}; + use super::ProjectsDal; - impl AsyncTestContext for Client { + impl AsyncTestContext for ServicesApiClient { async fn setup() -> Self { let server = get_mocked_gateway_server().await; - Client::new(server.uri().parse().unwrap(), server.uri().parse().unwrap()) + ServicesApiClient::new(server.uri().parse().unwrap()) } async fn teardown(self) {} } - #[test_context(Client)] + #[test_context(ServicesApiClient)] #[tokio::test] - async fn get_user_projects(client: &mut Client) { + async fn get_user_projects(client: &mut ServicesApiClient) { let res = client.get_user_projects("user-1").await.unwrap(); assert_eq!( @@ -155,9 +125,9 @@ mod tests { ) } - #[test_context(Client)] + #[test_context(ServicesApiClient)] #[tokio::test] - async fn get_user_project_ids(client: &mut Client) { + async fn get_user_project_ids(client: &mut ServicesApiClient) { let res = client.get_user_project_ids("user-2").await.unwrap(); assert_eq!(res, vec!["00000000000000000000000003"]) diff --git a/common/src/backends/client/mod.rs b/common/src/backends/client/mod.rs index 5b7ca299a..dc435dfc2 100644 --- a/common/src/backends/client/mod.rs +++ b/common/src/backends/client/mod.rs @@ -1,9 +1,10 @@ +use std::time::Duration; + use bytes::Bytes; -use headers::{ContentType, Header, HeaderMapExt}; -use http::{Method, Request, StatusCode, Uri}; -use hyper::{body, client::HttpConnector, Body, Client}; +use http::{header::AUTHORIZATION, HeaderMap, HeaderValue, Method, StatusCode, Uri}; use opentelemetry::global; use opentelemetry_http::HeaderInjector; +use reqwest::{Client, ClientBuilder}; use serde::{de::DeserializeOwned, Serialize}; use thiserror::Error; use tracing::{trace, Span}; @@ -17,94 +18,104 @@ pub use resource_recorder::ResourceDal; #[derive(Error, Debug)] pub enum Error { - #[error("Hyper error: {0}")] - Hyper(#[from] hyper::Error), + #[error("Reqwest error: {0}")] + Reqwest(#[from] reqwest::Error), #[error("Serde JSON error: {0}")] SerdeJson(#[from] serde_json::Error), - #[error("Hyper error: {0}")] - Http(#[from] hyper::http::Error), #[error("Request did not return correctly. Got status code: {0}")] RequestError(StatusCode), #[error("GRpc request did not return correctly. Got status code: {0}")] GrpcError(#[from] tonic::Status), } -/// `Hyper` wrapper to make request to RESTful Shuttle services easy +/// `reqwest` wrapper to make requests to other services easy #[derive(Clone)] pub struct ServicesApiClient { - client: Client, + client: Client, base: Uri, } impl ServicesApiClient { - fn new(base: Uri) -> Self { + fn _builder() -> ClientBuilder { + Client::builder().timeout(Duration::from_secs(60)) + } + + pub fn new(base: Uri) -> Self { + Self { + client: Self::_builder().build().unwrap(), + base, + } + } + + pub fn new_with_bearer(base: Uri, token: &str) -> Self { Self { - client: Client::new(), + client: Self::_builder() + .default_headers(header_map_with_bearer(token)) + .build() + .unwrap(), base, } } - pub async fn request( + pub async fn request( &self, method: Method, path: &str, body: Option, - extra_header: Option, + headers: Option>, ) -> Result { - let bytes = self.request_raw(method, path, body, extra_header).await?; + let bytes = self.request_raw(method, path, body, headers).await?; let json = serde_json::from_slice(&bytes)?; Ok(json) } - pub async fn request_raw( + pub async fn request_raw( &self, method: Method, path: &str, body: Option, - extra_header: Option, + headers: Option>, ) -> Result { let uri = format!("{}{path}", self.base); trace!(uri, "calling inner service"); - let mut req = Request::builder().method(method).uri(uri); - let headers = req - .headers_mut() - .expect("new request to have mutable headers"); - if let Some(extra_header) = extra_header { - headers.typed_insert(extra_header); - } - if body.is_some() { - headers.typed_insert(ContentType::json()); - } - + let mut h = headers.unwrap_or_default(); let cx = Span::current().context(); global::get_text_map_propagator(|propagator| { - propagator.inject_context(&cx, &mut HeaderInjector(req.headers_mut().unwrap())) + propagator.inject_context(&cx, &mut HeaderInjector(&mut h)) }); - + let req = self.client.request(method, uri).headers(h); let req = if let Some(body) = body { - req.body(Body::from(serde_json::to_vec(&body)?)) + req.json(&body) } else { - req.body(Body::empty()) + req }; - let resp = self.client.request(req?).await?; + let resp = req.send().await?; trace!(response = ?resp, "Load response"); if resp.status() != StatusCode::OK { return Err(Error::RequestError(resp.status())); } - let bytes = body::to_bytes(resp.into_body()).await?; + let bytes = resp.bytes().await?; Ok(bytes) } } +pub fn header_map_with_bearer(token: &str) -> HeaderMap { + let mut h = HeaderMap::new(); + h.append( + AUTHORIZATION, + format!("Bearer {token}").parse().expect("valid token"), + ); + h +} + #[cfg(test)] mod tests { - use headers::{authorization::Bearer, Authorization}; use http::{Method, StatusCode}; use crate::models; @@ -120,12 +131,7 @@ mod tests { let client = ServicesApiClient::new(server.uri().parse().unwrap()); let err = client - .request::<_, Vec, _>( - Method::GET, - "projects", - None::<()>, - None::>, - ) + .request::<_, Vec>(Method::GET, "projects", None::<()>, None) .await .unwrap_err(); diff --git a/deployer/src/deployment/gateway_client.rs b/deployer/src/deployment/gateway_client.rs index fbc1258df..2f85637ac 100644 --- a/deployer/src/deployment/gateway_client.rs +++ b/deployer/src/deployment/gateway_client.rs @@ -1,8 +1,7 @@ -use axum::headers::{authorization::Bearer, Authorization}; use hyper::Method; use shuttle_common::{ - backends::client::{gateway, Error}, - models::{self}, + backends::client::{Error, ServicesApiClient}, + models, }; use uuid::Uuid; @@ -17,17 +16,11 @@ pub trait BuildQueueClient: Clone + Send + Sync + 'static { } #[async_trait::async_trait] -impl BuildQueueClient for gateway::Client { +impl BuildQueueClient for ServicesApiClient { async fn get_slot(&self, deployment_id: Uuid) -> Result { let body = models::stats::LoadRequest { id: deployment_id }; let load: models::stats::LoadResponse = self - .public_client() - .request( - Method::POST, - "stats/load", - Some(body), - None::>, - ) + .request(Method::POST, "stats/load", Some(body), None) .await?; Ok(load.has_capacity) @@ -36,13 +29,7 @@ impl BuildQueueClient for gateway::Client { async fn release_slot(&self, deployment_id: Uuid) -> Result<(), Error> { let body = models::stats::LoadRequest { id: deployment_id }; let _load: models::stats::LoadResponse = self - .public_client() - .request( - Method::DELETE, - "stats/load", - Some(body), - None::>, - ) + .request(Method::DELETE, "stats/load", Some(body), None) .await?; Ok(()) diff --git a/deployer/src/lib.rs b/deployer/src/lib.rs index 087ae1d10..da61c8829 100644 --- a/deployer/src/lib.rs +++ b/deployer/src/lib.rs @@ -2,7 +2,7 @@ use std::sync::Arc; pub use persistence::Persistence; pub use runtime_manager::RuntimeManager; -use shuttle_common::log::LogRecorder; +use shuttle_common::{backends::client::ServicesApiClient, log::LogRecorder}; use shuttle_proto::{logger, provisioner}; use tokio::sync::Mutex; use tracing::info; @@ -18,7 +18,6 @@ mod runtime_manager; pub use crate::args::Args; pub use crate::deployment::state_change_layer::StateChangeLayer; use crate::deployment::{Built, DeploymentManager}; -use shuttle_common::backends::client::gateway; const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -40,10 +39,7 @@ pub async fn start( .runtime(runtime_manager) .resource_manager(persistence.clone()) .provisioner_client(provisioner::get_client(args.provisioner_address).await) - .queue_client(gateway::Client::new( - args.gateway_uri.clone(), - args.gateway_uri, - )) + .queue_client(ServicesApiClient::new(args.gateway_uri)) .log_fetcher(log_fetcher) .build(); diff --git a/provisioner/src/lib.rs b/provisioner/src/lib.rs index 70f0483d9..6a6253f92 100644 --- a/provisioner/src/lib.rs +++ b/provisioner/src/lib.rs @@ -12,7 +12,7 @@ pub use error::Error; use mongodb::{bson::doc, options::ClientOptions}; use rand::Rng; use shuttle_common::backends::auth::VerifyClaim; -use shuttle_common::backends::client::gateway; +use shuttle_common::backends::client::ServicesApiClient; use shuttle_common::backends::ClaimExt; use shuttle_common::claims::{Claim, Scope}; use shuttle_common::models::project::ProjectName; @@ -44,7 +44,7 @@ pub struct ShuttleProvisioner { internal_pg_address: String, internal_mongodb_address: String, rr_client: Arc>, - gateway_client: gateway::Client, + gateway_client: ServicesApiClient, } impl ShuttleProvisioner { @@ -81,7 +81,7 @@ impl ShuttleProvisioner { let rr_client = resource_recorder::get_client(resource_recorder_uri).await; - let gateway_client = gateway::Client::new(gateway_uri.clone(), gateway_uri); + let gateway_client = ServicesApiClient::new(gateway_uri); Ok(Self { pool, diff --git a/resource-recorder/src/lib.rs b/resource-recorder/src/lib.rs index 072f897f8..35b91dba7 100644 --- a/resource-recorder/src/lib.rs +++ b/resource-recorder/src/lib.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use dal::{Dal, DalError, Resource}; use prost_types::TimestampError; use shuttle_common::{ - backends::{auth::VerifyClaim, client::gateway, ClaimExt}, + backends::{auth::VerifyClaim, client::ServicesApiClient, ClaimExt}, claims::{Claim, Scope}, }; use shuttle_proto::resource_recorder::{ @@ -45,14 +45,14 @@ impl From for Error { pub struct Service { dal: D, - gateway_client: gateway::Client, + gateway_client: ServicesApiClient, } impl Service where D: Dal + Send + Sync + 'static, { - pub fn new(dal: D, gateway_client: gateway::Client) -> Self { + pub fn new(dal: D, gateway_client: ServicesApiClient) -> Self { Self { dal, gateway_client, diff --git a/resource-recorder/src/main.rs b/resource-recorder/src/main.rs index 066c66060..f5a7e1e98 100644 --- a/resource-recorder/src/main.rs +++ b/resource-recorder/src/main.rs @@ -4,7 +4,7 @@ use clap::Parser; use shuttle_common::{ backends::{ auth::{AuthPublicKey, JwtAuthenticationLayer}, - client::gateway, + client::ServicesApiClient, trace::setup_tracing, }, extract_propagation::ExtractPropagationLayer, @@ -30,7 +30,7 @@ async fn main() { .layer(JwtAuthenticationLayer::new(AuthPublicKey::new(auth_uri))) .layer(ExtractPropagationLayer); - let gateway_client = gateway::Client::new(gateway_uri.clone(), gateway_uri); + let gateway_client = ServicesApiClient::new(gateway_uri); let db_path = state.join("resource-recorder.sqlite"); let svc = Service::new( diff --git a/resource-recorder/tests/integration.rs b/resource-recorder/tests/integration.rs index ad3700b38..7c57a5d5a 100644 --- a/resource-recorder/tests/integration.rs +++ b/resource-recorder/tests/integration.rs @@ -4,7 +4,7 @@ use portpicker::pick_unused_port; use pretty_assertions::{assert_eq, assert_ne}; use serde_json::json; use shuttle_common::{ - backends::client::gateway::Client, claims::Scope, test_utils::get_mocked_gateway_server, + backends::client::ServicesApiClient, claims::Scope, test_utils::get_mocked_gateway_server, }; use shuttle_common_tests::JwtScopesLayer; use shuttle_proto::resource_recorder::{ @@ -22,7 +22,7 @@ async fn manage_resources() { let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), port); let server = get_mocked_gateway_server().await; - let client = Client::new(server.uri().parse().unwrap(), server.uri().parse().unwrap()); + let client = ServicesApiClient::new(server.uri().parse().unwrap()); let server_future = async { Server::builder() From 10f0371d38babfe226e9c61c0bf3694e4bc001f4 Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Tue, 19 Mar 2024 19:50:23 +0100 Subject: [PATCH 2/6] wip: add project and org ops to permit client --- common/src/backends/client/mod.rs | 32 ++- common/src/backends/client/permit.rs | 364 +++++++++++++++++++++++++++ 2 files changed, 394 insertions(+), 2 deletions(-) create mode 100644 common/src/backends/client/permit.rs diff --git a/common/src/backends/client/mod.rs b/common/src/backends/client/mod.rs index dc435dfc2..3b1a3cc30 100644 --- a/common/src/backends/client/mod.rs +++ b/common/src/backends/client/mod.rs @@ -11,6 +11,7 @@ use tracing::{trace, Span}; use tracing_opentelemetry::OpenTelemetrySpanExt; pub mod gateway; +pub mod permit; mod resource_recorder; pub use gateway::ProjectsDal; @@ -57,6 +58,33 @@ impl ServicesApiClient { } } + pub async fn get( + &self, + path: &str, + headers: Option>, + ) -> Result { + self.request(Method::GET, path, None::<()>, headers).await + } + + pub async fn post( + &self, + path: &str, + body: B, + headers: Option>, + ) -> Result { + self.request(Method::POST, path, Some(body), headers).await + } + + pub async fn delete( + &self, + path: &str, + body: B, + headers: Option>, + ) -> Result { + self.request(Method::DELETE, path, Some(body), headers) + .await + } + pub async fn request( &self, method: Method, @@ -93,9 +121,9 @@ impl ServicesApiClient { }; let resp = req.send().await?; - trace!(response = ?resp, "Load response"); + trace!(response = ?resp, "service response"); - if resp.status() != StatusCode::OK { + if !resp.status().is_success() { return Err(Error::RequestError(resp.status())); } diff --git a/common/src/backends/client/permit.rs b/common/src/backends/client/permit.rs new file mode 100644 index 000000000..6ca9c9a45 --- /dev/null +++ b/common/src/backends/client/permit.rs @@ -0,0 +1,364 @@ +use std::collections::HashMap; + +use http::{Method, Uri}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +use super::{Error, ServicesApiClient}; + +#[derive(Clone)] +pub struct Permit { + /// The Permit.io API + api: ServicesApiClient, + /// The local Permit PDP (Policy decision point) API + pdp: ServicesApiClient, + /// The project id used in API paths + proj_id: String, + /// The environment id used in API paths + env_id: String, +} + +impl Permit { + // "https://api.eu-central-1.permit.io".parse().unwrap(), + // "http://localhost:7000".parse().unwrap(), + pub fn new(api_uri: Uri, pdp_uri: Uri, proj_id: String, env_id: String, api_key: &str) -> Self { + Self { + api: ServicesApiClient::new_with_bearer(api_uri, api_key), + pdp: ServicesApiClient::new(pdp_uri), + proj_id, + env_id, + } + } + + fn endpoint(&self, base: &str, rest: &str) -> String { + format!("/v2/{}/{}/{}/{}", base, self.proj_id, self.env_id, rest) + } + fn resource_instances(&self) -> String { + self.endpoint("facts", "resource_instances") + } + fn role_assignments(&self) -> String { + self.endpoint("facts", "role_assignments") + } + + /// Creates a Project resource and assigns the user as admin for that project + pub async fn create_project(&self, user_id: &str, project_id: &str) -> Result<(), Error> { + self.api + .post( + &self.resource_instances(), + json!({ + "key": project_id, + "tenant": "default", + "resource": "Project", + }), + None, + ) + .await?; + + self.api + .post( + &self.role_assignments(), + json!({ + "role": "admin", + "resource_instance": format!("Project:{project_id}"), + "tenant": "default", + "user": user_id, + }), + None, + ) + .await + } + + /// Unassigns the admin role for a user on a project + pub async fn delete_user_project(&self, user_id: &str, project_id: &str) -> Result<(), Error> { + self.api + .delete( + &self.role_assignments(), + json!({ + "role": "admin", + "resource_instance": format!("Project:{project_id}"), + "tenant": "default", + "user": user_id, + }), + None, + ) + .await + } + + /// Assigns a user to an org directly without creating the org first + pub async fn create_organization(&self, user_id: &str, org_name: &str) -> Result<(), Error> { + self.api + .post( + "v2/facts/default/poc/role_assignments", + json!({ + "role": "admin", + "resource_instance": format!("Organization:{org_name}"), + "tenant": "default", + "user": user_id, + }), + None, + ) + .await + } + + pub async fn delete_organization(&self, organization_id: &str) -> Result<(), Error> { + self.api + .request( + Method::DELETE, + &format!("v2/facts/default/poc/resource_instances/{organization_id}"), + None::<()>, + None, + ) + .await + } + + pub async fn get_organizations(&self, user_id: &str) -> Result<(), Error> { + self.api + .get( + &format!( + "v2/facts/default/poc/role_assignments?user={user_id}&resource=Organization" + ), + None, + ) + .await + } + + pub async fn is_organization_admin( + &self, + user_id: &str, + org_name: &str, + ) -> Result { + let res: Vec = self + .api + .get( + &format!("v2/facts/default/poc/role_assignments?user={user_id}&resource_instance=Organization:{org_name}"), + None, + ) + .await?; + + Ok(res[0].as_object().unwrap()["role"].as_str().unwrap() == "admin") + } + + pub async fn create_organization_project( + &self, + org_name: &str, + project_id: &str, + ) -> Result<(), Error> { + self.api + .post( + "v2/facts/default/poc/relationship_tuples", + json!({ + "subject": format!("Organization:{org_name}"), + "tenant": "default", + "relation": "parent", + "object": format!("Project:{project_id}"), + }), + None, + ) + .await + } + + pub async fn delete_organization_project( + &self, + org_name: &str, + project_id: &str, + ) -> Result<(), Error> { + self.api + .delete( + "v2/facts/default/poc/relationship_tuples", + json!({ + "subject": format!("Organization:{org_name}"), + "relation": "parent", + "object": format!("Project:{project_id}"), + }), + None, + ) + .await + } + + pub async fn get_organization_projects( + &self, + org_name: &str, + ) -> Result, Error> { + self.api + .get( + &format!("v2/facts/default/poc/relationship_tuples?subject=Organization:{org_name}&detailed=true"), + None, + ) + .await + } + + pub async fn get_organization_members(&self, org_name: &str) -> Result, Error> { + self.api + .get( + &format!("v2/facts/default/poc/role_assignments?resource_instance=Organization:{org_name}&role=member"), + None, + ) + .await + } + + pub async fn create_organization_member( + &self, + org_name: &str, + user_id: &str, + ) -> Result<(), Error> { + self.api + .post( + "v2/facts/default/poc/role_assignments", + json!({ + "role": "member", + "resource_instance": format!("Organization:{org_name}"), + "tenant": "default", + "user": user_id, + }), + None, + ) + .await + } + + pub async fn delete_organization_member( + &self, + org_name: &str, + user_id: &str, + ) -> Result<(), Error> { + self.api + .delete( + "v2/facts/default/poc/role_assignments", + json!({ + "role": "member", + "resource_instance": format!("Organization:{org_name}"), + "tenant": "default", + "user": user_id, + }), + None, + ) + .await + } + + pub async fn get_user_projects(&self, user_id: &str) -> Result, Error> { + let perms: HashMap = self + .pdp + .post( + "user-permissions", + json!({ + "user": {"key": user_id}, + "resource_types": ["Project"], + }), + None, + ) + .await?; + + Ok(perms.into_values().collect()) + } + + pub async fn allowed( + &self, + user_id: &str, + project_id: &str, + action: &str, + ) -> Result { + let res: Value = self + .pdp + .post( + "allowed", + json!({ + "user": {"key": user_id}, + "action": action, + "resource": {"type": "Project", "key": project_id, "tenant": "default"}, + }), + None, + ) + .await?; + + Ok(res["allow"].as_bool().unwrap()) + } +} + +/// Struct to hold the following relationship tuple from permit +/// +/// ```json +/// { +/// "subject": "Organization:London", +/// "relation": "parent", +/// "object": "Project:01HRAER7SMNPYZR3RYPAGHMFYW", +/// "id": "dfb57d795ba1432192a5b0ffd0293cae", +/// "tenant": "default", +/// "subject_id": "6eb3094331694b09ac1596fdb7834be5", +/// "relation_id": "cc1bf6e3e51e4b588c36a04552427461", +/// "object_id": "0af595f5ce834c7cad1cca513a1a6fd2", +/// "tenant_id": "4da8b268e96644609978dd62041b5fc6", +/// "organization_id": "5f504714eee841aaaef0d9546d2fd998", +/// "project_id": "b3492c78ccf44f7fb72615bdbfa58027", +/// "environment_id": "b3d12e0fd440433c8ba480bde8cb6cd2", +/// "created_at": "2024-03-07T15:27:59+00:00", +/// "updated_at": "2024-03-07T15:27:59+00:00", +/// "subject_details": { +/// "key": "London", +/// "tenant": "default", +/// "resource": "Organization", +/// "attributes": {} +/// }, +/// "relation_details": { +/// "key": "parent", +/// "name": "parent", +/// "description": "Relation expresses possible 'parent' relation between subject of type 'Organization' to object of type 'Project'" +/// }, +/// "object_details": { +/// "key": "01HRAER7SMNPYZR3RYPAGHMFYW", +/// "tenant": "default", +/// "resource": "Project", +/// "attributes": {} +/// }, +/// "tenant_details": { +/// "key": "default", +/// "name": "Default Tenant", +/// "description": null, +/// "attributes": null +/// } +/// } +/// ``` +#[derive(Debug, Serialize, Deserialize)] +pub struct OrganizationResource { + pub subject: String, + pub relation: String, + pub object: String, + pub id: String, + + /// The project which this organization is the parent of + pub object_details: ObjectDetails, + + #[serde(flatten)] + pub extra: HashMap, +} + +/// Struct to hold the following +/// ```json +/// { +/// "key": "01HRAER7SMNPYZR3RYPAGHMFYW", +/// "tenant": "default", +/// "resource": "Project", +/// "attributes": {} +/// } +/// ``` +#[derive(Debug, Serialize, Deserialize)] +pub struct ObjectDetails { + pub key: String, + #[serde(default)] + pub name: String, + pub tenant: String, + pub resource: String, + pub attributes: Value, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ProjectPermissions { + pub resource: SimpleResource, + pub permissions: Vec, + pub roles: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct SimpleResource { + pub key: String, + pub r#type: String, + pub attributes: Value, +} From d9a84afc430838154b2badc348142fdb34c01b61 Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Wed, 20 Mar 2024 12:15:34 +0100 Subject: [PATCH 3/6] chore: isolate reqwest changes --- common/src/backends/client/mod.rs | 1 - common/src/backends/client/permit.rs | 364 --------------------------- 2 files changed, 365 deletions(-) delete mode 100644 common/src/backends/client/permit.rs diff --git a/common/src/backends/client/mod.rs b/common/src/backends/client/mod.rs index 3b1a3cc30..4c162dedd 100644 --- a/common/src/backends/client/mod.rs +++ b/common/src/backends/client/mod.rs @@ -11,7 +11,6 @@ use tracing::{trace, Span}; use tracing_opentelemetry::OpenTelemetrySpanExt; pub mod gateway; -pub mod permit; mod resource_recorder; pub use gateway::ProjectsDal; diff --git a/common/src/backends/client/permit.rs b/common/src/backends/client/permit.rs deleted file mode 100644 index 6ca9c9a45..000000000 --- a/common/src/backends/client/permit.rs +++ /dev/null @@ -1,364 +0,0 @@ -use std::collections::HashMap; - -use http::{Method, Uri}; -use serde::{Deserialize, Serialize}; -use serde_json::{json, Value}; - -use super::{Error, ServicesApiClient}; - -#[derive(Clone)] -pub struct Permit { - /// The Permit.io API - api: ServicesApiClient, - /// The local Permit PDP (Policy decision point) API - pdp: ServicesApiClient, - /// The project id used in API paths - proj_id: String, - /// The environment id used in API paths - env_id: String, -} - -impl Permit { - // "https://api.eu-central-1.permit.io".parse().unwrap(), - // "http://localhost:7000".parse().unwrap(), - pub fn new(api_uri: Uri, pdp_uri: Uri, proj_id: String, env_id: String, api_key: &str) -> Self { - Self { - api: ServicesApiClient::new_with_bearer(api_uri, api_key), - pdp: ServicesApiClient::new(pdp_uri), - proj_id, - env_id, - } - } - - fn endpoint(&self, base: &str, rest: &str) -> String { - format!("/v2/{}/{}/{}/{}", base, self.proj_id, self.env_id, rest) - } - fn resource_instances(&self) -> String { - self.endpoint("facts", "resource_instances") - } - fn role_assignments(&self) -> String { - self.endpoint("facts", "role_assignments") - } - - /// Creates a Project resource and assigns the user as admin for that project - pub async fn create_project(&self, user_id: &str, project_id: &str) -> Result<(), Error> { - self.api - .post( - &self.resource_instances(), - json!({ - "key": project_id, - "tenant": "default", - "resource": "Project", - }), - None, - ) - .await?; - - self.api - .post( - &self.role_assignments(), - json!({ - "role": "admin", - "resource_instance": format!("Project:{project_id}"), - "tenant": "default", - "user": user_id, - }), - None, - ) - .await - } - - /// Unassigns the admin role for a user on a project - pub async fn delete_user_project(&self, user_id: &str, project_id: &str) -> Result<(), Error> { - self.api - .delete( - &self.role_assignments(), - json!({ - "role": "admin", - "resource_instance": format!("Project:{project_id}"), - "tenant": "default", - "user": user_id, - }), - None, - ) - .await - } - - /// Assigns a user to an org directly without creating the org first - pub async fn create_organization(&self, user_id: &str, org_name: &str) -> Result<(), Error> { - self.api - .post( - "v2/facts/default/poc/role_assignments", - json!({ - "role": "admin", - "resource_instance": format!("Organization:{org_name}"), - "tenant": "default", - "user": user_id, - }), - None, - ) - .await - } - - pub async fn delete_organization(&self, organization_id: &str) -> Result<(), Error> { - self.api - .request( - Method::DELETE, - &format!("v2/facts/default/poc/resource_instances/{organization_id}"), - None::<()>, - None, - ) - .await - } - - pub async fn get_organizations(&self, user_id: &str) -> Result<(), Error> { - self.api - .get( - &format!( - "v2/facts/default/poc/role_assignments?user={user_id}&resource=Organization" - ), - None, - ) - .await - } - - pub async fn is_organization_admin( - &self, - user_id: &str, - org_name: &str, - ) -> Result { - let res: Vec = self - .api - .get( - &format!("v2/facts/default/poc/role_assignments?user={user_id}&resource_instance=Organization:{org_name}"), - None, - ) - .await?; - - Ok(res[0].as_object().unwrap()["role"].as_str().unwrap() == "admin") - } - - pub async fn create_organization_project( - &self, - org_name: &str, - project_id: &str, - ) -> Result<(), Error> { - self.api - .post( - "v2/facts/default/poc/relationship_tuples", - json!({ - "subject": format!("Organization:{org_name}"), - "tenant": "default", - "relation": "parent", - "object": format!("Project:{project_id}"), - }), - None, - ) - .await - } - - pub async fn delete_organization_project( - &self, - org_name: &str, - project_id: &str, - ) -> Result<(), Error> { - self.api - .delete( - "v2/facts/default/poc/relationship_tuples", - json!({ - "subject": format!("Organization:{org_name}"), - "relation": "parent", - "object": format!("Project:{project_id}"), - }), - None, - ) - .await - } - - pub async fn get_organization_projects( - &self, - org_name: &str, - ) -> Result, Error> { - self.api - .get( - &format!("v2/facts/default/poc/relationship_tuples?subject=Organization:{org_name}&detailed=true"), - None, - ) - .await - } - - pub async fn get_organization_members(&self, org_name: &str) -> Result, Error> { - self.api - .get( - &format!("v2/facts/default/poc/role_assignments?resource_instance=Organization:{org_name}&role=member"), - None, - ) - .await - } - - pub async fn create_organization_member( - &self, - org_name: &str, - user_id: &str, - ) -> Result<(), Error> { - self.api - .post( - "v2/facts/default/poc/role_assignments", - json!({ - "role": "member", - "resource_instance": format!("Organization:{org_name}"), - "tenant": "default", - "user": user_id, - }), - None, - ) - .await - } - - pub async fn delete_organization_member( - &self, - org_name: &str, - user_id: &str, - ) -> Result<(), Error> { - self.api - .delete( - "v2/facts/default/poc/role_assignments", - json!({ - "role": "member", - "resource_instance": format!("Organization:{org_name}"), - "tenant": "default", - "user": user_id, - }), - None, - ) - .await - } - - pub async fn get_user_projects(&self, user_id: &str) -> Result, Error> { - let perms: HashMap = self - .pdp - .post( - "user-permissions", - json!({ - "user": {"key": user_id}, - "resource_types": ["Project"], - }), - None, - ) - .await?; - - Ok(perms.into_values().collect()) - } - - pub async fn allowed( - &self, - user_id: &str, - project_id: &str, - action: &str, - ) -> Result { - let res: Value = self - .pdp - .post( - "allowed", - json!({ - "user": {"key": user_id}, - "action": action, - "resource": {"type": "Project", "key": project_id, "tenant": "default"}, - }), - None, - ) - .await?; - - Ok(res["allow"].as_bool().unwrap()) - } -} - -/// Struct to hold the following relationship tuple from permit -/// -/// ```json -/// { -/// "subject": "Organization:London", -/// "relation": "parent", -/// "object": "Project:01HRAER7SMNPYZR3RYPAGHMFYW", -/// "id": "dfb57d795ba1432192a5b0ffd0293cae", -/// "tenant": "default", -/// "subject_id": "6eb3094331694b09ac1596fdb7834be5", -/// "relation_id": "cc1bf6e3e51e4b588c36a04552427461", -/// "object_id": "0af595f5ce834c7cad1cca513a1a6fd2", -/// "tenant_id": "4da8b268e96644609978dd62041b5fc6", -/// "organization_id": "5f504714eee841aaaef0d9546d2fd998", -/// "project_id": "b3492c78ccf44f7fb72615bdbfa58027", -/// "environment_id": "b3d12e0fd440433c8ba480bde8cb6cd2", -/// "created_at": "2024-03-07T15:27:59+00:00", -/// "updated_at": "2024-03-07T15:27:59+00:00", -/// "subject_details": { -/// "key": "London", -/// "tenant": "default", -/// "resource": "Organization", -/// "attributes": {} -/// }, -/// "relation_details": { -/// "key": "parent", -/// "name": "parent", -/// "description": "Relation expresses possible 'parent' relation between subject of type 'Organization' to object of type 'Project'" -/// }, -/// "object_details": { -/// "key": "01HRAER7SMNPYZR3RYPAGHMFYW", -/// "tenant": "default", -/// "resource": "Project", -/// "attributes": {} -/// }, -/// "tenant_details": { -/// "key": "default", -/// "name": "Default Tenant", -/// "description": null, -/// "attributes": null -/// } -/// } -/// ``` -#[derive(Debug, Serialize, Deserialize)] -pub struct OrganizationResource { - pub subject: String, - pub relation: String, - pub object: String, - pub id: String, - - /// The project which this organization is the parent of - pub object_details: ObjectDetails, - - #[serde(flatten)] - pub extra: HashMap, -} - -/// Struct to hold the following -/// ```json -/// { -/// "key": "01HRAER7SMNPYZR3RYPAGHMFYW", -/// "tenant": "default", -/// "resource": "Project", -/// "attributes": {} -/// } -/// ``` -#[derive(Debug, Serialize, Deserialize)] -pub struct ObjectDetails { - pub key: String, - #[serde(default)] - pub name: String, - pub tenant: String, - pub resource: String, - pub attributes: Value, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct ProjectPermissions { - pub resource: SimpleResource, - pub permissions: Vec, - pub roles: Vec, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct SimpleResource { - pub key: String, - pub r#type: String, - pub attributes: Value, -} From f6066c733801a7e06b2e92f5a23bb040e46ac3b9 Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Wed, 20 Mar 2024 12:21:37 +0100 Subject: [PATCH 4/6] refactor bytes response (unused) --- common/src/backends/client/mod.rs | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/common/src/backends/client/mod.rs b/common/src/backends/client/mod.rs index 4c162dedd..9539d1e29 100644 --- a/common/src/backends/client/mod.rs +++ b/common/src/backends/client/mod.rs @@ -4,7 +4,7 @@ use bytes::Bytes; use http::{header::AUTHORIZATION, HeaderMap, HeaderValue, Method, StatusCode, Uri}; use opentelemetry::global; use opentelemetry_http::HeaderInjector; -use reqwest::{Client, ClientBuilder}; +use reqwest::{Client, ClientBuilder, Response}; use serde::{de::DeserializeOwned, Serialize}; use thiserror::Error; use tracing::{trace, Span}; @@ -91,19 +91,35 @@ impl ServicesApiClient { body: Option, headers: Option>, ) -> Result { - let bytes = self.request_raw(method, path, body, headers).await?; - let json = serde_json::from_slice(&bytes)?; + Ok(self + .request_raw(method, path, body, headers) + .await? + .json() + .await?) + } - Ok(json) + pub async fn request_bytes( + &self, + method: Method, + path: &str, + body: Option, + headers: Option>, + ) -> Result { + Ok(self + .request_raw(method, path, body, headers) + .await? + .bytes() + .await?) } + // can be used for explicit HEAD requests (ignores body) pub async fn request_raw( &self, method: Method, path: &str, body: Option, headers: Option>, - ) -> Result { + ) -> Result { let uri = format!("{}{path}", self.base); trace!(uri, "calling inner service"); @@ -126,9 +142,7 @@ impl ServicesApiClient { return Err(Error::RequestError(resp.status())); } - let bytes = resp.bytes().await?; - - Ok(bytes) + Ok(resp) } } From ef99a38453d4070fb7ae954adec2e73a8525f4ef Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:16:44 +0100 Subject: [PATCH 5/6] nits --- common/src/backends/client/gateway.rs | 13 +++---------- common/src/backends/client/mod.rs | 18 ++++++++---------- deployer/src/deployment/gateway_client.rs | 15 ++++++++++----- examples | 2 +- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/common/src/backends/client/gateway.rs b/common/src/backends/client/gateway.rs index a73ce759d..ec1e25bfa 100644 --- a/common/src/backends/client/gateway.rs +++ b/common/src/backends/client/gateway.rs @@ -44,10 +44,8 @@ impl ProjectsDal for ServicesApiClient { user_token: &str, project_name: &str, ) -> Result { - self.request( - Method::GET, + self.get( format!("projects/{}", project_name).as_str(), - None::<()>, Some(header_map_with_bearer(user_token)), ) .await @@ -71,13 +69,8 @@ impl ProjectsDal for ServicesApiClient { &self, user_token: &str, ) -> Result, Error> { - self.request( - Method::GET, - "projects", - None::<()>, - Some(header_map_with_bearer(user_token)), - ) - .await + self.get("projects", Some(header_map_with_bearer(user_token))) + .await } } diff --git a/common/src/backends/client/mod.rs b/common/src/backends/client/mod.rs index 9539d1e29..f65d08d1c 100644 --- a/common/src/backends/client/mod.rs +++ b/common/src/backends/client/mod.rs @@ -1,7 +1,8 @@ use std::time::Duration; use bytes::Bytes; -use http::{header::AUTHORIZATION, HeaderMap, HeaderValue, Method, StatusCode, Uri}; +use headers::{Authorization, HeaderMapExt}; +use http::{HeaderMap, HeaderValue, Method, StatusCode, Uri}; use opentelemetry::global; use opentelemetry_http::HeaderInjector; use reqwest::{Client, ClientBuilder, Response}; @@ -36,20 +37,20 @@ pub struct ServicesApiClient { } impl ServicesApiClient { - fn _builder() -> ClientBuilder { + pub fn builder() -> ClientBuilder { Client::builder().timeout(Duration::from_secs(60)) } pub fn new(base: Uri) -> Self { Self { - client: Self::_builder().build().unwrap(), + client: Self::builder().build().unwrap(), base, } } pub fn new_with_bearer(base: Uri, token: &str) -> Self { Self { - client: Self::_builder() + client: Self::builder() .default_headers(header_map_with_bearer(token)) .build() .unwrap(), @@ -148,16 +149,13 @@ impl ServicesApiClient { pub fn header_map_with_bearer(token: &str) -> HeaderMap { let mut h = HeaderMap::new(); - h.append( - AUTHORIZATION, - format!("Bearer {token}").parse().expect("valid token"), - ); + h.typed_insert(Authorization::bearer(token).expect("valid token")); h } #[cfg(test)] mod tests { - use http::{Method, StatusCode}; + use http::StatusCode; use crate::models; use crate::test_utils::get_mocked_gateway_server; @@ -172,7 +170,7 @@ mod tests { let client = ServicesApiClient::new(server.uri().parse().unwrap()); let err = client - .request::<_, Vec>(Method::GET, "projects", None::<()>, None) + .get::>("projects", None) .await .unwrap_err(); diff --git a/deployer/src/deployment/gateway_client.rs b/deployer/src/deployment/gateway_client.rs index 2f85637ac..f988714ff 100644 --- a/deployer/src/deployment/gateway_client.rs +++ b/deployer/src/deployment/gateway_client.rs @@ -1,4 +1,3 @@ -use hyper::Method; use shuttle_common::{ backends::client::{Error, ServicesApiClient}, models, @@ -18,18 +17,24 @@ pub trait BuildQueueClient: Clone + Send + Sync + 'static { #[async_trait::async_trait] impl BuildQueueClient for ServicesApiClient { async fn get_slot(&self, deployment_id: Uuid) -> Result { - let body = models::stats::LoadRequest { id: deployment_id }; let load: models::stats::LoadResponse = self - .request(Method::POST, "stats/load", Some(body), None) + .post( + "stats/load", + models::stats::LoadRequest { id: deployment_id }, + None, + ) .await?; Ok(load.has_capacity) } async fn release_slot(&self, deployment_id: Uuid) -> Result<(), Error> { - let body = models::stats::LoadRequest { id: deployment_id }; let _load: models::stats::LoadResponse = self - .request(Method::DELETE, "stats/load", Some(body), None) + .delete( + "stats/load", + models::stats::LoadRequest { id: deployment_id }, + None, + ) .await?; Ok(()) diff --git a/examples b/examples index d0872d676..b1ae18580 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit d0872d6761b0d50cfdbb6c3a5bad9ffb65a8699a +Subproject commit b1ae18580f6ba12af2e5d88d38d6bc74729f0e05 From 9e55ccb966554a21632cecbfd4270b616736769e Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Wed, 20 Mar 2024 14:19:18 +0100 Subject: [PATCH 6/6] fix: json feature --- common/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/Cargo.toml b/common/Cargo.toml index c08e8fac7..4ef89bcc2 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -62,7 +62,7 @@ backend = [ "opentelemetry-appender-tracing", "opentelemetry-otlp", "models", - "reqwest", + "reqwest/json", "rustrict", # only ProjectName model uses it "thiserror", "tokio",