Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refine the ExecuteQuery grpc interface #790

Merged
merged 2 commits into from
May 30, 2023
Merged

Refine the ExecuteQuery grpc interface #790

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3438188
Decouple session related operations from the execute_query interface
kyotoYaho May 29, 2023
ecb609b
Add failure details for grpc interface execute_query
kyotoYaho May 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 3 additions & 7 deletions ballista/client/src/context.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ use std::sync::Arc;

use ballista_core::config::BallistaConfig;
use ballista_core::serde::protobuf::scheduler_grpc_client::SchedulerGrpcClient;
use ballista_core::serde::protobuf::{ExecuteQueryParams, KeyValuePair};
use ballista_core::serde::protobuf::{CreateSessionParams, KeyValuePair};
use ballista_core::utils::{
create_df_ctx_with_ballista_query_planner, create_grpc_client_connection,
};
Expand Down Expand Up @@ -103,8 +103,7 @@ impl BallistaContext {
let mut scheduler = SchedulerGrpcClient::new(connection);

let remote_session_id = scheduler
.execute_query(ExecuteQueryParams {
query: None,
.create_session(CreateSessionParams {
settings: config
.settings()
.iter()
Expand All @@ -113,7 +112,6 @@ impl BallistaContext {
value: v.to_owned(),
})
.collect::<Vec<_>>(),
optional_session_id: None,
})
.await
.map_err(|e| DataFusionError::Execution(format!("{e:?}")))?
Expand Down Expand Up @@ -162,8 +160,7 @@ impl BallistaContext {
};

let remote_session_id = scheduler
.execute_query(ExecuteQueryParams {
query: None,
.create_session(CreateSessionParams {
settings: config
.settings()
.iter()
Expand All @@ -172,7 +169,6 @@ impl BallistaContext {
value: v.to_owned(),
})
.collect::<Vec<_>>(),
optional_session_id: None,
})
.await
.map_err(|e| DataFusionError::Execution(format!("{e:?}")))?
Expand Down
46 changes: 46 additions & 0 deletions ballista/core/proto/ballista.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,15 +536,55 @@ message ExecuteQueryParams {
repeated KeyValuePair settings = 4;
}

message CreateSessionParams {
repeated KeyValuePair settings = 1;
}

message CreateSessionResult {
string session_id = 1;
}

message UpdateSessionParams {
string session_id = 1;
repeated KeyValuePair settings = 2;
}

message UpdateSessionResult {
bool success = 1;
}

message RemoveSessionParams {
string session_id = 1;
}

message RemoveSessionResult {
bool success = 1;
}

message ExecuteSqlParams {
string sql = 1;
}

message ExecuteQueryResult {
oneof result {
ExecuteQuerySuccessResult success = 1;
ExecuteQueryFailureResult failure = 2;
}
}

message ExecuteQuerySuccessResult {
string job_id = 1;
string session_id = 2;
}

message ExecuteQueryFailureResult {
oneof failure {
string session_not_found = 1;
string plan_parsing_failure = 2;
string sql_parsing_failure = 3;
}
}

message GetJobStatusParams {
string job_id = 1;
}
Expand Down Expand Up @@ -676,6 +716,12 @@ service SchedulerGrpc {

rpc GetFileMetadata (GetFileMetadataParams) returns (GetFileMetadataResult) {}

rpc CreateSession (CreateSessionParams) returns (CreateSessionResult) {}

rpc UpdateSession (UpdateSessionParams) returns (UpdateSessionResult) {}

rpc RemoveSession (RemoveSessionParams) returns (RemoveSessionResult) {}

rpc ExecuteQuery (ExecuteQueryParams) returns (ExecuteQueryResult) {}

rpc GetJobStatus (GetJobStatusParams) returns (GetJobStatusResult) {}
Expand Down
25 changes: 13 additions & 12 deletions ballista/core/src/execution_plans/distributed_query.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ use crate::client::BallistaClient;
use crate::config::BallistaConfig;
use crate::serde::protobuf::execute_query_params::OptionalSessionId;
use crate::serde::protobuf::{
execute_query_params::Query, job_status, scheduler_grpc_client::SchedulerGrpcClient,
ExecuteQueryParams, GetJobStatusParams, GetJobStatusResult, KeyValuePair,
PartitionLocation,
execute_query_params::Query, execute_query_result, job_status,
scheduler_grpc_client::SchedulerGrpcClient, ExecuteQueryParams, GetJobStatusParams,
GetJobStatusResult, PartitionLocation,
};
use crate::utils::create_grpc_client_connection;
use datafusion::arrow::datatypes::SchemaRef;
Expand Down Expand Up @@ -175,15 +175,7 @@ impl<T: 'static + AsLogicalPlan> ExecutionPlan for DistributedQueryExec<T> {

let query = ExecuteQueryParams {
query: Some(Query::LogicalPlan(buf)),
settings: self
.config
.settings()
.iter()
.map(|(k, v)| KeyValuePair {
key: k.to_owned(),
value: v.to_owned(),
})
.collect::<Vec<_>>(),
settings: vec![],
optional_session_id: Some(OptionalSessionId::SessionId(
self.session_id.clone(),
)),
Expand Down Expand Up @@ -242,6 +234,15 @@ async fn execute_query(
.map_err(|e| DataFusionError::Execution(format!("{e:?}")))?
.into_inner();

let query_result = match query_result.result.unwrap() {
execute_query_result::Result::Success(success_result) => success_result,
execute_query_result::Result::Failure(failure_result) => {
return Err(DataFusionError::Execution(format!(
"Fail to execute query due to {failure_result:?}"
)));
}
};

assert_eq!(
session_id, query_result.session_id,
"Session id inconsistent between Client and Server side in DistributedQueryExec."
Expand Down
Loading