Skip to content

Commit c8ab5a4

Browse files
authored
Refactor Ballista executor so that FlightService delegates to an Executor struct (#450)
* Refactor Ballista executor so that FlightService delegates to an Executor struct * Simplify code * Pass executor to execution_loop * clippy * use arrow via datafusion
1 parent 80abb09 commit c8ab5a4

6 files changed

Lines changed: 135 additions & 91 deletions

File tree

‎ballista/rust/executor/src/execution_loop.rs‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,16 @@ use log::{debug, error, info, warn};
2525
use tonic::transport::Channel;
2626

2727
use ballista_core::serde::protobuf::ExecutorRegistration;
28-
use ballista_core::{
29-
client::BallistaClient,
30-
serde::protobuf::{
31-
self, scheduler_grpc_client::SchedulerGrpcClient, task_status, FailedTask,
32-
PartitionId, PollWorkParams, PollWorkResult, TaskDefinition, TaskStatus,
33-
},
28+
use ballista_core::serde::protobuf::{
29+
self, scheduler_grpc_client::SchedulerGrpcClient, task_status, FailedTask,
30+
PartitionId, PollWorkParams, PollWorkResult, TaskDefinition, TaskStatus,
3431
};
32+
use ballista_executor::executor::Executor;
3533
use protobuf::CompletedTask;
3634

3735
pub async fn poll_loop(
3836
mut scheduler: SchedulerGrpcClient<Channel>,
39-
executor_client: BallistaClient,
37+
executor: Arc<Executor>,
4038
executor_meta: ExecutorRegistration,
4139
concurrent_tasks: usize,
4240
) {
@@ -67,7 +65,7 @@ pub async fn poll_loop(
6765
Ok(result) => {
6866
if let Some(task) = result.into_inner().task {
6967
run_received_tasks(
70-
executor_client.clone(),
68+
executor.clone(),
7169
executor_meta.id.clone(),
7270
available_tasks_slots.clone(),
7371
task_status_sender,
@@ -86,7 +84,7 @@ pub async fn poll_loop(
8684
}
8785

8886
async fn run_received_tasks(
89-
mut executor_client: BallistaClient,
87+
executor: Arc<Executor>,
9088
executor_id: String,
9189
available_tasks_slots: Arc<AtomicUsize>,
9290
task_status_sender: Sender<TaskStatus>,
@@ -96,15 +94,13 @@ async fn run_received_tasks(
9694
available_tasks_slots.fetch_sub(1, Ordering::SeqCst);
9795
let plan: Arc<dyn ExecutionPlan> = (&task.plan.unwrap()).try_into().unwrap();
9896
let task_id = task.task_id.unwrap();
99-
// TODO: This is a convoluted way of executing the task. We should move the task
100-
// execution code outside of the FlightService (data plane) into the control plane.
10197

10298
tokio::spawn(async move {
103-
let execution_result = executor_client
99+
let execution_result = executor
104100
.execute_partition(
105101
task_id.job_id.clone(),
106102
task_id.stage_id as usize,
107-
vec![task_id.partition_id as usize],
103+
task_id.partition_id as usize,
108104
plan,
109105
)
110106
.await;
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Ballista executor logic
19+
20+
use std::path::PathBuf;
21+
use std::sync::Arc;
22+
use std::time::Instant;
23+
24+
use ballista_core::error::BallistaError;
25+
use ballista_core::utils;
26+
use datafusion::arrow::array::{ArrayRef, StringBuilder};
27+
use datafusion::arrow::datatypes::{DataType, Field, Schema};
28+
use datafusion::arrow::record_batch::RecordBatch;
29+
use datafusion::physical_plan::ExecutionPlan;
30+
use log::info;
31+
32+
/// Ballista executor
33+
pub struct Executor {
34+
/// Directory for storing partial results
35+
work_dir: String,
36+
}
37+
38+
impl Executor {
39+
/// Create a new executor instance
40+
pub fn new(work_dir: &str) -> Self {
41+
Self {
42+
work_dir: work_dir.to_owned(),
43+
}
44+
}
45+
}
46+
47+
impl Executor {
48+
/// Execute one partition of a query stage and persist the result to disk in IPC format. On
49+
/// success, return a RecordBatch containing metadata about the results, including path
50+
/// and statistics.
51+
pub async fn execute_partition(
52+
&self,
53+
job_id: String,
54+
stage_id: usize,
55+
part: usize,
56+
plan: Arc<dyn ExecutionPlan>,
57+
) -> Result<RecordBatch, BallistaError> {
58+
let mut path = PathBuf::from(&self.work_dir);
59+
path.push(&job_id);
60+
path.push(&format!("{}", stage_id));
61+
path.push(&format!("{}", part));
62+
std::fs::create_dir_all(&path)?;
63+
64+
path.push("data.arrow");
65+
let path = path.to_str().unwrap();
66+
info!("Writing results to {}", path);
67+
68+
let now = Instant::now();
69+
70+
// execute the query partition
71+
let mut stream = plan.execute(part).await?;
72+
73+
// stream results to disk
74+
let stats = utils::write_stream_to_disk(&mut stream, &path).await?;
75+
76+
info!(
77+
"Executed partition {} in {} seconds. Statistics: {:?}",
78+
part,
79+
now.elapsed().as_secs(),
80+
stats
81+
);
82+
83+
let schema = Arc::new(Schema::new(vec![
84+
Field::new("path", DataType::Utf8, false),
85+
stats.arrow_struct_repr(),
86+
]));
87+
88+
// build result set with summary of the partition execution status
89+
let mut c0 = StringBuilder::new(1);
90+
c0.append_value(&path).unwrap();
91+
let path: ArrayRef = Arc::new(c0.finish());
92+
93+
let stats: ArrayRef = stats.to_arrow_arrayref()?;
94+
RecordBatch::try_new(schema, vec![path, stats]).map_err(BallistaError::ArrowError)
95+
}
96+
97+
pub fn work_dir(&self) -> &str {
98+
&self.work_dir
99+
}
100+
}

‎ballista/rust/executor/src/flight_service.rs‎

Lines changed: 16 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -21,27 +21,24 @@ use std::fs::File;
2121
use std::path::PathBuf;
2222
use std::pin::Pin;
2323
use std::sync::Arc;
24-
use std::time::Instant;
2524

25+
use crate::executor::Executor;
2626
use ballista_core::error::BallistaError;
2727
use ballista_core::serde::decode_protobuf;
2828
use ballista_core::serde::scheduler::{Action as BallistaAction, PartitionStats};
29-
use ballista_core::utils;
3029

3130
use arrow_flight::{
3231
flight_service_server::FlightService, Action, ActionType, Criteria, Empty,
3332
FlightData, FlightDescriptor, FlightInfo, HandshakeRequest, HandshakeResponse,
3433
PutResult, SchemaResult, Ticket,
3534
};
3635
use datafusion::arrow::{
37-
array::{ArrayRef, StringBuilder},
3836
datatypes::{DataType, Field, Schema},
3937
error::ArrowError,
4038
ipc::reader::FileReader,
4139
ipc::writer::IpcWriteOptions,
4240
record_batch::RecordBatch,
4341
};
44-
use datafusion::error::DataFusionError;
4542
use datafusion::physical_plan::displayable;
4643
use futures::{Stream, StreamExt};
4744
use log::{info, warn};
@@ -61,12 +58,13 @@ type FlightDataReceiver = Receiver<Result<FlightData, Status>>;
6158
/// Service implementing the Apache Arrow Flight Protocol
6259
#[derive(Clone)]
6360
pub struct BallistaFlightService {
64-
work_dir: String,
61+
/// Executor
62+
executor: Arc<Executor>,
6563
}
6664

6765
impl BallistaFlightService {
68-
pub fn new(work_dir: String) -> Self {
69-
Self { work_dir }
66+
pub fn new(executor: Arc<Executor>) -> Self {
67+
Self { executor }
7068
}
7169
}
7270

@@ -105,59 +103,22 @@ impl FlightService for BallistaFlightService {
105103

106104
let mut tasks: Vec<JoinHandle<Result<_, BallistaError>>> = vec![];
107105
for &part in &partition.partition_id {
108-
let mut path = PathBuf::from(&self.work_dir);
109106
let partition = partition.clone();
107+
let executor = self.executor.clone();
110108
tasks.push(tokio::spawn(async move {
111-
path.push(partition.job_id);
112-
path.push(&format!("{}", partition.stage_id));
113-
path.push(&format!("{}", part));
114-
std::fs::create_dir_all(&path)?;
115-
116-
path.push("data.arrow");
117-
let path = path.to_str().unwrap();
118-
info!("Writing results to {}", path);
119-
120-
let now = Instant::now();
121-
122-
// execute the query partition
123-
let mut stream = partition
124-
.plan
125-
.execute(part)
126-
.await
127-
.map_err(|e| from_datafusion_err(&e))?;
128-
129-
// stream results to disk
130-
let stats = utils::write_stream_to_disk(&mut stream, &path)
131-
.await
132-
.map_err(|e| from_ballista_err(&e))?;
133-
134-
info!(
135-
"Executed partition {} in {} seconds. Statistics: {:?}",
136-
part,
137-
now.elapsed().as_secs(),
138-
stats
139-
);
109+
let results = executor
110+
.execute_partition(
111+
partition.job_id.clone(),
112+
partition.stage_id,
113+
part,
114+
partition.plan.clone(),
115+
)
116+
.await?;
117+
let results = vec![results];
140118

141119
let mut flights: Vec<Result<FlightData, Status>> = vec![];
142120
let options = arrow::ipc::writer::IpcWriteOptions::default();
143121

144-
let schema = Arc::new(Schema::new(vec![
145-
Field::new("path", DataType::Utf8, false),
146-
stats.arrow_struct_repr(),
147-
]));
148-
149-
// build result set with summary of the partition execution status
150-
let mut c0 = StringBuilder::new(1);
151-
c0.append_value(&path).unwrap();
152-
let path: ArrayRef = Arc::new(c0.finish());
153-
154-
let stats: ArrayRef = stats.to_arrow_arrayref()?;
155-
let results = vec![RecordBatch::try_new(
156-
schema,
157-
vec![path, stats],
158-
)
159-
.unwrap()];
160-
161122
let mut batches: Vec<Result<FlightData, Status>> = results
162123
.iter()
163124
.flat_map(|batch| create_flight_iter(batch, &options))
@@ -208,7 +169,7 @@ impl FlightService for BallistaFlightService {
208169
// fetch a partition that was previously executed by this executor
209170
info!("FetchPartition {:?}", partition_id);
210171

211-
let mut path = PathBuf::from(&self.work_dir);
172+
let mut path = PathBuf::from(self.executor.work_dir());
212173
path.push(&partition_id.job_id);
213174
path.push(&format!("{}", partition_id.stage_id));
214175
path.push(&format!("{}", partition_id.partition_id));
@@ -368,7 +329,3 @@ fn from_arrow_err(e: &ArrowError) -> Status {
368329
fn from_ballista_err(e: &ballista_core::error::BallistaError) -> Status {
369330
Status::internal(format!("Ballista Error: {:?}", e))
370331
}
371-
372-
fn from_datafusion_err(e: &DataFusionError) -> Status {
373-
Status::internal(format!("DataFusion Error: {:?}", e))
374-
}

‎ballista/rust/executor/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@
1818
//! Core executor logic for executing queries and storing results in memory.
1919
2020
pub mod collect;
21+
pub mod executor;
2122
pub mod flight_service;

‎ballista/rust/executor/src/main.rs‎

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,15 @@ use tempfile::TempDir;
3030
use tonic::transport::Server;
3131
use uuid::Uuid;
3232

33-
use ballista_core::{
34-
client::BallistaClient,
35-
serde::protobuf::{
36-
executor_registration, scheduler_grpc_client::SchedulerGrpcClient,
37-
ExecutorRegistration,
38-
},
33+
use ballista_core::serde::protobuf::{
34+
executor_registration, scheduler_grpc_client::SchedulerGrpcClient,
35+
ExecutorRegistration,
3936
};
4037
use ballista_core::{
4138
print_version, serde::protobuf::scheduler_grpc_server::SchedulerGrpcServer,
4239
BALLISTA_VERSION,
4340
};
41+
use ballista_executor::executor::Executor;
4442
use ballista_executor::flight_service::BallistaFlightService;
4543
use ballista_scheduler::{state::StandaloneClient, SchedulerServer};
4644
use config::prelude::*;
@@ -166,27 +164,20 @@ async fn main() -> Result<()> {
166164
let scheduler = SchedulerGrpcClient::connect(scheduler_url)
167165
.await
168166
.context("Could not connect to scheduler")?;
169-
let service = BallistaFlightService::new(work_dir);
167+
168+
let executor = Arc::new(Executor::new(&work_dir));
169+
170+
let service = BallistaFlightService::new(executor.clone());
170171

171172
let server = FlightServiceServer::new(service);
172173
info!(
173174
"Ballista v{} Rust Executor listening on {:?}",
174175
BALLISTA_VERSION, addr
175176
);
176177
let server_future = tokio::spawn(Server::builder().add_service(server).serve(addr));
177-
let client_host = external_host.as_deref().unwrap_or_else(|| {
178-
if bind_host == "0.0.0.0" {
179-
// If the executor is being bound to "0.0.0.0" (which means use all ips in all eth devices)
180-
// then use "localhost" to connect to itself through the BallistaClient
181-
"localhost"
182-
} else {
183-
&bind_host
184-
}
185-
});
186-
let client = BallistaClient::try_new(client_host, port).await?;
187178
tokio::spawn(execution_loop::poll_loop(
188179
scheduler,
189-
client,
180+
executor,
190181
executor_meta,
191182
opt.concurrent_tasks,
192183
));

‎ballista/rust/scheduler/Cargo.toml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@ tonic = "0.4"
5252
tower = { version = "0.4" }
5353
warp = "0.3"
5454

55-
arrow = { version = "4.0" }
5655
datafusion = { path = "../../../datafusion" }
5756

5857
[dev-dependencies]

0 commit comments

Comments
 (0)