@@ -21,27 +21,24 @@ use std::fs::File;
2121use std:: path:: PathBuf ;
2222use std:: pin:: Pin ;
2323use std:: sync:: Arc ;
24- use std:: time:: Instant ;
2524
25+ use crate :: executor:: Executor ;
2626use ballista_core:: error:: BallistaError ;
2727use ballista_core:: serde:: decode_protobuf;
2828use ballista_core:: serde:: scheduler:: { Action as BallistaAction , PartitionStats } ;
29- use ballista_core:: utils;
3029
3130use arrow_flight:: {
3231 flight_service_server:: FlightService , Action , ActionType , Criteria , Empty ,
3332 FlightData , FlightDescriptor , FlightInfo , HandshakeRequest , HandshakeResponse ,
3433 PutResult , SchemaResult , Ticket ,
3534} ;
3635use datafusion:: arrow:: {
37- array:: { ArrayRef , StringBuilder } ,
3836 datatypes:: { DataType , Field , Schema } ,
3937 error:: ArrowError ,
4038 ipc:: reader:: FileReader ,
4139 ipc:: writer:: IpcWriteOptions ,
4240 record_batch:: RecordBatch ,
4341} ;
44- use datafusion:: error:: DataFusionError ;
4542use datafusion:: physical_plan:: displayable;
4643use futures:: { Stream , StreamExt } ;
4744use log:: { info, warn} ;
@@ -61,12 +58,13 @@ type FlightDataReceiver = Receiver<Result<FlightData, Status>>;
6158/// Service implementing the Apache Arrow Flight Protocol
6259#[ derive( Clone ) ]
6360pub struct BallistaFlightService {
64- work_dir : String ,
61+ /// Executor
62+ executor : Arc < Executor > ,
6563}
6664
6765impl BallistaFlightService {
68- pub fn new ( work_dir : String ) -> Self {
69- Self { work_dir }
66+ pub fn new ( executor : Arc < Executor > ) -> Self {
67+ Self { executor }
7068 }
7169}
7270
@@ -105,59 +103,22 @@ impl FlightService for BallistaFlightService {
105103
106104 let mut tasks: Vec < JoinHandle < Result < _ , BallistaError > > > = vec ! [ ] ;
107105 for & part in & partition. partition_id {
108- let mut path = PathBuf :: from ( & self . work_dir ) ;
109106 let partition = partition. clone ( ) ;
107+ let executor = self . executor . clone ( ) ;
110108 tasks. push ( tokio:: spawn ( async move {
111- path. push ( partition. job_id ) ;
112- path. push ( & format ! ( "{}" , partition. stage_id) ) ;
113- path. push ( & format ! ( "{}" , part) ) ;
114- std:: fs:: create_dir_all ( & path) ?;
115-
116- path. push ( "data.arrow" ) ;
117- let path = path. to_str ( ) . unwrap ( ) ;
118- info ! ( "Writing results to {}" , path) ;
119-
120- let now = Instant :: now ( ) ;
121-
122- // execute the query partition
123- let mut stream = partition
124- . plan
125- . execute ( part)
126- . await
127- . map_err ( |e| from_datafusion_err ( & e) ) ?;
128-
129- // stream results to disk
130- let stats = utils:: write_stream_to_disk ( & mut stream, & path)
131- . await
132- . map_err ( |e| from_ballista_err ( & e) ) ?;
133-
134- info ! (
135- "Executed partition {} in {} seconds. Statistics: {:?}" ,
136- part,
137- now. elapsed( ) . as_secs( ) ,
138- stats
139- ) ;
109+ let results = executor
110+ . execute_partition (
111+ partition. job_id . clone ( ) ,
112+ partition. stage_id ,
113+ part,
114+ partition. plan . clone ( ) ,
115+ )
116+ . await ?;
117+ let results = vec ! [ results] ;
140118
141119 let mut flights: Vec < Result < FlightData , Status > > = vec ! [ ] ;
142120 let options = arrow:: ipc:: writer:: IpcWriteOptions :: default ( ) ;
143121
144- let schema = Arc :: new ( Schema :: new ( vec ! [
145- Field :: new( "path" , DataType :: Utf8 , false ) ,
146- stats. arrow_struct_repr( ) ,
147- ] ) ) ;
148-
149- // build result set with summary of the partition execution status
150- let mut c0 = StringBuilder :: new ( 1 ) ;
151- c0. append_value ( & path) . unwrap ( ) ;
152- let path: ArrayRef = Arc :: new ( c0. finish ( ) ) ;
153-
154- let stats: ArrayRef = stats. to_arrow_arrayref ( ) ?;
155- let results = vec ! [ RecordBatch :: try_new(
156- schema,
157- vec![ path, stats] ,
158- )
159- . unwrap( ) ] ;
160-
161122 let mut batches: Vec < Result < FlightData , Status > > = results
162123 . iter ( )
163124 . flat_map ( |batch| create_flight_iter ( batch, & options) )
@@ -208,7 +169,7 @@ impl FlightService for BallistaFlightService {
208169 // fetch a partition that was previously executed by this executor
209170 info ! ( "FetchPartition {:?}" , partition_id) ;
210171
211- let mut path = PathBuf :: from ( & self . work_dir ) ;
172+ let mut path = PathBuf :: from ( self . executor . work_dir ( ) ) ;
212173 path. push ( & partition_id. job_id ) ;
213174 path. push ( & format ! ( "{}" , partition_id. stage_id) ) ;
214175 path. push ( & format ! ( "{}" , partition_id. partition_id) ) ;
@@ -368,7 +329,3 @@ fn from_arrow_err(e: &ArrowError) -> Status {
368329fn from_ballista_err ( e : & ballista_core:: error:: BallistaError ) -> Status {
369330 Status :: internal ( format ! ( "Ballista Error: {:?}" , e) )
370331}
371-
372- fn from_datafusion_err ( e : & DataFusionError ) -> Status {
373- Status :: internal ( format ! ( "DataFusion Error: {:?}" , e) )
374- }
0 commit comments