Skip to content

Commit c5e4d9e

Browse files
blasrodriDarksonn
andauthored
runtime: keep io_uring op alive when submission fails after push (#8185)
Co-authored-by: Alice Ryhl <[email protected]> Fixes: #8255
1 parent bb28a94 commit c5e4d9e

13 files changed

Lines changed: 209 additions & 31 deletions

File tree

‎tokio/src/fs/file.rs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1038,14 +1038,21 @@ impl Inner {
10381038
let (res, r_fd, r_buf) =
10391039
// u64::MAX to use and advance the file position
10401040
Op::read_at(fd, buf, max_buf_size, u64::MAX).await;
1041-
match res {
1042-
Err(e) if e.kind() == io::ErrorKind::Interrupted => {
1041+
match (res, r_fd, r_buf) {
1042+
// Retry an interrupted read with the resources the op handed back.
1043+
(Err(e), Some(r_fd), Some(r_buf)) if e.kind() == io::ErrorKind::Interrupted => {
10431044
buf = r_buf;
10441045
fd = r_fd;
10451046
continue;
10461047
}
1047-
Err(e) => break (Operation::Read(Err(e)), r_buf),
1048-
Ok(n) => break (Operation::Read(Ok(n as usize)), r_buf),
1048+
// A cancelled op keeps its buffer in the driver, so the idle slot
1049+
// gets a fresh empty `Buf` instead.
1050+
(Err(e), _, r_buf) => {
1051+
let buf = r_buf.unwrap_or_else(|| Buf::with_capacity(0));
1052+
break (Operation::Read(Err(e)), buf);
1053+
}
1054+
(Ok(n), _, Some(r_buf)) => break (Operation::Read(Ok(n as usize)), r_buf),
1055+
(Ok(_), ..) => unreachable!("a successful read returns its buffer"),
10491056
}
10501057
}
10511058
}

‎tokio/src/fs/read_uring.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -135,17 +135,20 @@ async fn op_read(
135135
loop {
136136
let (res, r_fd, r_buf) = Op::read_at(fd, buf, read_len as usize, *offset).await;
137137

138-
match res {
139-
Err(e) if e.kind() == ErrorKind::Interrupted => {
138+
match (res, r_fd, r_buf) {
139+
// Retry an interrupted read with the buffer the op handed back. A
140+
// cancelled op returns no buffer, so it falls through as terminal.
141+
(Err(e), Some(r_fd), Some(r_buf)) if e.kind() == ErrorKind::Interrupted => {
140142
buf = r_buf;
141143
fd = r_fd;
142144
}
143-
Err(e) => return Err(e),
144-
Ok(size_read) => {
145+
(Err(e), ..) => return Err(e),
146+
(Ok(size_read), Some(r_fd), Some(r_buf)) => {
145147
*offset += size_read as u64;
146148

147149
return Ok((r_fd, r_buf, size_read == 0));
148150
}
151+
(Ok(_), ..) => unreachable!("a successful read returns its fd and buffer"),
149152
}
150153
}
151154
}

‎tokio/src/fs/write.rs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,13 @@ async fn write_uring(path: &Path, mut buf: OwnedBuf) -> io::Result<()> {
7575
let mut buf_offset: usize = 0;
7676
let mut file_offset: u64 = 0;
7777
while buf_offset < total {
78-
let (res, _buf, _fd) = Op::write_at(fd, buf, buf_offset, file_offset)?.await;
78+
let (res, r_buf, r_fd) = Op::write_at(fd, buf, buf_offset, file_offset)?.await;
79+
80+
// A cancelled op returns no resources to continue with, so its error is
81+
// terminal; otherwise an interrupted write retries from where it stopped.
82+
let (Some(r_buf), Some(r_fd)) = (r_buf, r_fd) else {
83+
return Err(res.expect_err("a cancelled op completes with an error"));
84+
};
7985

8086
let n = match res {
8187
Ok(0) => return Err(io::ErrorKind::WriteZero.into()),
@@ -84,8 +90,8 @@ async fn write_uring(path: &Path, mut buf: OwnedBuf) -> io::Result<()> {
8490
Err(e) => return Err(e),
8591
};
8692

87-
buf = _buf;
88-
fd = _fd;
93+
buf = r_buf;
94+
fd = r_fd;
8995
buf_offset += n as usize;
9096
file_offset += n as u64;
9197
}

‎tokio/src/io/uring/open.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ impl Completable for Open {
2727
fn complete_with_error(self, err: Error) -> Self::Output {
2828
Err(err)
2929
}
30+
31+
fn complete_after_cancel(err: Error) -> Self::Output {
32+
Err(err)
33+
}
3034
}
3135

3236
impl Cancellable for Open {

‎tokio/src/io/uring/read.rs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,19 +62,25 @@ impl<B: fmt::Debug, F> fmt::Debug for Read<B, F> {
6262
}
6363

6464
impl<B: ReadBuffer, F: UringFd> Completable for Read<B, F> {
65-
type Output = (io::Result<u32>, F, B);
65+
// The fd and buffer are `None` only when the op was cancelled and they were
66+
// kept in the driver; otherwise they are returned to the caller.
67+
type Output = (io::Result<u32>, Option<F>, Option<B>);
6668

6769
fn complete(self, cqe: CqeResult) -> Self::Output {
6870
let mut buf = self.buf;
6971
if let Ok(len) = cqe.result {
7072
// SAFETY: kernel wrote exactly `len` bytes into the prepared buffer.
7173
unsafe { buf.uring_read_complete(len) };
7274
}
73-
(cqe.result, self.fd, buf)
75+
(cqe.result, Some(self.fd), Some(buf))
7476
}
7577

7678
fn complete_with_error(self, err: Error) -> Self::Output {
77-
(Err(err), self.fd, self.buf)
79+
(Err(err), Some(self.fd), Some(self.buf))
80+
}
81+
82+
fn complete_after_cancel(err: Error) -> Self::Output {
83+
(Err(err), None, None)
7884
}
7985
}
8086

‎tokio/src/io/uring/rename.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@ impl Completable for Rename {
2525
fn complete_with_error(self, error: io::Error) -> Self::Output {
2626
Err(error)
2727
}
28+
29+
fn complete_after_cancel(error: io::Error) -> Self::Output {
30+
Err(error)
31+
}
2832
}
2933

3034
impl Cancellable for Rename {

‎tokio/src/io/uring/statx.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ impl Completable for Statx {
6868
fn complete_with_error(self, error: io::Error) -> Self::Output {
6969
Err(error)
7070
}
71+
72+
fn complete_after_cancel(error: io::Error) -> Self::Output {
73+
Err(error)
74+
}
7175
}
7276

7377
impl Cancellable for Statx {

‎tokio/src/io/uring/write.rs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,19 @@ pub(crate) struct Write {
1212
}
1313

1414
impl Completable for Write {
15-
type Output = (io::Result<u32>, OwnedBuf, OwnedFd);
15+
// The buffer and fd are `None` only when the op was cancelled and they were
16+
// kept in the driver; otherwise they are returned to the caller.
17+
type Output = (io::Result<u32>, Option<OwnedBuf>, Option<OwnedFd>);
1618
fn complete(self, cqe: CqeResult) -> Self::Output {
17-
(cqe.result, self.buf, self.fd)
19+
(cqe.result, Some(self.buf), Some(self.fd))
1820
}
1921

2022
fn complete_with_error(self, err: Error) -> Self::Output {
21-
(Err(err), self.buf, self.fd)
23+
(Err(err), Some(self.buf), Some(self.fd))
24+
}
25+
26+
fn complete_after_cancel(err: Error) -> Self::Output {
27+
(Err(err), None, None)
2228
}
2329
}
2430

‎tokio/src/runtime/driver/op.rs‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ use crate::io::uring::rename::Rename;
55
use crate::io::uring::utils::ArcFd;
66
use crate::io::uring::write::Write;
77

8+
use crate::runtime::io::RegisterResult;
89
use crate::runtime::Handle;
910

1011
#[cfg(
@@ -152,6 +153,10 @@ pub(crate) trait Completable {
152153
// The `Op` type that implements this trait can return the passed error
153154
// upstream by embedding it in the `Output`.
154155
fn complete_with_error(self, error: io::Error) -> Self::Output;
156+
157+
// Like `complete_with_error`, but for an op cancelled after the SQE reached
158+
// the kernel: its resources stay with the driver and are not returned.
159+
fn complete_after_cancel(error: io::Error) -> Self::Output;
155160
}
156161

157162
/// Extracts the `CancelData` needed to safely cancel an in-flight io_uring operation.
@@ -166,18 +171,22 @@ impl<T: Cancellable + Completable + Send> Future for Op<T> {
166171

167172
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
168173
let this = self.get_mut();
169-
let handle = &mut this.handle;
170-
let driver = handle.inner.driver().io();
171174

172175
match &mut this.state {
173176
State::Initialize(entry_opt) => {
174177
let entry = entry_opt.take().expect("Entry must be present");
175178
let waker = cx.waker().clone();
176179

180+
let driver = this.handle.inner.driver().io();
177181
// SAFETY: entry is valid for the entire duration of the operation
178-
match unsafe { driver.register_op(entry, waker) } {
179-
Ok(idx) => this.state = State::Polled(idx),
180-
Err(err) => {
182+
let result = unsafe { driver.register_op(entry, waker) };
183+
184+
match result {
185+
RegisterResult::Submitted(idx) => this.state = State::Polled(idx),
186+
187+
// The kernel never saw the entry, so the resources go back to
188+
// the caller with the error.
189+
RegisterResult::NotPushed(err) => {
181190
let data = this
182191
.take_data()
183192
.expect("Data must be present on Initialization");
@@ -186,12 +195,27 @@ impl<T: Cancellable + Completable + Send> Future for Op<T> {
186195

187196
return Poll::Ready(data.complete_with_error(err));
188197
}
198+
199+
// The kernel may have seen the entry, so cancel to keep the
200+
// resources alive until completion instead of returning them.
201+
RegisterResult::Pushed { index, error } => {
202+
let data = this
203+
.take_data()
204+
.expect("Data must be present on Initialization");
205+
206+
this.handle.inner.driver().io().cancel_op(index, Some(data));
207+
208+
this.state = State::Complete;
209+
210+
return Poll::Ready(T::complete_after_cancel(error));
211+
}
189212
};
190213

191214
Poll::Pending
192215
}
193216

194217
State::Polled(idx) => {
218+
let driver = this.handle.inner.driver().io();
195219
let mut ctx = driver.get_uring().lock();
196220
let lifecycle = ctx.ops.get_mut(*idx).expect("Lifecycle must be present");
197221

‎tokio/src/runtime/io/driver.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ cfg_signal_internal_and_unix! {
55
cfg_io_uring! {
66
mod uring;
77
use uring::UringContext;
8+
pub(crate) use uring::RegisterResult;
89
use crate::sync::OnceCell;
910
}
1011

0 commit comments

Comments
 (0)