Skip to content

Commit 0ced41b

Browse files
lun3xritchie46
authored andcommitted
fix: Wrap IoError in PathIoError such that callers can extract untruncated path (#29128)
(cherry picked from commit f611a6d)
1 parent 3b4075b commit 0ced41b

4 files changed

Lines changed: 248 additions & 38 deletions

File tree

Lines changed: 44 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,54 @@
11
use std::io;
2+
use std::path::{Path, PathBuf};
23

4+
use polars_error::PolarsResult;
5+
use polars_utils::io::_limit_path_len_io_err;
36
use polars_utils::pl_path::PlRefPath;
47

5-
pub fn mkdir_recursive(path: &PlRefPath) -> io::Result<()> {
6-
if !path.has_scheme() {
7-
std::fs::DirBuilder::new().recursive(true).create(
8-
path.parent()
9-
.ok_or(io::Error::other("path is not a file"))?,
10-
)?;
11-
}
8+
use crate::path_utils::resolve_homedir;
9+
10+
/// Creates the directory `path` will be written into.
11+
///
12+
/// Does nothing for paths with a scheme; object stores have no directories to create.
13+
pub fn mkdir_recursive(path: &PlRefPath) -> PolarsResult<()> {
14+
let Some(parent) = writable_parent(path)? else {
15+
return Ok(());
16+
};
17+
18+
std::fs::DirBuilder::new()
19+
.recursive(true)
20+
.create(&parent)
21+
.map_err(|err| _limit_path_len_io_err(&parent, err))
22+
}
1223

13-
Ok(())
24+
/// Async counterpart of [`mkdir_recursive`].
25+
pub async fn tokio_mkdir_recursive(path: &PlRefPath) -> PolarsResult<()> {
26+
let Some(parent) = writable_parent(path)? else {
27+
return Ok(());
28+
};
29+
30+
tokio::fs::DirBuilder::new()
31+
.recursive(true)
32+
.create(&parent)
33+
.await
34+
.map_err(|err| _limit_path_len_io_err(&parent, err))
1435
}
1536

16-
pub async fn tokio_mkdir_recursive(path: &PlRefPath) -> io::Result<()> {
17-
if !path.has_scheme() {
18-
tokio::fs::DirBuilder::new()
19-
.recursive(true)
20-
.create(
21-
path.parent()
22-
.ok_or(io::Error::other("path is not a file"))?,
23-
)
24-
.await?;
37+
/// The local directory that must exist before `path` can be opened for writing, or `None`
38+
/// when there is nothing to create.
39+
fn writable_parent(path: &PlRefPath) -> PolarsResult<Option<PathBuf>> {
40+
if path.has_scheme() {
41+
return Ok(None);
42+
}
43+
44+
let Some(parent) = path.parent() else {
45+
return Err(io::Error::other(format!("path is not a file: {path}")).into());
46+
};
47+
48+
// A path with a single component has no directory to create.
49+
if parent.is_empty() {
50+
return Ok(None);
2551
}
2652

27-
Ok(())
53+
Ok(Some(resolve_homedir(Path::new(parent)).into_owned()))
2854
}

‎crates/polars-stream/src/nodes/io_sinks/components/file_provider.rs‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -79,16 +79,10 @@ impl FileProvider {
7979
"provided path '{provided_path}' contained parent dir component '..'"
8080
);
8181

82-
if !path.has_scheme()
83-
&& let Some(path) = path.parent()
84-
{
85-
// Ignore errors from directory creation - the `Writable::try_new()` below will raise
86-
// appropriate errors.
87-
let _ = tokio::fs::DirBuilder::new()
88-
.recursive(true)
89-
.create(path)
90-
.await;
91-
}
82+
// Raise errors from directory creation here, so the reason it could not be created is
83+
// reported, rather than raising "No such file or directory" when we try to open the
84+
// missing directory below.
85+
polars_io::utils::mkdir::tokio_mkdir_recursive(&path).await?;
9286

9387
if let Some(path_info_entry) = &path_info_entry {
9488
path_info_entry.set_path(path.clone());

‎crates/polars-utils/src/io.rs‎

Lines changed: 116 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,62 @@
11
use std::fs::File;
2-
use std::io;
3-
use std::path::Path;
2+
use std::path::{Path, PathBuf};
3+
use std::{fmt, io};
44

55
use polars_error::*;
66

7+
/// An IO error together with the full path it occurred on.
8+
///
9+
/// The `Display` output truncates long paths so they stay readable in error messages, but
10+
/// the full path remains available to consumers that downcast the payload of the
11+
/// [`io::Error`] built by [`_limit_path_len_io_err`].
12+
#[derive(Debug)]
13+
pub struct PathIoError {
14+
pub path: PathBuf,
15+
pub source: io::Error,
16+
}
17+
18+
impl PathIoError {
19+
const MAX_DISPLAYED_PATH_CHARS: usize = 88;
20+
}
21+
22+
impl fmt::Display for PathIoError {
23+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
24+
let path = self.path.to_string_lossy();
25+
let n_chars = path.chars().count();
26+
if n_chars > Self::MAX_DISPLAYED_PATH_CHARS && !polars_config::config().verbose() {
27+
let truncated_path: String = path
28+
.chars()
29+
.skip(n_chars - Self::MAX_DISPLAYED_PATH_CHARS)
30+
.collect();
31+
write!(
32+
f,
33+
"{}: ...{truncated_path} (set POLARS_VERBOSE=1 to see full path)",
34+
self.source
35+
)
36+
} else {
37+
write!(f, "{}: {path}", self.source)
38+
}
39+
}
40+
}
41+
42+
impl std::error::Error for PathIoError {
43+
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
44+
Some(&self.source)
45+
}
46+
}
47+
48+
/// Attaches `path` to `err`, keeping its [`io::ErrorKind`].
49+
///
50+
/// The path is available in full through [`PathIoError`], and truncated in the message.
751
pub fn _limit_path_len_io_err(path: &Path, err: io::Error) -> PolarsError {
8-
let path = path.to_string_lossy();
9-
let msg = if path.len() > 88 && !polars_config::config().verbose() {
10-
let truncated_path: String = path.chars().skip(path.len() - 88).collect();
11-
format!("{err}: ...{truncated_path} (set POLARS_VERBOSE=1 to see full path)")
12-
} else {
13-
format!("{err}: {path}")
14-
};
15-
io::Error::new(err.kind(), msg).into()
52+
io::Error::new(
53+
err.kind(),
54+
PathIoError {
55+
path: path.to_path_buf(),
56+
source: err,
57+
},
58+
)
59+
.into()
1660
}
1761

1862
pub fn open_file(path: &Path) -> PolarsResult<File> {
@@ -31,3 +75,65 @@ pub fn open_file_write(path: &Path) -> PolarsResult<File> {
3175
pub fn create_file(path: &Path) -> PolarsResult<File> {
3276
File::create(path).map_err(|err| _limit_path_len_io_err(path, err))
3377
}
78+
79+
#[cfg(test)]
80+
mod tests {
81+
use super::*;
82+
83+
fn permission_denied() -> io::Error {
84+
io::Error::new(
85+
io::ErrorKind::PermissionDenied,
86+
"Permission denied (os error 13)",
87+
)
88+
}
89+
90+
fn io_error_of(err: &PolarsError) -> &io::Error {
91+
match err {
92+
PolarsError::IO { error, .. } => error,
93+
other => panic!("expected an IO error, got {other:?}"),
94+
}
95+
}
96+
97+
#[test]
98+
fn keeps_kind_and_full_path() {
99+
let path = PathBuf::from(format!("/{}/file.parquet", "a".repeat(200)));
100+
let err = _limit_path_len_io_err(&path, permission_denied());
101+
let io_err = io_error_of(&err);
102+
103+
assert_eq!(io_err.kind(), io::ErrorKind::PermissionDenied);
104+
105+
let with_path = io_err
106+
.get_ref()
107+
.and_then(|e| e.downcast_ref::<PathIoError>())
108+
.expect("payload should be a PathIoError");
109+
assert_eq!(with_path.path, path);
110+
assert_eq!(with_path.source.kind(), io::ErrorKind::PermissionDenied);
111+
}
112+
113+
#[test]
114+
fn display_truncates_long_paths() {
115+
let path = PathBuf::from(format!("/{}/file.parquet", "a".repeat(200)));
116+
let msg = _limit_path_len_io_err(&path, permission_denied()).to_string();
117+
118+
assert!(
119+
msg.starts_with("Permission denied (os error 13): ..."),
120+
"{msg}"
121+
);
122+
assert!(
123+
msg.ends_with("(set POLARS_VERBOSE=1 to see full path)"),
124+
"{msg}"
125+
);
126+
assert!(!msg.contains(path.to_str().unwrap()), "{msg}");
127+
assert!(msg.contains("aaa/file.parquet"), "{msg}");
128+
}
129+
130+
#[test]
131+
fn display_keeps_short_paths() {
132+
let path = PathBuf::from("/tmp/out/file.parquet");
133+
let msg = _limit_path_len_io_err(&path, permission_denied()).to_string();
134+
assert_eq!(
135+
msg,
136+
"Permission denied (os error 13): /tmp/out/file.parquet"
137+
);
138+
}
139+
}

‎py-polars/tests/unit/io/test_sink.py‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import io
44
import os
5+
import re
56
import subprocess
67
import sys
78
from functools import partial
@@ -67,6 +68,89 @@ def test_write_mkdir(tmp_path: Path) -> None:
6768
assert_frame_equal(pl.read_parquet(f), df)
6869

6970

71+
skip_if_permissions_unenforced = pytest.mark.skipif(
72+
sys.platform == "win32" or (hasattr(os, "geteuid") and os.geteuid() == 0),
73+
reason="file permissions are not enforced here",
74+
)
75+
76+
77+
def _read_only_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
78+
"""A read-only directory, reachable by a relative path from the working directory.
79+
80+
Paths over 88 characters are truncated out of IO error messages, which `tmp_path`
81+
alone already exceeds on some platforms. Sinking relative to it keeps the paths
82+
short enough that the assertions below see them in full.
83+
"""
84+
d = tmp_path / "read-only"
85+
d.mkdir()
86+
d.chmod(0o555)
87+
monkeypatch.chdir(tmp_path)
88+
return Path("read-only")
89+
90+
91+
@skip_if_permissions_unenforced
92+
@pytest.mark.write_disk
93+
def test_mkdir_permission_denied_names_the_directory(
94+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
95+
) -> None:
96+
df = pl.DataFrame({"a": [1, 2, 3]})
97+
d = _read_only_dir(tmp_path, monkeypatch) / "a"
98+
99+
with pytest.raises(PermissionError) as exc_info:
100+
df.lazy().sink_parquet(d / "file", mkdir=True)
101+
102+
# The directory that could not be created, not the file underneath it.
103+
assert str(exc_info.value).endswith(str(d))
104+
105+
106+
@skip_if_permissions_unenforced
107+
@pytest.mark.write_disk
108+
def test_partitioned_mkdir_permission_denied(
109+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
110+
) -> None:
111+
"""The partitioned sink must report why the directory could not be made.
112+
113+
It used to discard that error and let the file open fail afterwards, reporting a
114+
missing directory instead of the permissions that stopped it being created.
115+
"""
116+
df = pl.DataFrame({"a": [1, 2, 3]})
117+
base = _read_only_dir(tmp_path, monkeypatch) / "a"
118+
119+
with pytest.raises(PermissionError) as exc_info:
120+
df.lazy().sink_parquet(pl.PartitionBy(base, key="a"))
121+
122+
# A partition directory, not a file inside one - opening the file is what used to
123+
# raise, and it named `<base>/a=<key>/<part>.parquet`. Which key fails first is not
124+
# deterministic.
125+
assert re.search(rf"{re.escape(str(base))}/a=\d+$", str(exc_info.value))
126+
127+
128+
@pytest.mark.parametrize("partitioned", [False, True])
129+
@pytest.mark.write_disk
130+
def test_mkdir_resolves_homedir(
131+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, partitioned: bool
132+
) -> None:
133+
"""`mkdir` must create the resolved directory, not a literal `~`."""
134+
monkeypatch.setenv("HOME", str(tmp_path))
135+
monkeypatch.setenv("USERPROFILE", str(tmp_path))
136+
# Any literal `~` is created relative to the working directory, so keep that inside
137+
# `tmp_path` - a regression must not leave one behind in the repository.
138+
cwd = tmp_path / "cwd"
139+
cwd.mkdir()
140+
monkeypatch.chdir(cwd)
141+
df = pl.DataFrame({"a": [1, 2, 3]})
142+
143+
if partitioned:
144+
df.lazy().sink_parquet(pl.PartitionBy("~/sink-homedir", key="a"))
145+
out = pl.read_parquet(tmp_path / "sink-homedir").sort("a")
146+
else:
147+
df.lazy().sink_parquet("~/sink-homedir/file.parquet", mkdir=True)
148+
out = pl.read_parquet(tmp_path / "sink-homedir" / "file.parquet")
149+
150+
assert_frame_equal(out, df)
151+
assert not (cwd / "~").exists()
152+
153+
70154
@pytest.mark.parametrize(("scan", "sink"), SINKS)
71155
@pytest.mark.parametrize("engine", ["in-memory", "streaming"])
72156
@pytest.mark.write_disk

0 commit comments

Comments
 (0)