Skip to content

Commit c76f272

Browse files
abendrothjsylvestre
authored andcommitted
sort: keep to its own temporary directory and files
Temporary files were created, reopened and deleted by path, and cleanup emptied the temporary directory by path. Once the directory's path led to another directory, sort read the files there as its own and deleted them. The directory and each file are identified by device and inode when created. Files are created and deleted through a descriptor of the directory that is checked first, and reopened only if they are still the file sort created. Cleanup deletes only the names sort created. Redox and non-Unix systems keep the path-based code. The directory descriptor is closed after each use, so with merges creating their output before opening their inputs, sort needs no more file descriptors than before.
1 parent 238abf6 commit c76f272

5 files changed

Lines changed: 467 additions & 95 deletions

File tree

‎src/uu/sort/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ ctrlc = { workspace = true }
4444

4545
[target.'cfg(unix)'.dependencies]
4646
libc = { workspace = true }
47+
rustix = { workspace = true, features = ["fs"] }
4748

4849
[dev-dependencies]
4950
divan = { workspace = true }

‎src/uu/sort/src/ext_sort/threaded.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ use std::cmp::Ordering;
1010
use std::fs::File;
1111
use std::io::{Read, Write, stderr};
1212
use std::iter;
13-
use std::path::PathBuf;
1413
use std::sync::mpsc::{Receiver, SyncSender};
1514
use std::thread;
1615

@@ -22,7 +21,7 @@ use crate::chunks::RecycledChunk;
2221
use crate::merge::WriteableCompressedTmpFile;
2322
use crate::merge::WriteablePlainTmpFile;
2423
use crate::merge::WriteableTmpFile;
25-
use crate::tmp_dir::TmpDirWrapper;
24+
use crate::tmp_dir::{TmpDirWrapper, TmpPath};
2625
use crate::{
2726
GlobalSettings, Line,
2827
chunks::{self, Chunk},
@@ -290,7 +289,7 @@ fn read_write_loop<I: WriteableTmpFile>(
290289
/// `compress_prog` is used to optionally compress file contents.
291290
fn write<I: WriteableTmpFile>(
292291
chunk: &Chunk,
293-
file: (File, PathBuf),
292+
file: (File, TmpPath),
294293
compress_prog: Option<&str>,
295294
separator: u8,
296295
) -> UResult<I::Closed> {

‎src/uu/sort/src/merge.rs‎

Lines changed: 94 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,10 @@ use std::{
1616
cmp::Ordering,
1717
collections::BinaryHeap,
1818
ffi::{OsStr, OsString},
19-
fs::{self, File},
19+
fs::File,
2020
io::{self, BufWriter, Read, Write},
2121
iter,
22-
path::{Path, PathBuf},
22+
path::Path,
2323
process::{Child, ChildStdin, ChildStdout, Command, Stdio},
2424
rc::Rc,
2525
sync::mpsc::{Receiver, Sender, SyncSender, TryRecvError, channel, sync_channel},
@@ -32,25 +32,25 @@ use crate::{
3232
GlobalSettings, Output, SortError,
3333
chunks::{self, Chunk, RecycledChunk},
3434
current_open_fd_count, fd_soft_limit, merge_compare, open,
35-
tmp_dir::TmpDirWrapper,
35+
tmp_dir::{TmpDirWrapper, TmpPath},
3636
};
3737

3838
/// If the output file occurs in the input files as well, copy the contents of the output file
39-
/// and replace its occurrences in the inputs with that copy.
39+
/// and replace its occurrences in the inputs with that copy, which is returned.
4040
fn replace_output_file_in_input_files(
4141
files: &mut [OsString],
4242
output: Option<&OsStr>,
4343
tmp_dir: &mut TmpDirWrapper,
44-
) -> UResult<()> {
45-
let mut copy: Option<PathBuf> = None;
44+
) -> UResult<Option<TmpPath>> {
45+
let mut copy: Option<TmpPath> = None;
4646
if let Some(Ok(output_path)) = output.map(|path| Path::new(path).canonicalize()) {
4747
for file in files {
4848
if Path::new(file)
4949
.canonicalize()
5050
.is_ok_and(|file_path| file_path == output_path)
5151
{
5252
if let Some(copy) = &copy {
53-
*file = copy.clone().into_os_string();
53+
*file = copy.path().into();
5454
} else {
5555
// Write through the descriptor `next_file` just opened rather
5656
// than `fs::copy`, which would put the source's permission bits
@@ -60,13 +60,25 @@ fn replace_output_file_in_input_files(
6060
.map_err(|error| SortError::OpenTmpFileFailed { error })?;
6161
io::copy(&mut source, &mut copy_file)
6262
.map_err(|error| SortError::OpenTmpFileFailed { error })?;
63-
*file = copy_path.clone().into_os_string();
63+
*file = copy_path.path().into();
6464
copy = Some(copy_path);
6565
}
6666
}
6767
}
6868
}
69-
Ok(())
69+
Ok(copy)
70+
}
71+
72+
/// Open the merge input `file`. `copy` is the copy of the output file returned by
73+
/// [`replace_output_file_in_input_files`].
74+
fn open_input(file: &OsStr, copy: Option<&TmpPath>) -> UResult<Box<dyn Read + Send>> {
75+
match copy {
76+
Some(copy) if Path::new(file) == copy.path() => Ok(Box::new(
77+
copy.open()
78+
.map_err(|error| SortError::OpenTmpFileFailed { error })?,
79+
)),
80+
_ => open(file),
81+
}
7082
}
7183

7284
/// Determine the effective merge batch size, enforcing a minimum and respecting the
@@ -106,10 +118,10 @@ pub fn merge(
106118
output: Output,
107119
tmp_dir: &mut TmpDirWrapper,
108120
) -> UResult<()> {
109-
replace_output_file_in_input_files(files, output.as_output_name(), tmp_dir)?;
121+
let copy = replace_output_file_in_input_files(files, output.as_output_name(), tmp_dir)?;
110122
let files = files
111123
.iter()
112-
.map(|file| open(file).map(|file| PlainMergeInput { inner: file }));
124+
.map(|file| open_input(file, copy.as_ref()).map(|inner| PlainMergeInput { inner }));
113125
if settings.compress_prog.is_none() {
114126
merge_with_file_limit::<_, _, WriteablePlainTmpFile>(files, settings, output, tmp_dir)
115127
} else {
@@ -463,7 +475,7 @@ fn check_child_success(mut child: Child, program: &str) -> UResult<()> {
463475
pub trait WriteableTmpFile: Sized {
464476
type Closed: ClosedTmpFile;
465477
type InnerWrite: Write;
466-
fn create(file: (File, PathBuf), compress_prog: Option<&str>) -> UResult<Self>;
478+
fn create(file: (File, TmpPath), compress_prog: Option<&str>) -> UResult<Self>;
467479
/// Closes the temporary file.
468480
fn finished_writing(self) -> UResult<Self::Closed>;
469481
fn as_write(&mut self) -> &mut Self::InnerWrite;
@@ -484,21 +496,21 @@ pub trait MergeInput: Send {
484496
}
485497

486498
pub struct WriteablePlainTmpFile {
487-
path: PathBuf,
499+
path: TmpPath,
488500
file: BufWriter<File>,
489501
}
490502
pub struct ClosedPlainTmpFile {
491-
path: PathBuf,
503+
path: TmpPath,
492504
}
493505
pub struct PlainTmpMergeInput {
494-
path: PathBuf,
506+
path: TmpPath,
495507
file: File,
496508
}
497509
impl WriteableTmpFile for WriteablePlainTmpFile {
498510
type Closed = ClosedPlainTmpFile;
499511
type InnerWrite = BufWriter<File>;
500512

501-
fn create((file, path): (File, PathBuf), _: Option<&str>) -> UResult<Self> {
513+
fn create((file, path): (File, TmpPath), _: Option<&str>) -> UResult<Self> {
502514
Ok(Self {
503515
file: BufWriter::new(file),
504516
path,
@@ -517,7 +529,10 @@ impl ClosedTmpFile for ClosedPlainTmpFile {
517529
type Reopened = PlainTmpMergeInput;
518530
fn reopen(self) -> UResult<Self::Reopened> {
519531
Ok(PlainTmpMergeInput {
520-
file: File::open(&self.path).map_err(|error| SortError::OpenTmpFileFailed { error })?,
532+
file: self
533+
.path
534+
.open()
535+
.map_err(|error| SortError::OpenTmpFileFailed { error })?,
521536
path: self.path,
522537
})
523538
}
@@ -529,7 +544,9 @@ impl MergeInput for PlainTmpMergeInput {
529544
// we ignore failures to delete the temporary file,
530545
// because there is a race at the end of the execution and the whole
531546
// temporary directory might already be gone.
532-
let _ = fs::remove_file(self.path);
547+
// Close the file first: removing it briefly takes another descriptor.
548+
drop(self.file);
549+
self.path.remove();
533550
Ok(())
534551
}
535552

@@ -539,17 +556,17 @@ impl MergeInput for PlainTmpMergeInput {
539556
}
540557

541558
pub struct WriteableCompressedTmpFile {
542-
path: PathBuf,
559+
path: TmpPath,
543560
compress_prog: String,
544561
child: Child,
545562
child_stdin: BufWriter<ChildStdin>,
546563
}
547564
pub struct ClosedCompressedTmpFile {
548-
path: PathBuf,
565+
path: TmpPath,
549566
compress_prog: String,
550567
}
551568
pub struct CompressedTmpMergeInput {
552-
path: PathBuf,
569+
path: TmpPath,
553570
compress_prog: String,
554571
child: Child,
555572
child_stdout: ChildStdout,
@@ -558,7 +575,7 @@ impl WriteableTmpFile for WriteableCompressedTmpFile {
558575
type Closed = ClosedCompressedTmpFile;
559576
type InnerWrite = BufWriter<ChildStdin>;
560577

561-
fn create((file, path): (File, PathBuf), compress_prog: Option<&str>) -> UResult<Self> {
578+
fn create((file, path): (File, TmpPath), compress_prog: Option<&str>) -> UResult<Self> {
562579
let compress_prog = compress_prog.unwrap();
563580
let mut command = Command::new(compress_prog);
564581
command.stdin(Stdio::piped()).stdout(file);
@@ -596,8 +613,10 @@ impl ClosedTmpFile for ClosedCompressedTmpFile {
596613
fn reopen(self) -> UResult<Self::Reopened> {
597614
let mut command = Command::new(&self.compress_prog);
598615
// mirroring what is done for ClosedPlainTmpFile
599-
let file =
600-
File::open(&self.path).map_err(|error| SortError::OpenTmpFileFailed { error })?;
616+
let file = self
617+
.path
618+
.open()
619+
.map_err(|error| SortError::OpenTmpFileFailed { error })?;
601620
command.stdin(file).stdout(Stdio::piped()).arg("-d");
602621
let mut child = command
603622
.spawn()
@@ -622,7 +641,7 @@ impl MergeInput for CompressedTmpMergeInput {
622641
#[allow(clippy::drop_non_drop)]
623642
drop(self.child_stdout);
624643
check_child_success(self.child, &self.compress_prog)?;
625-
let _ = fs::remove_file(self.path);
644+
self.path.remove();
626645
Ok(())
627646
}
628647

@@ -647,6 +666,7 @@ impl<R: Read + Send> MergeInput for PlainMergeInput<R> {
647666
#[cfg(all(test, unix))]
648667
mod tests {
649668
use super::*;
669+
use std::fs;
650670
use std::os::unix::fs::PermissionsExt;
651671

652672
/// When the output file is also an input it is copied to a temporary file
@@ -672,4 +692,53 @@ mod tests {
672692
);
673693
assert_eq!(fs::read(copy).unwrap(), b"a\n");
674694
}
695+
696+
/// The copy of the output file is only read while it is the file sort
697+
/// created: once the temporary directory's path leads elsewhere, the file
698+
/// found at its name there is not read.
699+
#[test]
700+
#[cfg(not(target_os = "redox"))]
701+
fn output_copy_is_not_read_through_a_replaced_tmp_dir() {
702+
let dir = tempfile::tempdir().unwrap();
703+
let other = tempfile::tempdir().unwrap();
704+
fs::write(other.path().join("0"), b"other\n").unwrap();
705+
let out = dir.path().join("out");
706+
fs::write(&out, b"a\n").unwrap();
707+
708+
let mut tmp_dir = TmpDirWrapper::new(dir.path().to_owned());
709+
let mut files = vec![out.clone().into_os_string()];
710+
let copy =
711+
replace_output_file_in_input_files(&mut files, Some(out.as_os_str()), &mut tmp_dir)
712+
.unwrap();
713+
let tmp = copy.as_ref().unwrap().path().parent().unwrap();
714+
fs::rename(tmp, dir.path().join("moved")).unwrap();
715+
std::os::unix::fs::symlink(other.path(), tmp).unwrap();
716+
717+
assert!(open_input(&files[0], copy.as_ref()).is_err());
718+
}
719+
720+
/// A temporary file that is being merged when the directory's path is made
721+
/// to lead elsewhere: removing it afterwards leaves the file of the same
722+
/// name there alone.
723+
#[test]
724+
#[cfg(not(target_os = "redox"))]
725+
fn finished_input_is_not_removed_through_a_replaced_tmp_dir() {
726+
let dir = tempfile::tempdir().unwrap();
727+
let other = tempfile::tempdir().unwrap();
728+
fs::write(other.path().join("0"), b"other\n").unwrap();
729+
730+
let mut tmp_dir = TmpDirWrapper::new(dir.path().to_owned());
731+
let input = WriteablePlainTmpFile::create(tmp_dir.next_file().unwrap(), None)
732+
.unwrap()
733+
.finished_writing()
734+
.unwrap()
735+
.reopen()
736+
.unwrap();
737+
let tmp = input.path.path().parent().unwrap().to_owned();
738+
fs::rename(&tmp, dir.path().join("moved")).unwrap();
739+
std::os::unix::fs::symlink(other.path(), &tmp).unwrap();
740+
input.finished_reading().unwrap();
741+
742+
assert_eq!(fs::read(other.path().join("0")).unwrap(), b"other\n");
743+
}
675744
}

0 commit comments

Comments
 (0)