Skip to content

Commit e8423e5

Browse files
committed
Fix custom SSH remote job cleanup paths (apache#69813)
1 parent c49e1c9 commit e8423e5

3 files changed

Lines changed: 34 additions & 9 deletions

File tree

‎providers/ssh/src/airflow/providers/ssh/operators/ssh_remote_job.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -451,10 +451,11 @@ def _cleanup_remote_job(self, job_dir: str, remote_os: str) -> None:
451451
leave the directory rather than failing the (already finished) task.
452452
"""
453453
self.log.info("Cleaning up remote job directory: %s", job_dir)
454+
expected_base = self._paths.base_dir if self._paths else self.remote_base_dir
454455
if remote_os == "posix":
455-
cleanup_cmd = build_posix_cleanup_command(job_dir)
456+
cleanup_cmd = build_posix_cleanup_command(job_dir, expected_base=expected_base)
456457
else:
457-
cleanup_cmd = build_windows_cleanup_command(job_dir)
458+
cleanup_cmd = build_windows_cleanup_command(job_dir, expected_base=expected_base)
458459

459460
last_error: Exception | None = None
460461
for attempt in range(1, self.cleanup_retries + 1):

‎providers/ssh/src/airflow/providers/ssh/utils/remote_job.py‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,18 +30,25 @@
3030
WINDOWS_DEFAULT_BASE_DIR = "$env:TEMP\\airflow-ssh-jobs"
3131

3232

33-
def _validate_job_dir(job_dir: str, remote_os: Literal["posix", "windows"]) -> None:
33+
def _validate_job_dir(
34+
job_dir: str,
35+
remote_os: Literal["posix", "windows"],
36+
expected_base: str | None = None,
37+
) -> None:
3438
"""
3539
Validate that job_dir is under the expected base directory.
3640
3741
:param job_dir: The job directory path to validate
3842
:param remote_os: Operating system type
43+
:param expected_base: Expected base directory for the job
3944
:raises ValueError: If job_dir doesn't start with the expected base path
4045
"""
4146
if remote_os == "posix":
42-
expected_prefix = POSIX_DEFAULT_BASE_DIR + "/"
47+
expected_base = expected_base or POSIX_DEFAULT_BASE_DIR
48+
expected_prefix = expected_base + "/"
4349
else:
44-
expected_prefix = WINDOWS_DEFAULT_BASE_DIR + "\\"
50+
expected_base = expected_base or WINDOWS_DEFAULT_BASE_DIR
51+
expected_prefix = expected_base + "\\"
4552

4653
if not job_dir.startswith(expected_prefix):
4754
raise ValueError(
@@ -452,27 +459,29 @@ def build_windows_kill_command(pid_file: str) -> str:
452459
return f"powershell.exe -NoProfile -NonInteractive -EncodedCommand {encoded_script}"
453460

454461

455-
def build_posix_cleanup_command(job_dir: str) -> str:
462+
def build_posix_cleanup_command(job_dir: str, expected_base: str | None = None) -> str:
456463
"""
457464
Build a POSIX command to clean up the job directory.
458465
459466
:param job_dir: Path to the job directory
467+
:param expected_base: Expected base directory for the job
460468
:return: Shell command to remove the directory
461469
:raises ValueError: If job_dir is not under the expected base directory
462470
"""
463-
_validate_job_dir(job_dir, "posix")
471+
_validate_job_dir(job_dir, "posix", expected_base)
464472
return f"rm -rf '{job_dir}'"
465473

466474

467-
def build_windows_cleanup_command(job_dir: str) -> str:
475+
def build_windows_cleanup_command(job_dir: str, expected_base: str | None = None) -> str:
468476
"""
469477
Build a PowerShell command to clean up the job directory.
470478
471479
:param job_dir: Path to the job directory
480+
:param expected_base: Expected base directory for the job
472481
:return: PowerShell command to remove the directory
473482
:raises ValueError: If job_dir is not under the expected base directory
474483
"""
475-
_validate_job_dir(job_dir, "windows")
484+
_validate_job_dir(job_dir, "windows", expected_base)
476485
escaped_path = job_dir.replace("'", "''")
477486
script = f"Remove-Item -Recurse -Force -Path '{escaped_path}' -ErrorAction SilentlyContinue"
478487
script_bytes = script.encode("utf-16-le")

‎providers/ssh/tests/unit/ssh/utils/test_remote_job.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,14 @@ def test_posix_cleanup(self):
394394
assert "rm -rf" in cmd
395395
assert "/tmp/airflow-ssh-jobs/job_123" in cmd
396396

397+
def test_posix_cleanup_with_custom_base(self):
398+
"""Test POSIX cleanup under a custom base directory."""
399+
cmd = build_posix_cleanup_command(
400+
"/tmp-data/airflow-ssh-jobs/job_123", expected_base="/tmp-data/airflow-ssh-jobs"
401+
)
402+
assert "rm -rf" in cmd
403+
assert "/tmp-data/airflow-ssh-jobs/job_123" in cmd
404+
397405
def test_windows_cleanup(self):
398406
"""Test Windows cleanup command."""
399407
cmd = build_windows_cleanup_command("$env:TEMP\\airflow-ssh-jobs\\job_123")
@@ -409,6 +417,13 @@ def test_posix_cleanup_rejects_invalid_path(self):
409417
with pytest.raises(ValueError, match="Invalid job directory"):
410418
build_posix_cleanup_command("/tmp/other_dir")
411419

420+
def test_posix_cleanup_rejects_path_outside_custom_base(self):
421+
"""Test POSIX cleanup rejects paths outside a custom base directory."""
422+
with pytest.raises(ValueError, match="Invalid job directory"):
423+
build_posix_cleanup_command(
424+
"/tmp/airflow-ssh-jobs/job_123", expected_base="/tmp-data/airflow-ssh-jobs"
425+
)
426+
412427
def test_windows_cleanup_rejects_invalid_path(self):
413428
"""Test Windows cleanup rejects paths outside expected base directory."""
414429
with pytest.raises(ValueError, match="Invalid job directory"):

0 commit comments

Comments
 (0)