Skip to content

Commit c629fa3

Browse files
[Jobs] Fix hf jobs run losing logs and ignoring Ctrl+C (#5162)
* [Jobs] Fix hf jobs run losing logs and ignoring Ctrl+C * fix duplicated lines when following logs with tail * fix extra reconnect when the log stream ends with the job * fix tail resume dropping logs when the last tailed line is missing * fix 2s delay when following a finished job
1 parent 7858680 commit c629fa3

2 files changed

Lines changed: 47 additions & 14 deletions

File tree

‎src/huggingface_hub/cli/jobs.py‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -684,11 +684,19 @@ def _wait_while_scheduling(api: HfApi, job: JobInfo) -> None:
684684

685685
def _stream_logs_and_check_status(api: HfApi, job: JobInfo) -> None:
686686
"""Stream Job logs until the Job ends, then fail the command if the Job did not complete successfully."""
687-
_wait_while_scheduling(api, job)
688-
for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True):
689-
out.text(log)
690-
# The log stream can end while the Job is still scheduling or shutting down: settle the final state.
691-
final = api.wait_for_job(job_id=job.id, namespace=job.owner.name)
687+
try:
688+
_wait_while_scheduling(api, job)
689+
for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True):
690+
out.text(log)
691+
# The stream can still end before the Job does (e.g. on a malformed response): settle the final state.
692+
final = api.wait_for_job(job_id=job.id, namespace=job.owner.name)
693+
except KeyboardInterrupt:
694+
job_ref = f"{job.owner.name}/{job.id}"
695+
out.hint(
696+
f"Job {job_ref} was not canceled. Use `hf jobs logs -f {job_ref}` to follow it again,"
697+
f" or `hf jobs cancel {job_ref}` to cancel it."
698+
)
699+
raise
692700
if final.status.stage != JobStage.COMPLETED:
693701
message = f": {final.status.message}" if final.status.message else ""
694702
raise CLIError(f"Job {final.id} finished with stage '{final.status.stage}'{message}")

‎src/huggingface_hub/hf_api.py‎

Lines changed: 34 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8599,14 +8599,17 @@ def _stream_sse_events(
85998599
# Used by Spaces logs and Jobs logs/metrics. Two retry styles:
86008600
# - on_iteration_end is None: retries are the only backstop (Spaces).
86018601
# - on_iteration_end is set: it polls authoritative state after every
8602-
# failed iteration; ReadTimeouts/tolerated errors fall through to it
8603-
# instead of consuming retries (Jobs).
8602+
# iteration, including clean stream ends; ReadTimeouts/tolerated errors
8603+
# fall through to it instead of consuming retries (Jobs).
86048604
nb_tries = 0
86058605
max_retries = 5 if follow else 0
86068606
min_wait_time = 1
86078607
max_wait_time = 10
86088608
sleep_time = 0
86098609
start_event_idx = 0
8610+
last_event: dict[str, Any] | None = None
8611+
# Last event of a `tail` request, whose index in the full log is unknown
8612+
resume_after: dict[str, Any] | None = None
86108613
error_to_retry: Exception | None = None
86118614
while True:
86128615
if error_to_retry is not None:
@@ -8631,6 +8634,15 @@ def _stream_sse_events(
86318634
for line in response.iter_lines():
86328635
if line and line.startswith("data: {"):
86338636
event_idx += 1
8637+
if resume_after is not None:
8638+
event = json.loads(line[len("data: ") :])
8639+
if event == resume_after:
8640+
resume_after, start_event_idx = None, event_idx + 1
8641+
continue
8642+
if parse_datetime(event["timestamp"]) <= parse_datetime(resume_after["timestamp"]):
8643+
continue
8644+
# The last `tail` event is missing from the replay: resume at the first later one
8645+
resume_after, start_event_idx = None, event_idx
86348646
if event_idx >= start_event_idx:
86358647
if skip_previous_events_on_retry:
86368648
start_event_idx += 1
@@ -8641,8 +8653,13 @@ def _stream_sse_events(
86418653
and parse_datetime(ts) > replay_end
86428654
):
86438655
break
8656+
# New data: retries only count consecutive failures
8657+
nb_tries = sleep_time = 0
8658+
last_event = event
86448659
yield event
8645-
break
8660+
# Jobs: the stream can end before the Job does, so the stage check below decides
8661+
if on_iteration_end is None:
8662+
break
86468663
elif response.status_code not in tolerated_status_codes:
86478664
hf_raise_for_status(response)
86488665
except HfHubHTTPError:
@@ -8651,8 +8668,6 @@ def _stream_sse_events(
86518668
except httpx2.DecodingError:
86528669
# Response ended prematurely.
86538670
break
8654-
except KeyboardInterrupt:
8655-
break
86568671
except (httpx2.HTTPError, httpcore2.TimeoutException) as err:
86578672
is_no_new_line_timeout = isinstance(err, (httpx2.ReadTimeout, httpcore2.ReadTimeout))
86588673
if is_no_new_line_timeout and not follow:
@@ -8680,12 +8695,22 @@ def _stream_sse_events(
86808695
nb_tries += 1
86818696
sleep_time = min(max_wait_time, max(min_wait_time, sleep_time * 2))
86828697
error_to_retry = err
8683-
# Drop params after the first attempt: the start_event_idx dedup
8684-
# requires a stable replay prefix, which `tail` would break.
8698+
# Drop params after the first attempt: the full replay resumes after the last `tail` event
8699+
if params is not None:
8700+
resume_after = last_event
86858701
params = None
86868702
# no-follow: a single attempt, never loop back
8687-
if not follow or (on_iteration_end is not None and on_iteration_end()):
8703+
if not follow:
86888704
break
8705+
if on_iteration_end is not None:
8706+
if on_iteration_end():
8707+
break
8708+
if error_to_retry is None:
8709+
# Stream ended or went silent: back off, the Job's stage lags ~1.3s behind its last log line
8710+
sleep_time = min(max_wait_time, max(2 * min_wait_time, sleep_time * 2))
8711+
time.sleep(sleep_time)
8712+
if on_iteration_end():
8713+
break
86898714

86908715
def _fetch_space_logs_sse(
86918716
self,
@@ -12243,7 +12268,7 @@ def has_job_finished() -> bool:
1224312268
)
1224412269
hf_raise_for_status(job_status_response)
1224512270
job_status = job_status_response.json()
12246-
return "status" in job_status and job_status["status"]["stage"] not in ("RUNNING", "UPDATING")
12271+
return "status" in job_status and job_status["status"]["stage"] in TERMINAL_JOB_STAGES
1224712272

1224812273
yield from self._stream_sse_events(
1224912274
url=f"{self.endpoint}/api/jobs/{namespace}/{job_id}/{route}",

0 commit comments

Comments
 (0)