@@ -8599,14 +8599,17 @@ def _stream_sse_events(
85998599 # Used by Spaces logs and Jobs logs/metrics. Two retry styles:
86008600 # - on_iteration_end is None: retries are the only backstop (Spaces).
86018601 # - on_iteration_end is set: it polls authoritative state after every
8602- # failed iteration; ReadTimeouts/tolerated errors fall through to it
8603- # instead of consuming retries (Jobs).
8602+ # iteration, including clean stream ends ; ReadTimeouts/tolerated errors
8603+ # fall through to it instead of consuming retries (Jobs).
86048604 nb_tries = 0
86058605 max_retries = 5 if follow else 0
86068606 min_wait_time = 1
86078607 max_wait_time = 10
86088608 sleep_time = 0
86098609 start_event_idx = 0
8610+ last_event: dict[str, Any] | None = None
8611+ # Last event of a `tail` request, whose index in the full log is unknown
8612+ resume_after: dict[str, Any] | None = None
86108613 error_to_retry: Exception | None = None
86118614 while True:
86128615 if error_to_retry is not None:
@@ -8631,6 +8634,15 @@ def _stream_sse_events(
86318634 for line in response.iter_lines():
86328635 if line and line.startswith("data: {"):
86338636 event_idx += 1
8637+ if resume_after is not None:
8638+ event = json.loads(line[len("data: ") :])
8639+ if event == resume_after:
8640+ resume_after, start_event_idx = None, event_idx + 1
8641+ continue
8642+ if parse_datetime(event["timestamp"]) <= parse_datetime(resume_after["timestamp"]):
8643+ continue
8644+ # The last `tail` event is missing from the replay: resume at the first later one
8645+ resume_after, start_event_idx = None, event_idx
86348646 if event_idx >= start_event_idx:
86358647 if skip_previous_events_on_retry:
86368648 start_event_idx += 1
@@ -8641,8 +8653,13 @@ def _stream_sse_events(
86418653 and parse_datetime(ts) > replay_end
86428654 ):
86438655 break
8656+ # New data: retries only count consecutive failures
8657+ nb_tries = sleep_time = 0
8658+ last_event = event
86448659 yield event
8645- break
8660+ # Jobs: the stream can end before the Job does, so the stage check below decides
8661+ if on_iteration_end is None:
8662+ break
86468663 elif response.status_code not in tolerated_status_codes:
86478664 hf_raise_for_status(response)
86488665 except HfHubHTTPError:
@@ -8651,8 +8668,6 @@ def _stream_sse_events(
86518668 except httpx2.DecodingError:
86528669 # Response ended prematurely.
86538670 break
8654- except KeyboardInterrupt:
8655- break
86568671 except (httpx2.HTTPError, httpcore2.TimeoutException) as err:
86578672 is_no_new_line_timeout = isinstance(err, (httpx2.ReadTimeout, httpcore2.ReadTimeout))
86588673 if is_no_new_line_timeout and not follow:
@@ -8680,12 +8695,22 @@ def _stream_sse_events(
86808695 nb_tries += 1
86818696 sleep_time = min(max_wait_time, max(min_wait_time, sleep_time * 2))
86828697 error_to_retry = err
8683- # Drop params after the first attempt: the start_event_idx dedup
8684- # requires a stable replay prefix, which `tail` would break.
8698+ # Drop params after the first attempt: the full replay resumes after the last `tail` event
8699+ if params is not None:
8700+ resume_after = last_event
86858701 params = None
86868702 # no-follow: a single attempt, never loop back
8687- if not follow or (on_iteration_end is not None and on_iteration_end()) :
8703+ if not follow:
86888704 break
8705+ if on_iteration_end is not None:
8706+ if on_iteration_end():
8707+ break
8708+ if error_to_retry is None:
8709+ # Stream ended or went silent: back off, the Job's stage lags ~1.3s behind its last log line
8710+ sleep_time = min(max_wait_time, max(2 * min_wait_time, sleep_time * 2))
8711+ time.sleep(sleep_time)
8712+ if on_iteration_end():
8713+ break
86898714
86908715 def _fetch_space_logs_sse(
86918716 self,
@@ -12243,7 +12268,7 @@ def has_job_finished() -> bool:
1224312268 )
1224412269 hf_raise_for_status(job_status_response)
1224512270 job_status = job_status_response.json()
12246- return "status" in job_status and job_status["status"]["stage"] not in ("RUNNING", "UPDATING")
12271+ return "status" in job_status and job_status["status"]["stage"] in TERMINAL_JOB_STAGES
1224712272
1224812273 yield from self._stream_sse_events(
1224912274 url=f"{self.endpoint}/api/jobs/{namespace}/{job_id}/{route}",
0 commit comments