|
28 | 28 | from typing import Annotated, Any, TypeVar |
29 | 29 | from urllib.parse import urlsplit |
30 | 30 |
|
| 31 | +import httpx2 |
| 32 | + |
31 | 33 | from huggingface_hub import HfApi, JobHardware, JobInfo, JobNetwork, JobStage, Volume, constants |
32 | 34 | from huggingface_hub._jobs_api import ( |
33 | 35 | DEFAULT_UV_IMAGE, |
@@ -662,8 +664,27 @@ def _format_job_value(key: str, value: Any, from_script: Collection[str]) -> str |
662 | 664 | jobs_cli = typer_factory(help="Run and manage Jobs on the Hub.") |
663 | 665 |
|
664 | 666 |
|
| 667 | +def _wait_while_scheduling(api: HfApi, job: JobInfo) -> None: |
| 668 | + """Wait while the Job is scheduling, showing why (e.g. waiting for hardware) and for how long.""" |
| 669 | + status = out.status() |
| 670 | + start = time.monotonic() |
| 671 | + try: |
| 672 | + while job.status.stage == JobStage.SCHEDULING: |
| 673 | + for _ in range(2): # refresh the timer every second, poll the Job every 2s |
| 674 | + status.update( |
| 675 | + f"{job.status.message or 'Scheduling'} ({format_duration(int(time.monotonic() - start))})" |
| 676 | + ) |
| 677 | + time.sleep(1) |
| 678 | + job = api.inspect_job(job_id=job.id, namespace=job.owner.name) |
| 679 | + except httpx2.HTTPError: |
| 680 | + pass # best effort: if a poll fails, start streaming the logs right away |
| 681 | + finally: |
| 682 | + status.update("") |
| 683 | + |
| 684 | + |
665 | 685 | def _stream_logs_and_check_status(api: HfApi, job: JobInfo) -> None: |
666 | 686 | """Stream Job logs until the Job ends, then fail the command if the Job did not complete successfully.""" |
| 687 | + _wait_while_scheduling(api, job) |
667 | 688 | for log in api.fetch_job_logs(job_id=job.id, namespace=job.owner.name, follow=True): |
668 | 689 | out.text(log) |
669 | 690 | # The log stream can end while the Job is still scheduling or shutting down: settle the final state. |
|
0 commit comments