Skip to content

Commit 3edbfd7

Browse files
committed
Depend on httpx2 instead of httpx in the Task SDK
httpx has stalled: fixes that landed in httpx2 (a fork of httpx 0.28.1) have not shipped upstream, and starlette has already migrated, so FastAPI following is a matter of time. Moving the Execution API client now keeps the SDK on a maintained HTTP stack. Objects do not cross the httpx/httpx2 boundary, so the in-process Execution API transports move with the client that consumes them; the rest of airflow-core stays on httpx and migrates separately. related: apache#70522
1 parent 2678bc8 commit 3edbfd7

18 files changed

Lines changed: 568 additions & 486 deletions

File tree

‎airflow-core/docs/security/jwt_token_authentication.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ The token flows through the execution stack as follows:
300300
2. The workload JSON is passed to the worker process (via the executor-specific mechanism:
301301
Celery message, Kubernetes Pod spec, local subprocess arguments, etc.).
302302
3. The worker's ``execute_workload()`` function reads the workload JSON and extracts the token.
303-
4. The ``supervise_task()`` function receives the token and creates an ``httpx.Client`` instance
303+
4. The ``supervise_task()`` function receives the token and creates an ``httpx2.Client`` instance
304304
with ``BearerAuth(token)`` for all Execution API HTTP requests.
305305
5. The worker calls the ``/run`` endpoint with the ``workload``-scoped token to mark the task
306306
as running. The server responds with a fresh ``execution``-scoped token in the

‎airflow-core/pyproject.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,9 @@ dependencies = [
102102
"uvicorn>=0.37.0",
103103
"starlette>=1.0.1",
104104
"httpx>=0.25.0",
105+
# Only for InProcessExecutionAPI's transports, which feed httpx2-based Task SDK clients.
106+
# The rest of airflow-core is still on httpx; tracked at https://github.com/apache/airflow/issues/70522
107+
"httpx2>=2.0.0",
105108
'importlib_metadata>=6.5;python_version<"3.12"',
106109
'importlib_metadata>=7.0;python_version>="3.12"',
107110
"itsdangerous>=2.0",

‎airflow-core/src/airflow/api_fastapi/execution_api/app.py‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
)
4747

4848
if TYPE_CHECKING:
49-
import httpx
49+
import httpx2
5050

5151
import structlog
5252
from structlog.contextvars import bind_contextvars
@@ -378,7 +378,7 @@ class InProcessExecutionAPI:
378378
A helper class to make it possible to run the ExecutionAPI "in-process".
379379
380380
The sync version of this makes use of a2wsgi which runs the async loop in a separate thread. This is
381-
needed so that we can use the sync httpx client
381+
needed so that we can use the sync httpx2 client
382382
"""
383383

384384
_app: FastAPI | None = None
@@ -422,9 +422,11 @@ async def always_allow(request: Request):
422422

423423
return self._app
424424

425+
# httpx2, not httpx: the only consumers are Task SDK clients, which subclass httpx2.Client,
426+
# and transports do not cross the httpx/httpx2 package boundary.
425427
@cached_property
426-
def transport(self) -> httpx.WSGITransport:
427-
import httpx
428+
def transport(self) -> httpx2.WSGITransport:
429+
import httpx2
428430
from a2wsgi import ASGIMiddleware
429431

430432
# We choose to own the event loop + executor thread here so that we can have explicit control over
@@ -445,7 +447,7 @@ async def start_lifespan(cm: AsyncExitStack, app: FastAPI):
445447
# safely aclose() a context whose __aenter__ has actually run.
446448
asyncio.run_coroutine_threadsafe(start_lifespan(cm, self.app), loop).result()
447449

448-
transport = httpx.WSGITransport(app=middleware) # type: ignore[arg-type]
450+
transport = httpx2.WSGITransport(app=middleware) # type: ignore[arg-type]
449451

450452
# Stop the loop + thread and unwind the lifespan when the *transport* is garbage collected, not
451453
# this InProcessExecutionAPI instance. Callers commonly build a Client from ``.transport`` and drop
@@ -457,7 +459,7 @@ async def start_lifespan(cm: AsyncExitStack, app: FastAPI):
457459
return transport
458460

459461
@cached_property
460-
def atransport(self) -> httpx.ASGITransport:
461-
import httpx
462+
def atransport(self) -> httpx2.ASGITransport:
463+
import httpx2
462464

463-
return httpx.ASGITransport(app=self.app)
465+
return httpx2.ASGITransport(app=self.app)

‎airflow-core/tests/unit/api_fastapi/execution_api/test_app.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
from unittest import mock
2323
from uuid import UUID
2424

25-
import httpx
25+
import httpx2
2626
import pytest
2727
from fastapi import Request, status
2828
from fastapi.params import Security as SecurityParam
@@ -158,7 +158,7 @@ def test_routes_with_task_instance_id_param_enforce_ti_self(client):
158158
def test_in_process_execution_api_runs_without_jwt_secret():
159159
"""The in-process API must not require ``api_auth/jwt_secret`` to be configured."""
160160
api = InProcessExecutionAPI()
161-
with httpx.Client(transport=api.transport) as client:
161+
with httpx2.Client(transport=api.transport) as client:
162162
response = client.get("http://localhost/health")
163163
assert response.status_code == 200
164164

‎devel-common/src/tests_common/test_utils/in_process_taskrun.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -57,25 +57,25 @@
5757

5858
def _remembering_handler(store: dict, run_context_json: bytes) -> Callable:
5959
"""A dry-run transport handler: valid run-context + XCom round-trip from ``store``, else no-op."""
60-
import httpx
60+
import httpx2
6161

6262
from airflow.sdk.api.client import noop_handler
6363

64-
def handler(request: httpx.Request) -> httpx.Response:
64+
def handler(request: httpx2.Request) -> httpx2.Response:
6565
path = request.url.path
6666
if path.startswith("/task-instances/") and path.endswith("/run"):
67-
return httpx.Response(200, content=run_context_json)
67+
return httpx2.Response(200, content=run_context_json)
6868
parts = path.strip("/").split("/")
6969
if len(parts) == _XCOM_PATH_PARTS and parts[0] == "xcoms":
7070
dag_id, run_id, task_id, key = parts[1:]
7171
sig = (dag_id, run_id, task_id, key)
7272
if request.method == "POST":
7373
store[sig] = json.loads(request.content)
74-
return httpx.Response(201, json={"ok": True})
74+
return httpx2.Response(201, json={"ok": True})
7575
if request.method == "GET":
7676
if sig in store:
77-
return httpx.Response(200, json={"key": key, "value": store[sig]})
78-
return httpx.Response(404, json={"detail": "XCom not found"})
77+
return httpx2.Response(200, json={"key": key, "value": store[sig]})
78+
return httpx2.Response(404, json={"detail": "XCom not found"})
7979
return noop_handler(request)
8080

8181
return handler
@@ -87,7 +87,7 @@ def build_in_memory_client(ti_context) -> Client:
8787
``ti_context`` (a ``TIRunContext``) is replayed for the task-start request. Pushed XCom
8888
values are exposed as ``client.pushed_xcoms`` keyed by ``(dag_id, run_id, task_id, key)``.
8989
"""
90-
import httpx
90+
import httpx2
9191

9292
from airflow.sdk.api.client import Client
9393

@@ -96,7 +96,7 @@ def build_in_memory_client(ti_context) -> Client:
9696
base_url=None,
9797
dry_run=True,
9898
token="",
99-
transport=httpx.MockTransport(_remembering_handler(store, ti_context.model_dump_json().encode())),
99+
transport=httpx2.MockTransport(_remembering_handler(store, ti_context.model_dump_json().encode())),
100100
)
101101
client.pushed_xcoms = store # type: ignore[attr-defined]
102102
return client

‎scripts/ci/prek/generate_execution_api_schema.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,13 @@
2727
os.environ["_AIRFLOW__AS_LIBRARY"] = "1"
2828
sys.path.insert(0, str(Path("airflow-core/src").resolve()))
2929

30-
import httpx
30+
import httpx2
3131

3232
from airflow.api_fastapi.execution_api.app import InProcessExecutionAPI
3333

3434
app = InProcessExecutionAPI()
3535
version = app.app.versions.version_values[0]
36-
client = httpx.Client(transport=app.transport)
36+
client = httpx2.Client(transport=app.transport)
3737
response = client.get(f"http://localhost/openapi.json?version={version}")
3838
response.raise_for_status()
3939
print(json.dumps(response.json()))

‎shared/logging/src/airflow_shared/logging/structlog.py‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -420,22 +420,16 @@ def structlog_processors(
420420
# structlog to ignore.
421421

422422
import contextlib
423+
import importlib
423424

424425
import click
425426

426427
suppress: tuple[ModuleType, ...] = (click, contextlib)
427-
try:
428-
import httpcore
429-
430-
suppress = (*suppress, httpcore)
431-
except ImportError:
432-
pass
433-
try:
434-
import httpx
435-
436-
suppress = (*suppress, httpx)
437-
except ImportError:
438-
pass
428+
# airflow-core is still on httpx while the Task SDK is on httpx2, so either (or both) may
429+
# be installed; tracked at https://github.com/apache/airflow/issues/70522
430+
for module_name in ("httpcore", "httpcore2", "httpx", "httpx2"):
431+
with contextlib.suppress(ImportError):
432+
suppress = (*suppress, importlib.import_module(module_name))
439433

440434
if json_output:
441435
dict_exc_formatter = structlog.tracebacks.ExceptionDictTransformer(
@@ -690,6 +684,7 @@ def is_atty():
690684
"airflow": {"level": log_level.upper()},
691685
# These ones are too chatty even at info
692686
"httpx": {"level": "WARN"},
687+
"httpx2": {"level": "WARN"},
693688
"sqlalchemy.engine": {"level": "WARN"},
694689
"alembic.runtime.plugins": {"level": "WARN"},
695690
}
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
---
18+
id: no-httpx-import
19+
language: python
20+
severity: error
21+
message: Import 'httpx2' instead of 'httpx' -- see https://github.com/apache/airflow/issues/70522
22+
note: >-
23+
The Task SDK's Execution API client is an 'httpx2.Client'. Requests, responses,
24+
transports and exceptions do not cross the 'httpx'/'httpx2' boundary, so a stray
25+
'httpx' import here silently breaks mocking, retries and error handling.
26+
files:
27+
- "**/*.py"
28+
rule:
29+
all:
30+
- kind: dotted_name
31+
- regex: ^httpx($|\.)
32+
- any:
33+
- inside:
34+
kind: import_statement
35+
stopBy: end
36+
- inside:
37+
kind: import_from_statement
38+
field: module_name

‎task-sdk/.ast-grep/sgconfig.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
---
18+
# NOTE: this file configures `ast-grep`, which is used for enforcing custom lint / auto-formatting
19+
# across a number of different languages. Currently, we use this in a pre-commit hook to avoid
20+
# regressions of reintroducing `httpx` in the Task SDK, which is fully migrated to `httpx2`.
21+
#
22+
# If new ast-grep rules are required, they may be added in the rules directory listed below.
23+
#
24+
# See also:
25+
# https://github.com/ast-grep/ast-grep
26+
# https://ast-grep.github.io/guide/introduction
27+
ruleDirs:
28+
- rules

‎task-sdk/.pre-commit-config.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,12 @@ repos:
6565
pass_filenames: false
6666
files: ^.*\.py$
6767
require_serial: true
68+
- repo: https://github.com/boidolr/ast-grep-pre-commit
69+
rev: 0.45.2
70+
hooks:
71+
- id: ast-grep
72+
name: Check task-sdk imports httpx2 rather than httpx
73+
types: [python]
74+
# Pinned to the task-sdk config rather than relying on ast-grep walking up
75+
# from the working directory, so a future root sgconfig.yml cannot shadow it.
76+
args: ["--config", "./.ast-grep/sgconfig.yml"]

0 commit comments

Comments
 (0)