Skip to content

Commit cc4519f

Browse files
committed
Depend on httpx2 instead of httpx in the Task SDK
httpx has stalled: fixes that landed in httpx2 (a fork of httpx 0.28.1) have not shipped upstream, and starlette has already migrated, so FastAPI following is a matter of time. Moving the Execution API client now keeps the SDK on a maintained HTTP stack. Objects do not cross the httpx/httpx2 boundary, so the in-process Execution API transports move with the client that consumes them; the rest of airflow-core stays on httpx and migrates separately. related: apache#70522
1 parent f6f267d commit cc4519f

18 files changed

Lines changed: 604 additions & 522 deletions

File tree

‎airflow-core/docs/security/jwt_token_authentication.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ The token flows through the execution stack as follows:
300300
2. The workload JSON is passed to the worker process (via the executor-specific mechanism:
301301
Celery message, Kubernetes Pod spec, local subprocess arguments, etc.).
302302
3. The worker's ``execute_workload()`` function reads the workload JSON and extracts the token.
303-
4. The ``supervise_task()`` function receives the token and creates an ``httpx.Client`` instance
303+
4. The ``supervise_task()`` function receives the token and creates an ``httpx2.Client`` instance
304304
with ``BearerAuth(token)`` for all Execution API HTTP requests.
305305
5. The worker calls the ``/run`` endpoint with the ``workload``-scoped token to mark the task
306306
as running. The server responds with a fresh ``execution``-scoped token in the

‎airflow-core/pyproject.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,9 @@ dependencies = [
103103
# parsed with no field-count or field-size bound at all.
104104
"starlette>=1.3.1",
105105
"httpx>=0.25.0",
106+
# Only for InProcessExecutionAPI's transports, which feed httpx2-based Task SDK clients.
107+
# The rest of airflow-core is still on httpx; tracked at https://github.com/apache/airflow/issues/70522
108+
"httpx2>=2.0.0",
106109
'importlib_metadata>=6.5;python_version<"3.12"',
107110
'importlib_metadata>=7.0;python_version>="3.12"',
108111
"itsdangerous>=2.0",

‎airflow-core/src/airflow/api_fastapi/execution_api/in_process.py‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
from starlette.requests import Request
3131

3232
if TYPE_CHECKING:
33-
import httpx
33+
import httpx2
3434
from fastapi import FastAPI
3535

3636
logger = structlog.get_logger(logger_name=__name__)
@@ -86,7 +86,7 @@ class InProcessExecutionAPI:
8686
A helper class to make it possible to run the ExecutionAPI "in-process".
8787
8888
The sync version of this makes use of a2wsgi which runs the async loop in a separate thread. This is
89-
needed so that we can use the sync httpx client
89+
needed so that we can use the sync httpx2 client
9090
"""
9191

9292
_app: FastAPI | None = None
@@ -109,9 +109,11 @@ def app(self):
109109

110110
return self._app
111111

112+
# httpx2, not httpx: the only consumers are Task SDK clients, which subclass httpx2.Client,
113+
# and transports do not cross the httpx/httpx2 package boundary.
112114
@cached_property
113-
def transport(self) -> httpx.WSGITransport:
114-
import httpx
115+
def transport(self) -> httpx2.WSGITransport:
116+
import httpx2
115117
from a2wsgi import ASGIMiddleware
116118

117119
from airflow import settings
@@ -133,7 +135,7 @@ async def start_lifespan(cm: AsyncExitStack, app: FastAPI):
133135
# Wait for startup so callers see a ready app and the finalizer can close an entered context.
134136
asyncio.run_coroutine_threadsafe(start_lifespan(cm, self.app), loop).result()
135137

136-
transport = httpx.WSGITransport(app=middleware) # type: ignore[arg-type]
138+
transport = httpx2.WSGITransport(app=middleware) # type: ignore[arg-type]
137139

138140
# Callers can retain the transport after dropping this instance; finalizing on self would
139141
# stop the loop while requests still need it.
@@ -142,7 +144,7 @@ async def start_lifespan(cm: AsyncExitStack, app: FastAPI):
142144
return transport
143145

144146
@cached_property
145-
def atransport(self) -> httpx.ASGITransport:
146-
import httpx
147+
def atransport(self) -> httpx2.ASGITransport:
148+
import httpx2
147149

148-
return httpx.ASGITransport(app=self.app)
150+
return httpx2.ASGITransport(app=self.app)

‎airflow-core/tests/unit/api_fastapi/execution_api/test_app.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
from unittest import mock
2424
from uuid import UUID
2525

26-
import httpx
26+
import httpx2
2727
import pytest
2828
from fastapi import FastAPI, Request, status
2929
from fastapi.params import Security as SecurityParam
@@ -81,7 +81,7 @@ def test_server_and_in_process_apps_isolate_auth(in_process_first, monkeypatch):
8181

8282
with (
8383
TestClient(server_app) as server,
84-
httpx.Client(transport=api.transport, base_url="http://in-process.invalid") as local,
84+
httpx2.Client(transport=api.transport, base_url="http://in-process.invalid") as local,
8585
):
8686
response = local.get("/variables/execution_api_hosting")
8787
assert response.status_code == 200
@@ -208,7 +208,7 @@ def test_routes_with_task_instance_id_param_enforce_ti_self(client):
208208
def test_in_process_execution_api_runs_without_jwt_secret():
209209
"""The in-process API must not require ``api_auth/jwt_secret`` to be configured."""
210210
api = InProcessExecutionAPI()
211-
with httpx.Client(transport=api.transport) as client:
211+
with httpx2.Client(transport=api.transport) as client:
212212
response = client.get("http://localhost/health")
213213
assert response.status_code == 200
214214

‎devel-common/src/tests_common/test_utils/in_process_taskrun.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -57,25 +57,25 @@
5757

5858
def _remembering_handler(store: dict, run_context_json: bytes) -> Callable:
5959
"""A dry-run transport handler: valid run-context + XCom round-trip from ``store``, else no-op."""
60-
import httpx
60+
import httpx2
6161

6262
from airflow.sdk.api.client import noop_handler
6363

64-
def handler(request: httpx.Request) -> httpx.Response:
64+
def handler(request: httpx2.Request) -> httpx2.Response:
6565
path = request.url.path
6666
if path.startswith("/task-instances/") and path.endswith("/run"):
67-
return httpx.Response(200, content=run_context_json)
67+
return httpx2.Response(200, content=run_context_json)
6868
parts = path.strip("/").split("/")
6969
if len(parts) == _XCOM_PATH_PARTS and parts[0] == "xcoms":
7070
dag_id, run_id, task_id, key = parts[1:]
7171
sig = (dag_id, run_id, task_id, key)
7272
if request.method == "POST":
7373
store[sig] = json.loads(request.content)
74-
return httpx.Response(201, json={"ok": True})
74+
return httpx2.Response(201, json={"ok": True})
7575
if request.method == "GET":
7676
if sig in store:
77-
return httpx.Response(200, json={"key": key, "value": store[sig]})
78-
return httpx.Response(404, json={"detail": "XCom not found"})
77+
return httpx2.Response(200, json={"key": key, "value": store[sig]})
78+
return httpx2.Response(404, json={"detail": "XCom not found"})
7979
return noop_handler(request)
8080

8181
return handler
@@ -87,7 +87,7 @@ def build_in_memory_client(ti_context) -> Client:
8787
``ti_context`` (a ``TIRunContext``) is replayed for the task-start request. Pushed XCom
8888
values are exposed as ``client.pushed_xcoms`` keyed by ``(dag_id, run_id, task_id, key)``.
8989
"""
90-
import httpx
90+
import httpx2
9191

9292
from airflow.sdk.api.client import Client
9393

@@ -96,7 +96,7 @@ def build_in_memory_client(ti_context) -> Client:
9696
base_url=None,
9797
dry_run=True,
9898
token="",
99-
transport=httpx.MockTransport(_remembering_handler(store, ti_context.model_dump_json().encode())),
99+
transport=httpx2.MockTransport(_remembering_handler(store, ti_context.model_dump_json().encode())),
100100
)
101101
client.pushed_xcoms = store # type: ignore[attr-defined]
102102
return client

‎scripts/ci/prek/generate_execution_api_schema.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@
2727
os.environ["_AIRFLOW__AS_LIBRARY"] = "1"
2828
sys.path.insert(0, str(Path("airflow-core/src").resolve()))
2929

30-
import httpx
30+
import httpx2
3131

3232
# Keep this import compatible with the target branch: the version check runs this
3333
# generator against both checkouts.
3434
from airflow.api_fastapi.execution_api.app import InProcessExecutionAPI
3535

3636
app = InProcessExecutionAPI()
3737
version = app.app.versions.version_values[0]
38-
client = httpx.Client(transport=app.transport)
38+
client = httpx2.Client(transport=app.transport)
3939
response = client.get(f"http://localhost/openapi.json?version={version}")
4040
response.raise_for_status()
4141
print(json.dumps(response.json()))

‎shared/logging/src/airflow_shared/logging/structlog.py‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -420,22 +420,16 @@ def structlog_processors(
420420
# structlog to ignore.
421421

422422
import contextlib
423+
import importlib
423424

424425
import click
425426

426427
suppress: tuple[ModuleType, ...] = (click, contextlib)
427-
try:
428-
import httpcore
429-
430-
suppress = (*suppress, httpcore)
431-
except ImportError:
432-
pass
433-
try:
434-
import httpx
435-
436-
suppress = (*suppress, httpx)
437-
except ImportError:
438-
pass
428+
# airflow-core is still on httpx while the Task SDK is on httpx2, so either (or both) may
429+
# be installed; tracked at https://github.com/apache/airflow/issues/70522
430+
for module_name in ("httpcore", "httpcore2", "httpx", "httpx2"):
431+
with contextlib.suppress(ImportError):
432+
suppress = (*suppress, importlib.import_module(module_name))
439433

440434
if json_output:
441435
dict_exc_formatter = structlog.tracebacks.ExceptionDictTransformer(
@@ -690,6 +684,7 @@ def is_atty():
690684
"airflow": {"level": log_level.upper()},
691685
# These ones are too chatty even at info
692686
"httpx": {"level": "WARN"},
687+
"httpx2": {"level": "WARN"},
693688
"sqlalchemy.engine": {"level": "WARN"},
694689
"alembic.runtime.plugins": {"level": "WARN"},
695690
}
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
---
18+
id: no-httpx-import
19+
language: python
20+
severity: error
21+
message: Import 'httpx2' instead of 'httpx' -- see https://github.com/apache/airflow/issues/70522
22+
note: >-
23+
The Task SDK's Execution API client is an 'httpx2.Client'. Requests, responses,
24+
transports and exceptions do not cross the 'httpx'/'httpx2' boundary, so a stray
25+
'httpx' import here silently breaks mocking, retries and error handling.
26+
files:
27+
- "**/*.py"
28+
rule:
29+
all:
30+
- kind: dotted_name
31+
- regex: ^httpx($|\.)
32+
- any:
33+
- inside:
34+
kind: import_statement
35+
stopBy: end
36+
- inside:
37+
kind: import_from_statement
38+
field: module_name

‎task-sdk/.ast-grep/sgconfig.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
---
18+
# NOTE: this file configures `ast-grep`, which is used for enforcing custom lint / auto-formatting
19+
# across a number of different languages. Currently, we use this in a pre-commit hook to avoid
20+
# regressions of reintroducing `httpx` in the Task SDK, which is fully migrated to `httpx2`.
21+
#
22+
# If new ast-grep rules are required, they may be added in the rules directory listed below.
23+
#
24+
# See also:
25+
# https://github.com/ast-grep/ast-grep
26+
# https://ast-grep.github.io/guide/introduction
27+
ruleDirs:
28+
- rules

‎task-sdk/.pre-commit-config.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,3 +65,12 @@ repos:
6565
pass_filenames: false
6666
files: ^.*\.py$
6767
require_serial: true
68+
- repo: https://github.com/boidolr/ast-grep-pre-commit
69+
rev: 0.45.2
70+
hooks:
71+
- id: ast-grep
72+
name: Check task-sdk imports httpx2 rather than httpx
73+
types: [python]
74+
# Pinned to the task-sdk config rather than relying on ast-grep walking up
75+
# from the working directory, so a future root sgconfig.yml cannot shadow it.
76+
args: ["--config", "./.ast-grep/sgconfig.yml"]

0 commit comments

Comments
 (0)