Skip to content

Commit 4dadf0f

Browse files
committed
Switch to direct httpx2 imports and keep both httpx + httpx2 deps
Per review feedback by @Prab-27 on apache#71130: - Replace 'import httpx2 as httpx' with direct 'import httpx2' for better type identity and logger compatibility - Keep httpx as a dependency alongside httpx2 to avoid breaking transitive resolution for other packages Closes apache#70522
1 parent 61ae220 commit 4dadf0f

7 files changed

Lines changed: 442 additions & 441 deletions

File tree

‎task-sdk/dev/generate_task_sdk_models.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import sys
2222
from pathlib import Path
2323

24-
import httpx2 as httpx
24+
import httpx2
2525
from datamodel_code_generator import (
2626
DataModelType,
2727
DatetimeClassType,
@@ -81,7 +81,7 @@ def generate_file():
8181
app = InProcessExecutionAPI()
8282

8383
latest_version = app.app.versions.version_values[0]
84-
client = httpx.Client(transport=app.transport)
84+
client = httpx2.Client(transport=app.transport)
8585
openapi_schema = (
8686
client.get(f"http://localhost/openapi.json?version={latest_version}").raise_for_status().text
8787
)

‎task-sdk/pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ dependencies = [
5454
"attrs>=24.2.0, !=25.2.0",
5555
"babel>=2.17.0",
5656
"fsspec>=2023.10.0",
57+
"httpx>=0.27.0",
5758
"httpx2>=0.28.1",
5859
"jinja2>=3.1.5",
5960
"methodtools>=0.4.7",

‎task-sdk/src/airflow/sdk/api/client.py‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
from urllib.parse import quote
2929

3030
import certifi
31-
import httpx2 as httpx
31+
import httpx2
3232
import msgspec
3333
import structlog
3434
from opentelemetry import trace
@@ -190,22 +190,22 @@ def getuser() -> str:
190190
]
191191

192192

193-
def get_json_error(response: httpx.Response):
193+
def get_json_error(response: httpx2.Response):
194194
"""Raise a ServerResponseError if we can extract error info from the error."""
195195
err = ServerResponseError.from_response(response)
196196
if err:
197197
raise err
198198

199199

200-
def raise_on_4xx_5xx(response: httpx.Response):
200+
def raise_on_4xx_5xx(response: httpx2.Response):
201201
return get_json_error(response) or response.raise_for_status()
202202

203203

204204
# Py 3.11+ version
205-
def raise_on_4xx_5xx_with_note(response: httpx.Response):
205+
def raise_on_4xx_5xx_with_note(response: httpx2.Response):
206206
try:
207207
return get_json_error(response) or response.raise_for_status()
208-
except httpx.HTTPStatusError as e:
208+
except httpx2.HTTPStatusError as e:
209209
if TYPE_CHECKING:
210210
assert hasattr(e, "add_note")
211211
e.add_note(
@@ -225,11 +225,11 @@ def raise_on_4xx_5xx_with_note(response: httpx.Response):
225225
raise_on_4xx_5xx = raise_on_4xx_5xx_with_note
226226

227227

228-
def add_correlation_id(request: httpx.Request):
228+
def add_correlation_id(request: httpx2.Request):
229229
request.headers["correlation-id"] = str(uuid7())
230230

231231

232-
def inject_trace_context(request: httpx.Request) -> None:
232+
def inject_trace_context(request: httpx2.Request) -> None:
233233
_trace_propagator.inject(request.headers)
234234

235235

@@ -1108,25 +1108,25 @@ def run(self, callback_id: uuid.UUID) -> None:
11081108
self.client.patch(f"callbacks/{callback_id}/run")
11091109

11101110

1111-
class BearerAuth(httpx.Auth):
1111+
class BearerAuth(httpx2.Auth):
11121112
def __init__(self, token: str):
11131113
self.token: str = token
11141114

1115-
def auth_flow(self, request: httpx.Request):
1115+
def auth_flow(self, request: httpx2.Request):
11161116
if self.token:
11171117
request.headers["Authorization"] = "Bearer " + self.token
11181118
yield request
11191119

11201120

11211121
# This exists as an aid for debugging or local running via the `dry_run` argument to Client. It doesn't make
11221122
# sense for returning connections etc.
1123-
def noop_handler(request: httpx.Request) -> httpx.Response:
1123+
def noop_handler(request: httpx2.Request) -> httpx2.Response:
11241124
path = request.url.path
11251125
log.debug("Dry-run request", method=request.method, path=path)
11261126

11271127
if path.startswith("/task-instances/") and path.endswith("/run"):
11281128
# Return a fake context
1129-
return httpx.Response(
1129+
return httpx2.Response(
11301130
200,
11311131
json={
11321132
"dag_run": {
@@ -1142,7 +1142,7 @@ def noop_handler(request: httpx.Request) -> httpx.Response:
11421142
"should_retry": False,
11431143
},
11441144
)
1145-
return httpx.Response(200, json={"text": "Hello, world!"})
1145+
return httpx2.Response(200, json={"text": "Hello, world!"})
11461146

11471147

11481148
# Note: Given defaults make attempts after 1, 3, 7, 15 and fails after 31seconds
@@ -1159,13 +1159,13 @@ def noop_handler(request: httpx.Request) -> httpx.Response:
11591159

11601160
def _should_retry_api_request(exception: BaseException) -> bool:
11611161
"""Determine if an API request should be retried based on the exception type."""
1162-
if isinstance(exception, httpx.HTTPStatusError):
1162+
if isinstance(exception, httpx2.HTTPStatusError):
11631163
return exception.response.status_code >= 500
11641164

1165-
return isinstance(exception, httpx.RequestError)
1165+
return isinstance(exception, httpx2.RequestError)
11661166

11671167

1168-
class Client(httpx.Client):
1168+
class Client(httpx2.Client):
11691169
@lru_cache()
11701170
@staticmethod
11711171
def _get_ssl_context_cached(ca_file: str | None = None, ca_path: str | None = None) -> ssl.SSLContext:
@@ -1193,7 +1193,7 @@ def __init__(self, *, base_url: str | None, dry_run: bool = False, token: str, *
11931193
if dry_run:
11941194
# If dry run is requested, install a no op handler so that simple tasks can "heartbeat" using a
11951195
# real client, but just don't make any HTTP requests
1196-
kwargs.setdefault("transport", httpx.MockTransport(noop_handler))
1196+
kwargs.setdefault("transport", httpx2.MockTransport(noop_handler))
11971197
kwargs.setdefault("base_url", "dry-run://server")
11981198
else:
11991199
kwargs["base_url"] = base_url
@@ -1223,7 +1223,7 @@ def __init__(self, *, base_url: str | None, dry_run: bool = False, token: str, *
12231223
**kwargs,
12241224
)
12251225

1226-
def _update_auth(self, response: httpx.Response):
1226+
def _update_auth(self, response: httpx2.Response):
12271227
if new_token := response.headers.get("Refreshed-API-Token"):
12281228
log.debug("Execution API issued us a refreshed Task token")
12291229
self.auth = BearerAuth(new_token)
@@ -1236,7 +1236,7 @@ def _update_auth(self, response: httpx.Response):
12361236
reraise=True,
12371237
)
12381238
def request(self, *args, **kwargs):
1239-
"""Implement a convenience for httpx.Client.request with a retry layer."""
1239+
"""Implement a convenience for httpx2.Client.request with a retry layer."""
12401240
# Set content type as convenience if not already set
12411241
if kwargs.get("content", None) is not None and "content-type" not in (
12421242
kwargs.get("headers", {}) or {}
@@ -1336,8 +1336,8 @@ def __repr__(self):
13361336
return repr(self.detail)
13371337

13381338

1339-
class ServerResponseError(httpx.HTTPStatusError):
1340-
def __init__(self, message: str, *, request: httpx.Request, response: httpx.Response):
1339+
class ServerResponseError(httpx2.HTTPStatusError):
1340+
def __init__(self, message: str, *, request: httpx2.Request, response: httpx2.Response):
13411341
super().__init__(message, request=request, response=response)
13421342

13431343
detail: list[RemoteValidationError] | str | dict[str, Any] | None
@@ -1347,7 +1347,7 @@ def __reduce__(self) -> tuple[Any, ...]:
13471347
return Exception.__new__, (type(self),) + self.args, self.__dict__
13481348

13491349
@classmethod
1350-
def from_response(cls, response: httpx.Response) -> ServerResponseError | None:
1350+
def from_response(cls, response: httpx2.Response) -> ServerResponseError | None:
13511351
if response.is_success:
13521352
return None
13531353
# 4xx or 5xx error?

‎task-sdk/src/airflow/sdk/execution_time/supervisor.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
from uuid import UUID
4444

4545
import attrs
46-
import httpx2 as httpx
46+
import httpx2
4747
import msgspec
4848
import psutil
4949
import structlog
@@ -983,8 +983,8 @@ def handle_requests(self, log: FilteringBoundLogger) -> Generator[None, _Request
983983
request_id=request.id,
984984
)
985985
except Exception as e:
986-
# Generic exception handling so a transient network error (httpx.ConnectError /
987-
# httpx.TimeoutException) or any other exception
986+
# Generic exception handling so a transient network error (httpx2.ConnectError /
987+
# httpx2.TimeoutException) or any other exception
988988
# doesn't crash this generator and crash the IPC communication between supervisor and task.
989989
log.exception(
990990
"Unhandled exception while handling task request",
@@ -1290,7 +1290,7 @@ def _ensure_client(
12901290
yield client
12911291
return
12921292

1293-
limits = httpx.Limits(max_keepalive_connections=1, max_connections=10)
1293+
limits = httpx2.Limits(max_keepalive_connections=1, max_connections=10)
12941294
new_client = Client(base_url=server or "", limits=limits, dry_run=dry_run, token=token)
12951295
log.debug("Connecting to execution API server", server=server)
12961296
try:

‎task-sdk/tests/task_sdk/__init__.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,15 @@
1616
# under the License.
1717
from __future__ import annotations
1818

19-
import httpx2 as httpx
19+
import httpx2
2020

2121
from airflow.sdk.api.client import Client
2222
from airflow.sdk.execution_time.comms import BundleInfo
2323

2424
FAKE_BUNDLE = BundleInfo(name="anything", version="any")
2525

2626

27-
def make_client(transport: httpx.MockTransport) -> Client:
27+
def make_client(transport: httpx2.MockTransport) -> Client:
2828
"""Get a client with a custom transport."""
2929
return Client(base_url="test://server", token="", transport=transport)
3030

@@ -34,12 +34,12 @@ def make_client_w_dry_run() -> Client:
3434
return Client(base_url=None, dry_run=True, token="")
3535

3636

37-
def make_client_w_responses(responses: list[httpx.Response]) -> Client:
37+
def make_client_w_responses(responses: list[httpx2.Response]) -> Client:
3838
"""Get a client with custom responses."""
3939

40-
def handle_request(request: httpx.Request) -> httpx.Response:
40+
def handle_request(request: httpx2.Request) -> httpx2.Response:
4141
return responses.pop(0)
4242

4343
return Client(
44-
base_url=None, dry_run=True, token="", mounts={"'http://": httpx.MockTransport(handle_request)}
44+
base_url=None, dry_run=True, token="", mounts={"'http://": httpx2.MockTransport(handle_request)}
4545
)

0 commit comments

Comments
 (0)