Skip to content

Commit cf4bdeb

Browse files
committed
Switch to direct httpx2 imports and keep both httpx + httpx2 deps
Per review feedback by @Prab-27 on apache#71130: - Replace 'import httpx2 as httpx' with direct 'import httpx2' for better type identity and logger compatibility - Keep httpx as a dependency alongside httpx2 to avoid breaking transitive resolution for other packages Closes apache#70522
1 parent 4036382 commit cf4bdeb

7 files changed

Lines changed: 442 additions & 441 deletions

File tree

‎task-sdk/dev/generate_task_sdk_models.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import sys
2222
from pathlib import Path
2323

24-
import httpx2 as httpx
24+
import httpx2
2525
from datamodel_code_generator import (
2626
DataModelType,
2727
DatetimeClassType,
@@ -81,7 +81,7 @@ def generate_file():
8181
app = InProcessExecutionAPI()
8282

8383
latest_version = app.app.versions.version_values[0]
84-
client = httpx.Client(transport=app.transport)
84+
client = httpx2.Client(transport=app.transport)
8585
openapi_schema = (
8686
client.get(f"http://localhost/openapi.json?version={latest_version}").raise_for_status().text
8787
)

‎task-sdk/pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ dependencies = [
5454
"attrs>=24.2.0, !=25.2.0",
5555
"babel>=2.17.0",
5656
"fsspec>=2023.10.0",
57+
"httpx>=0.27.0",
5758
"httpx2>=0.28.1",
5859
"jinja2>=3.1.5",
5960
"methodtools>=0.4.7",

‎task-sdk/src/airflow/sdk/api/client.py‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
from urllib.parse import quote
2929

3030
import certifi
31-
import httpx2 as httpx
31+
import httpx2
3232
import msgspec
3333
import structlog
3434
from opentelemetry import trace
@@ -190,22 +190,22 @@ def getuser() -> str:
190190
]
191191

192192

193-
def get_json_error(response: httpx.Response):
193+
def get_json_error(response: httpx2.Response):
194194
"""Raise a ServerResponseError if we can extract error info from the error."""
195195
err = ServerResponseError.from_response(response)
196196
if err:
197197
raise err
198198

199199

200-
def raise_on_4xx_5xx(response: httpx.Response):
200+
def raise_on_4xx_5xx(response: httpx2.Response):
201201
return get_json_error(response) or response.raise_for_status()
202202

203203

204204
# Py 3.11+ version
205-
def raise_on_4xx_5xx_with_note(response: httpx.Response):
205+
def raise_on_4xx_5xx_with_note(response: httpx2.Response):
206206
try:
207207
return get_json_error(response) or response.raise_for_status()
208-
except httpx.HTTPStatusError as e:
208+
except httpx2.HTTPStatusError as e:
209209
if TYPE_CHECKING:
210210
assert hasattr(e, "add_note")
211211
e.add_note(
@@ -225,11 +225,11 @@ def raise_on_4xx_5xx_with_note(response: httpx.Response):
225225
raise_on_4xx_5xx = raise_on_4xx_5xx_with_note
226226

227227

228-
def add_correlation_id(request: httpx.Request):
228+
def add_correlation_id(request: httpx2.Request):
229229
request.headers["correlation-id"] = str(uuid7())
230230

231231

232-
def inject_trace_context(request: httpx.Request) -> None:
232+
def inject_trace_context(request: httpx2.Request) -> None:
233233
_trace_propagator.inject(request.headers)
234234

235235

@@ -1097,25 +1097,25 @@ def update_state(
10971097
self.client.patch(f"connection-tests/{id}", content=body.model_dump_json())
10981098

10991099

1100-
class BearerAuth(httpx.Auth):
1100+
class BearerAuth(httpx2.Auth):
11011101
def __init__(self, token: str):
11021102
self.token: str = token
11031103

1104-
def auth_flow(self, request: httpx.Request):
1104+
def auth_flow(self, request: httpx2.Request):
11051105
if self.token:
11061106
request.headers["Authorization"] = "Bearer " + self.token
11071107
yield request
11081108

11091109

11101110
# This exists as an aid for debugging or local running via the `dry_run` argument to Client. It doesn't make
11111111
# sense for returning connections etc.
1112-
def noop_handler(request: httpx.Request) -> httpx.Response:
1112+
def noop_handler(request: httpx2.Request) -> httpx2.Response:
11131113
path = request.url.path
11141114
log.debug("Dry-run request", method=request.method, path=path)
11151115

11161116
if path.startswith("/task-instances/") and path.endswith("/run"):
11171117
# Return a fake context
1118-
return httpx.Response(
1118+
return httpx2.Response(
11191119
200,
11201120
json={
11211121
"dag_run": {
@@ -1131,7 +1131,7 @@ def noop_handler(request: httpx.Request) -> httpx.Response:
11311131
"should_retry": False,
11321132
},
11331133
)
1134-
return httpx.Response(200, json={"text": "Hello, world!"})
1134+
return httpx2.Response(200, json={"text": "Hello, world!"})
11351135

11361136

11371137
# Note: Given defaults make attempts after 1, 3, 7, 15 and fails after 31seconds
@@ -1148,13 +1148,13 @@ def noop_handler(request: httpx.Request) -> httpx.Response:
11481148

11491149
def _should_retry_api_request(exception: BaseException) -> bool:
11501150
"""Determine if an API request should be retried based on the exception type."""
1151-
if isinstance(exception, httpx.HTTPStatusError):
1151+
if isinstance(exception, httpx2.HTTPStatusError):
11521152
return exception.response.status_code >= 500
11531153

1154-
return isinstance(exception, httpx.RequestError)
1154+
return isinstance(exception, httpx2.RequestError)
11551155

11561156

1157-
class Client(httpx.Client):
1157+
class Client(httpx2.Client):
11581158
@lru_cache()
11591159
@staticmethod
11601160
def _get_ssl_context_cached(ca_file: str | None = None, ca_path: str | None = None) -> ssl.SSLContext:
@@ -1182,7 +1182,7 @@ def __init__(self, *, base_url: str | None, dry_run: bool = False, token: str, *
11821182
if dry_run:
11831183
# If dry run is requested, install a no op handler so that simple tasks can "heartbeat" using a
11841184
# real client, but just don't make any HTTP requests
1185-
kwargs.setdefault("transport", httpx.MockTransport(noop_handler))
1185+
kwargs.setdefault("transport", httpx2.MockTransport(noop_handler))
11861186
kwargs.setdefault("base_url", "dry-run://server")
11871187
else:
11881188
kwargs["base_url"] = base_url
@@ -1212,7 +1212,7 @@ def __init__(self, *, base_url: str | None, dry_run: bool = False, token: str, *
12121212
**kwargs,
12131213
)
12141214

1215-
def _update_auth(self, response: httpx.Response):
1215+
def _update_auth(self, response: httpx2.Response):
12161216
if new_token := response.headers.get("Refreshed-API-Token"):
12171217
log.debug("Execution API issued us a refreshed Task token")
12181218
self.auth = BearerAuth(new_token)
@@ -1225,7 +1225,7 @@ def _update_auth(self, response: httpx.Response):
12251225
reraise=True,
12261226
)
12271227
def request(self, *args, **kwargs):
1228-
"""Implement a convenience for httpx.Client.request with a retry layer."""
1228+
"""Implement a convenience for httpx2.Client.request with a retry layer."""
12291229
# Set content type as convenience if not already set
12301230
if kwargs.get("content", None) is not None and "content-type" not in (
12311231
kwargs.get("headers", {}) or {}
@@ -1319,8 +1319,8 @@ def __repr__(self):
13191319
return repr(self.detail)
13201320

13211321

1322-
class ServerResponseError(httpx.HTTPStatusError):
1323-
def __init__(self, message: str, *, request: httpx.Request, response: httpx.Response):
1322+
class ServerResponseError(httpx2.HTTPStatusError):
1323+
def __init__(self, message: str, *, request: httpx2.Request, response: httpx2.Response):
13241324
super().__init__(message, request=request, response=response)
13251325

13261326
detail: list[RemoteValidationError] | str | dict[str, Any] | None
@@ -1330,7 +1330,7 @@ def __reduce__(self) -> tuple[Any, ...]:
13301330
return Exception.__new__, (type(self),) + self.args, self.__dict__
13311331

13321332
@classmethod
1333-
def from_response(cls, response: httpx.Response) -> ServerResponseError | None:
1333+
def from_response(cls, response: httpx2.Response) -> ServerResponseError | None:
13341334
if response.is_success:
13351335
return None
13361336
# 4xx or 5xx error?

‎task-sdk/src/airflow/sdk/execution_time/supervisor.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
from uuid import UUID
4444

4545
import attrs
46-
import httpx2 as httpx
46+
import httpx2
4747
import msgspec
4848
import psutil
4949
import structlog
@@ -983,8 +983,8 @@ def handle_requests(self, log: FilteringBoundLogger) -> Generator[None, _Request
983983
request_id=request.id,
984984
)
985985
except Exception as e:
986-
# Generic exception handling so a transient network error (httpx.ConnectError /
987-
# httpx.TimeoutException) or any other exception
986+
# Generic exception handling so a transient network error (httpx2.ConnectError /
987+
# httpx2.TimeoutException) or any other exception
988988
# doesn't crash this generator and crash the IPC communication between supervisor and task.
989989
log.exception(
990990
"Unhandled exception while handling task request",
@@ -1290,7 +1290,7 @@ def _ensure_client(
12901290
yield client
12911291
return
12921292

1293-
limits = httpx.Limits(max_keepalive_connections=1, max_connections=10)
1293+
limits = httpx2.Limits(max_keepalive_connections=1, max_connections=10)
12941294
new_client = Client(base_url=server or "", limits=limits, dry_run=dry_run, token=token)
12951295
log.debug("Connecting to execution API server", server=server)
12961296
try:

‎task-sdk/tests/task_sdk/__init__.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,15 @@
1616
# under the License.
1717
from __future__ import annotations
1818

19-
import httpx2 as httpx
19+
import httpx2
2020

2121
from airflow.sdk.api.client import Client
2222
from airflow.sdk.execution_time.comms import BundleInfo
2323

2424
FAKE_BUNDLE = BundleInfo(name="anything", version="any")
2525

2626

27-
def make_client(transport: httpx.MockTransport) -> Client:
27+
def make_client(transport: httpx2.MockTransport) -> Client:
2828
"""Get a client with a custom transport."""
2929
return Client(base_url="test://server", token="", transport=transport)
3030

@@ -34,12 +34,12 @@ def make_client_w_dry_run() -> Client:
3434
return Client(base_url=None, dry_run=True, token="")
3535

3636

37-
def make_client_w_responses(responses: list[httpx.Response]) -> Client:
37+
def make_client_w_responses(responses: list[httpx2.Response]) -> Client:
3838
"""Get a client with custom responses."""
3939

40-
def handle_request(request: httpx.Request) -> httpx.Response:
40+
def handle_request(request: httpx2.Request) -> httpx2.Response:
4141
return responses.pop(0)
4242

4343
return Client(
44-
base_url=None, dry_run=True, token="", mounts={"'http://": httpx.MockTransport(handle_request)}
44+
base_url=None, dry_run=True, token="", mounts={"'http://": httpx2.MockTransport(handle_request)}
4545
)

0 commit comments

Comments
 (0)