Skip to content

Commit bfb69f2

Browse files
committed
Switch to direct httpx2 imports and keep both httpx + httpx2 deps
Per review feedback by @Prab-27 on apache#71130: - Replace 'import httpx2 as httpx' with direct 'import httpx2' for better type identity and logger compatibility - Keep httpx as a dependency alongside httpx2 to avoid breaking transitive resolution for other packages Closes apache#70522
1 parent 9d88a3a commit bfb69f2

7 files changed

Lines changed: 442 additions & 441 deletions

File tree

‎task-sdk/dev/generate_task_sdk_models.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import sys
2222
from pathlib import Path
2323

24-
import httpx2 as httpx
24+
import httpx2
2525
from datamodel_code_generator import (
2626
DataModelType,
2727
DatetimeClassType,
@@ -81,7 +81,7 @@ def generate_file():
8181
app = InProcessExecutionAPI()
8282

8383
latest_version = app.app.versions.version_values[0]
84-
client = httpx.Client(transport=app.transport)
84+
client = httpx2.Client(transport=app.transport)
8585
openapi_schema = (
8686
client.get(f"http://localhost/openapi.json?version={latest_version}").raise_for_status().text
8787
)

‎task-sdk/pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ dependencies = [
5454
"attrs>=24.2.0, !=25.2.0",
5555
"babel>=2.17.0",
5656
"fsspec>=2023.10.0",
57+
"httpx>=0.27.0",
5758
"httpx2>=0.28.1",
5859
"jinja2>=3.1.5",
5960
"methodtools>=0.4.7",

‎task-sdk/src/airflow/sdk/api/client.py‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
from urllib.parse import quote
2929

3030
import certifi
31-
import httpx2 as httpx
31+
import httpx2
3232
import msgspec
3333
import structlog
3434
from opentelemetry import trace
@@ -190,22 +190,22 @@ def getuser() -> str:
190190
]
191191

192192

193-
def get_json_error(response: httpx.Response):
193+
def get_json_error(response: httpx2.Response):
194194
"""Raise a ServerResponseError if we can extract error info from the error."""
195195
err = ServerResponseError.from_response(response)
196196
if err:
197197
raise err
198198

199199

200-
def raise_on_4xx_5xx(response: httpx.Response):
200+
def raise_on_4xx_5xx(response: httpx2.Response):
201201
return get_json_error(response) or response.raise_for_status()
202202

203203

204204
# Py 3.11+ version
205-
def raise_on_4xx_5xx_with_note(response: httpx.Response):
205+
def raise_on_4xx_5xx_with_note(response: httpx2.Response):
206206
try:
207207
return get_json_error(response) or response.raise_for_status()
208-
except httpx.HTTPStatusError as e:
208+
except httpx2.HTTPStatusError as e:
209209
if TYPE_CHECKING:
210210
assert hasattr(e, "add_note")
211211
e.add_note(
@@ -219,11 +219,11 @@ def raise_on_4xx_5xx_with_note(response: httpx.Response):
219219
raise_on_4xx_5xx = raise_on_4xx_5xx_with_note
220220

221221

222-
def add_correlation_id(request: httpx.Request):
222+
def add_correlation_id(request: httpx2.Request):
223223
request.headers["correlation-id"] = str(uuid7())
224224

225225

226-
def inject_trace_context(request: httpx.Request) -> None:
226+
def inject_trace_context(request: httpx2.Request) -> None:
227227
_trace_propagator.inject(request.headers)
228228

229229

@@ -1091,25 +1091,25 @@ def update_state(
10911091
self.client.patch(f"connection-tests/{id}", content=body.model_dump_json())
10921092

10931093

1094-
class BearerAuth(httpx.Auth):
1094+
class BearerAuth(httpx2.Auth):
10951095
def __init__(self, token: str):
10961096
self.token: str = token
10971097

1098-
def auth_flow(self, request: httpx.Request):
1098+
def auth_flow(self, request: httpx2.Request):
10991099
if self.token:
11001100
request.headers["Authorization"] = "Bearer " + self.token
11011101
yield request
11021102

11031103

11041104
# This exists as an aid for debugging or local running via the `dry_run` argument to Client. It doesn't make
11051105
# sense for returning connections etc.
1106-
def noop_handler(request: httpx.Request) -> httpx.Response:
1106+
def noop_handler(request: httpx2.Request) -> httpx2.Response:
11071107
path = request.url.path
11081108
log.debug("Dry-run request", method=request.method, path=path)
11091109

11101110
if path.startswith("/task-instances/") and path.endswith("/run"):
11111111
# Return a fake context
1112-
return httpx.Response(
1112+
return httpx2.Response(
11131113
200,
11141114
json={
11151115
"dag_run": {
@@ -1125,7 +1125,7 @@ def noop_handler(request: httpx.Request) -> httpx.Response:
11251125
"should_retry": False,
11261126
},
11271127
)
1128-
return httpx.Response(200, json={"text": "Hello, world!"})
1128+
return httpx2.Response(200, json={"text": "Hello, world!"})
11291129

11301130

11311131
# Note: Given defaults make attempts after 1, 3, 7, 15 and fails after 31seconds
@@ -1142,13 +1142,13 @@ def noop_handler(request: httpx.Request) -> httpx.Response:
11421142

11431143
def _should_retry_api_request(exception: BaseException) -> bool:
11441144
"""Determine if an API request should be retried based on the exception type."""
1145-
if isinstance(exception, httpx.HTTPStatusError):
1145+
if isinstance(exception, httpx2.HTTPStatusError):
11461146
return exception.response.status_code >= 500
11471147

1148-
return isinstance(exception, httpx.RequestError)
1148+
return isinstance(exception, httpx2.RequestError)
11491149

11501150

1151-
class Client(httpx.Client):
1151+
class Client(httpx2.Client):
11521152
@lru_cache()
11531153
@staticmethod
11541154
def _get_ssl_context_cached(ca_file: str | None = None, ca_path: str | None = None) -> ssl.SSLContext:
@@ -1176,7 +1176,7 @@ def __init__(self, *, base_url: str | None, dry_run: bool = False, token: str, *
11761176
if dry_run:
11771177
# If dry run is requested, install a no op handler so that simple tasks can "heartbeat" using a
11781178
# real client, but just don't make any HTTP requests
1179-
kwargs.setdefault("transport", httpx.MockTransport(noop_handler))
1179+
kwargs.setdefault("transport", httpx2.MockTransport(noop_handler))
11801180
kwargs.setdefault("base_url", "dry-run://server")
11811181
else:
11821182
kwargs["base_url"] = base_url
@@ -1206,7 +1206,7 @@ def __init__(self, *, base_url: str | None, dry_run: bool = False, token: str, *
12061206
**kwargs,
12071207
)
12081208

1209-
def _update_auth(self, response: httpx.Response):
1209+
def _update_auth(self, response: httpx2.Response):
12101210
if new_token := response.headers.get("Refreshed-API-Token"):
12111211
log.debug("Execution API issued us a refreshed Task token")
12121212
self.auth = BearerAuth(new_token)
@@ -1219,7 +1219,7 @@ def _update_auth(self, response: httpx.Response):
12191219
reraise=True,
12201220
)
12211221
def request(self, *args, **kwargs):
1222-
"""Implement a convenience for httpx.Client.request with a retry layer."""
1222+
"""Implement a convenience for httpx2.Client.request with a retry layer."""
12231223
# Set content type as convenience if not already set
12241224
if kwargs.get("content", None) is not None and "content-type" not in (
12251225
kwargs.get("headers", {}) or {}
@@ -1313,8 +1313,8 @@ def __repr__(self):
13131313
return repr(self.detail)
13141314

13151315

1316-
class ServerResponseError(httpx.HTTPStatusError):
1317-
def __init__(self, message: str, *, request: httpx.Request, response: httpx.Response):
1316+
class ServerResponseError(httpx2.HTTPStatusError):
1317+
def __init__(self, message: str, *, request: httpx2.Request, response: httpx2.Response):
13181318
super().__init__(message, request=request, response=response)
13191319

13201320
detail: list[RemoteValidationError] | str | dict[str, Any] | None
@@ -1324,7 +1324,7 @@ def __reduce__(self) -> tuple[Any, ...]:
13241324
return Exception.__new__, (type(self),) + self.args, self.__dict__
13251325

13261326
@classmethod
1327-
def from_response(cls, response: httpx.Response) -> ServerResponseError | None:
1327+
def from_response(cls, response: httpx2.Response) -> ServerResponseError | None:
13281328
if response.is_success:
13291329
return None
13301330
# 4xx or 5xx error?

‎task-sdk/src/airflow/sdk/execution_time/supervisor.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
from uuid import UUID
4444

4545
import attrs
46-
import httpx2 as httpx
46+
import httpx2
4747
import msgspec
4848
import psutil
4949
import structlog
@@ -952,8 +952,8 @@ def handle_requests(self, log: FilteringBoundLogger) -> Generator[None, _Request
952952
request_id=request.id,
953953
)
954954
except Exception as e:
955-
# Generic exception handling so a transient network error (httpx.ConnectError /
956-
# httpx.TimeoutException) or any other exception
955+
# Generic exception handling so a transient network error (httpx2.ConnectError /
956+
# httpx2.TimeoutException) or any other exception
957957
# doesn't crash this generator and crash the IPC communication between supervisor and task.
958958
log.exception(
959959
"Unhandled exception while handling task request",
@@ -1233,7 +1233,7 @@ def _ensure_client(
12331233
yield client
12341234
return
12351235

1236-
limits = httpx.Limits(max_keepalive_connections=1, max_connections=10)
1236+
limits = httpx2.Limits(max_keepalive_connections=1, max_connections=10)
12371237
new_client = Client(base_url=server or "", limits=limits, dry_run=dry_run, token=token)
12381238
log.debug("Connecting to execution API server", server=server)
12391239
try:

‎task-sdk/tests/task_sdk/__init__.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,15 @@
1616
# under the License.
1717
from __future__ import annotations
1818

19-
import httpx2 as httpx
19+
import httpx2
2020

2121
from airflow.sdk.api.client import Client
2222
from airflow.sdk.execution_time.comms import BundleInfo
2323

2424
FAKE_BUNDLE = BundleInfo(name="anything", version="any")
2525

2626

27-
def make_client(transport: httpx.MockTransport) -> Client:
27+
def make_client(transport: httpx2.MockTransport) -> Client:
2828
"""Get a client with a custom transport."""
2929
return Client(base_url="test://server", token="", transport=transport)
3030

@@ -34,12 +34,12 @@ def make_client_w_dry_run() -> Client:
3434
return Client(base_url=None, dry_run=True, token="")
3535

3636

37-
def make_client_w_responses(responses: list[httpx.Response]) -> Client:
37+
def make_client_w_responses(responses: list[httpx2.Response]) -> Client:
3838
"""Get a client with custom responses."""
3939

40-
def handle_request(request: httpx.Request) -> httpx.Response:
40+
def handle_request(request: httpx2.Request) -> httpx2.Response:
4141
return responses.pop(0)
4242

4343
return Client(
44-
base_url=None, dry_run=True, token="", mounts={"'http://": httpx.MockTransport(handle_request)}
44+
base_url=None, dry_run=True, token="", mounts={"'http://": httpx2.MockTransport(handle_request)}
4545
)

0 commit comments

Comments
 (0)