Skip to content

Commit c4b5917

Browse files
sethmlarsonpquentin
sethmlarson
and
pquentin
authored
Add support for the new compression.zstd module in Python 3.14 (#3611)
* Add support for the new `compression.zstd` module in Python 3.14 * Add zstandard to default 'Accept-Encoding' header for stdlib, too * Fix all type hint issues --------- Co-authored-by: Quentin Pradet <[email protected]>
1 parent 47ac841 commit c4b5917

7 files changed

Lines changed: 99 additions & 53 deletions

File tree

‎changelog/3610.feature.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Add support for the ``compression.zstd`` module that is new in Python 3.14.
2+
See `PEP 784 <https://peps.python.org/pep-0784/>`_ for more information.

‎docs/advanced-usage.rst‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -561,11 +561,14 @@ Zstandard Encoding
561561
`Zstandard <https://datatracker.ietf.org/doc/html/rfc8878>`_
562562
is a compression algorithm created by Facebook with better compression
563563
than brotli, gzip and deflate (see `benchmarks <https://facebook.github.io/zstd/#benchmarks>`_)
564-
and is supported by urllib3 if the `zstandard package <https://pypi.org/project/zstandard/>`_ is installed.
564+
and is supported by urllib3 in Python 3.14+ using the `compression.zstd <https://peps.python.org/pep-0784/>`_ standard library module
565+
and for Python 3.13 and earlier if the `zstandard package <https://pypi.org/project/zstandard/>`_ is installed.
565566
You may also request the package be installed via the ``urllib3[zstd]`` extra:
566567

567568
.. code-block:: bash
568569
570+
# This is only necessary on Python 3.13 and earlier.
571+
# Otherwise zstandard support is included in the Python standard library.
569572
$ python -m pip install urllib3[zstd]
570573
571574
.. note::

‎pyproject.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@ brotli = [
4444
"brotli>=1.0.9; platform_python_implementation == 'CPython'",
4545
"brotlicffi>=0.8.0; platform_python_implementation != 'CPython'"
4646
]
47+
# Once we drop support for Python 3.13 this extra can be removed.
48+
# We'll need a deprecation period for the 'zstandard' module support
49+
# so that folks using Python without the 'compression.zstd' module
50+
# compiled will know to start doing so (although it'll likely be rare).
4751
zstd = [
4852
"zstandard>=0.18.0",
4953
]

‎src/urllib3/response.py‎

Lines changed: 47 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -26,23 +26,6 @@
2626
except ImportError:
2727
brotli = None
2828

29-
try:
30-
import zstandard as zstd
31-
except (AttributeError, ImportError, ValueError): # Defensive:
32-
HAS_ZSTD = False
33-
else:
34-
# The package 'zstandard' added the 'eof' property starting
35-
# in v0.18.0 which we require to ensure a complete and
36-
# valid zstd stream was fed into the ZstdDecoder.
37-
# See: https://github.com/urllib3/urllib3/pull/2624
38-
_zstd_version = tuple(
39-
map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups()) # type: ignore[union-attr]
40-
)
41-
if _zstd_version < (0, 18): # Defensive:
42-
HAS_ZSTD = False
43-
else:
44-
HAS_ZSTD = True
45-
4629
from . import util
4730
from ._base_connection import _TYPE_BODY
4831
from ._collections import HTTPHeaderDict
@@ -163,27 +146,69 @@ def flush(self) -> bytes:
163146
return b""
164147

165148

166-
if HAS_ZSTD:
149+
try:
150+
# Python 3.14+
151+
from compression import zstd # type: ignore[import-not-found] # noqa: F401
152+
153+
HAS_ZSTD = True
167154

168155
class ZstdDecoder(ContentDecoder):
169156
def __init__(self) -> None:
170-
self._obj = zstd.ZstdDecompressor().decompressobj()
157+
self._obj = zstd.ZstdDecompressor()
171158

172159
def decompress(self, data: bytes) -> bytes:
173160
if not data:
174161
return b""
175162
data_parts = [self._obj.decompress(data)]
176163
while self._obj.eof and self._obj.unused_data:
177164
unused_data = self._obj.unused_data
178-
self._obj = zstd.ZstdDecompressor().decompressobj()
165+
self._obj = zstd.ZstdDecompressor()
179166
data_parts.append(self._obj.decompress(unused_data))
180167
return b"".join(data_parts)
181168

182169
def flush(self) -> bytes:
183-
ret = self._obj.flush() # note: this is a no-op
184170
if not self._obj.eof:
185171
raise DecodeError("Zstandard data is incomplete")
186-
return ret
172+
return b""
173+
174+
except ImportError:
175+
try:
176+
# Python 3.13 and earlier require the 'zstandard' module.
177+
import zstandard as zstd
178+
179+
# The package 'zstandard' added the 'eof' property starting
180+
# in v0.18.0 which we require to ensure a complete and
181+
# valid zstd stream was fed into the ZstdDecoder.
182+
# See: https://github.com/urllib3/urllib3/pull/2624
183+
_zstd_version = tuple(
184+
map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups()) # type: ignore[union-attr]
185+
)
186+
if _zstd_version < (0, 18): # Defensive:
187+
raise ImportError("zstandard module doesn't have eof")
188+
except (AttributeError, ImportError, ValueError): # Defensive:
189+
HAS_ZSTD = False
190+
else:
191+
HAS_ZSTD = True
192+
193+
class ZstdDecoder(ContentDecoder): # type: ignore[no-redef]
194+
def __init__(self) -> None:
195+
self._obj = zstd.ZstdDecompressor().decompressobj()
196+
197+
def decompress(self, data: bytes) -> bytes:
198+
if not data:
199+
return b""
200+
data_parts = [self._obj.decompress(data)]
201+
while self._obj.eof and self._obj.unused_data:
202+
unused_data = self._obj.unused_data
203+
self._obj = zstd.ZstdDecompressor().decompressobj()
204+
data_parts.append(self._obj.decompress(unused_data))
205+
return b"".join(data_parts)
206+
207+
def flush(self) -> bytes:
208+
ret = self._obj.flush() # note: this is a no-op
209+
if not self._obj.eof:
210+
raise DecodeError("Zstandard data is incomplete")
211+
return ret # type: ignore[no-any-return]
187212

188213

189214
class MultiDecoder(ContentDecoder):

‎src/urllib3/util/request.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,20 @@
2828
pass
2929
else:
3030
ACCEPT_ENCODING += ",br"
31+
3132
try:
32-
import zstandard as _unused_module_zstd # noqa: F401
33-
except ImportError:
34-
pass
35-
else:
33+
from compression import ( # type: ignore[import-not-found] # noqa: F401
34+
zstd as _unused_module_zstd,
35+
)
36+
3637
ACCEPT_ENCODING += ",zstd"
38+
except ImportError:
39+
try:
40+
import zstandard as _unused_module_zstd # noqa: F401
41+
42+
ACCEPT_ENCODING += ",zstd"
43+
except ImportError:
44+
pass
3745

3846

3947
class _TYPE_FAILEDTELL(Enum):

‎test/__init__.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,18 @@
2626
brotli = None
2727

2828
try:
29-
import zstandard as _unused_module_zstd # noqa: F401
29+
# Python 3.14
30+
from compression import ( # type: ignore[import-not-found] # noqa: F401
31+
zstd as _unused_module_zstd,
32+
)
3033
except ImportError:
31-
HAS_ZSTD = False
34+
# Python 3.13 and earlier require the 'zstandard' module.
35+
try:
36+
import zstandard as _unused_module_zstd # noqa: F401
37+
except ImportError:
38+
HAS_ZSTD = False
39+
else:
40+
HAS_ZSTD = True
3241
else:
3342
HAS_ZSTD = True
3443

@@ -127,14 +136,15 @@ def notBrotli() -> typing.Callable[[_TestFuncT], _TestFuncT]:
127136

128137
def onlyZstd() -> typing.Callable[[_TestFuncT], _TestFuncT]:
129138
return pytest.mark.skipif(
130-
not HAS_ZSTD, reason="only run if a python-zstandard library is installed"
139+
not HAS_ZSTD,
140+
reason="only run if a python-zstandard library is installed or Python 3.14 and later",
131141
)
132142

133143

134144
def notZstd() -> typing.Callable[[_TestFuncT], _TestFuncT]:
135145
return pytest.mark.skipif(
136146
HAS_ZSTD,
137-
reason="only run if a python-zstandard library is not installed",
147+
reason="only run if a python-zstandard library is not installed or Python 3.13 and earlier",
138148
)
139149

140150

‎test/test_response.py‎

Lines changed: 16 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,14 @@
3535
from urllib3.util.retry import RequestHistory, Retry
3636

3737

38+
def zstd_compress(data: bytes) -> bytes:
39+
try:
40+
from compression import zstd # type: ignore[import-not-found] # noqa: F401
41+
except ImportError:
42+
import zstandard as zstd
43+
return zstd.compress(data) # type: ignore[no-any-return]
44+
45+
3846
class TestBytesQueueBuffer:
3947
def test_single_chunk(self) -> None:
4048
buffer = BytesQueueBuffer()
@@ -411,29 +419,25 @@ def test_decode_brotli_error(self) -> None:
411419

412420
@onlyZstd()
413421
def test_decode_zstd(self) -> None:
414-
import zstandard as zstd
415-
416-
data = zstd.compress(b"foo")
422+
data = zstd_compress(b"foo")
417423

418424
fp = BytesIO(data)
419425
r = HTTPResponse(fp, headers={"content-encoding": "zstd"})
420426
assert r.data == b"foo"
421427

422428
@onlyZstd()
423429
def test_decode_multiframe_zstd(self) -> None:
424-
import zstandard as zstd
425-
426430
data = (
427431
# Zstandard frame
428-
zstd.compress(b"foo")
432+
zstd_compress(b"foo")
429433
# skippable frame (must be ignored)
430434
+ bytes.fromhex(
431435
"50 2A 4D 18" # Magic_Number (little-endian)
432436
"07 00 00 00" # Frame_Size (little-endian)
433437
"00 00 00 00 00 00 00" # User_Data
434438
)
435439
# Zstandard frame
436-
+ zstd.compress(b"bar")
440+
+ zstd_compress(b"bar")
437441
)
438442

439443
fp = BytesIO(data)
@@ -442,9 +446,7 @@ def test_decode_multiframe_zstd(self) -> None:
442446

443447
@onlyZstd()
444448
def test_chunked_decoding_zstd(self) -> None:
445-
import zstandard as zstd
446-
447-
data = zstd.compress(b"foobarbaz")
449+
data = zstd_compress(b"foobarbaz")
448450

449451
fp = BytesIO(data)
450452
r = HTTPResponse(
@@ -475,9 +477,7 @@ def test_decode_zstd_error(self, data: bytes) -> None:
475477
@onlyZstd()
476478
@pytest.mark.parametrize("data", decode_param_set)
477479
def test_decode_zstd_incomplete_preload_content(self, data: bytes) -> None:
478-
import zstandard as zstd
479-
480-
data = zstd.compress(data)
480+
data = zstd_compress(data)
481481
fp = BytesIO(data[:-1])
482482

483483
with pytest.raises(DecodeError):
@@ -486,9 +486,7 @@ def test_decode_zstd_incomplete_preload_content(self, data: bytes) -> None:
486486
@onlyZstd()
487487
@pytest.mark.parametrize("data", decode_param_set)
488488
def test_decode_zstd_incomplete_read(self, data: bytes) -> None:
489-
import zstandard as zstd
490-
491-
data = zstd.compress(data)
489+
data = zstd_compress(data)
492490
fp = BytesIO(data[:-1]) # shorten the data to trigger DecodeError
493491

494492
# create response object without(!) reading/decoding the content
@@ -503,9 +501,7 @@ def test_decode_zstd_incomplete_read(self, data: bytes) -> None:
503501
@onlyZstd()
504502
@pytest.mark.parametrize("data", decode_param_set)
505503
def test_decode_zstd_incomplete_read1(self, data: bytes) -> None:
506-
import zstandard as zstd
507-
508-
data = zstd.compress(data)
504+
data = zstd_compress(data)
509505
fp = BytesIO(data[:-1])
510506

511507
r = HTTPResponse(
@@ -523,9 +519,7 @@ def test_decode_zstd_incomplete_read1(self, data: bytes) -> None:
523519
@onlyZstd()
524520
@pytest.mark.parametrize("data", decode_param_set)
525521
def test_decode_zstd_read1(self, data: bytes) -> None:
526-
import zstandard as zstd
527-
528-
encoded_data = zstd.compress(data)
522+
encoded_data = zstd_compress(data)
529523
fp = BytesIO(encoded_data)
530524

531525
r = HTTPResponse(

0 commit comments

Comments
 (0)