Skip to content

Commit 779d39a

Browse files
yuxi-liu-wiredgyscos
authored andcommitted
Add fallible constructors for prepared dictionaries
EncoderDictionary::copy/new and DecoderDictionary::copy/new panic when zstd cannot load the dictionary (ZSTD_createCDict / ZSTD_createDDict return NULL), for instance a dictionary that starts with the magic number but has a corrupt header. A service that accepts dictionaries from elsewhere has no way to reject such input short of catching the panic, while the non-prepared paths (Compressor/Decompressor::with_dictionary) already return an error. Add try_copy and try_new, returning io::Result, and document the panic on the existing constructors. In zstd-safe, add CDict/DDict:: try_create_by_reference next to the existing try_create, and document the panics there too. The panicking constructors are unchanged.
1 parent fc5d2b7 commit 779d39a

3 files changed

Lines changed: 181 additions & 8 deletions

File tree

‎src/dict.rs‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,40 @@ pub struct EncoderDictionary<'a> {
2727
cdict: CDict<'a>,
2828
}
2929

30+
/// The error for a dictionary zstd could not load.
31+
fn invalid_dictionary() -> std::io::Error {
32+
std::io::Error::new(
33+
std::io::ErrorKind::InvalidData,
34+
"zstd could not load the dictionary",
35+
)
36+
}
37+
3038
impl EncoderDictionary<'static> {
3139
/// Creates a prepared dictionary for compression.
3240
///
3341
/// This will copy the dictionary internally.
42+
///
43+
/// # Panics
44+
///
45+
/// If zstd cannot load the dictionary - for instance, it starts with the
46+
/// dictionary magic number but its header is corrupt. Use
47+
/// [`EncoderDictionary::try_copy`] for a dictionary you did not make
48+
/// yourself.
3449
pub fn copy(dictionary: &[u8], level: i32) -> Self {
3550
Self {
3651
cdict: zstd_safe::create_cdict(dictionary, level),
3752
}
3853
}
54+
55+
/// Creates a prepared dictionary for compression, or an error if zstd
56+
/// cannot load it.
57+
///
58+
/// This will copy the dictionary internally.
59+
pub fn try_copy(dictionary: &[u8], level: i32) -> std::io::Result<Self> {
60+
let cdict = zstd_safe::CDict::try_create(dictionary, level)
61+
.ok_or_else(invalid_dictionary)?;
62+
Ok(Self { cdict })
63+
}
3964
}
4065

4166
impl<'a> EncoderDictionary<'a> {
@@ -46,12 +71,31 @@ impl<'a> EncoderDictionary<'a> {
4671
/// A level of `0` uses zstd's default (currently `3`).
4772
///
4873
/// Only available with the `experimental` feature. Use `EncoderDictionary::copy` otherwise.
74+
///
75+
/// # Panics
76+
///
77+
/// If zstd cannot load the dictionary. Use [`EncoderDictionary::try_new`]
78+
/// for a dictionary you did not make yourself.
4979
pub fn new(dictionary: &'a [u8], level: i32) -> Self {
5080
Self {
5181
cdict: zstd_safe::CDict::create_by_reference(dictionary, level),
5282
}
5383
}
5484

85+
#[cfg(feature = "experimental")]
86+
#[cfg_attr(feature = "doc-cfg", doc(cfg(feature = "experimental")))]
87+
/// Create prepared dictionary for compression, or an error if zstd cannot
88+
/// load it.
89+
///
90+
/// Only available with the `experimental` feature. Use
91+
/// `EncoderDictionary::try_copy` otherwise.
92+
pub fn try_new(dictionary: &'a [u8], level: i32) -> std::io::Result<Self> {
93+
let cdict =
94+
zstd_safe::CDict::try_create_by_reference(dictionary, level)
95+
.ok_or_else(invalid_dictionary)?;
96+
Ok(Self { cdict })
97+
}
98+
5599
/// Returns reference to `CDict` inner object
56100
pub fn as_cdict(&self) -> &CDict<'a> {
57101
&self.cdict
@@ -67,11 +111,28 @@ impl DecoderDictionary<'static> {
67111
/// Create a prepared dictionary for decompression.
68112
///
69113
/// This will copy the dictionary internally.
114+
///
115+
/// # Panics
116+
///
117+
/// If zstd cannot load the dictionary - for instance, it starts with the
118+
/// dictionary magic number but its header is corrupt. Use
119+
/// [`DecoderDictionary::try_copy`] for a dictionary you did not make
120+
/// yourself.
70121
pub fn copy(dictionary: &[u8]) -> Self {
71122
Self {
72123
ddict: zstd_safe::DDict::create(dictionary),
73124
}
74125
}
126+
127+
/// Create a prepared dictionary for decompression, or an error if zstd
128+
/// cannot load it.
129+
///
130+
/// This will copy the dictionary internally.
131+
pub fn try_copy(dictionary: &[u8]) -> std::io::Result<Self> {
132+
let ddict = zstd_safe::DDict::try_create(dictionary)
133+
.ok_or_else(invalid_dictionary)?;
134+
Ok(Self { ddict })
135+
}
75136
}
76137

77138
impl<'a> DecoderDictionary<'a> {
@@ -80,12 +141,30 @@ impl<'a> DecoderDictionary<'a> {
80141
/// Create prepared dictionary for decompression
81142
///
82143
/// Only available with the `experimental` feature. Use `DecoderDictionary::copy` otherwise.
144+
///
145+
/// # Panics
146+
///
147+
/// If zstd cannot load the dictionary. Use [`DecoderDictionary::try_new`]
148+
/// for a dictionary you did not make yourself.
83149
pub fn new(dict: &'a [u8]) -> Self {
84150
Self {
85151
ddict: zstd_safe::DDict::create_by_reference(dict),
86152
}
87153
}
88154

155+
#[cfg(feature = "experimental")]
156+
#[cfg_attr(feature = "doc-cfg", doc(cfg(feature = "experimental")))]
157+
/// Create prepared dictionary for decompression, or an error if zstd
158+
/// cannot load it.
159+
///
160+
/// Only available with the `experimental` feature. Use
161+
/// `DecoderDictionary::try_copy` otherwise.
162+
pub fn try_new(dict: &'a [u8]) -> std::io::Result<Self> {
163+
let ddict = zstd_safe::DDict::try_create_by_reference(dict)
164+
.ok_or_else(invalid_dictionary)?;
165+
Ok(Self { ddict })
166+
}
167+
89168
/// Returns reference to `DDict` inner object
90169
pub fn as_ddict(&self) -> &DDict<'a> {
91170
&self.ddict

‎tests/invalid_dictionary.rs‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
//! A dictionary zstd cannot load: the dictionary magic number, then a corrupt header.
2+
//! `ZSTD_createCDict` / `ZSTD_createDDict` return NULL for it.
3+
4+
use zstd::dict::{DecoderDictionary, EncoderDictionary};
5+
6+
const CORRUPT: [u8; 16] = [
7+
0x37, 0xa4, 0x30, 0xec, 0x03, 0x47, 0x2f, 0x23, 0x5b, 0x2d, 0x10, 0x05, 0,
8+
0, 0, 0,
9+
];
10+
11+
#[test]
12+
fn try_copy_rejects_an_invalid_dictionary() {
13+
assert!(EncoderDictionary::try_copy(&CORRUPT, 3).is_err());
14+
assert!(DecoderDictionary::try_copy(&CORRUPT).is_err());
15+
}
16+
17+
#[cfg(feature = "experimental")]
18+
#[test]
19+
fn try_new_rejects_an_invalid_dictionary() {
20+
assert!(EncoderDictionary::try_new(&CORRUPT, 3).is_err());
21+
assert!(DecoderDictionary::try_new(&CORRUPT).is_err());
22+
}
23+
24+
#[test]
25+
fn try_copy_accepts_a_valid_dictionary() {
26+
let dict = b"a raw-content dictionary: any bytes without the magic number";
27+
let encoder = EncoderDictionary::try_copy(dict, 3).unwrap();
28+
let decoder = DecoderDictionary::try_copy(dict).unwrap();
29+
30+
let data = b"some data that shares words with the dictionary";
31+
let frame = zstd::bulk::Compressor::with_prepared_dictionary(&encoder)
32+
.unwrap()
33+
.compress(data)
34+
.unwrap();
35+
let back = zstd::bulk::Decompressor::with_prepared_dictionary(&decoder)
36+
.unwrap()
37+
.decompress(&frame, data.len())
38+
.unwrap();
39+
assert_eq!(back, data);
40+
}
41+
42+
#[test]
43+
#[should_panic]
44+
fn copy_still_panics_on_an_invalid_encoder_dictionary() {
45+
let _ = EncoderDictionary::copy(&CORRUPT, 3);
46+
}
47+
48+
#[test]
49+
#[should_panic]
50+
fn copy_still_panics_on_an_invalid_decoder_dictionary() {
51+
let _ = DecoderDictionary::copy(&CORRUPT);
52+
}

‎zstd-safe/src/lib.rs‎

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1527,12 +1527,32 @@ impl CDict<'static> {
15271527
}
15281528

15291529
impl<'a> CDict<'a> {
1530+
/// Prepare a dictionary to compress data, referencing `dict_buffer`.
1531+
///
1532+
/// # Panics
1533+
///
1534+
/// If loading this dictionary failed. See [`CDict::try_create_by_reference`].
15301535
#[cfg(feature = "experimental")]
15311536
#[cfg_attr(feature = "doc-cfg", doc(cfg(feature = "experimental")))]
15321537
pub fn create_by_reference(
15331538
dict_buffer: &'a [u8],
15341539
compression_level: CompressionLevel,
15351540
) -> Self {
1541+
Self::try_create_by_reference(dict_buffer, compression_level)
1542+
.expect("zstd returned null pointer")
1543+
}
1544+
1545+
/// Prepare a dictionary to compress data, referencing `dict_buffer`.
1546+
///
1547+
/// Returns `None` if zstd could not load the dictionary, for instance
1548+
/// because it starts with the dictionary magic number but its header is
1549+
/// corrupt.
1550+
#[cfg(feature = "experimental")]
1551+
#[cfg_attr(feature = "doc-cfg", doc(cfg(feature = "experimental")))]
1552+
pub fn try_create_by_reference(
1553+
dict_buffer: &'a [u8],
1554+
compression_level: CompressionLevel,
1555+
) -> Option<Self> {
15361556
#[cfg(feature = "with-rust-allocator")]
15371557
let ptr = unsafe {
15381558
rust_allocator::create_cdict(
@@ -1549,10 +1569,7 @@ impl<'a> CDict<'a> {
15491569
compression_level,
15501570
)
15511571
};
1552-
CDict(
1553-
NonNull::new(ptr).expect("zstd returned null pointer"),
1554-
PhantomData,
1555-
)
1572+
Some(CDict(NonNull::new(ptr)?, PhantomData))
15561573
}
15571574

15581575
/// Returns the _current_ memory usage of this dictionary.
@@ -1608,11 +1625,23 @@ pub fn compress_using_cdict(
16081625
pub struct DDict<'a>(NonNull<zstd_sys::ZSTD_DDict>, PhantomData<&'a ()>);
16091626

16101627
impl DDict<'static> {
1628+
/// Prepare a dictionary to decompress data.
1629+
///
1630+
/// The dictionary content will be copied internally, and does not need to be kept around.
1631+
///
1632+
/// # Panics
1633+
///
1634+
/// If loading this dictionary failed. See [`DDict::try_create`].
16111635
pub fn create(dict_buffer: &[u8]) -> Self {
16121636
Self::try_create(dict_buffer)
16131637
.expect("zstd returned null pointer when creating dict")
16141638
}
16151639

1640+
/// Prepare a dictionary to decompress data.
1641+
///
1642+
/// The dictionary content will be copied internally, and does not need to be kept around.
1643+
/// Returns `None` if zstd could not load the dictionary, for instance because it starts
1644+
/// with the dictionary magic number but its header is corrupt.
16161645
pub fn try_create(dict_buffer: &[u8]) -> Option<Self> {
16171646
#[cfg(feature = "with-rust-allocator")]
16181647
let ptr = unsafe {
@@ -1640,9 +1669,25 @@ impl<'a> DDict<'a> {
16401669
/// Wraps the `ZSTD_createDDict_byReference()` function.
16411670
///
16421671
/// The dictionary will keep referencing `dict_buffer`.
1672+
///
1673+
/// # Panics
1674+
///
1675+
/// If loading this dictionary failed. See [`DDict::try_create_by_reference`].
16431676
#[cfg(feature = "experimental")]
16441677
#[cfg_attr(feature = "doc-cfg", doc(cfg(feature = "experimental")))]
16451678
pub fn create_by_reference(dict_buffer: &'a [u8]) -> Self {
1679+
Self::try_create_by_reference(dict_buffer)
1680+
.expect("zstd returned null pointer")
1681+
}
1682+
1683+
/// Wraps the `ZSTD_createDDict_byReference()` function.
1684+
///
1685+
/// The dictionary will keep referencing `dict_buffer`. Returns `None` if
1686+
/// zstd could not load the dictionary, for instance because it starts with
1687+
/// the dictionary magic number but its header is corrupt.
1688+
#[cfg(feature = "experimental")]
1689+
#[cfg_attr(feature = "doc-cfg", doc(cfg(feature = "experimental")))]
1690+
pub fn try_create_by_reference(dict_buffer: &'a [u8]) -> Option<Self> {
16461691
#[cfg(feature = "with-rust-allocator")]
16471692
let ptr = unsafe {
16481693
rust_allocator::create_ddict(
@@ -1657,10 +1702,7 @@ impl<'a> DDict<'a> {
16571702
dict_buffer.len(),
16581703
)
16591704
};
1660-
DDict(
1661-
NonNull::new(ptr).expect("zstd returned null pointer"),
1662-
PhantomData,
1663-
)
1705+
Some(DDict(NonNull::new(ptr)?, PhantomData))
16641706
}
16651707

16661708
/// Returns the dictionary ID for this dict.

0 commit comments

Comments
 (0)