Skip to content

Commit c9a25b2

Browse files
authored
fix(encryption) [18/N] wire in ags1 file length for tamper proofing (#3236)
* feat(encryption) [18N] wire in ags1 file length for tamper proofing * remove debug * expect * fix test * fix opendal * make more clear * Add oversized tests * don't do unreachable * import fix * Assert length in streaming tests * DRAFT try key_metadata_with_length * fix(encryption): address key metadata length review
1 parent 830bb88 commit c9a25b2

20 files changed

Lines changed: 568 additions & 178 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,15 @@ All notable changes to this project will be documented in this file.
2424
The format is based on [Keep a Changelog](https://keepachangelog.com/)
2525
and this project adheres to [Semantic Versioning](https://semver.org/).
2626

27+
## [Unreleased]
28+
29+
### Breaking Changes
30+
31+
* `FileWrite::close`, `EncryptedOutputFile::write`, `ManifestListWriter::close`, and `PuffinWriter::close` now return `Result<io::FileMetadata>` instead of `Result<()>`. The returned size is the on-disk size, including encryption overhead. Custom `FileWrite` implementations must return the number of bytes stored.
32+
* `EncryptedOutputFile::key_metadata()` is replaced by `key_metadata_with_saved_file_metadata(&FileMetadata)`. Pass the metadata returned by `write()` or the writer's `close()` to include the stored length before encoding key metadata.
33+
* `EncryptedInputFile::metadata()` is now synchronous and derives the plaintext size from key metadata without a storage stat. Remove `.await` from calls to this method.
34+
* AGS1 readers now require `StandardKeyMetadata::file_length` and reject missing or invalid lengths without falling back to a storage stat. AGS1-encrypted manifests, manifest lists, and Puffin files written by earlier development builds without this field must be rewritten using a build that can still read them before upgrading. This matches the Java client's read contract.
35+
2736
## [v0.10.1] - 2026-07-28
2837

2938
* chore: Update crossbeam-epoch to 0.9.20, pin CI ruff version to 0.15.22 by @dannycjones in https://github.com/apache/iceberg-rust/pull/2911

‎crates/iceberg/public-api.txt‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -273,15 +273,15 @@ pub struct iceberg::encryption::AesGcmFileWrite
273273
impl iceberg::encryption::AesGcmFileWrite
274274
pub fn iceberg::encryption::AesGcmFileWrite::new(inner: alloc::boxed::Box<dyn iceberg::io::FileWrite>, cipher: alloc::sync::Arc<iceberg::encryption::AesGcmCipher>, aad_prefix: impl core::convert::Into<alloc::boxed::Box<[u8]>>) -> Self
275275
impl iceberg::io::FileWrite for iceberg::encryption::AesGcmFileWrite
276-
pub fn iceberg::encryption::AesGcmFileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = iceberg::Result<()>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait
276+
pub fn iceberg::encryption::AesGcmFileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = iceberg::Result<iceberg::io::FileMetadata>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait
277277
pub fn iceberg::encryption::AesGcmFileWrite::write<'life0, 'async_trait>(&'life0 mut self, bs: bytes::bytes::Bytes) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = iceberg::Result<()>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait
278278
pub struct iceberg::encryption::EncryptedInputFile
279279
impl iceberg::encryption::EncryptedInputFile
280280
pub async fn iceberg::encryption::EncryptedInputFile::exists(&self) -> iceberg::Result<bool>
281281
pub fn iceberg::encryption::EncryptedInputFile::into_inner(self) -> iceberg::io::InputFile
282282
pub fn iceberg::encryption::EncryptedInputFile::key_metadata(&self) -> &iceberg::encryption::StandardKeyMetadata
283283
pub fn iceberg::encryption::EncryptedInputFile::location(&self) -> &str
284-
pub async fn iceberg::encryption::EncryptedInputFile::metadata(&self) -> iceberg::Result<iceberg::io::FileMetadata>
284+
pub fn iceberg::encryption::EncryptedInputFile::metadata(&self) -> iceberg::Result<iceberg::io::FileMetadata>
285285
pub fn iceberg::encryption::EncryptedInputFile::new(inner: iceberg::io::InputFile, key_metadata: iceberg::encryption::StandardKeyMetadata) -> Self
286286
pub async fn iceberg::encryption::EncryptedInputFile::read(&self) -> iceberg::Result<bytes::bytes::Bytes>
287287
pub async fn iceberg::encryption::EncryptedInputFile::reader(&self) -> iceberg::Result<alloc::boxed::Box<dyn iceberg::io::FileRead>>
@@ -291,10 +291,10 @@ pub struct iceberg::encryption::EncryptedOutputFile
291291
impl iceberg::encryption::EncryptedOutputFile
292292
pub async fn iceberg::encryption::EncryptedOutputFile::delete(&self) -> iceberg::Result<()>
293293
pub fn iceberg::encryption::EncryptedOutputFile::into_inner(self) -> iceberg::io::OutputFile
294-
pub fn iceberg::encryption::EncryptedOutputFile::key_metadata(&self) -> &iceberg::encryption::StandardKeyMetadata
294+
pub fn iceberg::encryption::EncryptedOutputFile::key_metadata_with_saved_file_metadata(&self, file_metadata: &iceberg::io::FileMetadata) -> iceberg::encryption::StandardKeyMetadata
295295
pub fn iceberg::encryption::EncryptedOutputFile::location(&self) -> &str
296296
pub fn iceberg::encryption::EncryptedOutputFile::new(inner: iceberg::io::OutputFile, key_metadata: iceberg::encryption::StandardKeyMetadata) -> Self
297-
pub async fn iceberg::encryption::EncryptedOutputFile::write(&self, bs: bytes::bytes::Bytes) -> iceberg::Result<()>
297+
pub async fn iceberg::encryption::EncryptedOutputFile::write(&self, bs: bytes::bytes::Bytes) -> iceberg::Result<iceberg::io::FileMetadata>
298298
pub async fn iceberg::encryption::EncryptedOutputFile::writer(&self) -> iceberg::Result<alloc::boxed::Box<dyn iceberg::io::FileWrite>>
299299
impl core::fmt::Debug for iceberg::encryption::EncryptedOutputFile
300300
pub fn iceberg::encryption::EncryptedOutputFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
@@ -1046,10 +1046,10 @@ pub fn iceberg::encryption::AesGcmFileRead::read<'life0, 'async_trait>(&'life0 s
10461046
impl<T: core::convert::AsRef<dyn iceberg::io::FileRead> + core::marker::Send + core::marker::Sync + core::marker::Unpin + 'static> iceberg::io::FileRead for T
10471047
pub fn T::read<'life0, 'async_trait>(&'life0 self, range: core::ops::range::Range<u64>) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = iceberg::Result<bytes::bytes::Bytes>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait
10481048
pub trait iceberg::io::FileWrite: core::marker::Send + core::marker::Unpin + 'static
1049-
pub fn iceberg::io::FileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = iceberg::Result<()>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait
1049+
pub fn iceberg::io::FileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = iceberg::Result<iceberg::io::FileMetadata>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait
10501050
pub fn iceberg::io::FileWrite::write<'life0, 'async_trait>(&'life0 mut self, bs: bytes::bytes::Bytes) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = iceberg::Result<()>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait
10511051
impl iceberg::io::FileWrite for iceberg::encryption::AesGcmFileWrite
1052-
pub fn iceberg::encryption::AesGcmFileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = iceberg::Result<()>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait
1052+
pub fn iceberg::encryption::AesGcmFileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = iceberg::Result<iceberg::io::FileMetadata>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait
10531053
pub fn iceberg::encryption::AesGcmFileWrite::write<'life0, 'async_trait>(&'life0 mut self, bs: bytes::bytes::Bytes) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = iceberg::Result<()>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait
10541054
pub trait iceberg::io::Storage: core::fmt::Debug + core::marker::Send + core::marker::Sync + typetag::Serialize + typetag::Deserialize
10551055
pub fn iceberg::io::Storage::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output = iceberg::Result<()>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait
@@ -1267,7 +1267,7 @@ pub async fn iceberg::puffin::PuffinReader::new_from_encrypted(encrypted_input:
12671267
pub struct iceberg::puffin::PuffinWriter
12681268
impl iceberg::puffin::PuffinWriter
12691269
pub async fn iceberg::puffin::PuffinWriter::add(&mut self, blob: iceberg::puffin::Blob, compression_codec: iceberg::compression::CompressionCodec) -> iceberg::Result<()>
1270-
pub async fn iceberg::puffin::PuffinWriter::close(self) -> iceberg::Result<()>
1270+
pub async fn iceberg::puffin::PuffinWriter::close(self) -> iceberg::Result<iceberg::io::FileMetadata>
12711271
pub async fn iceberg::puffin::PuffinWriter::new(output_file: &iceberg::io::OutputFile, properties: std::collections::hash::map::HashMap<alloc::string::String, alloc::string::String>, compress_footer: bool) -> iceberg::Result<Self>
12721272
pub async fn iceberg::puffin::PuffinWriter::new_from_encrypted(encrypted_output: &iceberg::encryption::EncryptedOutputFile, properties: std::collections::hash::map::HashMap<alloc::string::String, alloc::string::String>, compress_footer: bool) -> iceberg::Result<Self>
12731273
pub const iceberg::puffin::APACHE_DATASKETCHES_THETA_V1: &str
@@ -2138,7 +2138,7 @@ pub async fn iceberg::spec::ManifestListReader::load(&self) -> iceberg::Result<i
21382138
pub struct iceberg::spec::ManifestListWriter
21392139
impl iceberg::spec::ManifestListWriter
21402140
pub fn iceberg::spec::ManifestListWriter::add_manifests(&mut self, manifests: impl core::iter::traits::iterator::Iterator<Item = iceberg::spec::ManifestFile>) -> iceberg::Result<()>
2141-
pub async fn iceberg::spec::ManifestListWriter::close(self) -> iceberg::Result<()>
2141+
pub async fn iceberg::spec::ManifestListWriter::close(self) -> iceberg::Result<iceberg::io::FileMetadata>
21422142
pub fn iceberg::spec::ManifestListWriter::next_row_id(&self) -> core::option::Option<u64>
21432143
pub fn iceberg::spec::ManifestListWriter::v1(writer: alloc::boxed::Box<dyn iceberg::io::FileWrite>, snapshot_id: i64, parent_snapshot_id: core::option::Option<i64>) -> Self
21442144
pub fn iceberg::spec::ManifestListWriter::v2(writer: alloc::boxed::Box<dyn iceberg::io::FileWrite>, snapshot_id: i64, parent_snapshot_id: core::option::Option<i64>, sequence_number: i64) -> Self

‎crates/iceberg/src/arrow/caching_delete_file_loader.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1621,13 +1621,16 @@ mod tests {
16211621
let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
16221622
.unwrap()
16231623
.with_aad_prefix(b"test-aad-prefix!");
1624-
let encoded_key_metadata = key_metadata.encode().unwrap();
16251624

16261625
let blob = encode_dv_blob([2u64, 4]);
16271626
let plaintext_size = blob.len() as i64;
16281627
let dv_path = format!("{table_location}/deletes.puffin");
16291628
let output = EncryptedOutputFile::new(file_io.new_output(&dv_path).unwrap(), key_metadata);
1630-
output.write(Bytes::from(blob)).await.unwrap();
1629+
let file_metadata = output.write(Bytes::from(blob)).await.unwrap();
1630+
let encoded_key_metadata = output
1631+
.key_metadata_with_saved_file_metadata(&file_metadata)
1632+
.encode()
1633+
.unwrap();
16311634

16321635
// content_offset / content_size_in_bytes are in the plaintext coordinate space, distinct
16331636
// from the ciphertext's on-disk size (header, nonce, and tag overhead).

0 commit comments

Comments
 (0)