Skip to content

Commit 13e370a

Browse files
authored
feat: support input changelog for primary key writes (#318)
1 parent 63a5374 commit 13e370a

8 files changed

Lines changed: 1032 additions & 101 deletions

File tree

‎crates/integrations/datafusion/tests/pk_tables.rs‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1460,9 +1460,9 @@ async fn test_pk_partitioned_multi_bucket() {
14601460

14611461
// ======================= Error Cases =======================
14621462

1463-
/// PK table with changelog-producer=input should be rejected.
1463+
/// PK table with changelog-producer=input should write through DataFusion SQL.
14641464
#[tokio::test]
1465-
async fn test_pk_reject_changelog_producer_input() {
1465+
async fn test_pk_input_changelog_write_read() {
14661466
let (_tmp, sql_context) = setup_sql_context().await;
14671467

14681468
sql_context
@@ -1475,18 +1475,24 @@ async fn test_pk_reject_changelog_producer_input() {
14751475
.await
14761476
.unwrap();
14771477

1478-
let result = sql_context
1479-
.sql("INSERT INTO paimon.test_db.t_changelog VALUES (1, 'alice')")
1480-
.await;
1478+
sql_context
1479+
.sql(
1480+
"INSERT INTO paimon.test_db.t_changelog VALUES
1481+
(1, 'alice'), (1, 'bob'), (2, 'carol')",
1482+
)
1483+
.await
1484+
.unwrap()
1485+
.collect()
1486+
.await
1487+
.unwrap();
14811488

1482-
let is_err = match result {
1483-
Err(_) => true,
1484-
Ok(df) => df.collect().await.is_err(),
1485-
};
1486-
assert!(
1487-
is_err,
1488-
"PK table with changelog-producer=input should reject writes"
1489-
);
1489+
let rows = collect_id_name(
1490+
&sql_context,
1491+
"SELECT id, name FROM paimon.test_db.t_changelog ORDER BY id",
1492+
)
1493+
.await;
1494+
1495+
assert_eq!(rows, vec![(1, "bob".to_string()), (2, "carol".to_string())]);
14901496
}
14911497

14921498
// ======================= String Primary Key =======================

‎crates/paimon/src/spec/core_options.rs‎

Lines changed: 165 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ const COMMIT_MAX_RETRY_WAIT_OPTION: &str = "commit.max-retry-wait";
4040
const FILE_COMPRESSION_OPTION: &str = "file.compression";
4141
const FILE_COMPRESSION_ZSTD_LEVEL_OPTION: &str = "file.compression.zstd-level";
4242
const FILE_FORMAT_OPTION: &str = "file.format";
43+
const CHANGELOG_FILE_PREFIX_OPTION: &str = "changelog-file.prefix";
44+
const CHANGELOG_FILE_FORMAT_OPTION: &str = "changelog-file.format";
45+
const CHANGELOG_FILE_COMPRESSION_OPTION: &str = "changelog-file.compression";
46+
const CHANGELOG_FILE_STATS_MODE_OPTION: &str = "changelog-file.stats-mode";
4347
const ROW_TRACKING_ENABLED_OPTION: &str = "row-tracking.enabled";
4448
const WRITE_PARQUET_BUFFER_SIZE_OPTION: &str = "write.parquet-buffer-size";
4549
const SEQUENCE_FIELD_OPTION: &str = "sequence.field";
@@ -55,6 +59,7 @@ pub const SCAN_VERSION_OPTION: &str = "scan.version";
5559
const DEFAULT_SOURCE_SPLIT_TARGET_SIZE: i64 = 128 * 1024 * 1024;
5660
const DEFAULT_SOURCE_SPLIT_OPEN_FILE_COST: i64 = 4 * 1024 * 1024;
5761
const DEFAULT_PARTITION_DEFAULT_NAME: &str = "__DEFAULT_PARTITION__";
62+
const DEFAULT_CHANGELOG_FILE_PREFIX: &str = "changelog-";
5863
const DEFAULT_TARGET_FILE_SIZE: i64 = 256 * 1024 * 1024;
5964
const DEFAULT_WRITE_PARQUET_BUFFER_SIZE: i64 = 256 * 1024 * 1024;
6065
const DYNAMIC_BUCKET_TARGET_ROW_NUM_OPTION: &str = "dynamic-bucket.target-row-num";
@@ -75,6 +80,32 @@ pub enum MergeEngine {
7580
FirstRow,
7681
}
7782

83+
/// Changelog producer for table writes.
84+
///
85+
/// Reference: Java `CoreOptions.ChangelogProducer`.
86+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
87+
pub enum ChangelogProducer {
88+
/// No changelog file.
89+
None,
90+
/// Double write input rows to changelog files.
91+
Input,
92+
/// Generate changelog files during full compaction.
93+
FullCompaction,
94+
/// Generate changelog files through lookup compaction.
95+
Lookup,
96+
}
97+
98+
impl ChangelogProducer {
99+
pub fn as_str(&self) -> &'static str {
100+
match self {
101+
Self::None => "none",
102+
Self::Input => "input",
103+
Self::FullCompaction => "full-compaction",
104+
Self::Lookup => "lookup",
105+
}
106+
}
107+
}
108+
78109
/// Format the bucket directory name for a given bucket number.
79110
/// Returns `"bucket-postpone"` for `POSTPONE_BUCKET` (-2), otherwise `"bucket-{N}"`.
80111
pub fn bucket_dir_name(bucket: i32) -> String {
@@ -138,14 +169,30 @@ impl<'a> CoreOptions<'a> {
138169
}
139170
}
140171

141-
/// Changelog producer setting. Default is "none".
172+
/// Raw changelog producer setting. Default is `"none"`.
142173
pub fn changelog_producer(&self) -> &str {
143174
self.options
144175
.get(CHANGELOG_PRODUCER_OPTION)
145176
.map(String::as_str)
146177
.unwrap_or("none")
147178
}
148179

180+
/// Typed changelog producer setting. Default is `None`.
181+
pub fn try_changelog_producer(&self) -> crate::Result<ChangelogProducer> {
182+
match self.options.get(CHANGELOG_PRODUCER_OPTION) {
183+
None => Ok(ChangelogProducer::None),
184+
Some(v) => match v.to_ascii_lowercase().as_str() {
185+
"none" => Ok(ChangelogProducer::None),
186+
"input" => Ok(ChangelogProducer::Input),
187+
"full-compaction" => Ok(ChangelogProducer::FullCompaction),
188+
"lookup" => Ok(ChangelogProducer::Lookup),
189+
other => Err(crate::Error::Unsupported {
190+
message: format!("Unsupported changelog-producer: '{other}'"),
191+
}),
192+
},
193+
}
194+
}
195+
149196
/// The `rowkind.field` option: a user column whose value encodes the row kind.
150197
pub fn rowkind_field(&self) -> Option<&str> {
151198
self.options.get(ROWKIND_FIELD_OPTION).map(String::as_str)
@@ -363,6 +410,43 @@ impl<'a> CoreOptions<'a> {
363410
.unwrap_or(1)
364411
}
365412

413+
/// File name prefix for changelog files. Default is `"changelog-"`.
414+
pub fn changelog_file_prefix(&self) -> &str {
415+
self.options
416+
.get(CHANGELOG_FILE_PREFIX_OPTION)
417+
.map(String::as_str)
418+
.unwrap_or(DEFAULT_CHANGELOG_FILE_PREFIX)
419+
}
420+
421+
/// Effective file format for changelog files.
422+
///
423+
/// When `changelog-file.format` is not configured, Java Paimon falls back
424+
/// to the table `file.format`.
425+
pub fn changelog_file_format(&self) -> &str {
426+
self.options
427+
.get(CHANGELOG_FILE_FORMAT_OPTION)
428+
.map(String::as_str)
429+
.unwrap_or_else(|| self.file_format())
430+
}
431+
432+
/// Effective compression codec for changelog files.
433+
///
434+
/// When `changelog-file.compression` is not configured, Java Paimon falls
435+
/// back to the table `file.compression`.
436+
pub fn changelog_file_compression(&self) -> &str {
437+
self.options
438+
.get(CHANGELOG_FILE_COMPRESSION_OPTION)
439+
.map(String::as_str)
440+
.unwrap_or_else(|| self.file_compression())
441+
}
442+
443+
/// Metadata stats collection mode for changelog files, if configured.
444+
pub fn changelog_file_stats_mode(&self) -> Option<&str> {
445+
self.options
446+
.get(CHANGELOG_FILE_STATS_MODE_OPTION)
447+
.map(String::as_str)
448+
}
449+
366450
/// Parquet writer in-progress buffer size limit. Default is 256MB.
367451
/// When the buffered data exceeds this, the writer flushes the current row group.
368452
pub fn write_parquet_buffer_size(&self) -> i64 {
@@ -546,6 +630,86 @@ mod tests {
546630
assert_eq!(core.merge_engine().unwrap(), MergeEngine::PartialUpdate);
547631
}
548632

633+
#[test]
634+
fn test_changelog_producer_defaults_to_none() {
635+
let options = HashMap::new();
636+
let core = CoreOptions::new(&options);
637+
638+
assert_eq!(core.changelog_producer(), "none");
639+
assert_eq!(
640+
core.try_changelog_producer().unwrap(),
641+
ChangelogProducer::None
642+
);
643+
}
644+
645+
#[test]
646+
fn test_changelog_producer_accepts_known_values() {
647+
for (value, expected) in [
648+
("none", ChangelogProducer::None),
649+
("input", ChangelogProducer::Input),
650+
("full-compaction", ChangelogProducer::FullCompaction),
651+
("lookup", ChangelogProducer::Lookup),
652+
("INPUT", ChangelogProducer::Input),
653+
] {
654+
let options = HashMap::from([(CHANGELOG_PRODUCER_OPTION.to_string(), value.into())]);
655+
let core = CoreOptions::new(&options);
656+
657+
assert_eq!(core.try_changelog_producer().unwrap(), expected);
658+
}
659+
}
660+
661+
#[test]
662+
fn test_changelog_producer_rejects_unknown_values() {
663+
let options = HashMap::from([(CHANGELOG_PRODUCER_OPTION.to_string(), "other".into())]);
664+
let core = CoreOptions::new(&options);
665+
666+
let err = core
667+
.try_changelog_producer()
668+
.expect_err("unknown producer should fail");
669+
assert!(
670+
matches!(err, crate::Error::Unsupported { message } if message.contains("Unsupported changelog-producer"))
671+
);
672+
}
673+
674+
#[test]
675+
fn test_changelog_file_options_defaults_and_overrides() {
676+
let default_options = HashMap::from([
677+
(FILE_FORMAT_OPTION.to_string(), "avro".to_string()),
678+
(FILE_COMPRESSION_OPTION.to_string(), "snappy".to_string()),
679+
]);
680+
let default_core = CoreOptions::new(&default_options);
681+
682+
assert_eq!(default_core.changelog_file_prefix(), "changelog-");
683+
assert_eq!(default_core.changelog_file_format(), "avro");
684+
assert_eq!(default_core.changelog_file_compression(), "snappy");
685+
assert_eq!(default_core.changelog_file_stats_mode(), None);
686+
687+
let custom_options = HashMap::from([
688+
(
689+
CHANGELOG_FILE_PREFIX_OPTION.to_string(),
690+
"custom-".to_string(),
691+
),
692+
(
693+
CHANGELOG_FILE_FORMAT_OPTION.to_string(),
694+
"parquet".to_string(),
695+
),
696+
(
697+
CHANGELOG_FILE_COMPRESSION_OPTION.to_string(),
698+
"zstd".to_string(),
699+
),
700+
(
701+
CHANGELOG_FILE_STATS_MODE_OPTION.to_string(),
702+
"counts".to_string(),
703+
),
704+
]);
705+
let custom_core = CoreOptions::new(&custom_options);
706+
707+
assert_eq!(custom_core.changelog_file_prefix(), "custom-");
708+
assert_eq!(custom_core.changelog_file_format(), "parquet");
709+
assert_eq!(custom_core.changelog_file_compression(), "zstd");
710+
assert_eq!(custom_core.changelog_file_stats_mode(), Some("counts"));
711+
}
712+
549713
#[test]
550714
fn test_commit_options_defaults() {
551715
let options = HashMap::new();

‎crates/paimon/src/table/commit_message.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ pub struct CommitMessage {
2929
pub bucket: i32,
3030
/// New data files to be added.
3131
pub new_files: Vec<DataFileMeta>,
32+
/// New changelog files to be added.
33+
pub new_changelog_files: Vec<DataFileMeta>,
3234
/// New index files to be added (used by dynamic bucket mode).
3335
pub new_index_files: Vec<IndexFileMeta>,
3436
/// Files to be deleted (copy-on-write rewrite: old files replaced by new_files).
@@ -41,6 +43,7 @@ impl CommitMessage {
4143
partition,
4244
bucket,
4345
new_files,
46+
new_changelog_files: Vec::new(),
4447
new_index_files: Vec::new(),
4548
deleted_files: Vec::new(),
4649
}

0 commit comments

Comments
 (0)