Skip to content

Commit 4cf3437

Browse files
authored
feat(datafusion): support clear_consumers (#1213)
1 parent dc049cb commit 4cf3437

5 files changed

Lines changed: 838 additions & 1 deletion

File tree

‎crates/integrations/datafusion/src/procedures.rs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
//! - `CALL sys.delete_tag(table => '...', tag => '...')`
2323
//! - `CALL sys.rename_tag(table => '...', tag => '...', target_tag => '...')`
2424
//! - `CALL sys.reset_consumer(table => '...', consumer_id => '...'[, next_snapshot_id => ...])`
25+
//! - `CALL sys.clear_consumers(table => '...'[, including_consumers => '...', excluding_consumers => '...'])`
2526
//! - `CALL sys.rollback_to(table => '...', snapshot_id => ... | tag => '...')`
2627
//! - `CALL sys.rollback_to_timestamp(table => '...', timestamp => ...)`
2728
//! - `CALL sys.create_tag_from_timestamp(table => '...', tag => '...', timestamp => ...)`
@@ -206,6 +207,7 @@ fn declared_parameters(proc_name: &str) -> Option<&'static [&'static str]> {
206207
"delete_tag" => &["table", "tag"],
207208
"rename_tag" => &["table", "tag", "target_tag"],
208209
"reset_consumer" => &["table", "consumer_id", "next_snapshot_id"],
210+
"clear_consumers" => &["table", "including_consumers", "excluding_consumers"],
209211
"rollback_to" => &["table", "snapshot_id", "tag"],
210212
"rollback_to_timestamp" => &["table", "timestamp"],
211213
"create_tag_from_timestamp" => &["table", "tag", "timestamp"],
@@ -318,6 +320,7 @@ pub async fn execute_call(
318320
"delete_tag" => proc_delete_tag(ctx, catalog, catalog_name, &args).await,
319321
"rename_tag" => proc_rename_tag(ctx, catalog, catalog_name, &args).await,
320322
"reset_consumer" => proc_reset_consumer(ctx, catalog, catalog_name, &args).await,
323+
"clear_consumers" => proc_clear_consumers(ctx, catalog, catalog_name, &args).await,
321324
"rollback_to" => proc_rollback_to(ctx, catalog, catalog_name, &args).await,
322325
"rollback_to_timestamp" => {
323326
proc_rollback_to_timestamp(ctx, catalog, catalog_name, &args).await
@@ -518,6 +521,34 @@ async fn proc_reset_consumer(
518521
ok_result(ctx)
519522
}
520523

524+
async fn proc_clear_consumers(
525+
ctx: &SessionContext,
526+
catalog: &Arc<dyn Catalog>,
527+
catalog_name: &str,
528+
args: &HashMap<String, String>,
529+
) -> DFResult<DataFrame> {
530+
let table = get_table(catalog, catalog_name, args).await?;
531+
if table.rest_env().is_some() {
532+
return Err(DataFusionError::NotImplemented(
533+
"clear_consumers is not supported for tables managed by a REST catalog".to_string(),
534+
));
535+
}
536+
if table.schema().core_options().is_format_table() {
537+
return Err(DataFusionError::NotImplemented(
538+
"clear_consumers requires a FileStoreTable; format-table is not supported".to_string(),
539+
));
540+
}
541+
table
542+
.consumer_manager()
543+
.clear(
544+
args.get("including_consumers").map(String::as_str),
545+
args.get("excluding_consumers").map(String::as_str),
546+
)
547+
.await
548+
.map_err(to_datafusion_error)?;
549+
ok_result(ctx)
550+
}
551+
521552
async fn proc_create_tag(
522553
ctx: &SessionContext,
523554
catalog: &Arc<dyn Catalog>,
Lines changed: 325 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,325 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
mod common;
19+
20+
use std::collections::BTreeMap;
21+
use std::path::{Path, PathBuf};
22+
23+
use common::{assert_sql_error, create_sql_context, create_test_env, exec, string_value};
24+
use paimon::catalog::Identifier;
25+
use paimon::spec::{DataType, IntType, Schema};
26+
use paimon::table::{BranchManager, Table};
27+
use paimon::Catalog;
28+
use paimon_datafusion::SQLContext;
29+
30+
async fn table_with_snapshot() -> (tempfile::TempDir, SQLContext, Table) {
31+
let (tmp, catalog) = create_test_env();
32+
let context = create_sql_context(catalog.clone()).await;
33+
exec(&context, "CREATE SCHEMA paimon.test_db").await;
34+
exec(&context, "CREATE TABLE paimon.test_db.events (id INT)").await;
35+
exec(&context, "INSERT INTO paimon.test_db.events VALUES (1)").await;
36+
let table = catalog
37+
.get_table(&Identifier::new("test_db", "events"))
38+
.await
39+
.unwrap();
40+
(tmp, context, table)
41+
}
42+
43+
async fn assert_ok(context: &SQLContext, sql: &str) {
44+
let batches = context.sql(sql).await.unwrap().collect().await.unwrap();
45+
assert_eq!(batches.len(), 1);
46+
assert_eq!(batches[0].num_rows(), 1);
47+
assert_eq!(batches[0].num_columns(), 1);
48+
assert_eq!(batches[0].schema().field(0).name(), "result");
49+
assert_eq!(string_value(batches[0].column(0).as_ref(), 0), "OK");
50+
}
51+
52+
fn table_files(root: &Path) -> BTreeMap<PathBuf, Vec<u8>> {
53+
let mut files = BTreeMap::new();
54+
for entry in std::fs::read_dir(root).unwrap() {
55+
let path = entry.unwrap().path();
56+
if path.is_dir() {
57+
files.extend(table_files(&path));
58+
} else {
59+
files.insert(path.clone(), std::fs::read(path).unwrap());
60+
}
61+
}
62+
files
63+
}
64+
65+
#[tokio::test]
66+
async fn clear_filters_ids_and_preserves_exceptions_other_files_and_branches() {
67+
let (tmp, context, table) = table_with_snapshot().await;
68+
let manager = table.consumer_manager();
69+
manager.reset("job-old", 1).await.unwrap();
70+
manager.reset("job-keep", 2).await.unwrap();
71+
let snapshot = table.snapshot_manager().get_snapshot(1).await.unwrap();
72+
table.tag_manager().create("base", &snapshot).await.unwrap();
73+
BranchManager::new(table.file_io().clone(), table.location().to_string())
74+
.create_branch_from_tag("dev", "base")
75+
.await
76+
.unwrap();
77+
let branch = table.copy_with_branch("dev").await.unwrap();
78+
branch.consumer_manager().reset("job-old", 3).await.unwrap();
79+
80+
let root = tmp.path().join("test_db.db/events");
81+
let directory = root.join("consumer");
82+
for name in [
83+
"consumer-job-old",
84+
"consumer-job-keep-extra",
85+
"consumer-other",
86+
] {
87+
std::fs::write(directory.join(name), "{").unwrap();
88+
}
89+
std::fs::write(
90+
directory.join("consumer-job-keep"),
91+
"{ \"nextSnapshot\": 2 }\n",
92+
)
93+
.unwrap();
94+
std::fs::write(directory.join("notes"), "keep").unwrap();
95+
std::fs::write(directory.join("consumer-pre-job-old"), "keep").unwrap();
96+
std::fs::create_dir(directory.join("consumer-job-directory")).unwrap();
97+
std::fs::write(directory.join("consumer-job-directory/checkpoint"), "keep").unwrap();
98+
let mut expected = table_files(&root);
99+
expected
100+
.remove(&directory.join("consumer-job-old"))
101+
.unwrap();
102+
expected
103+
.remove(&directory.join("consumer-job-keep-extra"))
104+
.unwrap();
105+
106+
for _ in 0..2 {
107+
assert_ok(&context, "CALL paimon.sys.clear_consumers(table => 'paimon.test_db.events', including_consumers => 'job-.*', excluding_consumers => 'job-keep')").await;
108+
assert_eq!(table_files(&root), expected);
109+
assert_eq!(manager.get("job-keep").await.unwrap(), Some(2));
110+
assert_eq!(
111+
branch.consumer_manager().get("job-old").await.unwrap(),
112+
Some(3)
113+
);
114+
}
115+
assert_sql_error(
116+
&context,
117+
"CALL sys.clear_consumers(table => 'test_db.events$branch_dev')",
118+
"does not exist",
119+
)
120+
.await;
121+
assert_eq!(table_files(&root), expected);
122+
}
123+
124+
#[tokio::test]
125+
async fn clear_accepts_defaults_and_does_not_create_a_missing_directory() {
126+
let (tmp, catalog) = create_test_env();
127+
let context = create_sql_context(catalog.clone()).await;
128+
exec(&context, "CREATE SCHEMA paimon.test_db").await;
129+
exec(&context, "CREATE TABLE paimon.test_db.events (id INT)").await;
130+
let table = catalog
131+
.get_table(&Identifier::new("test_db", "events"))
132+
.await
133+
.unwrap();
134+
let root = tmp.path().join("test_db.db/events");
135+
let original = table_files(&root);
136+
for _ in 0..2 {
137+
assert_ok(
138+
&context,
139+
"CALL sys.clear_consumers(table => 'test_db.events')",
140+
)
141+
.await;
142+
assert_eq!(table_files(&root), original);
143+
assert!(!root.join("consumer").exists());
144+
}
145+
for args in [
146+
"",
147+
", including_consumers => ''",
148+
", including_consumers => ' \t', excluding_consumers => ''",
149+
] {
150+
table.consumer_manager().reset("job", 2).await.unwrap();
151+
assert_ok(
152+
&context,
153+
&format!("CALL sys.clear_consumers(table => 'test_db.events'{args})"),
154+
)
155+
.await;
156+
assert_eq!(table.consumer_manager().get("job").await.unwrap(), None);
157+
assert_eq!(table_files(&root), original);
158+
}
159+
table.consumer_manager().reset("job", 2).await.unwrap();
160+
assert_ok(
161+
&context,
162+
"CALL sys.clear_consumers(table => 'test_db.events', excluding_consumers => 'job')",
163+
)
164+
.await;
165+
assert_eq!(table.consumer_manager().get("job").await.unwrap(), Some(2));
166+
}
167+
168+
#[tokio::test]
169+
async fn clear_rejects_invalid_arguments_before_changing_files() {
170+
let (tmp, context, table) = table_with_snapshot().await;
171+
table.consumer_manager().reset("job", 1).await.unwrap();
172+
let root = tmp.path().join("test_db.db/events");
173+
let original = table_files(&root);
174+
for (args, message) in [
175+
("", "Missing required argument: 'table'"),
176+
("'test_db.events'", "require named arguments"),
177+
("table => 'other.test_db.events'", "Catalog name mismatch"),
178+
("table => 'test_db.events', including_consumer => 'job'", "is unknown"),
179+
("table => 'test_db.events', excluding_consumer => 'job'", "is unknown"),
180+
("table => 'test_db.events', including_consumers => 'job', INCLUDING_CONSUMERS => 'other'", "is duplicated"),
181+
("table => 'test_db.events', excluding_consumers => 'job', excluding_consumers => 'other'", "is duplicated"),
182+
("table => 'test_db.events', including_consumers => NULL", "Unsupported argument value"),
183+
("table => 'test_db.events', excluding_consumers => NULL", "Unsupported argument value"),
184+
("table => 'test_db.events', including_consumers => '['", "Invalid including_consumers regex"),
185+
("table => 'test_db.events', excluding_consumers => '['", "Invalid excluding_consumers regex"),
186+
("table => 'test_db.events', including_consumers => 'no-match', excluding_consumers => '(?=job)job'", "Invalid excluding_consumers regex"),
187+
(r"table => 'test_db.events', including_consumers => '(job)\1'", "Invalid including_consumers regex"),
188+
] {
189+
assert_sql_error(&context, &format!("CALL sys.clear_consumers({args})"), message).await;
190+
assert_eq!(table_files(&root), original);
191+
}
192+
}
193+
194+
#[tokio::test]
195+
async fn clear_uses_whole_id_matching_with_rust_regex_flags() {
196+
let (_tmp, context, table) = table_with_snapshot().await;
197+
for (pattern, id, deleted) in [
198+
("job", "job-old", false),
199+
("a|ab", "ab", true),
200+
("(?m)^job$", "job\nother", false),
201+
("(?s)job.*", "job\nother", true),
202+
("(?i)job", "JOB", true),
203+
("(?x) job # trailing comment", "job", true),
204+
(" job", "job", false),
205+
] {
206+
table.consumer_manager().reset(id, 2).await.unwrap();
207+
assert_ok(&context, &format!("CALL sys.clear_consumers(table => 'test_db.events', including_consumers => '{pattern}')")).await;
208+
assert_eq!(
209+
table.consumer_manager().get(id).await.unwrap(),
210+
if deleted { None } else { Some(2) },
211+
"{pattern:?}: {id:?}"
212+
);
213+
table.consumer_manager().delete(id).await.unwrap();
214+
}
215+
}
216+
217+
#[tokio::test]
218+
async fn clear_rejects_format_tables_without_changing_files() {
219+
let (tmp, catalog) = create_test_env();
220+
catalog
221+
.create_database("test_db", false, Default::default())
222+
.await
223+
.unwrap();
224+
let identifier = Identifier::new("test_db", "events");
225+
let schema = Schema::builder()
226+
.column("id", DataType::Int(IntType::new()))
227+
.option("type", "format-table")
228+
.option("file.format", "parquet")
229+
.build()
230+
.unwrap();
231+
catalog
232+
.create_table(&identifier, schema, false)
233+
.await
234+
.unwrap();
235+
catalog
236+
.get_table(&identifier)
237+
.await
238+
.unwrap()
239+
.consumer_manager()
240+
.reset("job", 2)
241+
.await
242+
.unwrap();
243+
let context = create_sql_context(catalog).await;
244+
let root = tmp.path().join("test_db.db/events");
245+
let original = table_files(&root);
246+
assert_sql_error(
247+
&context,
248+
"CALL sys.clear_consumers(table => 'test_db.events')",
249+
"requires a FileStoreTable",
250+
)
251+
.await;
252+
assert_eq!(table_files(&root), original);
253+
}
254+
255+
#[tokio::test]
256+
async fn clear_rejects_engine_served_table_types_without_changing_files() {
257+
for table_type in ["object-table", "iceberg-table", "lance-table"] {
258+
let (tmp, catalog) = create_test_env();
259+
catalog
260+
.create_database("test_db", false, Default::default())
261+
.await
262+
.unwrap();
263+
let schema = Schema::builder()
264+
.column("id", DataType::Int(IntType::new()))
265+
.option("type", table_type)
266+
.build()
267+
.unwrap();
268+
catalog
269+
.create_table(&Identifier::new("test_db", "events"), schema, false)
270+
.await
271+
.unwrap();
272+
let root = tmp.path().join("test_db.db/events");
273+
std::fs::create_dir(root.join("consumer")).unwrap();
274+
std::fs::write(root.join("consumer/consumer-job"), "{").unwrap();
275+
let original = table_files(&root);
276+
let context = create_sql_context(catalog).await;
277+
assert_sql_error(
278+
&context,
279+
"CALL sys.clear_consumers(table => 'test_db.events')",
280+
"cannot be read as a Paimon table",
281+
)
282+
.await;
283+
assert_eq!(table_files(&root), original);
284+
}
285+
}
286+
287+
#[tokio::test]
288+
async fn clear_preserves_running_scan_progress_and_checkpoint_protocol() {
289+
let (_tmp, context, table) = table_with_snapshot().await;
290+
table.consumer_manager().reset("keep", 2).await.unwrap();
291+
let mut running = table.new_stream_scan().unwrap();
292+
running
293+
.with_consumer_id("job")
294+
.unwrap()
295+
.restore(Some(1))
296+
.unwrap();
297+
running.plan().await.unwrap().unwrap();
298+
running
299+
.notify_checkpoint_complete(running.checkpoint())
300+
.await
301+
.unwrap();
302+
assert_eq!(table.consumer_manager().get("job").await.unwrap(), Some(2));
303+
304+
assert_ok(
305+
&context,
306+
"CALL sys.clear_consumers(table => 'test_db.events', excluding_consumers => 'keep')",
307+
)
308+
.await;
309+
assert_eq!(table.consumer_manager().get("job").await.unwrap(), None);
310+
assert_eq!(table.consumer_manager().get("keep").await.unwrap(), Some(2));
311+
assert_eq!(running.checkpoint(), Some(2));
312+
assert!(running.plan().await.unwrap().is_none());
313+
let mut fresh = table.new_stream_scan().unwrap();
314+
fresh.with_consumer_id("job").unwrap();
315+
let plan = fresh.plan().await.unwrap().unwrap();
316+
assert_eq!(plan.snapshot_id(), Some(1));
317+
assert!(plan.splits().iter().all(|split| !split.is_streaming()));
318+
319+
running
320+
.notify_checkpoint_complete(running.checkpoint())
321+
.await
322+
.unwrap();
323+
assert_eq!(table.consumer_manager().get("job").await.unwrap(), Some(2));
324+
assert_eq!(table.consumer_manager().get("keep").await.unwrap(), Some(2));
325+
}

0 commit comments

Comments
 (0)