|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +mod common; |
| 19 | + |
| 20 | +use std::collections::BTreeMap; |
| 21 | +use std::path::{Path, PathBuf}; |
| 22 | + |
| 23 | +use common::{assert_sql_error, create_sql_context, create_test_env, exec, string_value}; |
| 24 | +use paimon::catalog::Identifier; |
| 25 | +use paimon::spec::{DataType, IntType, Schema}; |
| 26 | +use paimon::table::{BranchManager, Table}; |
| 27 | +use paimon::Catalog; |
| 28 | +use paimon_datafusion::SQLContext; |
| 29 | + |
| 30 | +async fn table_with_snapshot() -> (tempfile::TempDir, SQLContext, Table) { |
| 31 | + let (tmp, catalog) = create_test_env(); |
| 32 | + let context = create_sql_context(catalog.clone()).await; |
| 33 | + exec(&context, "CREATE SCHEMA paimon.test_db").await; |
| 34 | + exec(&context, "CREATE TABLE paimon.test_db.events (id INT)").await; |
| 35 | + exec(&context, "INSERT INTO paimon.test_db.events VALUES (1)").await; |
| 36 | + let table = catalog |
| 37 | + .get_table(&Identifier::new("test_db", "events")) |
| 38 | + .await |
| 39 | + .unwrap(); |
| 40 | + (tmp, context, table) |
| 41 | +} |
| 42 | + |
| 43 | +async fn assert_ok(context: &SQLContext, sql: &str) { |
| 44 | + let batches = context.sql(sql).await.unwrap().collect().await.unwrap(); |
| 45 | + assert_eq!(batches.len(), 1); |
| 46 | + assert_eq!(batches[0].num_rows(), 1); |
| 47 | + assert_eq!(batches[0].num_columns(), 1); |
| 48 | + assert_eq!(batches[0].schema().field(0).name(), "result"); |
| 49 | + assert_eq!(string_value(batches[0].column(0).as_ref(), 0), "OK"); |
| 50 | +} |
| 51 | + |
| 52 | +fn table_files(root: &Path) -> BTreeMap<PathBuf, Vec<u8>> { |
| 53 | + let mut files = BTreeMap::new(); |
| 54 | + for entry in std::fs::read_dir(root).unwrap() { |
| 55 | + let path = entry.unwrap().path(); |
| 56 | + if path.is_dir() { |
| 57 | + files.extend(table_files(&path)); |
| 58 | + } else { |
| 59 | + files.insert(path.clone(), std::fs::read(path).unwrap()); |
| 60 | + } |
| 61 | + } |
| 62 | + files |
| 63 | +} |
| 64 | + |
| 65 | +#[tokio::test] |
| 66 | +async fn clear_filters_ids_and_preserves_exceptions_other_files_and_branches() { |
| 67 | + let (tmp, context, table) = table_with_snapshot().await; |
| 68 | + let manager = table.consumer_manager(); |
| 69 | + manager.reset("job-old", 1).await.unwrap(); |
| 70 | + manager.reset("job-keep", 2).await.unwrap(); |
| 71 | + let snapshot = table.snapshot_manager().get_snapshot(1).await.unwrap(); |
| 72 | + table.tag_manager().create("base", &snapshot).await.unwrap(); |
| 73 | + BranchManager::new(table.file_io().clone(), table.location().to_string()) |
| 74 | + .create_branch_from_tag("dev", "base") |
| 75 | + .await |
| 76 | + .unwrap(); |
| 77 | + let branch = table.copy_with_branch("dev").await.unwrap(); |
| 78 | + branch.consumer_manager().reset("job-old", 3).await.unwrap(); |
| 79 | + |
| 80 | + let root = tmp.path().join("test_db.db/events"); |
| 81 | + let directory = root.join("consumer"); |
| 82 | + for name in [ |
| 83 | + "consumer-job-old", |
| 84 | + "consumer-job-keep-extra", |
| 85 | + "consumer-other", |
| 86 | + ] { |
| 87 | + std::fs::write(directory.join(name), "{").unwrap(); |
| 88 | + } |
| 89 | + std::fs::write( |
| 90 | + directory.join("consumer-job-keep"), |
| 91 | + "{ \"nextSnapshot\": 2 }\n", |
| 92 | + ) |
| 93 | + .unwrap(); |
| 94 | + std::fs::write(directory.join("notes"), "keep").unwrap(); |
| 95 | + std::fs::write(directory.join("consumer-pre-job-old"), "keep").unwrap(); |
| 96 | + std::fs::create_dir(directory.join("consumer-job-directory")).unwrap(); |
| 97 | + std::fs::write(directory.join("consumer-job-directory/checkpoint"), "keep").unwrap(); |
| 98 | + let mut expected = table_files(&root); |
| 99 | + expected |
| 100 | + .remove(&directory.join("consumer-job-old")) |
| 101 | + .unwrap(); |
| 102 | + expected |
| 103 | + .remove(&directory.join("consumer-job-keep-extra")) |
| 104 | + .unwrap(); |
| 105 | + |
| 106 | + for _ in 0..2 { |
| 107 | + assert_ok(&context, "CALL paimon.sys.clear_consumers(table => 'paimon.test_db.events', including_consumers => 'job-.*', excluding_consumers => 'job-keep')").await; |
| 108 | + assert_eq!(table_files(&root), expected); |
| 109 | + assert_eq!(manager.get("job-keep").await.unwrap(), Some(2)); |
| 110 | + assert_eq!( |
| 111 | + branch.consumer_manager().get("job-old").await.unwrap(), |
| 112 | + Some(3) |
| 113 | + ); |
| 114 | + } |
| 115 | + assert_sql_error( |
| 116 | + &context, |
| 117 | + "CALL sys.clear_consumers(table => 'test_db.events$branch_dev')", |
| 118 | + "does not exist", |
| 119 | + ) |
| 120 | + .await; |
| 121 | + assert_eq!(table_files(&root), expected); |
| 122 | +} |
| 123 | + |
| 124 | +#[tokio::test] |
| 125 | +async fn clear_accepts_defaults_and_does_not_create_a_missing_directory() { |
| 126 | + let (tmp, catalog) = create_test_env(); |
| 127 | + let context = create_sql_context(catalog.clone()).await; |
| 128 | + exec(&context, "CREATE SCHEMA paimon.test_db").await; |
| 129 | + exec(&context, "CREATE TABLE paimon.test_db.events (id INT)").await; |
| 130 | + let table = catalog |
| 131 | + .get_table(&Identifier::new("test_db", "events")) |
| 132 | + .await |
| 133 | + .unwrap(); |
| 134 | + let root = tmp.path().join("test_db.db/events"); |
| 135 | + let original = table_files(&root); |
| 136 | + for _ in 0..2 { |
| 137 | + assert_ok( |
| 138 | + &context, |
| 139 | + "CALL sys.clear_consumers(table => 'test_db.events')", |
| 140 | + ) |
| 141 | + .await; |
| 142 | + assert_eq!(table_files(&root), original); |
| 143 | + assert!(!root.join("consumer").exists()); |
| 144 | + } |
| 145 | + for args in [ |
| 146 | + "", |
| 147 | + ", including_consumers => ''", |
| 148 | + ", including_consumers => ' \t', excluding_consumers => ''", |
| 149 | + ] { |
| 150 | + table.consumer_manager().reset("job", 2).await.unwrap(); |
| 151 | + assert_ok( |
| 152 | + &context, |
| 153 | + &format!("CALL sys.clear_consumers(table => 'test_db.events'{args})"), |
| 154 | + ) |
| 155 | + .await; |
| 156 | + assert_eq!(table.consumer_manager().get("job").await.unwrap(), None); |
| 157 | + assert_eq!(table_files(&root), original); |
| 158 | + } |
| 159 | + table.consumer_manager().reset("job", 2).await.unwrap(); |
| 160 | + assert_ok( |
| 161 | + &context, |
| 162 | + "CALL sys.clear_consumers(table => 'test_db.events', excluding_consumers => 'job')", |
| 163 | + ) |
| 164 | + .await; |
| 165 | + assert_eq!(table.consumer_manager().get("job").await.unwrap(), Some(2)); |
| 166 | +} |
| 167 | + |
| 168 | +#[tokio::test] |
| 169 | +async fn clear_rejects_invalid_arguments_before_changing_files() { |
| 170 | + let (tmp, context, table) = table_with_snapshot().await; |
| 171 | + table.consumer_manager().reset("job", 1).await.unwrap(); |
| 172 | + let root = tmp.path().join("test_db.db/events"); |
| 173 | + let original = table_files(&root); |
| 174 | + for (args, message) in [ |
| 175 | + ("", "Missing required argument: 'table'"), |
| 176 | + ("'test_db.events'", "require named arguments"), |
| 177 | + ("table => 'other.test_db.events'", "Catalog name mismatch"), |
| 178 | + ("table => 'test_db.events', including_consumer => 'job'", "is unknown"), |
| 179 | + ("table => 'test_db.events', excluding_consumer => 'job'", "is unknown"), |
| 180 | + ("table => 'test_db.events', including_consumers => 'job', INCLUDING_CONSUMERS => 'other'", "is duplicated"), |
| 181 | + ("table => 'test_db.events', excluding_consumers => 'job', excluding_consumers => 'other'", "is duplicated"), |
| 182 | + ("table => 'test_db.events', including_consumers => NULL", "Unsupported argument value"), |
| 183 | + ("table => 'test_db.events', excluding_consumers => NULL", "Unsupported argument value"), |
| 184 | + ("table => 'test_db.events', including_consumers => '['", "Invalid including_consumers regex"), |
| 185 | + ("table => 'test_db.events', excluding_consumers => '['", "Invalid excluding_consumers regex"), |
| 186 | + ("table => 'test_db.events', including_consumers => 'no-match', excluding_consumers => '(?=job)job'", "Invalid excluding_consumers regex"), |
| 187 | + (r"table => 'test_db.events', including_consumers => '(job)\1'", "Invalid including_consumers regex"), |
| 188 | + ] { |
| 189 | + assert_sql_error(&context, &format!("CALL sys.clear_consumers({args})"), message).await; |
| 190 | + assert_eq!(table_files(&root), original); |
| 191 | + } |
| 192 | +} |
| 193 | + |
| 194 | +#[tokio::test] |
| 195 | +async fn clear_uses_whole_id_matching_with_rust_regex_flags() { |
| 196 | + let (_tmp, context, table) = table_with_snapshot().await; |
| 197 | + for (pattern, id, deleted) in [ |
| 198 | + ("job", "job-old", false), |
| 199 | + ("a|ab", "ab", true), |
| 200 | + ("(?m)^job$", "job\nother", false), |
| 201 | + ("(?s)job.*", "job\nother", true), |
| 202 | + ("(?i)job", "JOB", true), |
| 203 | + ("(?x) job # trailing comment", "job", true), |
| 204 | + (" job", "job", false), |
| 205 | + ] { |
| 206 | + table.consumer_manager().reset(id, 2).await.unwrap(); |
| 207 | + assert_ok(&context, &format!("CALL sys.clear_consumers(table => 'test_db.events', including_consumers => '{pattern}')")).await; |
| 208 | + assert_eq!( |
| 209 | + table.consumer_manager().get(id).await.unwrap(), |
| 210 | + if deleted { None } else { Some(2) }, |
| 211 | + "{pattern:?}: {id:?}" |
| 212 | + ); |
| 213 | + table.consumer_manager().delete(id).await.unwrap(); |
| 214 | + } |
| 215 | +} |
| 216 | + |
| 217 | +#[tokio::test] |
| 218 | +async fn clear_rejects_format_tables_without_changing_files() { |
| 219 | + let (tmp, catalog) = create_test_env(); |
| 220 | + catalog |
| 221 | + .create_database("test_db", false, Default::default()) |
| 222 | + .await |
| 223 | + .unwrap(); |
| 224 | + let identifier = Identifier::new("test_db", "events"); |
| 225 | + let schema = Schema::builder() |
| 226 | + .column("id", DataType::Int(IntType::new())) |
| 227 | + .option("type", "format-table") |
| 228 | + .option("file.format", "parquet") |
| 229 | + .build() |
| 230 | + .unwrap(); |
| 231 | + catalog |
| 232 | + .create_table(&identifier, schema, false) |
| 233 | + .await |
| 234 | + .unwrap(); |
| 235 | + catalog |
| 236 | + .get_table(&identifier) |
| 237 | + .await |
| 238 | + .unwrap() |
| 239 | + .consumer_manager() |
| 240 | + .reset("job", 2) |
| 241 | + .await |
| 242 | + .unwrap(); |
| 243 | + let context = create_sql_context(catalog).await; |
| 244 | + let root = tmp.path().join("test_db.db/events"); |
| 245 | + let original = table_files(&root); |
| 246 | + assert_sql_error( |
| 247 | + &context, |
| 248 | + "CALL sys.clear_consumers(table => 'test_db.events')", |
| 249 | + "requires a FileStoreTable", |
| 250 | + ) |
| 251 | + .await; |
| 252 | + assert_eq!(table_files(&root), original); |
| 253 | +} |
| 254 | + |
| 255 | +#[tokio::test] |
| 256 | +async fn clear_rejects_engine_served_table_types_without_changing_files() { |
| 257 | + for table_type in ["object-table", "iceberg-table", "lance-table"] { |
| 258 | + let (tmp, catalog) = create_test_env(); |
| 259 | + catalog |
| 260 | + .create_database("test_db", false, Default::default()) |
| 261 | + .await |
| 262 | + .unwrap(); |
| 263 | + let schema = Schema::builder() |
| 264 | + .column("id", DataType::Int(IntType::new())) |
| 265 | + .option("type", table_type) |
| 266 | + .build() |
| 267 | + .unwrap(); |
| 268 | + catalog |
| 269 | + .create_table(&Identifier::new("test_db", "events"), schema, false) |
| 270 | + .await |
| 271 | + .unwrap(); |
| 272 | + let root = tmp.path().join("test_db.db/events"); |
| 273 | + std::fs::create_dir(root.join("consumer")).unwrap(); |
| 274 | + std::fs::write(root.join("consumer/consumer-job"), "{").unwrap(); |
| 275 | + let original = table_files(&root); |
| 276 | + let context = create_sql_context(catalog).await; |
| 277 | + assert_sql_error( |
| 278 | + &context, |
| 279 | + "CALL sys.clear_consumers(table => 'test_db.events')", |
| 280 | + "cannot be read as a Paimon table", |
| 281 | + ) |
| 282 | + .await; |
| 283 | + assert_eq!(table_files(&root), original); |
| 284 | + } |
| 285 | +} |
| 286 | + |
| 287 | +#[tokio::test] |
| 288 | +async fn clear_preserves_running_scan_progress_and_checkpoint_protocol() { |
| 289 | + let (_tmp, context, table) = table_with_snapshot().await; |
| 290 | + table.consumer_manager().reset("keep", 2).await.unwrap(); |
| 291 | + let mut running = table.new_stream_scan().unwrap(); |
| 292 | + running |
| 293 | + .with_consumer_id("job") |
| 294 | + .unwrap() |
| 295 | + .restore(Some(1)) |
| 296 | + .unwrap(); |
| 297 | + running.plan().await.unwrap().unwrap(); |
| 298 | + running |
| 299 | + .notify_checkpoint_complete(running.checkpoint()) |
| 300 | + .await |
| 301 | + .unwrap(); |
| 302 | + assert_eq!(table.consumer_manager().get("job").await.unwrap(), Some(2)); |
| 303 | + |
| 304 | + assert_ok( |
| 305 | + &context, |
| 306 | + "CALL sys.clear_consumers(table => 'test_db.events', excluding_consumers => 'keep')", |
| 307 | + ) |
| 308 | + .await; |
| 309 | + assert_eq!(table.consumer_manager().get("job").await.unwrap(), None); |
| 310 | + assert_eq!(table.consumer_manager().get("keep").await.unwrap(), Some(2)); |
| 311 | + assert_eq!(running.checkpoint(), Some(2)); |
| 312 | + assert!(running.plan().await.unwrap().is_none()); |
| 313 | + let mut fresh = table.new_stream_scan().unwrap(); |
| 314 | + fresh.with_consumer_id("job").unwrap(); |
| 315 | + let plan = fresh.plan().await.unwrap().unwrap(); |
| 316 | + assert_eq!(plan.snapshot_id(), Some(1)); |
| 317 | + assert!(plan.splits().iter().all(|split| !split.is_streaming())); |
| 318 | + |
| 319 | + running |
| 320 | + .notify_checkpoint_complete(running.checkpoint()) |
| 321 | + .await |
| 322 | + .unwrap(); |
| 323 | + assert_eq!(table.consumer_manager().get("job").await.unwrap(), Some(2)); |
| 324 | + assert_eq!(table.consumer_manager().get("keep").await.unwrap(), Some(2)); |
| 325 | +} |
0 commit comments