Skip to content

Commit 46e6871

Browse files
authored
Complete authorized native read views and writer file planning (#1157)
1 parent 4a947ec commit 46e6871

29 files changed

Lines changed: 2521 additions & 360 deletions

‎bindings/python/python/pypaimon_rust/datafusion.pyi‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,9 @@ class TableScan:
179179
ranges of shuffled chunks. Row-position sharding is a separate API."""
180180
...
181181
def plan(self) -> Plan: ...
182+
def plan_all_files(self) -> Plan:
183+
"""Include L0 and all column files, retaining scan filters and limits. Requires unrestricted query authorization."""
184+
...
182185

183186
class StreamTableScan:
184187
"""Stateful continuous scan; checkpoint is the next snapshot to consume."""
@@ -335,10 +338,11 @@ class Table:
335338
def copy_with_resolved_schema(self, schema_json: str, *, branch: Optional[str] = None) -> "Table":
336339
"""Replace all fields/options, preserving FileIO, REST context and branch."""
337340
...
338-
def copy_with_pinned_snapshot(self, snapshot_json: Optional[str]) -> "Table":
339-
"""Preserve the read schema and pin Java Snapshot JSON, or an empty view with None."""
341+
def copy_with_pinned_snapshot(self, snapshot_json: Optional[str], *, with_schema: bool = False) -> "Table":
342+
"""Pin Java Snapshot JSON, optionally loading its schema; None pins an empty view."""
340343
...
341344

345+
def copy(self, options: Dict[str, str]) -> "Table": ...
342346
def identifier(self) -> str: ...
343347
def branch(self) -> str: ...
344348
def location(self) -> str: ...

‎bindings/python/src/read.rs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,6 +737,18 @@ impl PyTableScan {
737737
Ok(slf)
738738
}
739739

740+
fn plan_all_files(&self, py: Python<'_>) -> PyResult<PyPlan> {
741+
py.detach(|| {
742+
runtime().block_on(async {
743+
self.core_scan()?
744+
.plan_all_files()
745+
.await
746+
.map(PyPlan::from)
747+
.map_err(to_py_err)
748+
})
749+
})
750+
}
751+
740752
fn plan(&self, py: Python<'_>) -> PyResult<PyPlan> {
741753
py.detach(|| {
742754
runtime().block_on(async {
@@ -915,6 +927,9 @@ impl PyTableRead {
915927
// path so an invalid projection fails consistently regardless of
916928
// how many splits are passed.
917929
let read = builder.new_read().map_err(to_py_err)?;
930+
let splits = runtime()
931+
.block_on(read.authorize_splits(&splits))
932+
.map_err(to_py_err)?;
918933
if self.include_row_kind {
919934
read.to_arrow_with_row_kind(&splits).map_err(to_py_err)
920935
} else {

‎bindings/python/src/table.rs‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,15 @@ impl PyTable {
156156
Ok(Self::new(Arc::new(table)))
157157
}
158158

159+
/// Java Table.copy: apply dynamic options and resolve the selected schema.
160+
#[pyo3(signature = (options))]
161+
fn copy(&self, py: Python<'_>, options: HashMap<String, String>) -> PyResult<Self> {
162+
let table = py
163+
.detach(|| runtime().block_on(self.inner.copy_with_time_travel_strict(options)))
164+
.map_err(to_py_err)?;
165+
Ok(Self::new(Arc::new(table)))
166+
}
167+
159168
fn identifier(&self) -> String {
160169
let id = self.inner.identifier();
161170
format!("{}.{}", id.database(), id.object())
@@ -214,15 +223,24 @@ impl PyTable {
214223
}
215224

216225
/// Forward one captured Java-format snapshot, or an explicitly empty view.
217-
#[pyo3(signature = (snapshot_json))]
218-
fn copy_with_pinned_snapshot(&self, snapshot_json: Option<&str>) -> PyResult<Self> {
226+
#[pyo3(signature = (snapshot_json, *, with_schema=false))]
227+
fn copy_with_pinned_snapshot(
228+
&self,
229+
py: Python<'_>,
230+
snapshot_json: Option<&str>,
231+
with_schema: bool,
232+
) -> PyResult<Self> {
219233
let snapshot = snapshot_json
220234
.map(serde_json::from_str::<paimon::spec::Snapshot>)
221235
.transpose()
222236
.map_err(|error| PyValueError::new_err(format!("Invalid snapshot JSON: {error}")))?;
223-
Ok(Self {
224-
inner: Arc::new(self.inner.copy_with_pinned_snapshot(snapshot.as_ref())),
225-
})
237+
let table = match snapshot.as_ref().filter(|_| with_schema) {
238+
Some(snapshot) => py
239+
.detach(|| runtime().block_on(self.inner.copy_with_resolved_snapshot(snapshot)))
240+
.map_err(to_py_err)?,
241+
None => self.inner.copy_with_pinned_snapshot(snapshot.as_ref()),
242+
};
243+
Ok(Self::new(Arc::new(table)))
226244
}
227245

228246
fn new_vector_search_builder(&self) -> crate::vector_search::PyVectorSearchBuilder {
Lines changed: 286 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,286 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Resolve and validate the REST catalog's local FUSE mount.
19+
20+
use crate::api::rest_api::RESTApi;
21+
use crate::catalog::Identifier;
22+
use crate::common::{FuseOptions, Options};
23+
use crate::io::cache::LocalCache;
24+
use crate::io::{FileIO, FileIOCacheContext};
25+
use crate::{Error, Result};
26+
use std::sync::Arc;
27+
use tokio::sync::OnceCell;
28+
29+
/// One validation result per catalog, shared by tables and dependency reads.
30+
#[derive(Debug)]
31+
pub(crate) struct FusePathResolver {
32+
options: Options,
33+
validation: OnceCell<bool>,
34+
}
35+
36+
impl FusePathResolver {
37+
pub(crate) fn new(options: &Options) -> Self {
38+
Self {
39+
options: options.clone(),
40+
validation: OnceCell::new(),
41+
}
42+
}
43+
44+
pub(crate) async fn file_io(
45+
&self,
46+
original: &str,
47+
identifier: &Identifier,
48+
api: &RESTApi,
49+
local_cache: Option<Arc<LocalCache>>,
50+
file_io_cache: FileIOCacheContext,
51+
) -> Result<Option<FileIO>> {
52+
if !self
53+
.options
54+
.get(FuseOptions::ENABLED)
55+
.is_some_and(|v| v.eq_ignore_ascii_case("true"))
56+
{
57+
return Ok(None);
58+
}
59+
let local = self.resolve(original, identifier.database(), Some(identifier.object()))?;
60+
let validation_mode = self
61+
.options
62+
.get(FuseOptions::VALIDATION_MODE)
63+
.map(String::as_str)
64+
.unwrap_or("strict");
65+
if !matches!(validation_mode, "strict" | "warn" | "none") {
66+
return Err(Error::ConfigInvalid { message: format!("Invalid fuse.validation-mode: {validation_mode}; expected strict, warn or none") });
67+
}
68+
let valid = self
69+
.validation
70+
.get_or_try_init(|| self.validate(api, validation_mode))
71+
.await?;
72+
if !valid {
73+
return Ok(None);
74+
}
75+
#[cfg(feature = "storage-fs")]
76+
{
77+
let local = Self::local_path(&local)?;
78+
let local = local.to_string_lossy();
79+
let mut builder = FileIO::from_path(local.as_ref())?
80+
.with_props(self.options.to_map())
81+
.with_cache_context(file_io_cache);
82+
if let Some(cache) = local_cache {
83+
builder = builder.with_local_cache(cache);
84+
}
85+
Ok(Some(builder.build()?.with_fuse(original, &local)?))
86+
}
87+
#[cfg(not(feature = "storage-fs"))]
88+
{
89+
let _ = (local, local_cache, file_io_cache);
90+
Err(Error::Unsupported {
91+
message: "FUSE requires the storage-fs feature".into(),
92+
})
93+
}
94+
}
95+
96+
fn resolve(&self, original: &str, database: &str, table: Option<&str>) -> Result<String> {
97+
let root = self
98+
.options
99+
.get(FuseOptions::ROOT)
100+
.filter(|root| !root.is_empty())
101+
.ok_or_else(|| Error::ConfigInvalid {
102+
message: "FUSE is enabled but fuse.root is not configured".into(),
103+
})?;
104+
let root = root.trim_end_matches('/');
105+
match self
106+
.options
107+
.get(FuseOptions::MODE)
108+
.map(String::as_str)
109+
.unwrap_or("pvfs")
110+
{
111+
"pvfs" => Ok(match table {
112+
Some(table) => format!("{root}/{database}/{table}"),
113+
None => format!("{root}/{database}"),
114+
}),
115+
"raw" => {
116+
// Keep literal spaces, percent escapes and Unicode unchanged,
117+
// matching urllib.parse rather than URL-normalizing object keys.
118+
let original = original.split(['?', '#']).next().unwrap_or_default();
119+
let scheme = original.split_once(':').filter(|(scheme, _)| {
120+
scheme.starts_with(|c: char| c.is_ascii_alphabetic())
121+
&& scheme
122+
.chars()
123+
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
124+
});
125+
let mut path = scheme.map_or(original, |(_, rest)| rest);
126+
if let Some(authority) = path.strip_prefix("//") {
127+
path = authority.find('/').map_or("", |index| &authority[index..]);
128+
}
129+
let mut path = path.trim_start_matches('/');
130+
if scheme.is_none() {
131+
if let Some((_, suffix)) = path.split_once('/') {
132+
path = suffix;
133+
}
134+
}
135+
Ok(format!("{root}/{path}"))
136+
}
137+
mode => Err(Error::ConfigInvalid {
138+
message: format!("Invalid fuse.mode: {mode}; expected pvfs or raw"),
139+
}),
140+
}
141+
}
142+
143+
fn local_path(value: &str) -> Result<std::path::PathBuf> {
144+
let path = if value
145+
.get(..5)
146+
.is_some_and(|scheme| scheme.eq_ignore_ascii_case("file:"))
147+
{
148+
let uri = url::Url::parse(value).map_err(|error| Error::ConfigInvalid {
149+
message: format!("Invalid FUSE file URI '{value}': {error}"),
150+
})?;
151+
#[cfg(not(windows))]
152+
if let Some(host) = uri.host_str().filter(|host| *host != "localhost") {
153+
let path = percent_encoding::percent_decode_str(uri.path()).decode_utf8_lossy();
154+
return std::path::absolute(format!("//{host}{path}")).map_err(|error| {
155+
Error::UnexpectedError {
156+
message: "Cannot resolve the FUSE mount directory".into(),
157+
source: Some(Box::new(error)),
158+
}
159+
});
160+
}
161+
uri.to_file_path().map_err(|_| Error::ConfigInvalid {
162+
message: format!("Invalid FUSE file URI '{value}'"),
163+
})?
164+
} else {
165+
std::path::PathBuf::from(value)
166+
};
167+
std::path::absolute(path).map_err(|error| Error::UnexpectedError {
168+
message: "Cannot resolve the FUSE mount directory".into(),
169+
source: Some(Box::new(error)),
170+
})
171+
}
172+
173+
async fn validate(&self, api: &RESTApi, mode: &str) -> Result<bool> {
174+
if mode == "none" {
175+
return Ok(true);
176+
}
177+
// Even warn mode propagates REST errors; only a missing mount falls back.
178+
let database = api.get_database("default").await?;
179+
let Some(location) = database.location.filter(|path| !path.is_empty()) else {
180+
return Ok(true);
181+
};
182+
let local = self.resolve(&location, "default", None)?;
183+
let path = Self::local_path(&local)?;
184+
match tokio::fs::metadata(&path).await {
185+
Ok(_) => Ok(true),
186+
Err(error)
187+
if matches!(
188+
error.kind(),
189+
std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory
190+
) =>
191+
{
192+
let message = format!(
193+
"FUSE local path validation failed: '{local}' does not exist for '{location}'"
194+
);
195+
if mode == "warn" {
196+
log::warn!("{message}. Falling back to default FileIO.");
197+
Ok(false)
198+
} else {
199+
Err(Error::ConfigInvalid { message })
200+
}
201+
}
202+
Err(error) => Err(Error::UnexpectedError {
203+
message: format!("Cannot validate FUSE mount {local}"),
204+
source: Some(Box::new(error)),
205+
}),
206+
}
207+
}
208+
}
209+
210+
#[cfg(test)]
211+
mod tests {
212+
use super::*;
213+
214+
#[test]
215+
fn fuse_layouts_preserve_identifiers_and_literal_uri_paths() {
216+
let mut options = Options::new();
217+
options.set(FuseOptions::ROOT, "/mnt/fuse/");
218+
let resolver = FusePathResolver::new(&options);
219+
assert_eq!(
220+
resolver
221+
.resolve("oss://bucket/physical", "db 空格", Some("t$branch_dev"))
222+
.unwrap(),
223+
"/mnt/fuse/db 空格/t$branch_dev"
224+
);
225+
assert_eq!(
226+
resolver
227+
.resolve("oss://bucket/physical", "default", None)
228+
.unwrap(),
229+
"/mnt/fuse/default"
230+
);
231+
options.set(FuseOptions::MODE, "raw");
232+
let resolver = FusePathResolver::new(&options);
233+
for (original, expected) in [
234+
(
235+
"oss://bucket/a%2Fb/空 格?t=1#fragment",
236+
"/mnt/fuse/a%2Fb/空 格",
237+
),
238+
("memory:/warehouse/db/t", "/mnt/fuse/warehouse/db/t"),
239+
("catalog/db/t", "/mnt/fuse/db/t"),
240+
("/catalog/db/t", "/mnt/fuse/db/t"),
241+
("table", "/mnt/fuse/table"),
242+
] {
243+
assert_eq!(
244+
resolver
245+
.resolve(original, "ignored", Some("ignored"))
246+
.unwrap(),
247+
expected
248+
);
249+
}
250+
options.set(FuseOptions::ROOT, "/");
251+
assert_eq!(
252+
FusePathResolver::new(&options)
253+
.resolve("oss://bucket/db/t", "db", Some("t"))
254+
.unwrap(),
255+
"/db/t"
256+
);
257+
}
258+
259+
#[test]
260+
fn file_uri_mounts_decode_paths_but_plain_paths_keep_percent_escapes() {
261+
let root = tempfile::tempdir().unwrap();
262+
let mount = root.path().join("fuse mount%");
263+
let uri = url::Url::from_file_path(&mount).unwrap().to_string();
264+
assert_eq!(FusePathResolver::local_path(&uri).unwrap(), mount);
265+
let plain = root.path().join("fuse%20mount");
266+
assert_eq!(
267+
FusePathResolver::local_path(plain.to_str().unwrap()).unwrap(),
268+
plain
269+
);
270+
}
271+
272+
#[test]
273+
fn fuse_layout_requires_a_root_and_known_mode() {
274+
let mut options = Options::new();
275+
assert!(FusePathResolver::new(&options)
276+
.resolve("oss://bucket/t", "db", Some("t"))
277+
.is_err());
278+
options.set(FuseOptions::ROOT, "/mnt");
279+
options.set(FuseOptions::MODE, "invalid");
280+
assert!(FusePathResolver::new(&options)
281+
.resolve("oss://bucket/t", "db", Some("t"))
282+
.unwrap_err()
283+
.to_string()
284+
.contains("fuse.mode"));
285+
}
286+
}

‎crates/paimon/src/catalog/rest/mod.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,13 @@
2020
//! This module provides a REST-based catalog that communicates with
2121
//! a Paimon REST catalog server for metadata operations.
2222
23+
mod fuse_support;
2324
mod rest_catalog;
2425
mod rest_token;
2526
mod rest_token_file_io;
2627

2728
pub use rest_catalog::*;
2829
pub use rest_token::*;
2930
pub use rest_token_file_io::*;
31+
32+
pub(crate) use fuse_support::FusePathResolver;

0 commit comments

Comments
 (0)