|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +//! Resolve and validate the REST catalog's local FUSE mount. |
| 19 | +
|
| 20 | +use crate::api::rest_api::RESTApi; |
| 21 | +use crate::catalog::Identifier; |
| 22 | +use crate::common::{FuseOptions, Options}; |
| 23 | +use crate::io::cache::LocalCache; |
| 24 | +use crate::io::{FileIO, FileIOCacheContext}; |
| 25 | +use crate::{Error, Result}; |
| 26 | +use std::sync::Arc; |
| 27 | +use tokio::sync::OnceCell; |
| 28 | + |
| 29 | +/// One validation result per catalog, shared by tables and dependency reads. |
| 30 | +#[derive(Debug)] |
| 31 | +pub(crate) struct FusePathResolver { |
| 32 | + options: Options, |
| 33 | + validation: OnceCell<bool>, |
| 34 | +} |
| 35 | + |
| 36 | +impl FusePathResolver { |
| 37 | + pub(crate) fn new(options: &Options) -> Self { |
| 38 | + Self { |
| 39 | + options: options.clone(), |
| 40 | + validation: OnceCell::new(), |
| 41 | + } |
| 42 | + } |
| 43 | + |
| 44 | + pub(crate) async fn file_io( |
| 45 | + &self, |
| 46 | + original: &str, |
| 47 | + identifier: &Identifier, |
| 48 | + api: &RESTApi, |
| 49 | + local_cache: Option<Arc<LocalCache>>, |
| 50 | + file_io_cache: FileIOCacheContext, |
| 51 | + ) -> Result<Option<FileIO>> { |
| 52 | + if !self |
| 53 | + .options |
| 54 | + .get(FuseOptions::ENABLED) |
| 55 | + .is_some_and(|v| v.eq_ignore_ascii_case("true")) |
| 56 | + { |
| 57 | + return Ok(None); |
| 58 | + } |
| 59 | + let local = self.resolve(original, identifier.database(), Some(identifier.object()))?; |
| 60 | + let validation_mode = self |
| 61 | + .options |
| 62 | + .get(FuseOptions::VALIDATION_MODE) |
| 63 | + .map(String::as_str) |
| 64 | + .unwrap_or("strict"); |
| 65 | + if !matches!(validation_mode, "strict" | "warn" | "none") { |
| 66 | + return Err(Error::ConfigInvalid { message: format!("Invalid fuse.validation-mode: {validation_mode}; expected strict, warn or none") }); |
| 67 | + } |
| 68 | + let valid = self |
| 69 | + .validation |
| 70 | + .get_or_try_init(|| self.validate(api, validation_mode)) |
| 71 | + .await?; |
| 72 | + if !valid { |
| 73 | + return Ok(None); |
| 74 | + } |
| 75 | + #[cfg(feature = "storage-fs")] |
| 76 | + { |
| 77 | + let local = Self::local_path(&local)?; |
| 78 | + let local = local.to_string_lossy(); |
| 79 | + let mut builder = FileIO::from_path(local.as_ref())? |
| 80 | + .with_props(self.options.to_map()) |
| 81 | + .with_cache_context(file_io_cache); |
| 82 | + if let Some(cache) = local_cache { |
| 83 | + builder = builder.with_local_cache(cache); |
| 84 | + } |
| 85 | + Ok(Some(builder.build()?.with_fuse(original, &local)?)) |
| 86 | + } |
| 87 | + #[cfg(not(feature = "storage-fs"))] |
| 88 | + { |
| 89 | + let _ = (local, local_cache, file_io_cache); |
| 90 | + Err(Error::Unsupported { |
| 91 | + message: "FUSE requires the storage-fs feature".into(), |
| 92 | + }) |
| 93 | + } |
| 94 | + } |
| 95 | + |
| 96 | + fn resolve(&self, original: &str, database: &str, table: Option<&str>) -> Result<String> { |
| 97 | + let root = self |
| 98 | + .options |
| 99 | + .get(FuseOptions::ROOT) |
| 100 | + .filter(|root| !root.is_empty()) |
| 101 | + .ok_or_else(|| Error::ConfigInvalid { |
| 102 | + message: "FUSE is enabled but fuse.root is not configured".into(), |
| 103 | + })?; |
| 104 | + let root = root.trim_end_matches('/'); |
| 105 | + match self |
| 106 | + .options |
| 107 | + .get(FuseOptions::MODE) |
| 108 | + .map(String::as_str) |
| 109 | + .unwrap_or("pvfs") |
| 110 | + { |
| 111 | + "pvfs" => Ok(match table { |
| 112 | + Some(table) => format!("{root}/{database}/{table}"), |
| 113 | + None => format!("{root}/{database}"), |
| 114 | + }), |
| 115 | + "raw" => { |
| 116 | + // Keep literal spaces, percent escapes and Unicode unchanged, |
| 117 | + // matching urllib.parse rather than URL-normalizing object keys. |
| 118 | + let original = original.split(['?', '#']).next().unwrap_or_default(); |
| 119 | + let scheme = original.split_once(':').filter(|(scheme, _)| { |
| 120 | + scheme.starts_with(|c: char| c.is_ascii_alphabetic()) |
| 121 | + && scheme |
| 122 | + .chars() |
| 123 | + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.')) |
| 124 | + }); |
| 125 | + let mut path = scheme.map_or(original, |(_, rest)| rest); |
| 126 | + if let Some(authority) = path.strip_prefix("//") { |
| 127 | + path = authority.find('/').map_or("", |index| &authority[index..]); |
| 128 | + } |
| 129 | + let mut path = path.trim_start_matches('/'); |
| 130 | + if scheme.is_none() { |
| 131 | + if let Some((_, suffix)) = path.split_once('/') { |
| 132 | + path = suffix; |
| 133 | + } |
| 134 | + } |
| 135 | + Ok(format!("{root}/{path}")) |
| 136 | + } |
| 137 | + mode => Err(Error::ConfigInvalid { |
| 138 | + message: format!("Invalid fuse.mode: {mode}; expected pvfs or raw"), |
| 139 | + }), |
| 140 | + } |
| 141 | + } |
| 142 | + |
| 143 | + fn local_path(value: &str) -> Result<std::path::PathBuf> { |
| 144 | + let path = if value |
| 145 | + .get(..5) |
| 146 | + .is_some_and(|scheme| scheme.eq_ignore_ascii_case("file:")) |
| 147 | + { |
| 148 | + let uri = url::Url::parse(value).map_err(|error| Error::ConfigInvalid { |
| 149 | + message: format!("Invalid FUSE file URI '{value}': {error}"), |
| 150 | + })?; |
| 151 | + #[cfg(not(windows))] |
| 152 | + if let Some(host) = uri.host_str().filter(|host| *host != "localhost") { |
| 153 | + let path = percent_encoding::percent_decode_str(uri.path()).decode_utf8_lossy(); |
| 154 | + return std::path::absolute(format!("//{host}{path}")).map_err(|error| { |
| 155 | + Error::UnexpectedError { |
| 156 | + message: "Cannot resolve the FUSE mount directory".into(), |
| 157 | + source: Some(Box::new(error)), |
| 158 | + } |
| 159 | + }); |
| 160 | + } |
| 161 | + uri.to_file_path().map_err(|_| Error::ConfigInvalid { |
| 162 | + message: format!("Invalid FUSE file URI '{value}'"), |
| 163 | + })? |
| 164 | + } else { |
| 165 | + std::path::PathBuf::from(value) |
| 166 | + }; |
| 167 | + std::path::absolute(path).map_err(|error| Error::UnexpectedError { |
| 168 | + message: "Cannot resolve the FUSE mount directory".into(), |
| 169 | + source: Some(Box::new(error)), |
| 170 | + }) |
| 171 | + } |
| 172 | + |
| 173 | + async fn validate(&self, api: &RESTApi, mode: &str) -> Result<bool> { |
| 174 | + if mode == "none" { |
| 175 | + return Ok(true); |
| 176 | + } |
| 177 | + // Even warn mode propagates REST errors; only a missing mount falls back. |
| 178 | + let database = api.get_database("default").await?; |
| 179 | + let Some(location) = database.location.filter(|path| !path.is_empty()) else { |
| 180 | + return Ok(true); |
| 181 | + }; |
| 182 | + let local = self.resolve(&location, "default", None)?; |
| 183 | + let path = Self::local_path(&local)?; |
| 184 | + match tokio::fs::metadata(&path).await { |
| 185 | + Ok(_) => Ok(true), |
| 186 | + Err(error) |
| 187 | + if matches!( |
| 188 | + error.kind(), |
| 189 | + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory |
| 190 | + ) => |
| 191 | + { |
| 192 | + let message = format!( |
| 193 | + "FUSE local path validation failed: '{local}' does not exist for '{location}'" |
| 194 | + ); |
| 195 | + if mode == "warn" { |
| 196 | + log::warn!("{message}. Falling back to default FileIO."); |
| 197 | + Ok(false) |
| 198 | + } else { |
| 199 | + Err(Error::ConfigInvalid { message }) |
| 200 | + } |
| 201 | + } |
| 202 | + Err(error) => Err(Error::UnexpectedError { |
| 203 | + message: format!("Cannot validate FUSE mount {local}"), |
| 204 | + source: Some(Box::new(error)), |
| 205 | + }), |
| 206 | + } |
| 207 | + } |
| 208 | +} |
| 209 | + |
| 210 | +#[cfg(test)] |
| 211 | +mod tests { |
| 212 | + use super::*; |
| 213 | + |
| 214 | + #[test] |
| 215 | + fn fuse_layouts_preserve_identifiers_and_literal_uri_paths() { |
| 216 | + let mut options = Options::new(); |
| 217 | + options.set(FuseOptions::ROOT, "/mnt/fuse/"); |
| 218 | + let resolver = FusePathResolver::new(&options); |
| 219 | + assert_eq!( |
| 220 | + resolver |
| 221 | + .resolve("oss://bucket/physical", "db 空格", Some("t$branch_dev")) |
| 222 | + .unwrap(), |
| 223 | + "/mnt/fuse/db 空格/t$branch_dev" |
| 224 | + ); |
| 225 | + assert_eq!( |
| 226 | + resolver |
| 227 | + .resolve("oss://bucket/physical", "default", None) |
| 228 | + .unwrap(), |
| 229 | + "/mnt/fuse/default" |
| 230 | + ); |
| 231 | + options.set(FuseOptions::MODE, "raw"); |
| 232 | + let resolver = FusePathResolver::new(&options); |
| 233 | + for (original, expected) in [ |
| 234 | + ( |
| 235 | + "oss://bucket/a%2Fb/空 格?t=1#fragment", |
| 236 | + "/mnt/fuse/a%2Fb/空 格", |
| 237 | + ), |
| 238 | + ("memory:/warehouse/db/t", "/mnt/fuse/warehouse/db/t"), |
| 239 | + ("catalog/db/t", "/mnt/fuse/db/t"), |
| 240 | + ("/catalog/db/t", "/mnt/fuse/db/t"), |
| 241 | + ("table", "/mnt/fuse/table"), |
| 242 | + ] { |
| 243 | + assert_eq!( |
| 244 | + resolver |
| 245 | + .resolve(original, "ignored", Some("ignored")) |
| 246 | + .unwrap(), |
| 247 | + expected |
| 248 | + ); |
| 249 | + } |
| 250 | + options.set(FuseOptions::ROOT, "/"); |
| 251 | + assert_eq!( |
| 252 | + FusePathResolver::new(&options) |
| 253 | + .resolve("oss://bucket/db/t", "db", Some("t")) |
| 254 | + .unwrap(), |
| 255 | + "/db/t" |
| 256 | + ); |
| 257 | + } |
| 258 | + |
| 259 | + #[test] |
| 260 | + fn file_uri_mounts_decode_paths_but_plain_paths_keep_percent_escapes() { |
| 261 | + let root = tempfile::tempdir().unwrap(); |
| 262 | + let mount = root.path().join("fuse mount%"); |
| 263 | + let uri = url::Url::from_file_path(&mount).unwrap().to_string(); |
| 264 | + assert_eq!(FusePathResolver::local_path(&uri).unwrap(), mount); |
| 265 | + let plain = root.path().join("fuse%20mount"); |
| 266 | + assert_eq!( |
| 267 | + FusePathResolver::local_path(plain.to_str().unwrap()).unwrap(), |
| 268 | + plain |
| 269 | + ); |
| 270 | + } |
| 271 | + |
| 272 | + #[test] |
| 273 | + fn fuse_layout_requires_a_root_and_known_mode() { |
| 274 | + let mut options = Options::new(); |
| 275 | + assert!(FusePathResolver::new(&options) |
| 276 | + .resolve("oss://bucket/t", "db", Some("t")) |
| 277 | + .is_err()); |
| 278 | + options.set(FuseOptions::ROOT, "/mnt"); |
| 279 | + options.set(FuseOptions::MODE, "invalid"); |
| 280 | + assert!(FusePathResolver::new(&options) |
| 281 | + .resolve("oss://bucket/t", "db", Some("t")) |
| 282 | + .unwrap_err() |
| 283 | + .to_string() |
| 284 | + .contains("fuse.mode")); |
| 285 | + } |
| 286 | +} |
0 commit comments