|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +//! Protobuf conversions for the file-scan leaf types owned by this crate: |
| 19 | +//! [`FileRange`], [`PartitionedFile`] and [`FileGroup`]. |
| 20 | +//! |
| 21 | +//! These are the single copy of that wire logic. `datafusion-proto`'s |
| 22 | +//! `TryFromProto` implementations for the same types are thin shims that |
| 23 | +//! delegate here, so the format cannot drift between the central serializer and |
| 24 | +//! the per-source `try_to_proto` hooks. |
| 25 | +//! |
| 26 | +//! None of these conversions need a codec or an encode/decode context: every |
| 27 | +//! field is plain data or goes through `datafusion-proto-common`. That is why |
| 28 | +//! they are plain [`TryFrom`] impls rather than the `try_to_proto(ctx)` / |
| 29 | +//! `try_from_proto(node, ctx)` hooks used for plans, expressions and scan |
| 30 | +//! configs: the standard trait can express a conversion that takes nothing but |
| 31 | +//! the value, and the orphan rule allows it here because one side of each |
| 32 | +//! conversion is a type this crate owns. |
| 33 | +
|
| 34 | +use std::sync::Arc; |
| 35 | + |
| 36 | +use chrono::{TimeZone, Utc}; |
| 37 | +use datafusion_common::{DataFusionError, Result, internal_datafusion_err}; |
| 38 | +use datafusion_proto_models::protobuf; |
| 39 | +use object_store::ObjectMeta; |
| 40 | +use object_store::path::Path; |
| 41 | + |
| 42 | +use crate::file_groups::FileGroup; |
| 43 | +use crate::{FileRange, PartitionedFile}; |
| 44 | + |
| 45 | +impl TryFrom<&FileRange> for protobuf::FileRange { |
| 46 | + type Error = DataFusionError; |
| 47 | + |
| 48 | + fn try_from(range: &FileRange) -> Result<Self> { |
| 49 | + Ok(protobuf::FileRange { |
| 50 | + start: range.start, |
| 51 | + end: range.end, |
| 52 | + }) |
| 53 | + } |
| 54 | +} |
| 55 | + |
| 56 | +impl TryFrom<&protobuf::FileRange> for FileRange { |
| 57 | + type Error = DataFusionError; |
| 58 | + |
| 59 | + fn try_from(range: &protobuf::FileRange) -> Result<Self> { |
| 60 | + Ok(FileRange { |
| 61 | + start: range.start, |
| 62 | + end: range.end, |
| 63 | + }) |
| 64 | + } |
| 65 | +} |
| 66 | + |
| 67 | +impl TryFrom<&PartitionedFile> for protobuf::PartitionedFile { |
| 68 | + type Error = DataFusionError; |
| 69 | + |
| 70 | + fn try_from(file: &PartitionedFile) -> Result<Self> { |
| 71 | + let last_modified = file.object_meta.last_modified; |
| 72 | + let last_modified_ns = last_modified.timestamp_nanos_opt().ok_or_else(|| { |
| 73 | + DataFusionError::Plan(format!( |
| 74 | + "Invalid timestamp on PartitionedFile::ObjectMeta: {last_modified}" |
| 75 | + )) |
| 76 | + })? as u64; |
| 77 | + Ok(protobuf::PartitionedFile { |
| 78 | + arrow_schema: file |
| 79 | + .arrow_schema |
| 80 | + .as_ref() |
| 81 | + .map(|s| s.as_ref().try_into()) |
| 82 | + .transpose()?, |
| 83 | + path: file.object_meta.location.as_ref().to_owned(), |
| 84 | + size: file.object_meta.size, |
| 85 | + last_modified_ns, |
| 86 | + partition_values: file |
| 87 | + .partition_values |
| 88 | + .iter() |
| 89 | + .map(|v| v.try_into()) |
| 90 | + .collect::<Result<Vec<_>, _>>()?, |
| 91 | + range: file.range.as_ref().map(TryInto::try_into).transpose()?, |
| 92 | + statistics: file.statistics.as_ref().map(|s| s.as_ref().into()), |
| 93 | + }) |
| 94 | + } |
| 95 | +} |
| 96 | + |
| 97 | +impl TryFrom<&protobuf::PartitionedFile> for PartitionedFile { |
| 98 | + type Error = DataFusionError; |
| 99 | + |
| 100 | + fn try_from(file: &protobuf::PartitionedFile) -> Result<Self> { |
| 101 | + let mut pf = PartitionedFile::new_from_meta(ObjectMeta { |
| 102 | + location: Path::parse(file.path.as_str()).map_err(|e| { |
| 103 | + internal_datafusion_err!("Invalid object_store path: {e}") |
| 104 | + })?, |
| 105 | + last_modified: Utc.timestamp_nanos(file.last_modified_ns as i64), |
| 106 | + size: file.size, |
| 107 | + e_tag: None, |
| 108 | + version: None, |
| 109 | + }) |
| 110 | + .with_partition_values( |
| 111 | + file.partition_values |
| 112 | + .iter() |
| 113 | + .map(|v| v.try_into()) |
| 114 | + .collect::<Result<Vec<_>, _>>()?, |
| 115 | + ); |
| 116 | + if let Some(proto_schema) = file.arrow_schema.as_ref() { |
| 117 | + pf = pf.with_arrow_schema(Arc::new( |
| 118 | + proto_schema.try_into().map_err(DataFusionError::from)?, |
| 119 | + )); |
| 120 | + } |
| 121 | + if let Some(range) = file.range.as_ref() { |
| 122 | + let range = FileRange::try_from(range)?; |
| 123 | + pf = pf.with_range(range.start, range.end); |
| 124 | + } |
| 125 | + if let Some(proto_stats) = file.statistics.as_ref() { |
| 126 | + // The wire format carries statistics for the full table schema (file + partition |
| 127 | + // columns), so assign directly — `with_statistics` would append the partition |
| 128 | + // column stats a second time. |
| 129 | + pf.statistics = Some(Arc::new(proto_stats.try_into()?)); |
| 130 | + } |
| 131 | + Ok(pf) |
| 132 | + } |
| 133 | +} |
| 134 | + |
| 135 | +impl TryFrom<&FileGroup> for protobuf::FileGroup { |
| 136 | + type Error = DataFusionError; |
| 137 | + |
| 138 | + fn try_from(group: &FileGroup) -> Result<Self> { |
| 139 | + Ok(protobuf::FileGroup { |
| 140 | + files: group |
| 141 | + .files() |
| 142 | + .iter() |
| 143 | + .map(TryInto::try_into) |
| 144 | + .collect::<Result<Vec<_>>>()?, |
| 145 | + }) |
| 146 | + } |
| 147 | +} |
| 148 | + |
| 149 | +impl TryFrom<&protobuf::FileGroup> for FileGroup { |
| 150 | + type Error = DataFusionError; |
| 151 | + |
| 152 | + fn try_from(group: &protobuf::FileGroup) -> Result<Self> { |
| 153 | + Ok(FileGroup::new( |
| 154 | + group |
| 155 | + .files |
| 156 | + .iter() |
| 157 | + .map(TryInto::try_into) |
| 158 | + .collect::<Result<Vec<_>>>()?, |
| 159 | + )) |
| 160 | + } |
| 161 | +} |
| 162 | + |
| 163 | +#[cfg(test)] |
| 164 | +mod tests { |
| 165 | + use arrow::datatypes::{DataType, Field, Schema}; |
| 166 | + use datafusion_common::{ScalarValue, Statistics}; |
| 167 | + |
| 168 | + use super::*; |
| 169 | + |
| 170 | + #[test] |
| 171 | + fn partitioned_file_roundtrip_preserves_all_fields() -> Result<()> { |
| 172 | + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); |
| 173 | + let pf = PartitionedFile::new_from_meta(ObjectMeta { |
| 174 | + location: Path::parse("foo/bar.parquet")?, |
| 175 | + last_modified: Utc.timestamp_nanos(1_000_000_000), |
| 176 | + size: 1234, |
| 177 | + e_tag: None, |
| 178 | + version: None, |
| 179 | + }) |
| 180 | + .with_partition_values(vec![ScalarValue::from("2024-01-01")]) |
| 181 | + .with_range(10, 20) |
| 182 | + .with_arrow_schema(Arc::clone(&schema)) |
| 183 | + .with_statistics(Arc::new(Statistics::new_unknown(&schema))); |
| 184 | + |
| 185 | + let encoded = protobuf::PartitionedFile::try_from(&pf)?; |
| 186 | + let decoded = PartitionedFile::try_from(&encoded)?; |
| 187 | + |
| 188 | + assert_eq!(decoded.object_meta.location, pf.object_meta.location); |
| 189 | + assert_eq!(decoded.object_meta.size, pf.object_meta.size); |
| 190 | + assert_eq!( |
| 191 | + decoded.object_meta.last_modified, |
| 192 | + pf.object_meta.last_modified |
| 193 | + ); |
| 194 | + assert_eq!(decoded.partition_values, pf.partition_values); |
| 195 | + assert_eq!(decoded.range, pf.range); |
| 196 | + assert_eq!(decoded.arrow_schema.as_deref(), Some(schema.as_ref())); |
| 197 | + // Statistics span the full table schema (file columns followed by one |
| 198 | + // entry per partition column), and survive the round trip intact. |
| 199 | + assert_eq!( |
| 200 | + pf.statistics.as_ref().unwrap().column_statistics.len(), |
| 201 | + schema.fields().len() + pf.partition_values.len() |
| 202 | + ); |
| 203 | + assert_eq!(decoded.statistics, pf.statistics); |
| 204 | + Ok(()) |
| 205 | + } |
| 206 | + |
| 207 | + #[test] |
| 208 | + fn partitioned_file_from_proto_rejects_invalid_path() { |
| 209 | + let proto = protobuf::PartitionedFile { |
| 210 | + path: "foo//bar.parquet".to_string(), |
| 211 | + ..Default::default() |
| 212 | + }; |
| 213 | + |
| 214 | + let err = PartitionedFile::try_from(&proto).unwrap_err(); |
| 215 | + assert!( |
| 216 | + err.to_string().contains("Invalid object_store path"), |
| 217 | + "unexpected error: {err}" |
| 218 | + ); |
| 219 | + } |
| 220 | + |
| 221 | + #[test] |
| 222 | + fn file_group_roundtrip() -> Result<()> { |
| 223 | + let group = FileGroup::new(vec![ |
| 224 | + PartitionedFile::new("a.parquet", 1), |
| 225 | + PartitionedFile::new("b.parquet", 2), |
| 226 | + ]); |
| 227 | + |
| 228 | + let encoded = protobuf::FileGroup::try_from(&group)?; |
| 229 | + let decoded = FileGroup::try_from(&encoded)?; |
| 230 | + |
| 231 | + assert_eq!(decoded.len(), 2); |
| 232 | + assert_eq!( |
| 233 | + decoded.files()[1].object_meta.location, |
| 234 | + group.files()[1].object_meta.location |
| 235 | + ); |
| 236 | + Ok(()) |
| 237 | + } |
| 238 | +} |
0 commit comments