Skip to content

Commit 1196e22

Browse files
haoxu0ntkathole
haoxu0
authored and
ntkathole
committed
feat: Add LanceFormat table format with version/tag pinning
Extends the existing TableFormat abstraction (#5650) with Lance rather than introducing a separate data source, so Lance is addressed the same way Iceberg, Delta and Hudi already are. Closes part of #6899. LanceFormat carries catalog/namespace addressing plus an optional pin to a dataset version or tag. Because SparkSource already drives its reader generically from table_format.format_type.value and table_format.properties, this works with SparkSource with no changes to it: format_type.value is "lance", and the pin is mirrored into properties as lance.version / lance.tag. version is validated as >= 1 so the Python and proto semantics agree. Lance dataset versions start at 1 and the proto treats 0 as unset, so without that guard version=0 would not round-trip, since to_proto/from_proto read 0 as absent. version and tag are mutually exclusive, because a tag already resolves to a version. Only DataFormat_pb2 is regenerated, using grpcio-tools 1.62.3 so the emitted gencode stays at the 4.25.1 level the other checked-in protos use. Regenerating with the pinned grpcio-tools 1.84.0 instead emits gencode that calls ValidateProtobufRuntimeVersion for protobuf 7.35.1, which would break the declared protobuf>=4.24.0 floor for that one module. DataSource_pb2 is deliberately left untouched. It is already stale against DataSource.proto on master, missing ConnectionRef entries, and regenerating it produces ~125 lines of churn unrelated to this change. Signed-off-by: hao-xu5 <[email protected]>
1 parent 298c3f6 commit 1196e22

5 files changed

Lines changed: 372 additions & 25 deletions

File tree

‎protos/feast/core/DataFormat.proto‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,33 @@ message TableFormat {
7575
map<string, string> properties = 4;
7676
}
7777

78+
// Defines options for the Lance table format
79+
message LanceFormat {
80+
// Optional catalog name for the Lance table
81+
string catalog = 1;
82+
83+
// Optional namespace within the catalog
84+
string namespace = 2;
85+
86+
// Pin reads to a specific Lance dataset version. Zero means unset.
87+
// Mutually exclusive with tag.
88+
uint64 version = 3;
89+
90+
// Pin reads to a named Lance tag, which resolves to a version.
91+
// Mutually exclusive with version.
92+
string tag = 4;
93+
94+
// Additional properties for Lance configuration
95+
// Examples: storage options, index parameters, etc.
96+
map<string, string> properties = 5;
97+
}
98+
7899
// Specifies the table format and format-specific options
79100
oneof format {
80101
IcebergFormat iceberg_format = 1;
81102
DeltaFormat delta_format = 2;
82103
HudiFormat hudi_format = 3;
104+
LanceFormat lance_format = 4;
83105
}
84106
}
85107

‎sdk/python/feast/protos/feast/core/DataFormat_pb2.py‎

Lines changed: 28 additions & 22 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎sdk/python/feast/protos/feast/core/DataFormat_pb2.pyi‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -191,25 +191,82 @@ class TableFormat(google.protobuf.message.Message):
191191
) -> None: ...
192192
def ClearField(self, field_name: typing_extensions.Literal["precombine_field", b"precombine_field", "properties", b"properties", "record_key", b"record_key", "table_type", b"table_type"]) -> None: ...
193193

194+
class LanceFormat(google.protobuf.message.Message):
195+
"""Defines options for the Lance table format"""
196+
197+
DESCRIPTOR: google.protobuf.descriptor.Descriptor
198+
199+
class PropertiesEntry(google.protobuf.message.Message):
200+
DESCRIPTOR: google.protobuf.descriptor.Descriptor
201+
202+
KEY_FIELD_NUMBER: builtins.int
203+
VALUE_FIELD_NUMBER: builtins.int
204+
key: builtins.str
205+
value: builtins.str
206+
def __init__(
207+
self,
208+
*,
209+
key: builtins.str = ...,
210+
value: builtins.str = ...,
211+
) -> None: ...
212+
def ClearField(self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]) -> None: ...
213+
214+
CATALOG_FIELD_NUMBER: builtins.int
215+
NAMESPACE_FIELD_NUMBER: builtins.int
216+
VERSION_FIELD_NUMBER: builtins.int
217+
TAG_FIELD_NUMBER: builtins.int
218+
PROPERTIES_FIELD_NUMBER: builtins.int
219+
catalog: builtins.str
220+
"""Optional catalog name for the Lance table"""
221+
namespace: builtins.str
222+
"""Optional namespace within the catalog"""
223+
version: builtins.int
224+
"""Pin reads to a specific Lance dataset version. Zero means unset.
225+
Mutually exclusive with tag.
226+
"""
227+
tag: builtins.str
228+
"""Pin reads to a named Lance tag, which resolves to a version.
229+
Mutually exclusive with version.
230+
"""
231+
@property
232+
def properties(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
233+
"""Additional properties for Lance configuration
234+
Examples: storage options, index parameters, etc.
235+
"""
236+
def __init__(
237+
self,
238+
*,
239+
catalog: builtins.str = ...,
240+
namespace: builtins.str = ...,
241+
version: builtins.int = ...,
242+
tag: builtins.str = ...,
243+
properties: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
244+
) -> None: ...
245+
def ClearField(self, field_name: typing_extensions.Literal["catalog", b"catalog", "namespace", b"namespace", "properties", b"properties", "tag", b"tag", "version", b"version"]) -> None: ...
246+
194247
ICEBERG_FORMAT_FIELD_NUMBER: builtins.int
195248
DELTA_FORMAT_FIELD_NUMBER: builtins.int
196249
HUDI_FORMAT_FIELD_NUMBER: builtins.int
250+
LANCE_FORMAT_FIELD_NUMBER: builtins.int
197251
@property
198252
def iceberg_format(self) -> global___TableFormat.IcebergFormat: ...
199253
@property
200254
def delta_format(self) -> global___TableFormat.DeltaFormat: ...
201255
@property
202256
def hudi_format(self) -> global___TableFormat.HudiFormat: ...
257+
@property
258+
def lance_format(self) -> global___TableFormat.LanceFormat: ...
203259
def __init__(
204260
self,
205261
*,
206262
iceberg_format: global___TableFormat.IcebergFormat | None = ...,
207263
delta_format: global___TableFormat.DeltaFormat | None = ...,
208264
hudi_format: global___TableFormat.HudiFormat | None = ...,
265+
lance_format: global___TableFormat.LanceFormat | None = ...,
209266
) -> None: ...
210-
def HasField(self, field_name: typing_extensions.Literal["delta_format", b"delta_format", "format", b"format", "hudi_format", b"hudi_format", "iceberg_format", b"iceberg_format"]) -> builtins.bool: ...
211-
def ClearField(self, field_name: typing_extensions.Literal["delta_format", b"delta_format", "format", b"format", "hudi_format", b"hudi_format", "iceberg_format", b"iceberg_format"]) -> None: ...
212-
def WhichOneof(self, oneof_group: typing_extensions.Literal["format", b"format"]) -> typing_extensions.Literal["iceberg_format", "delta_format", "hudi_format"] | None: ...
267+
def HasField(self, field_name: typing_extensions.Literal["delta_format", b"delta_format", "format", b"format", "hudi_format", b"hudi_format", "iceberg_format", b"iceberg_format", "lance_format", b"lance_format"]) -> builtins.bool: ...
268+
def ClearField(self, field_name: typing_extensions.Literal["delta_format", b"delta_format", "format", b"format", "hudi_format", b"hudi_format", "iceberg_format", b"iceberg_format", "lance_format", b"lance_format"]) -> None: ...
269+
def WhichOneof(self, oneof_group: typing_extensions.Literal["format", b"format"]) -> typing_extensions.Literal["iceberg_format", "delta_format", "hudi_format", "lance_format"] | None: ...
213270

214271
global___TableFormat = TableFormat
215272

0 commit comments

Comments
 (0)