Skip to content

Commit c0545f7

Browse files
authored
fix: Record which storage class wrote a custom_storage payload (feast-dev#6988)
1 parent d6cc1dc commit c0545f7

5 files changed

Lines changed: 300 additions & 9 deletions

File tree

‎sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse_source.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -174,9 +174,7 @@ def from_proto(storage_proto: SavedDatasetStorageProto) -> SavedDatasetStorage:
174174
)
175175

176176
def to_proto(self) -> SavedDatasetStorageProto:
177-
return SavedDatasetStorageProto(
178-
custom_storage=self.clickhouse_options.to_proto()
179-
)
177+
return self._custom_storage_proto(self.clickhouse_options.to_proto())
180178

181179
def to_data_source(self) -> DataSource:
182180
return ClickhouseSource(table=self.clickhouse_options._table)

‎sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/couchbase_source.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -343,9 +343,7 @@ def from_proto(storage_proto: SavedDatasetStorageProto) -> SavedDatasetStorage:
343343
)
344344

345345
def to_proto(self) -> SavedDatasetStorageProto:
346-
return SavedDatasetStorageProto(
347-
custom_storage=self.couchbase_options.to_proto()
348-
)
346+
return self._custom_storage_proto(self.couchbase_options.to_proto())
349347

350348
def to_data_source(self) -> DataSource:
351349
return CouchbaseColumnarSource(

‎sdk/python/feast/infra/offline_stores/contrib/postgres_offline_store/postgres_source.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ def from_proto(storage_proto: SavedDatasetStorageProto) -> SavedDatasetStorage:
234234
)
235235

236236
def to_proto(self) -> SavedDatasetStorageProto:
237-
return SavedDatasetStorageProto(custom_storage=self.postgres_options.to_proto())
237+
return self._custom_storage_proto(self.postgres_options.to_proto())
238238

239239
def to_data_source(self) -> DataSource:
240240
return PostgreSQLSource(table=self.postgres_options._table)

‎sdk/python/feast/saved_dataset.py‎

Lines changed: 91 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import json
12
from abc import abstractmethod
23
from datetime import datetime
34
from typing import TYPE_CHECKING, Dict, List, Optional, Type, cast
@@ -9,6 +10,7 @@
910
from feast.data_source import DataSource
1011
from feast.dqm.profilers.profiler import Profile, Profiler
1112
from feast.importer import import_class
13+
from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto
1214
from feast.protos.feast.core.SavedDataset_pb2 import SavedDataset as SavedDatasetProto
1315
from feast.protos.feast.core.SavedDataset_pb2 import SavedDatasetMeta, SavedDatasetSpec
1416
from feast.protos.feast.core.SavedDataset_pb2 import (
@@ -24,14 +26,63 @@
2426

2527
class _StorageRegistry(type):
2628
classes_by_proto_attr_name: Dict[str, Type["SavedDatasetStorage"]] = {}
29+
# Several storage classes share the ``custom_storage`` oneof field, so the
30+
# mapping above cannot identify them. Every claimant is recorded here in
31+
# registration order so that dispatch can tell "one obvious class" apart
32+
# from "several, pick deliberately".
33+
all_classes_by_proto_attr_name: Dict[str, List[Type["SavedDatasetStorage"]]] = {}
2734

2835
def __new__(cls, name, bases, dct):
2936
kls = type.__new__(cls, name, bases, dct)
30-
if dct.get("_proto_attr_name"):
31-
cls.classes_by_proto_attr_name[dct["_proto_attr_name"]] = kls
37+
proto_attr_name = dct.get("_proto_attr_name")
38+
if proto_attr_name:
39+
cls.classes_by_proto_attr_name[proto_attr_name] = kls
40+
claimants = cls.all_classes_by_proto_attr_name.setdefault(
41+
proto_attr_name, []
42+
)
43+
if kls not in claimants:
44+
claimants.append(kls)
3245
return kls
3346

3447

48+
# Key under which a custom_storage payload records the concrete storage class
49+
# that wrote it. CustomSourceOptions.configuration is an opaque byte string the
50+
# implementer owns, and every claimant already reads it as JSON by explicit key,
51+
# so an extra key is ignored by older readers.
52+
CUSTOM_STORAGE_CLASS_KEY = "saved_dataset_storage_class"
53+
54+
55+
def _class_path(cls: type) -> str:
56+
return f"{cls.__module__}.{cls.__qualname__}"
57+
58+
59+
def _read_custom_storage_class(
60+
options_proto: DataSourceProto.CustomSourceOptions,
61+
) -> Optional[Type["SavedDatasetStorage"]]:
62+
"""Resolve the class recorded in a custom_storage payload, if it has one.
63+
64+
Returns ``None`` for a payload written before this key existed, or one whose
65+
configuration is not the JSON object the claimants write, so the caller can
66+
fall back instead of failing on data it could previously read.
67+
"""
68+
try:
69+
config = json.loads(options_proto.configuration.decode("utf8"))
70+
except (UnicodeDecodeError, ValueError):
71+
return None
72+
if not isinstance(config, dict):
73+
return None
74+
class_path = config.get(CUSTOM_STORAGE_CLASS_KEY)
75+
if not isinstance(class_path, str) or not class_path:
76+
return None
77+
try:
78+
return get_saved_dataset_storage_class_from_path(class_path)
79+
except Exception:
80+
# A recorded path can be unresolvable here: the class may live in an
81+
# extra that is not installed, or may not be importable at all. Fall
82+
# back rather than turn a readable payload into an import failure.
83+
return None
84+
85+
3586
_DATA_SOURCE_TO_SAVED_DATASET_STORAGE = {
3687
"FileSource": "feast.infra.offline_stores.file_source.SavedDatasetFileStorage",
3788
"SparkSource": "feast.infra.offline_stores.contrib.spark_offline_store.spark_source.SavedDatasetSparkStorage",
@@ -49,10 +100,48 @@ class SavedDatasetStorage(metaclass=_StorageRegistry):
49100
@staticmethod
50101
def from_proto(storage_proto: SavedDatasetStorageProto) -> "SavedDatasetStorage":
51102
proto_attr_name = cast(str, storage_proto.WhichOneof("kind"))
103+
104+
if proto_attr_name == "custom_storage":
105+
recorded = _read_custom_storage_class(storage_proto.custom_storage)
106+
if recorded is not None:
107+
return recorded.from_proto(storage_proto)
108+
109+
claimants = _StorageRegistry.all_classes_by_proto_attr_name.get(
110+
proto_attr_name, []
111+
)
112+
if len(claimants) > 1:
113+
raise ValueError(
114+
f"{len(claimants)} storage classes claim the "
115+
f"'{proto_attr_name}' field and this payload does not record "
116+
f"which one wrote it, so it cannot be read back unambiguously: "
117+
f"{', '.join(sorted(_class_path(c) for c in claimants))}. "
118+
f"Saved datasets written after this version record their class "
119+
f"under the '{CUSTOM_STORAGE_CLASS_KEY}' key; this one predates "
120+
f"that, and the payloads of these classes are not otherwise "
121+
f"distinguishable. Call the intended class's from_proto directly."
122+
)
123+
52124
return _StorageRegistry.classes_by_proto_attr_name[proto_attr_name].from_proto(
53125
storage_proto
54126
)
55127

128+
def _custom_storage_proto(
129+
self, options_proto: DataSourceProto.CustomSourceOptions
130+
) -> SavedDatasetStorageProto:
131+
"""Wrap custom source options, recording which class wrote them.
132+
133+
Classes that share the ``custom_storage`` field must use this instead of
134+
constructing the proto directly, otherwise ``from_proto`` has nothing to
135+
dispatch on and has to guess from import order.
136+
"""
137+
config = json.loads(options_proto.configuration.decode("utf8"))
138+
config[CUSTOM_STORAGE_CLASS_KEY] = _class_path(type(self))
139+
return SavedDatasetStorageProto(
140+
custom_storage=DataSourceProto.CustomSourceOptions(
141+
configuration=json.dumps(config).encode()
142+
)
143+
)
144+
56145
@abstractmethod
57146
def to_proto(self) -> SavedDatasetStorageProto:
58147
pass
Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
import json
2+
import sys
3+
import types
4+
5+
import pytest
6+
7+
from feast.infra.offline_stores.file_source import SavedDatasetFileStorage
8+
from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto
9+
from feast.protos.feast.core.SavedDataset_pb2 import (
10+
SavedDatasetStorage as SavedDatasetStorageProto,
11+
)
12+
from feast.saved_dataset import (
13+
CUSTOM_STORAGE_CLASS_KEY,
14+
SavedDatasetStorage,
15+
_StorageRegistry,
16+
)
17+
18+
SYNTHETIC_MODULE = "feast_test_synthetic_storage"
19+
20+
# Two storage classes that share a proto field, in a real importable module so
21+
# that the class path recorded in a payload can actually be resolved.
22+
SYNTHETIC_SOURCE = """
23+
import json
24+
25+
from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto
26+
from feast.saved_dataset import SavedDatasetStorage
27+
28+
29+
def _options(**config):
30+
return DataSourceProto.CustomSourceOptions(
31+
configuration=json.dumps(config).encode()
32+
)
33+
34+
35+
class FirstSharedStorage(SavedDatasetStorage):
36+
_proto_attr_name = "custom_storage"
37+
38+
def to_proto(self):
39+
return self._custom_storage_proto(_options(table="first"))
40+
41+
@staticmethod
42+
def from_proto(storage_proto):
43+
return FirstSharedStorage()
44+
45+
def to_data_source(self):
46+
raise NotImplementedError
47+
48+
49+
class SecondSharedStorage(SavedDatasetStorage):
50+
_proto_attr_name = "custom_storage"
51+
52+
def to_proto(self):
53+
return self._custom_storage_proto(_options(table="second"))
54+
55+
@staticmethod
56+
def from_proto(storage_proto):
57+
return SecondSharedStorage()
58+
59+
def to_data_source(self):
60+
raise NotImplementedError
61+
62+
63+
"""
64+
65+
66+
@pytest.fixture
67+
def synthetic_storage():
68+
"""Register synthetic storage classes, then undo the registration.
69+
70+
The registry is process-global, so without the teardown these classes would
71+
remain claimants for every test that runs afterwards.
72+
"""
73+
by_name = dict(_StorageRegistry.classes_by_proto_attr_name)
74+
all_by_name = {
75+
key: list(value)
76+
for key, value in _StorageRegistry.all_classes_by_proto_attr_name.items()
77+
}
78+
79+
module = types.ModuleType(SYNTHETIC_MODULE)
80+
sys.modules[SYNTHETIC_MODULE] = module
81+
exec(compile(SYNTHETIC_SOURCE, SYNTHETIC_MODULE, "exec"), module.__dict__)
82+
83+
yield module
84+
85+
sys.modules.pop(SYNTHETIC_MODULE, None)
86+
_StorageRegistry.classes_by_proto_attr_name.clear()
87+
_StorageRegistry.classes_by_proto_attr_name.update(by_name)
88+
_StorageRegistry.all_classes_by_proto_attr_name.clear()
89+
_StorageRegistry.all_classes_by_proto_attr_name.update(all_by_name)
90+
91+
92+
def _custom_options(**config) -> DataSourceProto.CustomSourceOptions:
93+
return DataSourceProto.CustomSourceOptions(
94+
configuration=json.dumps(config).encode()
95+
)
96+
97+
98+
def test_a_field_with_one_claimant_is_unaffected():
99+
storage = SavedDatasetFileStorage(path="data/driver_stats.parquet")
100+
101+
read_back = SavedDatasetStorage.from_proto(storage.to_proto())
102+
103+
assert isinstance(read_back, SavedDatasetFileStorage)
104+
assert read_back.file_options.uri == "data/driver_stats.parquet"
105+
106+
107+
def test_sharing_a_field_round_trips_to_the_class_that_wrote_it(synthetic_storage):
108+
first = synthetic_storage.FirstSharedStorage()
109+
second = synthetic_storage.SecondSharedStorage()
110+
111+
# SecondSharedStorage was defined last, so the flat registry points at it.
112+
# Both must still read back as themselves.
113+
assert isinstance(
114+
SavedDatasetStorage.from_proto(first.to_proto()),
115+
synthetic_storage.FirstSharedStorage,
116+
)
117+
assert isinstance(
118+
SavedDatasetStorage.from_proto(second.to_proto()),
119+
synthetic_storage.SecondSharedStorage,
120+
)
121+
122+
123+
def test_the_written_proto_records_the_concrete_class(synthetic_storage):
124+
proto = synthetic_storage.FirstSharedStorage().to_proto()
125+
126+
config = json.loads(proto.custom_storage.configuration.decode("utf8"))
127+
128+
assert config[CUSTOM_STORAGE_CLASS_KEY] == (
129+
f"{SYNTHETIC_MODULE}.FirstSharedStorage"
130+
)
131+
# The payload the class itself wrote is preserved alongside the new key.
132+
assert config["table"] == "first"
133+
134+
135+
def test_an_untagged_payload_with_several_claimants_names_them(synthetic_storage):
136+
# What an older version wrote: no class recorded in the payload.
137+
legacy = _custom_options(table="first")
138+
assert CUSTOM_STORAGE_CLASS_KEY not in json.loads(
139+
legacy.configuration.decode("utf8")
140+
)
141+
142+
with pytest.raises(ValueError) as error:
143+
SavedDatasetStorage.from_proto(SavedDatasetStorageProto(custom_storage=legacy))
144+
145+
message = str(error.value)
146+
assert CUSTOM_STORAGE_CLASS_KEY in message
147+
# Every claimant is named so the reader knows which to call directly.
148+
assert f"{SYNTHETIC_MODULE}.FirstSharedStorage" in message
149+
assert f"{SYNTHETIC_MODULE}.SecondSharedStorage" in message
150+
151+
152+
def test_an_unresolvable_recorded_class_does_not_raise_an_import_error(
153+
synthetic_storage,
154+
):
155+
"""A recorded class whose module is not installed must fall back.
156+
157+
The fallback still cannot disambiguate, so this reports the ambiguity
158+
rather than leaking an import failure from the recorded path.
159+
"""
160+
payload = _custom_options(
161+
table="t", **{CUSTOM_STORAGE_CLASS_KEY: "not_a_module.NotAClass"}
162+
)
163+
164+
with pytest.raises(ValueError) as error:
165+
SavedDatasetStorage.from_proto(SavedDatasetStorageProto(custom_storage=payload))
166+
167+
assert CUSTOM_STORAGE_CLASS_KEY in str(error.value)
168+
169+
170+
@pytest.mark.parametrize(
171+
"module_path, class_name, kwargs",
172+
[
173+
(
174+
"feast.infra.offline_stores.contrib.postgres_offline_store.postgres_source",
175+
"SavedDatasetPostgreSQLStorage",
176+
{"table_ref": "public.saved_dataset"},
177+
),
178+
(
179+
"feast.infra.offline_stores.contrib.clickhouse_offline_store.clickhouse_source",
180+
"SavedDatasetClickhouseStorage",
181+
{"table_ref": "public.saved_dataset"},
182+
),
183+
(
184+
"feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase_source",
185+
"SavedDatasetCouchbaseColumnarStorage",
186+
{
187+
"database_ref": "db",
188+
"scope_ref": "scope",
189+
"collection_ref": "collection",
190+
},
191+
),
192+
],
193+
)
194+
def test_the_real_custom_storage_classes_round_trip(module_path, class_name, kwargs):
195+
"""The three in-tree classes that share ``custom_storage``.
196+
197+
Postgres and Clickhouse serialize identical payloads, so before the writing
198+
class was recorded these two could only be told apart by import order.
199+
"""
200+
module = pytest.importorskip(module_path)
201+
storage_class = getattr(module, class_name)
202+
203+
written = storage_class(**kwargs)
204+
read_back = SavedDatasetStorage.from_proto(written.to_proto())
205+
206+
assert type(read_back) is storage_class

0 commit comments

Comments
 (0)