Skip to content

Commit f416309

Browse files
shanalybshuchu
andauthored
fix: Apply on demand feature views in SparkRetrievalJob.to_spark_df() (feast-dev#6993)
* fix: Apply on demand feature views in SparkRetrievalJob.to_spark_df() SparkRetrievalJob.to_spark_df() returned the raw query result, so on demand feature views were applied only by to_df() and to_arrow(), and persist() and to_feast_df() left them out as well. Compute the requested ODFV features on the executors with mapInArrow, selecting them as RetrievalJob.to_arrow does. ODFVs are shipped as protos together with their projection, and outputs are cast to the declared feature types that have an Arrow mapping. SparkDAGRetrievalJob now overrides _to_spark_df_internal, so to_df() keeps reading the plan result. Signed-off-by: Denis Ivanov <[email protected]> * fix: Map PdfBytes, ImageBytes, Uuid, TimeUuid and ZonedTimestamp to Spark types from_feast_to_spark_type had no entries for PdfBytes, ImageBytes, Uuid, TimeUuid and ZonedTimestamp, so an on demand feature view declaring one of them failed in to_spark_df() before it ran. Map them to the Spark type of the Arrow values Feast produces for them: BinaryType, StringType and TimestampType. Signed-off-by: Denis Ivanov <[email protected]> * refactor: Move Spark on demand feature view application into a class Signed-off-by: Denis Ivanov <[email protected]> * refactor: Share the on demand feature selection between to_arrow and to_spark_df Move the requested ODFV feature lookup from RetrievalJob.to_arrow into RetrievalJob._get_odfv_feature_refs, so SparkOnDemandFeatureViewProcessor uses it instead of a copy. Signed-off-by: Denis Ivanov <[email protected]> --------- Signed-off-by: Denis Ivanov <[email protected]> Co-authored-by: Shuchu Han <[email protected]>
1 parent f61079f commit f416309

6 files changed

Lines changed: 322 additions & 32 deletions

File tree

‎sdk/python/feast/infra/compute_engines/spark/job.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ def _ensure_executed(self):
5151
result = self._plan.execute(self._context)
5252
self._spark_df = result.data
5353

54-
def to_spark_df(self) -> pyspark.sql.DataFrame:
54+
def _to_spark_df_internal(self) -> pyspark.sql.DataFrame:
5555
self._ensure_executed()
5656
assert self._spark_df is not None, "Execution plan did not produce a DataFrame"
5757
return self._spark_df

‎sdk/python/feast/infra/compute_engines/spark/nodes.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,13 +75,18 @@ def from_feast_to_spark_type(feast_type) -> Optional[SparkDataType]:
7575
if isinstance(feast_type, PrimitiveFeastType):
7676
mapping = {
7777
PrimitiveFeastType.BYTES: BinaryType(),
78+
PrimitiveFeastType.PDF_BYTES: BinaryType(),
79+
PrimitiveFeastType.IMAGE_BYTES: BinaryType(),
7880
PrimitiveFeastType.STRING: StringType(),
81+
PrimitiveFeastType.UUID: StringType(),
82+
PrimitiveFeastType.TIME_UUID: StringType(),
7983
PrimitiveFeastType.INT32: IntegerType(),
8084
PrimitiveFeastType.INT64: LongType(),
8185
PrimitiveFeastType.FLOAT64: DoubleType(),
8286
PrimitiveFeastType.FLOAT32: FloatType(),
8387
PrimitiveFeastType.BOOL: BooleanType(),
8488
PrimitiveFeastType.UNIX_TIMESTAMP: TimestampType(),
89+
PrimitiveFeastType.ZONED_TIMESTAMP: TimestampType(),
8590
PrimitiveFeastType.MAP: MapType(StringType(), StringType()),
8691
PrimitiveFeastType.JSON: StringType(),
8792
}

‎sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py‎

Lines changed: 161 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
Any,
1212
Callable,
1313
Dict,
14+
Iterable,
15+
Iterator,
1416
KeysView,
1517
List,
1618
Optional,
@@ -40,6 +42,8 @@
4042
from feast.dataframe import DataFrameEngine, FeastDataFrame
4143
from feast.errors import EntitySQLEmptyResults, InvalidEntityType
4244
from feast.feature_view import DUMMY_ENTITY_ID, DUMMY_ENTITY_VAL
45+
from feast.feature_view_projection import FeatureViewProjection
46+
from feast.field import Field
4347
from feast.infra.offline_stores import offline_utils
4448
from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import (
4549
SavedDatasetSparkStorage,
@@ -63,9 +67,16 @@
6367
normalize_monitoring_row,
6468
opt_float,
6569
)
70+
from feast.protos.feast.core.FeatureViewProjection_pb2 import (
71+
FeatureViewProjection as FeatureViewProjectionProto,
72+
)
73+
from feast.protos.feast.core.OnDemandFeatureView_pb2 import (
74+
OnDemandFeatureView as OnDemandFeatureViewProto,
75+
)
6676
from feast.repo_config import FeastConfigBaseModel, RepoConfig
6777
from feast.saved_dataset import SavedDatasetStorage
6878
from feast.type_map import spark_schema_to_np_dtypes
79+
from feast.types import from_feast_to_pyarrow_type
6980
from feast.utils import _get_fields_with_aliases, compute_non_entity_date_range
7081

7182
# Make sure spark warning doesn't raise more than once.
@@ -1030,13 +1041,29 @@ def on_demand_feature_views(self) -> List[OnDemandFeatureView]:
10301041
return self._on_demand_feature_views
10311042

10321043
def to_spark_df(self) -> pyspark.sql.DataFrame:
1044+
"""
1045+
Return the result as a Spark DataFrame.
1046+
1047+
On demand transformations will be executed on the Spark executors.
1048+
"""
1049+
spark_df = self._to_spark_df_internal()
1050+
if self.on_demand_feature_views:
1051+
return SparkOnDemandFeatureViewProcessor(
1052+
on_demand_feature_views=self.on_demand_feature_views,
1053+
full_feature_names=self.full_feature_names,
1054+
odfv_feature_refs=self._get_odfv_feature_refs(),
1055+
).transform(spark_df)
1056+
return spark_df
1057+
1058+
def _to_spark_df_internal(self) -> pyspark.sql.DataFrame:
1059+
"""Return dataset as Spark DataFrame, without on demand transformations"""
10331060
statements = self.query.split("---EOS---")
10341061
*_, last = map(self.spark_session.sql, statements)
10351062
return last
10361063

10371064
def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame:
10381065
"""Return dataset as Pandas DataFrame synchronously"""
1039-
return self.to_spark_df().toPandas()
1066+
return self._to_spark_df_internal().toPandas()
10401067

10411068
def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table:
10421069
"""Return dataset as pyarrow Table synchronously"""
@@ -1183,7 +1210,7 @@ def to_remote_storage(self) -> List[str]:
11831210
if not self.supports_remote_storage_export():
11841211
raise NotImplementedError()
11851212

1186-
sdf: pyspark.sql.DataFrame = self.to_spark_df()
1213+
sdf: pyspark.sql.DataFrame = self._to_spark_df_internal()
11871214
staging_location = self._config.offline_store.staging_location
11881215

11891216
if staging_location.startswith("/"):
@@ -1237,6 +1264,138 @@ def metadata(self) -> Optional[RetrievalMetadata]:
12371264
return self._metadata
12381265

12391266

1267+
class SparkOnDemandFeatureViewProcessor:
1268+
"""
1269+
Appends the requested on demand features to a Spark DataFrame on the executors.
1270+
1271+
The requested ODFV features are added under the names ``RetrievalJob.to_arrow``
1272+
gives them, or every declared feature of an ODFV when none of its features is
1273+
listed. Each executor rebuilds the ODFVs from their protos and applies them to
1274+
every Arrow batch, casting the results to the declared feature types where Feast
1275+
has an Arrow type for them.
1276+
"""
1277+
1278+
def __init__(
1279+
self,
1280+
on_demand_feature_views: List[OnDemandFeatureView],
1281+
full_feature_names: bool,
1282+
odfv_feature_refs: Dict[str, set[str]],
1283+
):
1284+
self.on_demand_feature_views = on_demand_feature_views
1285+
self.full_feature_names = full_feature_names
1286+
self.odfv_feature_refs = odfv_feature_refs
1287+
1288+
def transform(self, spark_df: pyspark.sql.DataFrame) -> pyspark.sql.DataFrame:
1289+
from pyspark.sql.types import StructField, StructType
1290+
1291+
from feast.infra.compute_engines.spark.nodes import from_feast_to_spark_type
1292+
1293+
output_features = self._get_requested_features()
1294+
1295+
spark_fields = list(spark_df.schema.fields)
1296+
# Feast-declared Arrow types to cast the ODFV outputs to. Types without one
1297+
# (e.g. Bytes) keep the type Arrow inferred.
1298+
output_pa_types: Dict[str, pyarrow.DataType] = {}
1299+
for name, feature in output_features.items():
1300+
spark_type = from_feast_to_spark_type(feature.dtype)
1301+
if spark_type is None:
1302+
raise ValueError(
1303+
f"Could not resolve Spark type for on demand feature '{name}' "
1304+
f"(dtype={feature.dtype})."
1305+
)
1306+
spark_fields.append(StructField(name, spark_type, nullable=True))
1307+
try:
1308+
pa_type = from_feast_to_pyarrow_type(feature.dtype)
1309+
except (ValueError, KeyError, TypeError):
1310+
continue
1311+
output_pa_types[name] = pa_type
1312+
1313+
# The ODFV proto does not carry its projection (alias, version tag), so ship
1314+
# the projection proto alongside and re-apply it with with_projection on the
1315+
# executors.
1316+
odfv_protos = [
1317+
(
1318+
odfv.to_proto().SerializeToString(),
1319+
odfv.projection.to_proto().SerializeToString(),
1320+
)
1321+
for odfv in self.on_demand_feature_views
1322+
]
1323+
# Keep self out of the closure, so the ODFVs reach the executors only as protos.
1324+
full_feature_names = self.full_feature_names
1325+
output_columns = list(output_features)
1326+
return spark_df.mapInArrow(
1327+
lambda x: SparkOnDemandFeatureViewProcessor._map_in_arrow(
1328+
x,
1329+
odfv_protos,
1330+
full_feature_names=full_feature_names,
1331+
output_columns=output_columns,
1332+
output_pa_types=output_pa_types,
1333+
),
1334+
StructType(spark_fields),
1335+
)
1336+
1337+
def _get_requested_features(self) -> Dict[str, Field]:
1338+
output_features: Dict[str, Field] = {}
1339+
for odfv in self.on_demand_feature_views:
1340+
requested_features_for_odfv = (
1341+
self.odfv_feature_refs.get(odfv.name)
1342+
if odfv.name in self.odfv_feature_refs
1343+
else self.odfv_feature_refs.get(odfv.projection.name_to_use())
1344+
)
1345+
for feature in odfv.features:
1346+
col = (
1347+
f"{odfv.projection.name_to_use()}__{feature.name}"
1348+
if self.full_feature_names
1349+
else feature.name
1350+
)
1351+
if (
1352+
requested_features_for_odfv is None
1353+
or col in requested_features_for_odfv
1354+
):
1355+
output_features[col] = feature
1356+
return output_features
1357+
1358+
@staticmethod
1359+
def _map_in_arrow(
1360+
iterator: Iterable[pyarrow.RecordBatch],
1361+
odfv_protos: List[Tuple[bytes, bytes]],
1362+
full_feature_names: bool,
1363+
output_columns: List[str],
1364+
output_pa_types: Dict[str, pyarrow.DataType],
1365+
) -> Iterator[pyarrow.RecordBatch]:
1366+
for batch in iterator:
1367+
features_table = pyarrow.Table.from_batches([batch])
1368+
1369+
odfvs = []
1370+
for serialized_odfv, serialized_projection in odfv_protos:
1371+
odfv_proto = OnDemandFeatureViewProto.FromString(serialized_odfv)
1372+
projection_proto = FeatureViewProjectionProto.FromString(
1373+
serialized_projection
1374+
)
1375+
odfvs.append(
1376+
OnDemandFeatureView.from_proto(odfv_proto).with_projection(
1377+
FeatureViewProjection.from_proto(projection_proto)
1378+
)
1379+
)
1380+
1381+
for odfv in odfvs:
1382+
transformed_arrow = odfv.transform_arrow(
1383+
features_table, full_feature_names
1384+
)
1385+
1386+
for col in transformed_arrow.column_names:
1387+
if col in output_columns:
1388+
column = transformed_arrow[col]
1389+
if col in output_pa_types:
1390+
column = column.cast(output_pa_types[col])
1391+
features_table = features_table.append_column(col, column)
1392+
1393+
# mapInArrow requires the declared column order.
1394+
yield from features_table.select(
1395+
batch.schema.names + output_columns
1396+
).to_batches()
1397+
1398+
12401399
def _register_iceberg_source_as_temp_view(
12411400
spark_session: SparkSession,
12421401
data_source: "DataSource",

‎sdk/python/feast/infra/offline_stores/offline_store.py‎

Lines changed: 33 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -217,33 +217,7 @@ def to_arrow(
217217
)
218218

219219
if self.on_demand_feature_views:
220-
# Build a mapping of ODFV name to requested feature names
221-
# This ensures we only return the features that were explicitly requested
222-
odfv_feature_refs: Dict[str, set[str]] = {}
223-
try:
224-
metadata = self.metadata
225-
except NotImplementedError:
226-
metadata = None
227-
228-
if metadata and metadata.features:
229-
for feature_ref in metadata.features:
230-
if ":" in feature_ref:
231-
view_name, feature_name = feature_ref.split(":", 1)
232-
# Check if this view_name matches any of the ODFVs
233-
for odfv in self.on_demand_feature_views:
234-
if (
235-
odfv.name == view_name
236-
or odfv.projection.name_to_use() == view_name
237-
):
238-
if view_name not in odfv_feature_refs:
239-
odfv_feature_refs[view_name] = set()
240-
# Store the feature name in the format that will appear in transformed_arrow
241-
expected_col_name = (
242-
f"{odfv.projection.name_to_use()}__{feature_name}"
243-
if self.full_feature_names
244-
else feature_name
245-
)
246-
odfv_feature_refs[view_name].add(expected_col_name)
220+
odfv_feature_refs = self._get_odfv_feature_refs()
247221

248222
for odfv in self.on_demand_feature_views:
249223
transformed_arrow = odfv.transform_arrow(
@@ -393,6 +367,38 @@ def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table:
393367
"""
394368
raise NotImplementedError
395369

370+
def _get_odfv_feature_refs(self) -> Dict[str, set[str]]:
371+
"""
372+
Returns a mapping of ODFV name to requested feature names, so that only the
373+
features that were explicitly requested are returned.
374+
"""
375+
odfv_feature_refs: Dict[str, set[str]] = {}
376+
try:
377+
metadata = self.metadata
378+
except NotImplementedError:
379+
metadata = None
380+
381+
if metadata and metadata.features:
382+
for feature_ref in metadata.features:
383+
if ":" in feature_ref:
384+
view_name, feature_name = feature_ref.split(":", 1)
385+
# Check if this view_name matches any of the ODFVs
386+
for odfv in self.on_demand_feature_views:
387+
if (
388+
odfv.name == view_name
389+
or odfv.projection.name_to_use() == view_name
390+
):
391+
if view_name not in odfv_feature_refs:
392+
odfv_feature_refs[view_name] = set()
393+
# Store the feature name in the format that will appear in transformed_arrow
394+
expected_col_name = (
395+
f"{odfv.projection.name_to_use()}__{feature_name}"
396+
if self.full_feature_names
397+
else feature_name
398+
)
399+
odfv_feature_refs[view_name].add(expected_col_name)
400+
return odfv_feature_refs
401+
396402
@property
397403
def full_feature_names(self) -> bool:
398404
"""Returns True if full feature names should be applied to the results of the query."""

0 commit comments

Comments
 (0)