|
11 | 11 | Any, |
12 | 12 | Callable, |
13 | 13 | Dict, |
| 14 | + Iterable, |
| 15 | + Iterator, |
14 | 16 | KeysView, |
15 | 17 | List, |
16 | 18 | Optional, |
|
40 | 42 | from feast.dataframe import DataFrameEngine, FeastDataFrame |
41 | 43 | from feast.errors import EntitySQLEmptyResults, InvalidEntityType |
42 | 44 | from feast.feature_view import DUMMY_ENTITY_ID, DUMMY_ENTITY_VAL |
| 45 | +from feast.feature_view_projection import FeatureViewProjection |
| 46 | +from feast.field import Field |
43 | 47 | from feast.infra.offline_stores import offline_utils |
44 | 48 | from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import ( |
45 | 49 | SavedDatasetSparkStorage, |
|
63 | 67 | normalize_monitoring_row, |
64 | 68 | opt_float, |
65 | 69 | ) |
| 70 | +from feast.protos.feast.core.FeatureViewProjection_pb2 import ( |
| 71 | + FeatureViewProjection as FeatureViewProjectionProto, |
| 72 | +) |
| 73 | +from feast.protos.feast.core.OnDemandFeatureView_pb2 import ( |
| 74 | + OnDemandFeatureView as OnDemandFeatureViewProto, |
| 75 | +) |
66 | 76 | from feast.repo_config import FeastConfigBaseModel, RepoConfig |
67 | 77 | from feast.saved_dataset import SavedDatasetStorage |
68 | 78 | from feast.type_map import spark_schema_to_np_dtypes |
| 79 | +from feast.types import from_feast_to_pyarrow_type |
69 | 80 | from feast.utils import _get_fields_with_aliases, compute_non_entity_date_range |
70 | 81 |
|
71 | 82 | # Make sure spark warning doesn't raise more than once. |
@@ -1030,13 +1041,29 @@ def on_demand_feature_views(self) -> List[OnDemandFeatureView]: |
1030 | 1041 | return self._on_demand_feature_views |
1031 | 1042 |
|
1032 | 1043 | def to_spark_df(self) -> pyspark.sql.DataFrame: |
| 1044 | + """ |
| 1045 | + Return the result as a Spark DataFrame. |
| 1046 | +
|
| 1047 | + On demand transformations will be executed on the Spark executors. |
| 1048 | + """ |
| 1049 | + spark_df = self._to_spark_df_internal() |
| 1050 | + if self.on_demand_feature_views: |
| 1051 | + return SparkOnDemandFeatureViewProcessor( |
| 1052 | + on_demand_feature_views=self.on_demand_feature_views, |
| 1053 | + full_feature_names=self.full_feature_names, |
| 1054 | + odfv_feature_refs=self._get_odfv_feature_refs(), |
| 1055 | + ).transform(spark_df) |
| 1056 | + return spark_df |
| 1057 | + |
| 1058 | + def _to_spark_df_internal(self) -> pyspark.sql.DataFrame: |
| 1059 | + """Return dataset as Spark DataFrame, without on demand transformations""" |
1033 | 1060 | statements = self.query.split("---EOS---") |
1034 | 1061 | *_, last = map(self.spark_session.sql, statements) |
1035 | 1062 | return last |
1036 | 1063 |
|
1037 | 1064 | def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: |
1038 | 1065 | """Return dataset as Pandas DataFrame synchronously""" |
1039 | | - return self.to_spark_df().toPandas() |
| 1066 | + return self._to_spark_df_internal().toPandas() |
1040 | 1067 |
|
1041 | 1068 | def _to_arrow_internal(self, timeout: Optional[int] = None) -> pyarrow.Table: |
1042 | 1069 | """Return dataset as pyarrow Table synchronously""" |
@@ -1183,7 +1210,7 @@ def to_remote_storage(self) -> List[str]: |
1183 | 1210 | if not self.supports_remote_storage_export(): |
1184 | 1211 | raise NotImplementedError() |
1185 | 1212 |
|
1186 | | - sdf: pyspark.sql.DataFrame = self.to_spark_df() |
| 1213 | + sdf: pyspark.sql.DataFrame = self._to_spark_df_internal() |
1187 | 1214 | staging_location = self._config.offline_store.staging_location |
1188 | 1215 |
|
1189 | 1216 | if staging_location.startswith("/"): |
@@ -1237,6 +1264,138 @@ def metadata(self) -> Optional[RetrievalMetadata]: |
1237 | 1264 | return self._metadata |
1238 | 1265 |
|
1239 | 1266 |
|
| 1267 | +class SparkOnDemandFeatureViewProcessor: |
| 1268 | + """ |
| 1269 | + Appends the requested on demand features to a Spark DataFrame on the executors. |
| 1270 | +
|
| 1271 | + The requested ODFV features are added under the names ``RetrievalJob.to_arrow`` |
| 1272 | + gives them, or every declared feature of an ODFV when none of its features is |
| 1273 | + listed. Each executor rebuilds the ODFVs from their protos and applies them to |
| 1274 | + every Arrow batch, casting the results to the declared feature types where Feast |
| 1275 | + has an Arrow type for them. |
| 1276 | + """ |
| 1277 | + |
| 1278 | + def __init__( |
| 1279 | + self, |
| 1280 | + on_demand_feature_views: List[OnDemandFeatureView], |
| 1281 | + full_feature_names: bool, |
| 1282 | + odfv_feature_refs: Dict[str, set[str]], |
| 1283 | + ): |
| 1284 | + self.on_demand_feature_views = on_demand_feature_views |
| 1285 | + self.full_feature_names = full_feature_names |
| 1286 | + self.odfv_feature_refs = odfv_feature_refs |
| 1287 | + |
| 1288 | + def transform(self, spark_df: pyspark.sql.DataFrame) -> pyspark.sql.DataFrame: |
| 1289 | + from pyspark.sql.types import StructField, StructType |
| 1290 | + |
| 1291 | + from feast.infra.compute_engines.spark.nodes import from_feast_to_spark_type |
| 1292 | + |
| 1293 | + output_features = self._get_requested_features() |
| 1294 | + |
| 1295 | + spark_fields = list(spark_df.schema.fields) |
| 1296 | + # Feast-declared Arrow types to cast the ODFV outputs to. Types without one |
| 1297 | + # (e.g. Bytes) keep the type Arrow inferred. |
| 1298 | + output_pa_types: Dict[str, pyarrow.DataType] = {} |
| 1299 | + for name, feature in output_features.items(): |
| 1300 | + spark_type = from_feast_to_spark_type(feature.dtype) |
| 1301 | + if spark_type is None: |
| 1302 | + raise ValueError( |
| 1303 | + f"Could not resolve Spark type for on demand feature '{name}' " |
| 1304 | + f"(dtype={feature.dtype})." |
| 1305 | + ) |
| 1306 | + spark_fields.append(StructField(name, spark_type, nullable=True)) |
| 1307 | + try: |
| 1308 | + pa_type = from_feast_to_pyarrow_type(feature.dtype) |
| 1309 | + except (ValueError, KeyError, TypeError): |
| 1310 | + continue |
| 1311 | + output_pa_types[name] = pa_type |
| 1312 | + |
| 1313 | + # The ODFV proto does not carry its projection (alias, version tag), so ship |
| 1314 | + # the projection proto alongside and re-apply it with with_projection on the |
| 1315 | + # executors. |
| 1316 | + odfv_protos = [ |
| 1317 | + ( |
| 1318 | + odfv.to_proto().SerializeToString(), |
| 1319 | + odfv.projection.to_proto().SerializeToString(), |
| 1320 | + ) |
| 1321 | + for odfv in self.on_demand_feature_views |
| 1322 | + ] |
| 1323 | + # Keep self out of the closure, so the ODFVs reach the executors only as protos. |
| 1324 | + full_feature_names = self.full_feature_names |
| 1325 | + output_columns = list(output_features) |
| 1326 | + return spark_df.mapInArrow( |
| 1327 | + lambda x: SparkOnDemandFeatureViewProcessor._map_in_arrow( |
| 1328 | + x, |
| 1329 | + odfv_protos, |
| 1330 | + full_feature_names=full_feature_names, |
| 1331 | + output_columns=output_columns, |
| 1332 | + output_pa_types=output_pa_types, |
| 1333 | + ), |
| 1334 | + StructType(spark_fields), |
| 1335 | + ) |
| 1336 | + |
| 1337 | + def _get_requested_features(self) -> Dict[str, Field]: |
| 1338 | + output_features: Dict[str, Field] = {} |
| 1339 | + for odfv in self.on_demand_feature_views: |
| 1340 | + requested_features_for_odfv = ( |
| 1341 | + self.odfv_feature_refs.get(odfv.name) |
| 1342 | + if odfv.name in self.odfv_feature_refs |
| 1343 | + else self.odfv_feature_refs.get(odfv.projection.name_to_use()) |
| 1344 | + ) |
| 1345 | + for feature in odfv.features: |
| 1346 | + col = ( |
| 1347 | + f"{odfv.projection.name_to_use()}__{feature.name}" |
| 1348 | + if self.full_feature_names |
| 1349 | + else feature.name |
| 1350 | + ) |
| 1351 | + if ( |
| 1352 | + requested_features_for_odfv is None |
| 1353 | + or col in requested_features_for_odfv |
| 1354 | + ): |
| 1355 | + output_features[col] = feature |
| 1356 | + return output_features |
| 1357 | + |
| 1358 | + @staticmethod |
| 1359 | + def _map_in_arrow( |
| 1360 | + iterator: Iterable[pyarrow.RecordBatch], |
| 1361 | + odfv_protos: List[Tuple[bytes, bytes]], |
| 1362 | + full_feature_names: bool, |
| 1363 | + output_columns: List[str], |
| 1364 | + output_pa_types: Dict[str, pyarrow.DataType], |
| 1365 | + ) -> Iterator[pyarrow.RecordBatch]: |
| 1366 | + for batch in iterator: |
| 1367 | + features_table = pyarrow.Table.from_batches([batch]) |
| 1368 | + |
| 1369 | + odfvs = [] |
| 1370 | + for serialized_odfv, serialized_projection in odfv_protos: |
| 1371 | + odfv_proto = OnDemandFeatureViewProto.FromString(serialized_odfv) |
| 1372 | + projection_proto = FeatureViewProjectionProto.FromString( |
| 1373 | + serialized_projection |
| 1374 | + ) |
| 1375 | + odfvs.append( |
| 1376 | + OnDemandFeatureView.from_proto(odfv_proto).with_projection( |
| 1377 | + FeatureViewProjection.from_proto(projection_proto) |
| 1378 | + ) |
| 1379 | + ) |
| 1380 | + |
| 1381 | + for odfv in odfvs: |
| 1382 | + transformed_arrow = odfv.transform_arrow( |
| 1383 | + features_table, full_feature_names |
| 1384 | + ) |
| 1385 | + |
| 1386 | + for col in transformed_arrow.column_names: |
| 1387 | + if col in output_columns: |
| 1388 | + column = transformed_arrow[col] |
| 1389 | + if col in output_pa_types: |
| 1390 | + column = column.cast(output_pa_types[col]) |
| 1391 | + features_table = features_table.append_column(col, column) |
| 1392 | + |
| 1393 | + # mapInArrow requires the declared column order. |
| 1394 | + yield from features_table.select( |
| 1395 | + batch.schema.names + output_columns |
| 1396 | + ).to_batches() |
| 1397 | + |
| 1398 | + |
1240 | 1399 | def _register_iceberg_source_as_temp_view( |
1241 | 1400 | spark_session: SparkSession, |
1242 | 1401 | data_source: "DataSource", |
|
0 commit comments