Last active
April 2, 2024 15:41
-
-
Save whoahbot/29255324f925447cc9b3aa6db3a4e9ef to your computer and use it in GitHub Desktop.
Updated Bytewax dataflow for Feast
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import logging | |
| import os | |
| from datetime import datetime | |
| from typing import Generator, List | |
| import bytewax.operators as op | |
| import pyarrow as pa | |
| import pyarrow.parquet as pq | |
| from bytewax.dataflow import Dataflow | |
| from bytewax.inputs import DynamicSource, StatelessSourcePartition | |
| from bytewax.outputs import DynamicSink, StatelessSinkPartition | |
| from bytewax.testing import run_main | |
| from feast import FeatureStore, FeatureView, RepoConfig | |
| from feast.utils import _convert_arrow_to_proto, _run_pyarrow_field_mapping | |
| from pyarrow import RecordBatch | |
| logger = logging.getLogger(__name__) | |
| DEFAULT_BATCH_SIZE = 1000 | |
| class FeastPartition(StatelessSourcePartition[RecordBatch]): | |
| def __init__( | |
| self, | |
| path: str, | |
| mini_batch_size: int, | |
| ): | |
| self._path = path | |
| self._mini_batch_size = mini_batch_size | |
| def next_batch(self, sched: datetime) -> Generator[list[RecordBatch], None, None]: | |
| logger.info(f"Processing path {self._path}") | |
| dataset = pq.ParquetDataset(self._path, use_legacy_dataset=False) | |
| for fragment in dataset.fragments: | |
| for batch in fragment.to_table().to_batches( | |
| max_chunksize=self._mini_batch_size | |
| ): | |
| yield [batch] | |
| class FeastSource(DynamicSource): | |
| def __init__(self, paths: list[str], job_index: int, mini_batch_size: int): | |
| self._paths = paths | |
| self._job_index = job_index | |
| self._mini_batch_size = mini_batch_size | |
| def build( | |
| self, now: datetime, _worker_index: int, _worker_count: int | |
| ) -> FeastPartition: | |
| return FeastPartition(self._paths[self._job_index], self._mini_batch_size) | |
| class FeastSinkPartition(StatelessSinkPartition): | |
| def __init__(self, config, feature_store: FeatureStore, feature_view: FeatureView): | |
| self._config = config | |
| self._feature_store = feature_store | |
| self._feature_view = feature_view | |
| def write_batch(self, mini_batch: list[RecordBatch]): | |
| table: pa.Table = pa.Table.from_batches(mini_batch) | |
| if self._feature_view.batch_source.field_mapping is not None: | |
| table = _run_pyarrow_field_mapping( | |
| table, self._feature_view.batch_source.field_mapping | |
| ) | |
| join_key_to_value_type = { | |
| entity.name: entity.dtype.to_value_type() | |
| for entity in self._feature_view.entity_columns | |
| } | |
| rows_to_write = _convert_arrow_to_proto( | |
| table, self._feature_view, join_key_to_value_type | |
| ) | |
| self._feature_store._get_provider().online_write_batch( | |
| config=self._config, | |
| table=self._feature_view, | |
| data=rows_to_write, | |
| progress=None, | |
| ) | |
| class FeastSink(DynamicSink): | |
| def __init__(self, config, feature_store: FeatureStore, feature_view: FeatureView): | |
| self._config = config | |
| self._feature_store = feature_store | |
| self._feature_view = feature_view | |
| def build(self, _worker_index, _worker_count): | |
| return FeastSinkPartition(self._config, self._feature_store, self._feature_view) | |
| class BytewaxMaterializationDataflow: | |
| def __init__( | |
| self, | |
| config: RepoConfig, | |
| feature_view: FeatureView, | |
| paths: List[str], | |
| job_index: int, | |
| ): | |
| feature_store = FeatureStore(config=config) | |
| mini_batch_size = int(os.getenv("BYTEWAX_MINI_BATCH_SIZE", DEFAULT_BATCH_SIZE)) | |
| flow = Dataflow("feast_materialization") | |
| feast_input = op.input( | |
| "inp", flow, FeastSource(paths, job_index, mini_batch_size) | |
| ) | |
| flow.output("out", feast_input, FeastSink(config, feature_store, feature_view)) | |
| self._run_dataflow(flow) | |
| def _run_dataflow(self, flow: Dataflow): | |
| run_main(flow) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import logging | |
| import os | |
| from datetime import datetime | |
| from typing import List, Generator | |
| import pyarrow as pa | |
| import pyarrow.parquet as pq | |
| from pyarrow import RecordBatch | |
| from feast import FeatureStore, FeatureView, RepoConfig | |
| from feast.utils import _convert_arrow_to_proto, _run_pyarrow_field_mapping | |
| logger = logging.getLogger(__name__) | |
| DEFAULT_BATCH_SIZE = 1000 | |
| def materialize(config: RepoConfig, feature_view: FeatureView, paths: List[str], job_index: int): | |
| # Create an instance of the Feature store from the supplied config | |
| feature_store = FeatureStore(config=config) | |
| mini_batch_size = int(os.getenv("BYTEWAX_MINI_BATCH_SIZE", DEFAULT_BATCH_SIZE)) | |
| path = paths[job_index] | |
| logger.info(f"Processing path {path}") | |
| # Create a ParquetDataset, using PyArrow | |
| dataset = pq.ParquetDataset(path, use_legacy_dataset=False) | |
| for fragment in dataset.fragments: | |
| for mini_batch in fragment.to_table().to_batches( | |
| max_chunksize=DEFAULT_BATCH_SIZE | |
| ): | |
| table: pa.Table = pa.Table.from_batches(mini_batch) | |
| if feature_view.batch_source.field_mapping is not None: | |
| table = _run_pyarrow_field_mapping( | |
| table, feature_view.batch_source.field_mapping | |
| ) | |
| join_key_to_value_type = { | |
| entity.name: entity.dtype.to_value_type() | |
| for entity in feature_view.entity_columns | |
| } | |
| rows_to_write = _convert_arrow_to_proto( | |
| table, feature_view, join_key_to_value_type | |
| ) | |
| feature_store._get_provider().online_write_batch( | |
| config=config, | |
| table=feature_view, | |
| data=rows_to_write, | |
| progress=None, | |
| ) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment