Skip to content

Instantly share code, notes, and snippets.

@whoahbot
Last active April 2, 2024 15:41
Show Gist options
  • Select an option

  • Save whoahbot/29255324f925447cc9b3aa6db3a4e9ef to your computer and use it in GitHub Desktop.

Select an option

Save whoahbot/29255324f925447cc9b3aa6db3a4e9ef to your computer and use it in GitHub Desktop.
Updated Bytewax dataflow for Feast
import logging
import os
from datetime import datetime
from typing import Generator, List
import bytewax.operators as op
import pyarrow as pa
import pyarrow.parquet as pq
from bytewax.dataflow import Dataflow
from bytewax.inputs import DynamicSource, StatelessSourcePartition
from bytewax.outputs import DynamicSink, StatelessSinkPartition
from bytewax.testing import run_main
from feast import FeatureStore, FeatureView, RepoConfig
from feast.utils import _convert_arrow_to_proto, _run_pyarrow_field_mapping
from pyarrow import RecordBatch
logger = logging.getLogger(__name__)
DEFAULT_BATCH_SIZE = 1000
class FeastPartition(StatelessSourcePartition[RecordBatch]):
def __init__(
self,
path: str,
mini_batch_size: int,
):
self._path = path
self._mini_batch_size = mini_batch_size
def next_batch(self, sched: datetime) -> Generator[list[RecordBatch], None, None]:
logger.info(f"Processing path {self._path}")
dataset = pq.ParquetDataset(self._path, use_legacy_dataset=False)
for fragment in dataset.fragments:
for batch in fragment.to_table().to_batches(
max_chunksize=self._mini_batch_size
):
yield [batch]
class FeastSource(DynamicSource):
def __init__(self, paths: list[str], job_index: int, mini_batch_size: int):
self._paths = paths
self._job_index = job_index
self._mini_batch_size = mini_batch_size
def build(
self, now: datetime, _worker_index: int, _worker_count: int
) -> FeastPartition:
return FeastPartition(self._paths[self._job_index], self._mini_batch_size)
class FeastSinkPartition(StatelessSinkPartition):
def __init__(self, config, feature_store: FeatureStore, feature_view: FeatureView):
self._config = config
self._feature_store = feature_store
self._feature_view = feature_view
def write_batch(self, mini_batch: list[RecordBatch]):
table: pa.Table = pa.Table.from_batches(mini_batch)
if self._feature_view.batch_source.field_mapping is not None:
table = _run_pyarrow_field_mapping(
table, self._feature_view.batch_source.field_mapping
)
join_key_to_value_type = {
entity.name: entity.dtype.to_value_type()
for entity in self._feature_view.entity_columns
}
rows_to_write = _convert_arrow_to_proto(
table, self._feature_view, join_key_to_value_type
)
self._feature_store._get_provider().online_write_batch(
config=self._config,
table=self._feature_view,
data=rows_to_write,
progress=None,
)
class FeastSink(DynamicSink):
def __init__(self, config, feature_store: FeatureStore, feature_view: FeatureView):
self._config = config
self._feature_store = feature_store
self._feature_view = feature_view
def build(self, _worker_index, _worker_count):
return FeastSinkPartition(self._config, self._feature_store, self._feature_view)
class BytewaxMaterializationDataflow:
def __init__(
self,
config: RepoConfig,
feature_view: FeatureView,
paths: List[str],
job_index: int,
):
feature_store = FeatureStore(config=config)
mini_batch_size = int(os.getenv("BYTEWAX_MINI_BATCH_SIZE", DEFAULT_BATCH_SIZE))
flow = Dataflow("feast_materialization")
feast_input = op.input(
"inp", flow, FeastSource(paths, job_index, mini_batch_size)
)
flow.output("out", feast_input, FeastSink(config, feature_store, feature_view))
self._run_dataflow(flow)
def _run_dataflow(self, flow: Dataflow):
run_main(flow)
import logging
import os
from datetime import datetime
from typing import List, Generator
import pyarrow as pa
import pyarrow.parquet as pq
from pyarrow import RecordBatch
from feast import FeatureStore, FeatureView, RepoConfig
from feast.utils import _convert_arrow_to_proto, _run_pyarrow_field_mapping
logger = logging.getLogger(__name__)
DEFAULT_BATCH_SIZE = 1000
def materialize(config: RepoConfig, feature_view: FeatureView, paths: List[str], job_index: int):
# Create an instance of the Feature store from the supplied config
feature_store = FeatureStore(config=config)
mini_batch_size = int(os.getenv("BYTEWAX_MINI_BATCH_SIZE", DEFAULT_BATCH_SIZE))
path = paths[job_index]
logger.info(f"Processing path {path}")
# Create a ParquetDataset, using PyArrow
dataset = pq.ParquetDataset(path, use_legacy_dataset=False)
for fragment in dataset.fragments:
for mini_batch in fragment.to_table().to_batches(
max_chunksize=DEFAULT_BATCH_SIZE
):
table: pa.Table = pa.Table.from_batches(mini_batch)
if feature_view.batch_source.field_mapping is not None:
table = _run_pyarrow_field_mapping(
table, feature_view.batch_source.field_mapping
)
join_key_to_value_type = {
entity.name: entity.dtype.to_value_type()
for entity in feature_view.entity_columns
}
rows_to_write = _convert_arrow_to_proto(
table, feature_view, join_key_to_value_type
)
feature_store._get_provider().online_write_batch(
config=config,
table=feature_view,
data=rows_to_write,
progress=None,
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment