Skip to content

Commit 456562e

Browse files
committed
1 parent 0691277 commit 456562e

2 files changed

Lines changed: 21 additions & 9 deletions

File tree

  • sdk/python
    • feast/infra/offline_stores/contrib/spark_offline_store
    • tests/unit/infra/offline_stores/contrib/spark_offline_store

‎sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -468,18 +468,24 @@ def _to_arrow_via_staging(self) -> pyarrow.Table:
468468
if not paths:
469469
return pyarrow.table({})
470470

471-
normalized_paths = self._normalize_staging_paths(paths)
471+
parquet_paths = _filter_parquet_files(paths)
472+
if not parquet_paths:
473+
return pyarrow.table({})
474+
475+
normalized_paths = self._normalize_staging_paths(parquet_paths)
472476
dataset = ds.dataset(normalized_paths, format="parquet")
473477
return dataset.to_table()
474478

475479
def _normalize_staging_paths(self, paths: List[str]) -> List[str]:
476-
"""Ensure each path has a URI scheme for PyArrow datasets."""
480+
"""Normalize staging paths for PyArrow datasets."""
477481
normalized = []
478482
for path in paths:
479-
if "://" in path:
483+
if path.startswith("file://"):
484+
normalized.append(path[len("file://") :])
485+
elif "://" in path:
480486
normalized.append(path)
481487
else:
482-
normalized.append(f"file://{path}")
488+
normalized.append(path)
483489
return normalized
484490

485491
def to_arrow_batches(
@@ -490,7 +496,11 @@ def to_arrow_batches(
490496
if not paths:
491497
return iter([])
492498

493-
normalized_paths = self._normalize_staging_paths(paths)
499+
parquet_paths = _filter_parquet_files(paths)
500+
if not parquet_paths:
501+
return iter([])
502+
503+
normalized_paths = self._normalize_staging_paths(parquet_paths)
494504
dataset = ds.dataset(normalized_paths, format="parquet")
495505
scanner = dataset.scan()
496506
return scanner.to_batches(batch_size=batch_size)
@@ -838,6 +848,10 @@ def _list_files_in_folder(folder):
838848
return files
839849

840850

851+
def _filter_parquet_files(paths: List[str]) -> List[str]:
852+
return [path for path in paths if path.endswith(".parquet")]
853+
854+
841855
def _list_hdfs_files(spark_session: SparkSession, uri: str) -> List[str]:
842856
jvm = spark_session._jvm
843857
jsc = spark_session._jsc

‎sdk/python/tests/unit/infra/offline_stores/contrib/spark_offline_store/test_spark.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -544,7 +544,7 @@ def test_to_arrow_uses_staging_when_enabled(monkeypatch, tmp_path):
544544

545545
job.to_remote_storage.assert_called_once()
546546
spark_module.ds.dataset.assert_called_once_with(
547-
["file:///test.parquet"], format="parquet"
547+
["/test.parquet"], format="parquet"
548548
)
549549
assert result.equals(expected_table)
550550

@@ -583,9 +583,7 @@ def test_to_arrow_normalizes_local_staging_paths(monkeypatch, tmp_path):
583583
result = job._to_arrow_internal()
584584

585585
job.to_remote_storage.assert_called_once()
586-
spark_module.ds.dataset.assert_called_once_with(
587-
[f"file://{raw_path}"], format="parquet"
588-
)
586+
spark_module.ds.dataset.assert_called_once_with([raw_path], format="parquet")
589587
assert result.equals(expected_table)
590588

591589

0 commit comments

Comments
 (0)