@@ -468,18 +468,24 @@ def _to_arrow_via_staging(self) -> pyarrow.Table:
468468 if not paths :
469469 return pyarrow .table ({})
470470
471- normalized_paths = self ._normalize_staging_paths (paths )
471+ parquet_paths = _filter_parquet_files (paths )
472+ if not parquet_paths :
473+ return pyarrow .table ({})
474+
475+ normalized_paths = self ._normalize_staging_paths (parquet_paths )
472476 dataset = ds .dataset (normalized_paths , format = "parquet" )
473477 return dataset .to_table ()
474478
475479 def _normalize_staging_paths (self , paths : List [str ]) -> List [str ]:
476- """Ensure each path has a URI scheme for PyArrow datasets."""
480+ """Normalize staging paths for PyArrow datasets."""
477481 normalized = []
478482 for path in paths :
479- if "://" in path :
483+ if path .startswith ("file://" ):
484+ normalized .append (path [len ("file://" ) :])
485+ elif "://" in path :
480486 normalized .append (path )
481487 else :
482- normalized .append (f"file:// { path } " )
488+ normalized .append (path )
483489 return normalized
484490
485491 def to_arrow_batches (
@@ -490,7 +496,11 @@ def to_arrow_batches(
490496 if not paths :
491497 return iter ([])
492498
493- normalized_paths = self ._normalize_staging_paths (paths )
499+ parquet_paths = _filter_parquet_files (paths )
500+ if not parquet_paths :
501+ return iter ([])
502+
503+ normalized_paths = self ._normalize_staging_paths (parquet_paths )
494504 dataset = ds .dataset (normalized_paths , format = "parquet" )
495505 scanner = dataset .scan ()
496506 return scanner .to_batches (batch_size = batch_size )
@@ -838,6 +848,10 @@ def _list_files_in_folder(folder):
838848 return files
839849
840850
851+ def _filter_parquet_files (paths : List [str ]) -> List [str ]:
852+ return [path for path in paths if path .endswith (".parquet" )]
853+
854+
841855def _list_hdfs_files (spark_session : SparkSession , uri : str ) -> List [str ]:
842856 jvm = spark_session ._jvm
843857 jsc = spark_session ._jsc
0 commit comments