Skip to content

Commit 81673d9

Browse files
ericdeloreficentkathole
authored andcommitted
fix: Report Dask monitoring read errors instead of no data
get_monitoring_max_timestamp returned None for any exception from pq.read_table, so an unreadable source was reported as having no data and metrics were saved without max_event_timestamp. Only a missing file now means no data; other errors reach the caller. Signed-off-by: Eric Del Orefice <[email protected]>
1 parent d169238 commit 81673d9

2 files changed

Lines changed: 74 additions & 1 deletion

File tree

‎sdk/python/feast/infra/offline_stores/dask.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -667,7 +667,8 @@ def get_monitoring_max_timestamp(
667667
)
668668
try:
669669
t = pq.read_table(path, filesystem=filesystem, columns=[timestamp_field])
670-
except Exception:
670+
except FileNotFoundError:
671+
# Nothing written to the source yet: no data to monitor.
671672
return None
672673
if t.num_rows == 0:
673674
return None
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
from datetime import datetime, timezone
2+
3+
import pandas as pd
4+
import pyarrow as pa
5+
import pytest
6+
7+
from feast.infra.offline_stores import dask as dask_mod
8+
from feast.infra.offline_stores.dask import DaskOfflineStore, DaskOfflineStoreConfig
9+
from feast.infra.offline_stores.file_source import FileSource
10+
from feast.repo_config import RepoConfig
11+
12+
13+
def _config(tmp_path) -> RepoConfig:
14+
return RepoConfig(
15+
project="test_project",
16+
registry="test_registry",
17+
provider="local",
18+
offline_store=DaskOfflineStoreConfig(type="dask"),
19+
repo_path=tmp_path,
20+
)
21+
22+
23+
def _max_ts(tmp_path, path):
24+
return DaskOfflineStore.get_monitoring_max_timestamp(
25+
config=_config(tmp_path),
26+
data_source=FileSource(path=str(path), timestamp_field="event_timestamp"),
27+
timestamp_field="event_timestamp",
28+
)
29+
30+
31+
def test_returns_the_latest_event_timestamp(tmp_path):
32+
path = tmp_path / "events.parquet"
33+
pd.DataFrame(
34+
{
35+
"event_timestamp": pd.to_datetime(
36+
["2026-09-01T00:00:00Z", "2026-09-30T12:00:00Z"]
37+
),
38+
"conv_rate": [0.1, 0.2],
39+
}
40+
).to_parquet(path)
41+
42+
assert _max_ts(tmp_path, path) == datetime(2026, 9, 30, 12, tzinfo=timezone.utc)
43+
44+
45+
def test_missing_file_is_still_no_data(tmp_path):
46+
assert _max_ts(tmp_path, tmp_path / "not_written_yet.parquet") is None
47+
48+
49+
def test_unreadable_file_raises(tmp_path):
50+
# A file that exists but cannot be read is not "no data for this feature view".
51+
path = tmp_path / "events.parquet"
52+
path.write_bytes(b"not a parquet file")
53+
54+
with pytest.raises(pa.ArrowInvalid, match="magic bytes"):
55+
_max_ts(tmp_path, path)
56+
57+
58+
def test_storage_error_raises(tmp_path, monkeypatch):
59+
path = tmp_path / "events.parquet"
60+
pd.DataFrame(
61+
{"event_timestamp": pd.to_datetime(["2026-09-30T12:00:00Z"])}
62+
).to_parquet(path)
63+
64+
def _unreachable(*args, **kwargs):
65+
raise OSError(
66+
"When reading information for key 'events.parquet': AWS Error NETWORK_CONNECTION"
67+
)
68+
69+
monkeypatch.setattr(dask_mod.pq, "read_table", _unreachable)
70+
71+
with pytest.raises(OSError, match="NETWORK_CONNECTION"):
72+
_max_ts(tmp_path, path)

0 commit comments

Comments
 (0)