Skip to content

Commit 6f408e1

Browse files
authored
Scope upstream asset events of a Dag run to assets and Dags the caller may read (#72924)
GET /dags/{dag_id}/dagRuns/{dag_run_id}/upstreamAssetEvents returned every asset event a run consumed as long as the caller could read the consuming Dag. Events of assets, or produced by source Dags, the caller is not allowed to read were exposed, including their name, uri, extra and source Dag/task ids, even though GET /assets/events hides the same events. Query the consumed events through the dagrun_asset_event table and apply the same per-asset and per-source-Dag readability filters as GET /assets/events, so hidden events are excluded from both the response and total_entries.
1 parent a1109f8 commit 6f408e1

2 files changed

Lines changed: 108 additions & 14 deletions

File tree

  • airflow-core

‎airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_run.py‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
from fastapi.responses import StreamingResponse
2727
from pydantic import ValidationError
2828
from sqlalchemy import select
29-
from sqlalchemy.orm import joinedload
29+
from sqlalchemy.orm import joinedload, subqueryload
3030

3131
from airflow.api_fastapi.app import get_auth_manager
3232
from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity, DagDetails
@@ -102,6 +102,8 @@
102102
from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc
103103
from airflow.api_fastapi.core_api.security import (
104104
GetUserDep,
105+
ReadableAssetEventsByAssetFilterDep,
106+
ReadableAssetEventsFilterDep,
105107
ReadableDagRunsFilterDep,
106108
requires_access_asset,
107109
requires_access_dag,
@@ -122,7 +124,7 @@
122124
from airflow.api_fastapi.logging.decorators import action_logging
123125
from airflow.exceptions import DagVersionNotFound, ParamValidationError
124126
from airflow.models import DagModel, DagRun
125-
from airflow.models.asset import AssetEvent
127+
from airflow.models.asset import AssetEvent, association_table as dagrun_asset_event_table
126128
from airflow.models.dag_version import DagVersion
127129
from airflow.utils.state import DagRunState
128130
from airflow.utils.types import DagRunTriggeredByType, DagRunType
@@ -282,26 +284,34 @@ def bulk_dag_runs(
282284
],
283285
)
284286
def get_upstream_asset_events(
285-
dag_id: str, dag_run_id: str, session: SessionDep
287+
dag_id: str,
288+
dag_run_id: str,
289+
readable_asset_events_filter: ReadableAssetEventsFilterDep,
290+
readable_asset_events_by_asset_filter: ReadableAssetEventsByAssetFilterDep,
291+
session: SessionDep,
286292
) -> AssetEventCollectionResponse:
287293
"""If dag run is asset-triggered, return the asset events that triggered it."""
288-
dag_run: DagRun | None = session.scalar(
289-
select(DagRun)
290-
.where(
294+
dag_run_pk = session.scalar(
295+
select(DagRun.id).where(
291296
DagRun.dag_id == dag_id,
292297
DagRun.run_id == dag_run_id,
293298
)
294-
.options(
295-
joinedload(DagRun.consumed_asset_events).joinedload(AssetEvent.asset),
296-
joinedload(DagRun.consumed_asset_events).subqueryload(AssetEvent.created_dagruns),
297-
)
298299
)
299-
if dag_run is None:
300+
if dag_run_pk is None:
300301
raise HTTPException(
301302
status.HTTP_404_NOT_FOUND,
302303
f"The DagRun with dag_id: `{dag_id}` and run_id: `{dag_run_id}` was not found",
303304
)
304-
events = dag_run.consumed_asset_events
305+
# Reading the consuming run must not expose events of assets or source Dags the caller may not read.
306+
events_select = apply_filters_to_select(
307+
statement=select(AssetEvent)
308+
.join(dagrun_asset_event_table, dagrun_asset_event_table.c.event_id == AssetEvent.id)
309+
.where(dagrun_asset_event_table.c.dag_run_id == dag_run_pk)
310+
.options(joinedload(AssetEvent.asset), subqueryload(AssetEvent.created_dagruns))
311+
.order_by(AssetEvent.id),
312+
filters=[readable_asset_events_filter, readable_asset_events_by_asset_filter],
313+
)
314+
events = session.scalars(events_select).unique().all()
305315
return AssetEventCollectionResponse(
306316
asset_events=serialize_asset_events(events, session=session),
307317
total_entries=len(events),

‎airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_run.py‎

Lines changed: 86 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2019,7 +2019,7 @@ def test_should_respond_200(
20192019
session.commit()
20202020
assert event.timestamp
20212021

2022-
with assert_queries_count(4):
2022+
with assert_queries_count(7):
20232023
response = test_client.get(
20242024
"/dags/TEST_DAG_ID/dagRuns/TEST_DAG_RUN_ID/upstreamAssetEvents",
20252025
)
@@ -2102,7 +2102,7 @@ def test_query_count_does_not_scale_with_consumed_events(
21022102
session.commit()
21032103

21042104
# Constant regardless of the number of consumed events (parametrized 1 vs 5).
2105-
with assert_queries_count(4):
2105+
with assert_queries_count(7):
21062106
response = test_client.get("/dags/TEST_DAG_ID/dagRuns/TEST_DAG_RUN_ID/upstreamAssetEvents")
21072107

21082108
assert response.status_code == 200
@@ -2116,6 +2116,90 @@ def test_query_count_does_not_scale_with_consumed_events(
21162116
assert triggering_by_event[newest_event_id] is True
21172117
assert all(value is False for eid, value in triggering_by_event.items() if eid != newest_event_id)
21182118

2119+
def _create_consuming_run_with_events(self, dag_maker, session):
2120+
"""
2121+
Create a run consuming three events: one per source Dag, one with no source Dag.
2122+
2123+
Returns the asset ids keyed by index and the event ids keyed by scenario.
2124+
"""
2125+
asset_ids = {}
2126+
for index in (1, 2):
2127+
asset = AssetModel(name=f"ds{index}", uri=f"file:///da{index}", group="asset")
2128+
session.add(asset)
2129+
session.flush()
2130+
asset_ids[index] = asset.id
2131+
events = {
2132+
"dag_a": AssetEvent(asset_id=asset_ids[1], source_dag_id="source_dag_a", timestamp=START_DATE1),
2133+
"dag_b": AssetEvent(asset_id=asset_ids[2], source_dag_id="source_dag_b", timestamp=START_DATE1),
2134+
"dagless": AssetEvent(asset_id=asset_ids[1], timestamp=START_DATE1),
2135+
}
2136+
session.add_all(events.values())
2137+
session.flush()
2138+
with dag_maker(
2139+
dag_id="TEST_DAG_ID", start_date=START_DATE1, schedule=timedelta(days=1), session=session
2140+
):
2141+
pass
2142+
dag_run = dag_maker.create_dagrun(run_id="TEST_DAG_RUN_ID", run_type=DagRunType.ASSET_TRIGGERED)
2143+
dag_run.consumed_asset_events.extend(events.values())
2144+
session.commit()
2145+
return asset_ids, {key: event.id for key, event in events.items()}
2146+
2147+
@pytest.mark.parametrize(
2148+
("readable_dags", "expected_events"),
2149+
[
2150+
pytest.param(["source_dag_a"], ["dag_a", "dagless"], id="one-readable-source-dag-plus-dagless"),
2151+
pytest.param(
2152+
["source_dag_a", "source_dag_b"],
2153+
["dag_a", "dag_b", "dagless"],
2154+
id="both-source-dags-readable",
2155+
),
2156+
pytest.param([], ["dagless"], id="no-readable-source-dags-still-sees-dagless"),
2157+
],
2158+
)
2159+
@mock.patch(
2160+
"airflow.api_fastapi.auth.managers.base_auth_manager.BaseAuthManager.get_authorized_dag_ids",
2161+
autospec=True,
2162+
)
2163+
def test_returns_only_events_from_source_dags_the_caller_may_read(
2164+
self, mock_get_authorized_dag_ids, test_client, dag_maker, session, readable_dags, expected_events
2165+
):
2166+
"""Reading the consuming run must not expose events produced by Dags the caller cannot read."""
2167+
mock_get_authorized_dag_ids.return_value = set(readable_dags)
2168+
_, event_ids = self._create_consuming_run_with_events(dag_maker, session)
2169+
expected_ids = [event_ids[key] for key in expected_events]
2170+
2171+
response = test_client.get("/dags/TEST_DAG_ID/dagRuns/TEST_DAG_RUN_ID/upstreamAssetEvents")
2172+
2173+
assert response.status_code == 200
2174+
body = response.json()
2175+
assert [event["id"] for event in body["asset_events"]] == expected_ids
2176+
# The count must be scoped too, so the existence of hidden events does not leak.
2177+
assert body["total_entries"] == len(expected_ids)
2178+
2179+
@mock.patch(
2180+
"airflow.api_fastapi.auth.managers.base_auth_manager.BaseAuthManager.get_authorized_dag_ids",
2181+
autospec=True,
2182+
)
2183+
@mock.patch(
2184+
"airflow.api_fastapi.auth.managers.base_auth_manager.BaseAuthManager.get_authorized_assets",
2185+
autospec=True,
2186+
)
2187+
def test_returns_only_events_of_assets_the_caller_may_read(
2188+
self, mock_get_authorized_assets, mock_get_authorized_dag_ids, test_client, dag_maker, session
2189+
):
2190+
"""An event of an asset the caller cannot read is hidden even when its source Dag is readable."""
2191+
asset_ids, event_ids = self._create_consuming_run_with_events(dag_maker, session)
2192+
mock_get_authorized_dag_ids.return_value = {"source_dag_a", "source_dag_b"}
2193+
mock_get_authorized_assets.return_value = {asset_ids[2]}
2194+
2195+
response = test_client.get("/dags/TEST_DAG_ID/dagRuns/TEST_DAG_RUN_ID/upstreamAssetEvents")
2196+
2197+
assert response.status_code == 200
2198+
body = response.json()
2199+
assert [event["id"] for event in body["asset_events"]] == [event_ids["dag_b"]]
2200+
assert {event["uri"] for event in body["asset_events"]} == {"file:///da2"}
2201+
assert body["total_entries"] == 1
2202+
21192203
def test_should_respond_401(self, unauthenticated_test_client):
21202204
response = unauthenticated_test_client.get(
21212205
"/dags/TEST_DAG_ID/dagRuns/TEST_DAG_RUN_ID/upstreamAssetEvents",

0 commit comments

Comments
 (0)