Skip to content

Commit a462d68

Browse files
authored
UI: Scope partitioned Dag run asset list to readable assets (#75213)
* Scope asset expressions to the assets the caller may read `next_run_assets` scopes the asset scheduling expression it serves to the assets the caller may read (#72864), and the dependencies graph followed (#73008). Four endpoints still served `DagModel.asset_expression` verbatim: the public Dag details endpoint, the UI Dag list, and both partitioned Dag run endpoints. A caller authorized on a Dag but not on its upstream assets could read every name and uri out of them, and the public endpoint needs no UI to reach. Asset names and uris commonly encode bucket, table or dataset names, which is why the rest of the asset API already scopes them. The sibling `assets` array of the pending partitioned Dag run response carries the same names; scoping that is a separate change with its own UI consumers. * UI: Scope partitioned Dag run asset list to readable assets The pending partitioned Dag run detail response carries two asset-bearing fields. `asset_expression` is scoped to the assets the caller may read; the sibling `assets` array was not, so one payload could blank an asset inside the boolean tree while spelling its name and uri out one key over. That defeated the redaction on this route, and the route's own asset dependency only checks whether the caller may read assets at all -- there is no asset id in its path for it to authorize against. The counts and partition keys of a blanked asset are kept. They describe the Dag's own schedule, which a caller who reached this endpoint can already read, and dropping them would leave a stuck run with no explanation at all. This endpoint is served only to the UI: the public v2 spec regenerates byte-identical, and no airflow-ctl model references the response.
1 parent da767b2 commit a462d68

13 files changed

Lines changed: 415 additions & 45 deletions

File tree

‎airflow-core/src/airflow/api_fastapi/core_api/datamodels/common.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,22 @@ def _coerce_unrecognized_expression_to_none(value: Any) -> Any:
220220
]
221221
"""``AssetExpression | None`` that degrades a legacy/unrecognized stored shape to ``None``."""
222222

223+
224+
_maybe_asset_expression_adapter: TypeAdapter[Union[AssetExpression, None]] = TypeAdapter(MaybeAssetExpression)
225+
226+
227+
def validate_asset_expression(value: Any) -> AssetExpression | None:
228+
"""
229+
Validate a raw ``asset_expression`` tree into the typed model, the way a response field does.
230+
231+
A route that rewrites the tree after its response model is built -- redacting the assets the
232+
caller may not read, for instance -- must assign the result of this rather than the raw tree.
233+
FastAPI does not validate the nested models of an already-built response, so a raw tree would
234+
bypass ``MaybeAssetExpression`` and reach the client verbatim, legacy shapes included.
235+
"""
236+
return _maybe_asset_expression_adapter.validate_python(value)
237+
238+
223239
# Common Bulk Data Models
224240
T = TypeVar("T")
225241
K = TypeVar("K")

‎airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/partitioned_dag_runs.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,15 @@ class PartitionedDagRunCollectionResponse(BaseModel):
4444
class PartitionedDagRunAssetResponse(BaseModel):
4545
"""Asset info within a partitioned Dag run detail."""
4646

47-
asset_id: int
48-
asset_name: str
49-
asset_uri: str
47+
asset_id: int | None
48+
asset_name: str | None
49+
asset_uri: str | None
50+
hidden: bool = False
51+
"""True when the caller may not read this asset: ``asset_id``, ``asset_name`` and ``asset_uri``
52+
are blanked and the UI must not link to the asset. The counts and partition keys stay, so the
53+
run's progress is still readable -- they describe the Dag's own schedule, which the caller can
54+
already read, rather than the asset's identity. Mirrors the redaction
55+
``redact_asset_expression`` applies to the sibling ``asset_expression`` field."""
5056
received: bool
5157
received_count: int
5258
required_count: int

‎airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5531,14 +5531,24 @@ components:
55315531
PartitionedDagRunAssetResponse:
55325532
properties:
55335533
asset_id:
5534-
type: integer
5534+
anyOf:
5535+
- type: integer
5536+
- type: 'null'
55355537
title: Asset Id
55365538
asset_name:
5537-
type: string
5539+
anyOf:
5540+
- type: string
5541+
- type: 'null'
55385542
title: Asset Name
55395543
asset_uri:
5540-
type: string
5544+
anyOf:
5545+
- type: string
5546+
- type: 'null'
55415547
title: Asset Uri
5548+
hidden:
5549+
type: boolean
5550+
title: Hidden
5551+
default: false
55425552
received:
55435553
type: boolean
55445554
title: Received

‎airflow-core/src/airflow/api_fastapi/core_api/routes/public/dags.py‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from sqlalchemy import delete, func, insert, select, update
2626

2727
from airflow.api.common import delete_dag as delete_dag_module
28+
from airflow.api_fastapi.common.asset_expression import redact_asset_expression
2829
from airflow.api_fastapi.common.dagbag import DagBagDep, get_latest_version_of_dag
2930
from airflow.api_fastapi.common.db.common import SessionDep, apply_filters_to_select, paginated_select
3031
from airflow.api_fastapi.common.db.dags import eager_load_teams, generate_dag_with_latest_run_query
@@ -58,7 +59,11 @@
5859
)
5960
from airflow.api_fastapi.common.router import AirflowRouter
6061
from airflow.api_fastapi.compat import HTTP_422_UNPROCESSABLE_CONTENT
61-
from airflow.api_fastapi.core_api.datamodels.common import BulkBody, BulkResponse
62+
from airflow.api_fastapi.core_api.datamodels.common import (
63+
BulkBody,
64+
BulkResponse,
65+
validate_asset_expression,
66+
)
6267
from airflow.api_fastapi.core_api.datamodels.dags import (
6368
BulkDAGBody,
6469
DAGCollectionResponse,
@@ -71,6 +76,7 @@
7176
from airflow.api_fastapi.core_api.security import (
7277
EditableDagsFilterDep,
7378
GetUserDep,
79+
ReadableAssetsFilterDep,
7480
ReadableDagsFilterDep,
7581
requires_access_dag,
7682
requires_access_dag_bulk,
@@ -239,7 +245,11 @@ def get_dag(
239245
dependencies=[Depends(requires_access_dag(method="GET"))],
240246
)
241247
def get_dag_details(
242-
dag_id: str, session: SessionDep, dag_bag: DagBagDep, user: GetUserDep
248+
dag_id: str,
249+
session: SessionDep,
250+
dag_bag: DagBagDep,
251+
user: GetUserDep,
252+
readable_assets_filter: ReadableAssetsFilterDep,
243253
) -> DAGDetailsResponse:
244254
"""Get details of Dag."""
245255
dag = get_latest_version_of_dag(dag_bag, dag_id, session)
@@ -280,7 +290,16 @@ def get_dag_details(
280290
setattr(dag_model, "active_runs_count", active_runs_count)
281291
setattr(dag_model, "latest_dag_version", latest_dag_version)
282292

283-
return DAGDetailsResponse.model_validate(dag_model)
293+
details = DAGDetailsResponse.model_validate(dag_model)
294+
# The redaction lands on the response, not on `dag_model`: that is a live ORM instance whose
295+
# session commits when the request ends, so assigning the blanked leaves back onto it would
296+
# persist them.
297+
details.asset_expression = validate_asset_expression(
298+
redact_asset_expression(
299+
dag_model.asset_expression, readable_asset_ids=readable_assets_filter.value or set()
300+
)
301+
)
302+
return details
284303

285304

286305
@dags_router.patch(

‎airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
from sqlalchemy.orm import defaultload
2727

2828
from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity
29+
from airflow.api_fastapi.common.asset_expression import redact_asset_expression
2930
from airflow.api_fastapi.common.dagbag import DagBagDep
3031
from airflow.api_fastapi.common.db.common import (
3132
SessionDep,
@@ -79,6 +80,7 @@
7980
from airflow.api_fastapi.core_api.routes.public.hitl import HITLDetailView
8081
from airflow.api_fastapi.core_api.security import (
8182
GetUserDep,
83+
ReadableAssetsFilterDep,
8284
ReadableDagsFilterDep,
8385
requires_access_dag,
8486
)
@@ -155,6 +157,7 @@ def get_dags(
155157
],
156158
has_pending_actions: QueryPendingActionsFilter,
157159
readable_dags_filter: ReadableDagsFilterDep,
160+
readable_assets_filter: ReadableAssetsFilterDep,
158161
session: SessionDep,
159162
dag_bag: DagBagDep,
160163
user: GetUserDep,
@@ -311,6 +314,7 @@ def get_dags(
311314
# Build the dict dynamically from DAGResponse.model_fields so that new fields
312315
# added to DAGResponse are picked up automatically without code changes here.
313316
dag_runs_by_dag_id: dict[str, DAGWithLatestDagRunsResponse] = {}
317+
readable_asset_ids = readable_assets_filter.value or set()
314318
for dag in dags:
315319
dag_data = {
316320
DAG_ALIAS_MAPPING.get(field_name, field_name): getattr(
@@ -320,7 +324,9 @@ def get_dags(
320324
}
321325
dag_data.update(
322326
{
323-
"asset_expression": dag.asset_expression,
327+
"asset_expression": redact_asset_expression(
328+
dag.asset_expression, readable_asset_ids=readable_asset_ids
329+
),
324330
"latest_dag_runs": [],
325331
"has_unfinished_runs": has_unfinished_runs_by_dag_id[dag.dag_id],
326332
"pending_actions": pending_actions_by_dag_id[dag.dag_id],

‎airflow-core/src/airflow/api_fastapi/core_api/routes/ui/partitioned_dag_runs.py‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
from fastapi import Depends, HTTPException, status
2323
from sqlalchemy import and_, select
2424

25+
from airflow.api_fastapi.common.asset_expression import redact_asset_expression
2526
from airflow.api_fastapi.common.db.common import SessionDep, apply_filters_to_select, paginated_select
2627
from airflow.api_fastapi.common.parameters import (
2728
QueryLimit,
@@ -42,6 +43,7 @@
4243
)
4344
from airflow.api_fastapi.core_api.openapi.exceptions import create_openapi_http_exception_doc
4445
from airflow.api_fastapi.core_api.security import (
46+
ReadableAssetsFilterDep,
4547
ReadableDagsFilterDep,
4648
requires_access_asset,
4749
requires_access_dag,
@@ -251,6 +253,7 @@ def get_partitioned_dag_runs(
251253
limit: QueryLimit,
252254
offset: QueryOffset,
253255
readable_dags_filter: ReadableDagsFilterDep,
256+
readable_assets_filter: ReadableAssetsFilterDep,
254257
dag_id: QueryPartitionedDagRunDagIdFilter,
255258
has_created_dag_run_id: QueryPartitionedDagRunHasCreatedDagRunIdFilter,
256259
) -> PartitionedDagRunCollectionResponse:
@@ -351,7 +354,11 @@ def get_partitioned_dag_runs(
351354

352355
asset_expressions: dict[str, dict | None] | None = None
353356
if dag_id.value is None:
354-
asset_expressions = {dm.dag_id: dm.asset_expression for dm in dag_models.values()}
357+
readable_asset_ids = readable_assets_filter.value or set()
358+
asset_expressions = {
359+
dm.dag_id: redact_asset_expression(dm.asset_expression, readable_asset_ids=readable_asset_ids)
360+
for dm in dag_models.values()
361+
}
355362

356363
model_data: dict[str, Any] = {
357364
"partitioned_dag_runs": results,
@@ -370,6 +377,7 @@ def get_pending_partitioned_dag_run(
370377
dag_id: str,
371378
partition_key: str,
372379
session: SessionDep,
380+
readable_assets_filter: ReadableAssetsFilterDep,
373381
partitioned_dag_run_id: int | None = None,
374382
) -> PartitionedDagRunDetailResponse:
375383
"""Return full details for pending PartitionedDagRun."""
@@ -437,8 +445,13 @@ def get_pending_partitioned_dag_run(
437445
has_rollup_mappers = dag_model is not None and dag_model.has_rollup_mappers
438446
rollup_timetable = load_partitioned_timetable(dag_id, session) if has_rollup_mappers else None
439447

448+
readable_asset_ids = readable_assets_filter.value or set()
449+
440450
assets = []
441451
for asset_row in asset_rows:
452+
# The row itself is still needed in full below: the rollup mapper is keyed by the asset's
453+
# real name and uri, and the received-key lookup by its real id. Only the response is blanked.
454+
is_hidden = asset_row.id not in readable_asset_ids
442455
received_keys = sorted(received_keys_by_asset.get(asset_row.id, set()))
443456
required_keys: list[str] = [partition_key]
444457
is_rollup = (
@@ -480,9 +493,10 @@ def get_pending_partitioned_dag_run(
480493
received_count = 1 if received_keys else 0
481494
assets.append(
482495
PartitionedDagRunAssetResponse(
483-
asset_id=asset_row.id,
484-
asset_name=asset_row.name,
485-
asset_uri=asset_row.uri,
496+
asset_id=None if is_hidden else asset_row.id,
497+
asset_name=None if is_hidden else asset_row.name,
498+
asset_uri=None if is_hidden else asset_row.uri,
499+
hidden=is_hidden,
486500
received=received_count >= required_count and required_count > 0,
487501
received_count=received_count,
488502
required_count=required_count,
@@ -496,7 +510,11 @@ def get_pending_partitioned_dag_run(
496510

497511
total_received = sum(a.received_count for a in assets)
498512
total_required = sum(a.required_count for a in assets)
499-
asset_expression = dag_model.asset_expression if dag_model is not None else None
513+
asset_expression = (
514+
redact_asset_expression(dag_model.asset_expression, readable_asset_ids=readable_asset_ids)
515+
if dag_model is not None
516+
else None
517+
)
500518

501519
model_data: dict[str, Any] = {
502520
"id": partitioned_dag_run.id,

‎airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12954,17 +12954,43 @@ export const $OklchColor = {
1295412954
export const $PartitionedDagRunAssetResponse = {
1295512955
properties: {
1295612956
asset_id: {
12957-
type: 'integer',
12957+
anyOf: [
12958+
{
12959+
type: 'integer'
12960+
},
12961+
{
12962+
type: 'null'
12963+
}
12964+
],
1295812965
title: 'Asset Id'
1295912966
},
1296012967
asset_name: {
12961-
type: 'string',
12968+
anyOf: [
12969+
{
12970+
type: 'string'
12971+
},
12972+
{
12973+
type: 'null'
12974+
}
12975+
],
1296212976
title: 'Asset Name'
1296312977
},
1296412978
asset_uri: {
12965-
type: 'string',
12979+
anyOf: [
12980+
{
12981+
type: 'string'
12982+
},
12983+
{
12984+
type: 'null'
12985+
}
12986+
],
1296612987
title: 'Asset Uri'
1296712988
},
12989+
hidden: {
12990+
type: 'boolean',
12991+
title: 'Hidden',
12992+
default: false
12993+
},
1296812994
received: {
1296912995
type: 'boolean',
1297012996
title: 'Received'

‎airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3317,9 +3317,10 @@ export type OklchColor = string;
33173317
* Asset info within a partitioned Dag run detail.
33183318
*/
33193319
export type PartitionedDagRunAssetResponse = {
3320-
asset_id: number;
3321-
asset_name: string;
3322-
asset_uri: string;
3320+
asset_id: number | null;
3321+
asset_name: string | null;
3322+
asset_uri: string | null;
3323+
hidden?: boolean;
33233324
received: boolean;
33243325
received_count: number;
33253326
required_count: number;

‎airflow-core/src/airflow/ui/src/components/AssetProgressCell.test.tsx‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,13 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19-
import { render, waitFor } from "@testing-library/react";
19+
import "@testing-library/jest-dom/vitest";
20+
import { fireEvent, render, screen, waitFor } from "@testing-library/react";
2021
import { expect, it, vi } from "vitest";
2122

2223
import { PartitionedDagRunService } from "openapi/requests/services.gen";
2324

25+
import i18n from "src/i18n/config";
2426
import { Wrapper } from "src/utils/Wrapper";
2527

2628
import { AssetProgressCell } from "./AssetProgressCell";
@@ -68,3 +70,62 @@ it("fetches separate details for pending runs sharing a partition key", async ()
6870
});
6971
});
7072
});
73+
74+
it("labels a hidden asset instead of linking to it, leaving readable assets linked", async () => {
75+
// The shape the pending-detail endpoint serves for an asset the caller may not read.
76+
vi.spyOn(PartitionedDagRunService, "getPendingPartitionedDagRun").mockResolvedValue({
77+
assets: [
78+
{
79+
asset_id: 1,
80+
asset_name: "sales_daily",
81+
asset_uri: "s3://bucket/sales",
82+
hidden: false,
83+
is_rollup: true,
84+
received: false,
85+
received_count: 0,
86+
received_keys: [],
87+
required_count: 1,
88+
required_keys: ["2024-07-01"],
89+
},
90+
{
91+
asset_id: null,
92+
asset_name: null,
93+
asset_uri: null,
94+
hidden: true,
95+
is_rollup: true,
96+
received: false,
97+
received_count: 0,
98+
received_keys: [],
99+
required_count: 1,
100+
required_keys: ["2024-07-01"],
101+
},
102+
],
103+
dag_id: "consumer",
104+
id: 1,
105+
partition_key: "2024-07-01",
106+
total_received: 0,
107+
total_required: 2,
108+
});
109+
110+
render(
111+
<Wrapper>
112+
<AssetProgressCell
113+
dagId="consumer"
114+
partitionedDagRunId={1}
115+
partitionKey="2024-07-01"
116+
totalReceived={0}
117+
totalRequired={2}
118+
/>
119+
</Wrapper>,
120+
);
121+
122+
const trigger = screen.getByRole("button");
123+
124+
// The trigger is disabled while the detail query is in flight, which swallows the click.
125+
await waitFor(() => expect(trigger).not.toBeDisabled());
126+
fireEvent.click(trigger);
127+
128+
await waitFor(() => expect(screen.getByText(i18n.t("common:expression.hiddenAsset"))).toBeInTheDocument());
129+
expect(screen.getByRole("link", { name: "sales_daily" })).toHaveAttribute("href", "/assets/1");
130+
expect(screen.getAllByRole("link")).toHaveLength(1);
131+
});

0 commit comments

Comments
 (0)