Skip to content

Commit ab918c9

Browse files
shivaampotiuk
andauthored
Auto-triage: flag area:UI PRs missing screenshots/demo from new contributors (#64516)
* Add UI demo/screenshot check to auto-triage for area:UI PRs Add a deterministic check to `breeze pr auto-triage` that flags `area:UI` PRs from non-collaborator contributors when their PR description lacks screenshots or demo videos. The check runs as part of the existing deterministic checks in `_assess_pr_deterministic()` alongside CI failure, merge conflict, and unresolved comment checks. PRs flagged only for missing UI demo receive a comment (not draft conversion), using a new soft-violation branch in `_compute_default_action`. Detection patterns cover GitHub drag-and-drop uploads (`<img>` tags, `user-attachments/assets/` URLs), markdown image syntax, and direct media file URLs (png, jpg, gif, mp4, etc). * Clarify that demo should be in PR description, not comments * Remove soft-violation special-case in _compute_default_action Per reviewer feedback from @pierrejeambrun and @potiuk: the separate elif branch for soft-only violations (e.g. missing UI demo) defaulting to COMMENT is unnecessary — maintainers can override DRAFT to COMMENT interactively. A future enhancement will let each check define its own suggested action severity. --------- Co-authored-by: Jarek Potiuk <[email protected]>
1 parent d0e41dd commit ab918c9

3 files changed

Lines changed: 303 additions & 3 deletions

File tree

‎dev/breeze/src/airflow_breeze/commands/pr_commands.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7249,6 +7249,7 @@ def _review_ready_prs_review_mode(
72497249
PRAssessment,
72507250
assess_pr_checks,
72517251
assess_pr_conflicts,
7252+
assess_pr_ui_demo,
72527253
assess_pr_unresolved_comments,
72537254
)
72547255

@@ -7350,8 +7351,9 @@ def _review_ready_prs_review_mode(
73507351
comments_assessment = assess_pr_unresolved_comments(
73517352
pr.number, pr.unresolved_review_comments, pr.unresolved_threads
73527353
)
7354+
ui_demo_assessment = assess_pr_ui_demo(pr.number, pr.labels, pr.body, pr.author_association)
73537355

7354-
if ci_assessment or conflict_assessment or comments_assessment:
7356+
if ci_assessment or conflict_assessment or comments_assessment or ui_demo_assessment:
73557357
violations = []
73567358
summaries = []
73577359
if conflict_assessment:
@@ -7363,6 +7365,9 @@ def _review_ready_prs_review_mode(
73637365
if comments_assessment:
73647366
violations.extend(comments_assessment.violations)
73657367
summaries.append(comments_assessment.summary)
7368+
if ui_demo_assessment:
7369+
violations.extend(ui_demo_assessment.violations)
7370+
summaries.append(ui_demo_assessment.summary)
73667371

73677372
assessment = PRAssessment(
73687373
should_flag=True,
@@ -8922,6 +8927,7 @@ def _assess_pr_deterministic(
89228927
from airflow_breeze.utils.github import (
89238928
assess_pr_checks,
89248929
assess_pr_conflicts,
8930+
assess_pr_ui_demo,
89258931
assess_pr_unresolved_comments,
89268932
)
89278933

@@ -8950,11 +8956,14 @@ def _assess_pr_deterministic(
89508956
comments_assessment = assess_pr_unresolved_comments(
89518957
pr.number, pr.unresolved_review_comments, pr.unresolved_threads
89528958
)
8959+
ui_demo_assessment = assess_pr_ui_demo(pr.number, pr.labels, pr.body, pr.author_association)
89538960

8954-
if ci_assessment or conflict_assessment or comments_assessment:
8961+
if ci_assessment or conflict_assessment or comments_assessment or ui_demo_assessment:
89558962
if pr.is_draft:
89568963
return DeterministicResult(category="draft_skipped")
8957-
merged = _merge_pr_assessments(conflict_assessment, ci_assessment, comments_assessment)
8964+
merged = _merge_pr_assessments(
8965+
conflict_assessment, ci_assessment, comments_assessment, ui_demo_assessment
8966+
)
89588967
return DeterministicResult(category="flagged", assessment=merged)
89598968

89608969
# No deterministic issues found — needs further categorization

‎dev/breeze/src/airflow_breeze/utils/github.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -646,3 +646,70 @@ def assess_pr_unresolved_comments(
646646
],
647647
summary=f"PR #{pr_number} has {unresolved_review_comments} unresolved review {thread_word}.",
648648
)
649+
650+
651+
# Patterns that indicate screenshots or video demos in a PR body.
652+
# Based on analysis of real Airflow UI PRs — GitHub drag-and-drop uploads
653+
# produce HTML <img> tags or bare URLs with user-attachments/assets/ paths.
654+
_DEMO_EVIDENCE_PATTERNS = [
655+
r"<img\s", # HTML <img> tag (GitHub drag-and-drop screenshots)
656+
r"https://github\.com/user-attachments/assets/", # GitHub-uploaded assets (images & videos)
657+
r"!\[", # Markdown image syntax (rare but valid)
658+
r"https?://\S+\.(?:png|jpg|jpeg|gif|webp|mp4|mov|webm)", # Direct media file URLs
659+
]
660+
661+
# authorAssociation values that indicate the author has write access
662+
_COLLABORATOR_ASSOCIATIONS_FOR_UI = {"COLLABORATOR", "MEMBER", "OWNER"}
663+
664+
665+
def _has_demo_evidence(body: str | None) -> bool:
666+
"""Check if PR body contains screenshots, images, or video links."""
667+
if not body:
668+
return False
669+
for pattern in _DEMO_EVIDENCE_PATTERNS:
670+
if re.search(pattern, body, re.IGNORECASE):
671+
return True
672+
return False
673+
674+
675+
def assess_pr_ui_demo(
676+
pr_number: int,
677+
labels: list[str],
678+
body: str | None,
679+
author_association: str,
680+
) -> PRAssessment | None:
681+
"""Flag UI PRs from new contributors that lack screenshots or demo videos.
682+
683+
Returns None if the PR does not have the ``area:UI`` label, the author is a
684+
collaborator/member/owner, or the PR body already contains demo evidence.
685+
"""
686+
if "area:UI" not in labels:
687+
return None
688+
689+
if author_association in _COLLABORATOR_ASSOCIATIONS_FOR_UI:
690+
return None
691+
692+
if _has_demo_evidence(body):
693+
return None
694+
695+
return PRAssessment(
696+
should_flag=True,
697+
violations=[
698+
Violation(
699+
category="Missing UI demo",
700+
explanation=(
701+
"This PR changes UI code but the description does not include "
702+
"screenshots or a demo video."
703+
),
704+
severity="warning",
705+
details=(
706+
"For UI changes, please add screenshots or a short screen recording "
707+
"directly to the PR description (not in comments) so reviewers can "
708+
"verify the visual changes. You can paste images directly into the "
709+
"GitHub PR description or drag-and-drop a screen recording. "
710+
f"See [pull request guidelines]({_CONTRIBUTING_DOCS_URL}/05_pull_requests.rst)."
711+
),
712+
)
713+
],
714+
summary=f"PR #{pr_number} changes UI code but has no screenshots or demo.",
715+
)
Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
from __future__ import annotations
18+
19+
import pytest
20+
21+
from airflow_breeze.commands.pr_commands import _compute_default_action
22+
from airflow_breeze.utils.confirm import TriageAction
23+
from airflow_breeze.utils.github import PRAssessment, Violation, _has_demo_evidence, assess_pr_ui_demo
24+
from airflow_breeze.utils.pr_models import PRData
25+
26+
27+
class TestHasDemoEvidence:
28+
"""Tests for _has_demo_evidence helper function."""
29+
30+
def test_empty_body(self):
31+
assert _has_demo_evidence("") is False
32+
33+
def test_none_body(self):
34+
assert _has_demo_evidence(None) is False
35+
36+
def test_text_only_body(self):
37+
assert _has_demo_evidence("This PR fixes a bug in the UI sidebar component.") is False
38+
39+
def test_html_img_tag_github_assets(self):
40+
body = (
41+
"## Changes\n"
42+
'<img width="1159" alt="image" '
43+
'src="https://github.com/user-attachments/assets/abc123-def456">'
44+
)
45+
assert _has_demo_evidence(body) is True
46+
47+
def test_bare_github_assets_url(self):
48+
body = "## Demo\n\nhttps://github.com/user-attachments/assets/abc123-def456-video\n"
49+
assert _has_demo_evidence(body) is True
50+
51+
def test_markdown_image_syntax(self):
52+
body = "![screenshot](https://example.com/image.png)"
53+
assert _has_demo_evidence(body) is True
54+
55+
def test_direct_image_url(self):
56+
body = "See the result at https://example.com/demo.png"
57+
assert _has_demo_evidence(body) is True
58+
59+
def test_direct_video_url(self):
60+
body = "Recording: https://example.com/screen.mp4"
61+
assert _has_demo_evidence(body) is True
62+
63+
@pytest.mark.parametrize(
64+
"extension",
65+
["png", "jpg", "jpeg", "gif", "webp", "mp4", "mov", "webm"],
66+
)
67+
def test_various_media_extensions(self, extension):
68+
body = f"https://example.com/file.{extension}"
69+
assert _has_demo_evidence(body) is True
70+
71+
def test_no_false_positive_on_code_mention(self):
72+
body = "Updated the image component to handle png files better."
73+
assert _has_demo_evidence(body) is False
74+
75+
def test_no_false_positive_on_url_without_media_extension(self):
76+
body = "See https://github.com/apache/airflow/pull/12345 for context."
77+
assert _has_demo_evidence(body) is False
78+
79+
80+
class TestAssessPrUiDemo:
81+
"""Tests for assess_pr_ui_demo deterministic check."""
82+
83+
def test_no_ui_label_returns_none(self):
84+
result = assess_pr_ui_demo(
85+
pr_number=123,
86+
labels=["area:API", "kind:bug"],
87+
body="No screenshots here.",
88+
author_association="CONTRIBUTOR",
89+
)
90+
assert result is None
91+
92+
@pytest.mark.parametrize("association", ["COLLABORATOR", "MEMBER", "OWNER"])
93+
def test_collaborator_association_returns_none(self, association):
94+
result = assess_pr_ui_demo(
95+
pr_number=123,
96+
labels=["area:UI"],
97+
body="No screenshots here.",
98+
author_association=association,
99+
)
100+
assert result is None
101+
102+
def test_contributor_with_demo_returns_none(self):
103+
body = '<img width="500" src="https://github.com/user-attachments/assets/abc123">'
104+
result = assess_pr_ui_demo(
105+
pr_number=123,
106+
labels=["area:UI"],
107+
body=body,
108+
author_association="CONTRIBUTOR",
109+
)
110+
assert result is None
111+
112+
def test_contributor_without_demo_returns_assessment(self):
113+
result = assess_pr_ui_demo(
114+
pr_number=123,
115+
labels=["area:UI"],
116+
body="This fixes a UI bug.",
117+
author_association="CONTRIBUTOR",
118+
)
119+
assert result is not None
120+
assert result.should_flag is True
121+
assert len(result.violations) == 1
122+
assert result.violations[0].category == "Missing UI demo"
123+
assert result.violations[0].severity == "warning"
124+
125+
def test_none_association_without_demo_returns_assessment(self):
126+
result = assess_pr_ui_demo(
127+
pr_number=456,
128+
labels=["area:UI", "kind:feature"],
129+
body="Added new sidebar component.",
130+
author_association="NONE",
131+
)
132+
assert result is not None
133+
assert result.should_flag is True
134+
assert "screenshots" in result.violations[0].explanation.lower()
135+
136+
def test_empty_body_returns_assessment(self):
137+
result = assess_pr_ui_demo(
138+
pr_number=789,
139+
labels=["area:UI"],
140+
body="",
141+
author_association="CONTRIBUTOR",
142+
)
143+
assert result is not None
144+
assert result.should_flag is True
145+
146+
def test_summary_includes_pr_number(self):
147+
result = assess_pr_ui_demo(
148+
pr_number=42,
149+
labels=["area:UI"],
150+
body="Some text",
151+
author_association="CONTRIBUTOR",
152+
)
153+
assert result is not None
154+
assert "42" in result.summary
155+
156+
157+
class TestComputeDefaultActionSoftViolations:
158+
"""Test default action for PRs with UI demo violations."""
159+
160+
@staticmethod
161+
def _make_pr(**overrides) -> PRData:
162+
defaults = {
163+
"number": 123,
164+
"title": "UI change",
165+
"body": "Some text",
166+
"url": "https://github.com/apache/airflow/pull/123",
167+
"created_at": "2026-01-01T00:00:00Z",
168+
"updated_at": "2026-01-01T00:00:00Z",
169+
"node_id": "PR_123",
170+
"author_login": "contributor",
171+
"author_association": "NONE",
172+
"head_sha": "abc123",
173+
"base_ref": "main",
174+
"check_summary": "",
175+
"checks_state": "SUCCESS",
176+
"failed_checks": [],
177+
"commits_behind": 10,
178+
"is_draft": False,
179+
"mergeable": "MERGEABLE",
180+
"labels": ["area:UI"],
181+
"unresolved_threads": [],
182+
}
183+
defaults.update(overrides)
184+
return PRData(**defaults)
185+
186+
@staticmethod
187+
def _make_assessment(summary: str) -> PRAssessment:
188+
return PRAssessment(
189+
should_flag=True,
190+
violations=[
191+
Violation(
192+
category="Missing UI demo",
193+
explanation="No screenshots",
194+
severity="warning",
195+
details="Add screenshots",
196+
)
197+
],
198+
summary=summary,
199+
)
200+
201+
def test_soft_violation_only_suggests_draft(self):
202+
"""A PR with only a soft violation (no CI failures, conflicts, or comments) gets DRAFT."""
203+
pr = self._make_pr()
204+
assessment = self._make_assessment("PR #123 changes UI code but has no screenshots or demo.")
205+
action, reason = _compute_default_action(pr, assessment, author_flagged_count={})
206+
assert action == TriageAction.DRAFT
207+
assert "draft" in reason
208+
209+
def test_soft_violation_with_ci_failures_suggests_draft(self):
210+
"""A PR with a soft violation AND multiple CI failures gets DRAFT (hard issue takes priority)."""
211+
pr = self._make_pr(
212+
failed_checks=["Tests / Core", "Tests / API", "Tests / CLI"],
213+
checks_state="FAILURE",
214+
)
215+
assessment = self._make_assessment("PR has CI failures and no demo.")
216+
action, _reason = _compute_default_action(pr, assessment, author_flagged_count={})
217+
assert action == TriageAction.DRAFT
218+
219+
def test_soft_violation_with_conflicts_suggests_draft(self):
220+
"""A PR with a soft violation AND merge conflicts gets DRAFT."""
221+
pr = self._make_pr(mergeable="CONFLICTING")
222+
assessment = self._make_assessment("PR has conflicts and no demo.")
223+
action, _reason = _compute_default_action(pr, assessment, author_flagged_count={})
224+
assert action == TriageAction.DRAFT

0 commit comments

Comments
 (0)