Skip to content

Commit 09d742d

Browse files
Merge branch 'main' into fix/apache#69262
2 parents c870944 + 4e65210 commit 09d742d

40 files changed

Lines changed: 1017 additions & 615 deletions

File tree

‎airflow-core/src/airflow/dag_processing/manager.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -751,11 +751,11 @@ def get_bundle_state(self, bundle_name: str, *, session: Session = NEW_SESSION)
751751
752752
Returns ``None`` if the bundle has no database record.
753753
"""
754-
row = session.scalar(
755-
select(DagBundleModel)
756-
.where(DagBundleModel.name == bundle_name)
757-
.options(load_only(DagBundleModel.last_refreshed, DagBundleModel.version))
758-
)
754+
row = session.execute(
755+
select(DagBundleModel.last_refreshed, DagBundleModel.version).where(
756+
DagBundleModel.name == bundle_name
757+
)
758+
).one_or_none()
759759
if row is None:
760760
return None
761761
return BundleState(last_refreshed=row.last_refreshed, version=row.version)
@@ -817,7 +817,6 @@ def _refresh_dag_bundles(self, known_files: dict[str, set[DagFileInfo]]):
817817
except AirflowException as e:
818818
self.log.exception("Error initializing bundle %s: %s", bundle.name, e)
819819
continue
820-
# TODO: AIP-66 test to make sure we get a fresh record from the db and it's not cached
821820
try:
822821
bundle_state = self.get_bundle_state(bundle.name)
823822
except Exception:

‎airflow-core/src/airflow/ui/src/components/renderStructuredLog.test.tsx‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,47 @@ describe("tiContextFields", () => {
3535
});
3636
});
3737

38+
describe("renderStructuredLog — traceback frame highlighting", () => {
39+
const sitepkgFile = "/usr/local/lib/python3.14/site-packages/airflow/sdk/bases/operator.py";
40+
const dagBundleFile = "/tmp/airflow/dag_bundles/astro/main/dags/non_alert/enrich_ticket.py";
41+
42+
const renderWithError = () => {
43+
const result = renderStructuredLog({
44+
index: 0,
45+
logLink: "",
46+
logMessage: {
47+
error_detail: [
48+
{
49+
exc_notes: [],
50+
exc_type: "TypeError",
51+
exc_value: "fromisoformat: argument must be str",
52+
frames: [
53+
{ filename: sitepkgFile, lineno: 445, name: "wrapper" },
54+
{ filename: dagBundleFile, lineno: 226, name: "retrieve_cluster" },
55+
],
56+
is_cause: false,
57+
syntax_error: null,
58+
},
59+
],
60+
event: "Task failed with exception",
61+
level: "error",
62+
timestamp: "2026-07-22T09:15:20Z",
63+
},
64+
renderingMode: "jsx",
65+
translate: translate as never,
66+
});
67+
68+
return render(<Wrapper>{result}</Wrapper>);
69+
};
70+
71+
it("marks a DAG-bundle frame as user code and a site-packages frame as library", () => {
72+
renderWithError();
73+
74+
expect(screen.getByText(JSON.stringify(dagBundleFile))).toHaveAttribute("data-frame-source", "user");
75+
expect(screen.getByText(JSON.stringify(sitepkgFile))).toHaveAttribute("data-frame-source", "library");
76+
});
77+
});
78+
3879
describe("renderStructuredLog — TI context field stripping", () => {
3980
it("does not render TI context fields as per-line structured attributes", () => {
4081
const result = renderStructuredLog({

‎airflow-core/src/airflow/ui/src/components/renderStructuredLog.tsx‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import type { StructuredLogMessage, TaskInstancesLogResponse } from "openapi/req
2626
import AnsiRenderer from "src/components/AnsiRenderer";
2727
import Time from "src/components/Time";
2828
import { urlRegex } from "src/constants/urlRegex";
29-
import { LogLevel, logLevelColorMapping } from "src/utils/logs";
29+
import { isUserCodeFrame, LogLevel, logLevelColorMapping } from "src/utils/logs";
3030

3131
type Frame = {
3232
filename: string;
@@ -255,11 +255,21 @@ const renderStructuredLogImpl = ({
255255
return ` ${translate("components:logs.file")} ${frame.filename}, ${translate("components:logs.location", { line: frame.lineno, name: frame.name })}\n`;
256256
}
257257

258+
// Highlight user-code frames (DAG bundle, plugins, local files) in the info blue,
259+
// and leave installed-package frames in the normal text color, so the frame the
260+
// error actually came from stands out from the framework frames around it.
261+
const userCode = isUserCodeFrame(frame.filename);
262+
258263
return (
259264
<chakra.p key={`frame-${frame.name}-${frame.filename}-${frame.lineno}`}>
260265
{translate("components:logs.file")}{" "}
261-
<chakra.span color="fg.info">{JSON.stringify(frame.filename)}</chakra.span>,{" "}
262-
{translate("components:logs.location", { line: frame.lineno, name: frame.name })}
266+
<chakra.span
267+
color={userCode ? "fg.info" : undefined}
268+
data-frame-source={userCode ? "user" : "library"}
269+
>
270+
{JSON.stringify(frame.filename)}
271+
</chakra.span>
272+
, {translate("components:logs.location", { line: frame.lineno, name: frame.name })}
263273
</chakra.p>
264274
);
265275
});

‎airflow-core/src/airflow/ui/src/utils/logs.test.ts‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,26 @@ import { describe, it, expect } from "vitest";
2020

2121
import type { TaskInstancesLogResponse } from "openapi/requests/types.gen";
2222

23-
import { parseStreamingLogContent } from "./logs";
23+
import { isUserCodeFrame, parseStreamingLogContent } from "./logs";
24+
25+
describe("isUserCodeFrame", () => {
26+
it.each([
27+
"/tmp/airflow/dag_bundles/astro/main/dags/non_alert/enrich_ticket.py",
28+
"/opt/airflow/dags/my_dag.py",
29+
"/home/user/plugins/my_plugin.py",
30+
"C:\\airflow\\dags\\my_dag.py",
31+
])("treats %s as user code", (filename) => {
32+
expect(isUserCodeFrame(filename)).toBe(true);
33+
});
34+
35+
it.each([
36+
"/usr/local/lib/python3.14/site-packages/airflow/sdk/execution_time/task_runner.py",
37+
"/usr/lib/python3/dist-packages/airflow/models/baseoperator.py",
38+
"C:\\Python314\\Lib\\site-packages\\airflow\\sdk\\bases\\operator.py",
39+
])("treats %s as library code", (filename) => {
40+
expect(isUserCodeFrame(filename)).toBe(false);
41+
});
42+
});
2443

2544
describe("parseStreamingLogContent", () => {
2645
it("returns content when data has content property", () => {

‎airflow-core/src/airflow/ui/src/utils/logs.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,11 @@ export const logLevelOptions = createListCollection<{
5454
],
5555
});
5656

57+
// A traceback frame is "user code" unless it lives in an installed-package directory
58+
// (site-packages / dist-packages). DAG bundle code, plugins, and local files are all user code.
59+
export const isUserCodeFrame = (filename: string): boolean =>
60+
!/[/\\](?:site|dist)-packages[/\\]/u.test(filename);
61+
5762
export const parseStreamingLogContent = (
5863
data: TaskInstancesLogResponse | undefined,
5964
): TaskInstancesLogResponse["content"] => {

‎airflow-core/tests/unit/dag_processing/test_manager.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2726,6 +2726,34 @@ def test_get_bundle_state_returns_correct_state(self, session):
27262726

27272727
assert state == BundleState(last_refreshed=refreshed_at, version="v1")
27282728

2729+
def test_get_bundle_state_reads_latest_database_values(self, session):
2730+
bundle_name = "test_fresh_state_bundle"
2731+
initial_refreshed_at = timezone.datetime(2024, 1, 15, 12, 0, 0)
2732+
refreshed_at = timezone.datetime(2024, 1, 16, 12, 0, 0)
2733+
model = DagBundleModel(name=bundle_name, version="old")
2734+
model.last_refreshed = initial_refreshed_at
2735+
session.add(model)
2736+
session.commit()
2737+
2738+
manager = DagFileProcessorManager(max_runs=1)
2739+
state = manager.get_bundle_state(bundle_name, session=session)
2740+
2741+
assert state == BundleState(last_refreshed=initial_refreshed_at, version="old")
2742+
2743+
with create_session(scoped=False) as update_session:
2744+
update_model = update_session.get(DagBundleModel, bundle_name)
2745+
assert update_model is not None
2746+
update_model.last_refreshed = refreshed_at
2747+
update_model.version = "fresh"
2748+
2749+
# End the read transaction started by the first get_bundle_state() call so we don't keep
2750+
# reading a stale snapshot on backends like SQLite.
2751+
session.commit()
2752+
2753+
state = manager.get_bundle_state(bundle_name, session=session)
2754+
2755+
assert state == BundleState(last_refreshed=refreshed_at, version="fresh")
2756+
27292757
def test_get_bundle_state_null_fields(self, session):
27302758
bundle_name = "test_null_state_bundle"
27312759
session.add(DagBundleModel(name=bundle_name))

‎dev/registry/extract_parameters.py‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -386,11 +386,29 @@ def load_resumable_job_mixin() -> type | None:
386386

387387

388388
def is_durable_capable(cls: type, resumable_mixin: type | None) -> bool:
389-
"""Return True if a class fully implements ResumableJobMixin's crash-recovery contract.
390-
391-
Inheriting the mixin is not sufficient: a complete override is inert unless
392-
execute() actually calls execute_resumable().
389+
"""Return True if a class implements durable/crash-safe execution.
390+
391+
Two ways to qualify:
392+
1. A class-level `__supports_durable_execution = True`
393+
declaration (for operators that implement this directly against
394+
task_state_store, without ResumableJobMixin -- e.g. KubernetesPodOperator,
395+
AgentOperator).
396+
2. Genuinely implementing ResumableJobMixin's contract.
397+
398+
The first path deliberately looks up the class prefixed attribute
399+
(`_{ClassName}__supports_durable_execution`) rather than a fixed string.
400+
A subclass that overrides execute() itself (e.g. SparkKubernetesOperator)
401+
may not preserve the parent's task_state_store reconnect behavior, so the
402+
declaration must not be inherited -- only the exact class that wrote
403+
`__supports_durable_execution` in its own body qualifies this way.
404+
405+
Inheriting the mixin alone is not sufficient for the second path: a
406+
complete override is inert unless execute() actually calls
407+
execute_resumable().
393408
"""
409+
if getattr(cls, f"_{cls.__name__}__supports_durable_execution", None) is True:
410+
return True
411+
394412
if resumable_mixin is None or resumable_mixin not in cls.__mro__:
395413
return False
396414

‎dev/registry/tests/test_extract_parameters.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,24 @@ def execute(self, context):
225225
return None
226226

227227

228+
class ManuallyDurableOperator:
229+
"""Implements durable execution directly (e.g. via task_state_store), without
230+
ResumableJobMixin -- mirrors KubernetesPodOperator/AgentOperator."""
231+
232+
__supports_durable_execution = True
233+
234+
def execute(self, context):
235+
return None
236+
237+
238+
class ManuallyDurableSubclass(ManuallyDurableOperator):
239+
"""Overrides execute() itself -- must NOT inherit the parent's declaration,
240+
since nothing here verifies it preserves the reconnect behavior."""
241+
242+
def execute(self, context):
243+
return "something else entirely"
244+
245+
228246
class TestIsDurableCapable:
229247
def test_fully_implemented_and_wired_qualifies(self):
230248
assert is_durable_capable(FullyImplementedResumableOperator, FakeResumableJobMixin) is True
@@ -241,6 +259,12 @@ def test_no_mixin_in_mro_disqualifies(self):
241259
def test_mixin_unavailable_disqualifies(self):
242260
assert is_durable_capable(FullyImplementedResumableOperator, None) is False
243261

262+
def test_manual_durable_marker_qualifies_without_mixin(self):
263+
assert is_durable_capable(ManuallyDurableOperator, None) is True
264+
265+
def test_subclass_not_redeclaring_marker_disqualifies(self):
266+
assert is_durable_capable(ManuallyDurableSubclass, FakeResumableJobMixin) is False
267+
244268

245269
# ---------------------------------------------------------------------------
246270
# Module dataclass

‎providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -433,7 +433,7 @@ def __init__(
433433
):
434434
super().__init__(**kwargs)
435435
self.name = name
436-
self.ruleset = ruleset.strip()
436+
self.ruleset = ruleset
437437
self.description = description
438438
self.update_rule_set = update_rule_set
439439
self.data_quality_ruleset_kwargs = data_quality_ruleset_kwargs or {}
@@ -449,6 +449,8 @@ def validate_inputs(self) -> None:
449449
raise AttributeError("Target table must have DatabaseName and TableName")
450450

451451
def execute(self, context: Context):
452+
# ruleset is a template field; strip the rendered value here, not in __init__.
453+
self.ruleset = self.ruleset.strip()
452454
self.validate_inputs()
453455

454456
config = {

‎providers/amazon/tests/unit/amazon/aws/operators/test_glue.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from __future__ import annotations
1818

1919
from collections.abc import Generator
20+
from datetime import datetime
2021
from typing import TYPE_CHECKING
2122
from unittest import mock
2223

@@ -25,6 +26,7 @@
2526
from boto3 import client
2627
from moto import mock_aws
2728

29+
from airflow.models.dag import DAG
2830
from airflow.providers.amazon.aws.hooks.glue import GlueDataQualityHook, GlueJobHook
2931
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
3032
from airflow.providers.amazon.aws.links.glue import GlueJobRunDetailsLink
@@ -813,6 +815,28 @@ def test_execute_create_rule(self, glue_data_quality_mock_conn):
813815
Ruleset=self.RULE_SET,
814816
)
815817

818+
@mock.patch.object(GlueDataQualityHook, "conn")
819+
def test_execute_strips_rendered_ruleset(self, glue_data_quality_mock_conn):
820+
# ruleset is a template field; execute strips the rendered value (rendering can add whitespace).
821+
with DAG("glue_dq_strip", schedule=None, start_date=datetime(2020, 1, 1)) as dag:
822+
self.operator = GlueDataQualityOperator(
823+
task_id="create_data_quality_ruleset",
824+
name=self.RULE_SET_NAME,
825+
ruleset="{{ params.rules }}",
826+
dag=dag,
827+
)
828+
self.operator.defer = mock.MagicMock()
829+
self.operator.render_template_fields({"params": {"rules": f" {self.RULE_SET} "}})
830+
assert self.operator.ruleset == f" {self.RULE_SET} "
831+
832+
self.operator.execute({})
833+
834+
glue_data_quality_mock_conn.create_data_quality_ruleset.assert_called_once_with(
835+
Description="AWS Glue Data Quality Rule Set With Airflow",
836+
Name=self.RULE_SET_NAME,
837+
Ruleset=self.RULE_SET,
838+
)
839+
816840
@mock.patch.object(GlueDataQualityHook, "conn")
817841
def test_execute_create_rule_should_fail_if_rule_already_exists(self, glue_data_quality_mock_conn):
818842
self.operator = GlueDataQualityOperator(

0 commit comments

Comments
 (0)