Skip to content

Commit 082bfd0

Browse files
committed
Allow embedding a Dag run in an external trace via run conf
A run triggered by an external system that already emits a trace (an upstream orchestrator, event pipeline, CI job, or another Airflow) currently always starts its own root trace, so the Airflow run cannot be seen as part of that larger workflow. A reserved run conf key, airflow/parent_trace_context, lets a run opt into embedding under a supplied W3C traceparent: the whole run rides the external trace and parent-based samplers inherit its sampling decision. Absent the key, the run stays a root trace as before. closes: apache#67210
1 parent 90b8ee4 commit 082bfd0

7 files changed

Lines changed: 327 additions & 21 deletions

File tree

‎airflow-core/docs/administration-and-deployment/logging-monitoring/traces.rst‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,35 @@ Custom spans created this way are automatically nested as children of the Airflo
8787
task span when tracing is enabled. When tracing is disabled, the no-op tracer provided by
8888
the OpenTelemetry API is used, so tasks run without any overhead.
8989

90+
Per-run trace controls (run conf)
91+
---------------------------------
92+
93+
A few reserved keys in a Dag run's ``conf`` let you control tracing for that individual run.
94+
They are read when the run is created (and when it is cleared), so set them when you trigger the
95+
run — from the UI/API "Trigger with config" dialog, the ``airflow dags trigger --conf`` CLI, or
96+
``TriggerDagRunOperator(conf=...)``.
97+
98+
``airflow/trace_sampled``
99+
Force the head-sampling decision for this run. ``true`` always traces the run and ``false``
100+
never does, regardless of the configured sampler; omit the key to let the sampler decide.
101+
Only an explicit boolean is honored.
102+
103+
``airflow/task_span_detail_level``
104+
Integer detail level controlling how many spans task execution emits for this run.
105+
106+
``airflow/parent_trace_context``
107+
Embed this run in an **external** trace instead of it being a root trace. Supply a W3C
108+
``traceparent`` string (optionally a mapping with ``traceparent`` and ``tracestate``) captured
109+
from the system that triggered the run — an upstream orchestrator, event pipeline, CI job, or
110+
another Airflow deployment. The whole run (the ``dag_run`` span and all task/worker spans) then
111+
lives inside that trace, and parent-based samplers inherit the external sampling decision. When
112+
the key is absent the run is a root trace, which is the default. A missing or malformed value is
113+
ignored (the run stays a root) rather than failing run creation.
114+
115+
.. code-block:: json
116+
117+
{"airflow/parent_trace_context": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}
118+
90119
Enable Https
91120
-----------------
92121

‎airflow-core/src/airflow/models/dagrun.py‎

Lines changed: 45 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464

6565
from airflow._shared.observability.metrics import stats
6666
from airflow._shared.observability.traces import (
67+
PARENT_TRACE_CONTEXT_KEY,
6768
TASK_SPAN_DETAIL_LEVEL_KEY,
6869
TRACE_SAMPLED_KEY,
6970
new_dagrun_trace_carrier,
@@ -194,6 +195,33 @@ def trace_sampled_override(conf) -> bool | None:
194195
return None
195196

196197

198+
def parent_trace_context(conf) -> context.Context | None:
199+
"""
200+
External parent trace context from the ``airflow/parent_trace_context`` run conf key.
201+
202+
Lets a run be embedded in a trace owned by an external system (an upstream
203+
orchestrator, event pipeline, CI job, another Airflow) instead of being a root
204+
trace. The value is a W3C ``traceparent`` string, or a carrier dict carrying
205+
``traceparent`` (and optionally ``tracestate``); anything else -- including a
206+
malformed traceparent -- is ignored so it can neither silently mis-parent the
207+
run nor fail run creation. Returns an OpenTelemetry ``Context`` when a valid
208+
parent is present, else None (root trace, the default).
209+
"""
210+
if not conf:
211+
return None
212+
raw = conf.get(PARENT_TRACE_CONTEXT_KEY)
213+
if isinstance(raw, str):
214+
carrier = {"traceparent": raw}
215+
elif isinstance(raw, dict) and isinstance(raw.get("traceparent"), str):
216+
carrier = {k: raw[k] for k in ("traceparent", "tracestate") if isinstance(raw.get(k), str)}
217+
else:
218+
return None
219+
ctx = TraceContextTextMapPropagator().extract(carrier)
220+
if not trace.get_current_span(ctx).get_span_context().is_valid:
221+
return None
222+
return ctx
223+
224+
197225
class DagRun(Base, LoggingMixin):
198226
"""
199227
Invocation instance of a DAG.
@@ -425,6 +453,7 @@ def __init__(
425453
task_span_detail_level=self.conf.get(TASK_SPAN_DETAIL_LEVEL_KEY, None),
426454
attributes=dagrun_trace_attributes(self), # these are for potential use by head sampler
427455
force_sampled=trace_sampled_override(self.conf),
456+
parent_context=parent_trace_context(self.conf),
428457
)
429458

430459
if not isinstance(partition_key, str | None):
@@ -1116,11 +1145,11 @@ def _emit_dagrun_span(self, state: DagRunState):
11161145
span_context = span.get_span_context()
11171146

11181147
# Skip if the run was head-sampled out. Unlike the task spans, this guard is
1119-
# required (not just an optimization): the span below is forced to be a root span
1120-
# (context=context.Context()), so the configured sampler never sees the carrier's
1121-
# flag. A valid-but-unsampled carrier means head-sampled out; an invalid/empty
1122-
# carrier (legacy DagRun) recorded no decision, so it falls through and still
1123-
# emits — prior behavior we may want to reconsider.
1148+
# required (not just an optimization): the span below is started as a root span
1149+
# (or under an external parent) rather than under the carrier, so the configured
1150+
# sampler is not driven by the carrier's flag. A valid-but-unsampled carrier means
1151+
# head-sampled out; an invalid/empty carrier (legacy DagRun) recorded no decision,
1152+
# so it falls through and still emits — prior behavior we may want to reconsider.
11241153
if span_context.is_valid and not span_context.trace_flags.sampled:
11251154
return
11261155

@@ -1143,18 +1172,21 @@ def _emit_dagrun_span(self, state: DagRunState):
11431172
if self.partition_date:
11441173
attributes["airflow.dag_run.partition_date"] = self.partition_date.isoformat()
11451174

1146-
# TODO: make the empty parent context optional. Default should be to
1147-
# nest the dag run span under the currently active parent span (by
1148-
# omitting `context` here); only use the empty `context.Context()` to
1149-
# force a root span when Airflow itself initiates the run (e.g. dag
1150-
# triggered via API, scheduler, or backfill). Today this forces a
1151-
# root span unconditionally.
1152-
# Tracked at https://github.com/apache/airflow/issues/67210
1175+
# Root by default: an Airflow-initiated run owns its own trace, so we do not
1176+
# attach to whatever span is incidentally active in the scheduler. When the run
1177+
# opted into an external trace via airflow/parent_trace_context, start under that
1178+
# parent so the run nests inside it. The carrier was already built riding the same
1179+
# external trace_id (both read the same conf), so the override_ids trace_id is a
1180+
# no-op here — only the span_id override takes effect and the parent link comes
1181+
# from parent_ctx. The emitted span's sampling then follows the external parent, so
1182+
# an unsampled external trace drops this dag_run span even when airflow/trace_sampled
1183+
# forced the head decision that the task/worker spans ride.
1184+
parent_ctx = parent_trace_context(self.conf)
11531185
span = tracer.start_span(
11541186
name=f"dag_run.{self.dag_id}",
11551187
start_time=int((self.queued_at or self.start_date or timezone.utcnow()).timestamp() * 1e9),
11561188
attributes=attributes,
1157-
context=context.Context(),
1189+
context=parent_ctx if parent_ctx is not None else context.Context(),
11581190
)
11591191
status_code = StatusCode.OK if state == DagRunState.SUCCESS else StatusCode.ERROR
11601192
span.set_status(status_code)

‎airflow-core/src/airflow/models/taskinstance.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,7 @@ def clear_task_instances(
409409
from airflow.models.dagrun import ( # Avoid circular import
410410
DagRun,
411411
dagrun_trace_attributes,
412+
parent_trace_context,
412413
trace_sampled_override,
413414
)
414415

@@ -435,6 +436,7 @@ def clear_task_instances(
435436
task_span_detail_level=dr.conf.get(TASK_SPAN_DETAIL_LEVEL_KEY) if dr.conf else None,
436437
attributes=dagrun_trace_attributes(dr),
437438
force_sampled=trace_sampled_override(dr.conf),
439+
parent_context=parent_trace_context(dr.conf),
438440
)
439441

440442
_recalculate_dagrun_queued_at_deadlines(dr, dr.queued_at, session)

‎airflow-core/tests/unit/models/test_dagrun.py‎

Lines changed: 126 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,10 @@
4545

4646
from airflow import settings
4747
from airflow._shared.observability.metrics.base_stats_logger import StatsLogger
48-
from airflow._shared.observability.traces import OverrideableRandomIdGenerator
48+
from airflow._shared.observability.traces import (
49+
PARENT_TRACE_CONTEXT_KEY,
50+
OverrideableRandomIdGenerator,
51+
)
4952
from airflow._shared.timezones import timezone
5053
from airflow.callbacks.callback_requests import DagCallbackRequest, DagRunContext
5154
from airflow.models.dag import DagModel, infer_automated_data_interval
@@ -4441,6 +4444,128 @@ def test_context_carrier_honors_trace_sampled_conf(self, dag_maker, flag):
44414444
span_ctx = trace.get_current_span(ctx).get_span_context()
44424445
assert span_ctx.trace_flags.sampled is flag
44434446

4447+
_EXTERNAL_TRACE_ID = "11111111111111111111111111111111"
4448+
_EXTERNAL_SPAN_ID = "2222222222222222"
4449+
4450+
@pytest.mark.parametrize(
4451+
("conf", "expected_trace_id"),
4452+
[
4453+
pytest.param(
4454+
{PARENT_TRACE_CONTEXT_KEY: f"00-{_EXTERNAL_TRACE_ID}-{_EXTERNAL_SPAN_ID}-01"},
4455+
_EXTERNAL_TRACE_ID,
4456+
id="traceparent-string",
4457+
),
4458+
pytest.param(
4459+
{
4460+
PARENT_TRACE_CONTEXT_KEY: {
4461+
"traceparent": f"00-{_EXTERNAL_TRACE_ID}-{_EXTERNAL_SPAN_ID}-01"
4462+
}
4463+
},
4464+
_EXTERNAL_TRACE_ID,
4465+
id="carrier-dict",
4466+
),
4467+
pytest.param({PARENT_TRACE_CONTEXT_KEY: "not-a-traceparent"}, None, id="malformed"),
4468+
pytest.param({PARENT_TRACE_CONTEXT_KEY: 123}, None, id="non-str"),
4469+
pytest.param({PARENT_TRACE_CONTEXT_KEY: {"nope": "x"}}, None, id="dict-without-traceparent"),
4470+
pytest.param({}, None, id="empty-conf"),
4471+
pytest.param(None, None, id="no-conf"),
4472+
pytest.param({"other": "x"}, None, id="unrelated-key"),
4473+
],
4474+
)
4475+
def test_parent_trace_context(self, conf, expected_trace_id):
4476+
"""Only a valid W3C traceparent (string or carrier dict) yields a parent context; else None."""
4477+
from airflow.models.dagrun import parent_trace_context
4478+
4479+
ctx = parent_trace_context(conf)
4480+
if expected_trace_id is None:
4481+
assert ctx is None
4482+
else:
4483+
span_ctx = otel_trace.get_current_span(ctx).get_span_context()
4484+
assert span_ctx.is_valid
4485+
assert format(span_ctx.trace_id, "032x") == expected_trace_id
4486+
4487+
def test_parent_trace_context_carrier_dict_preserves_tracestate(self):
4488+
"""A carrier dict may also carry tracestate, which is extracted alongside traceparent."""
4489+
from airflow.models.dagrun import parent_trace_context
4490+
4491+
conf = {
4492+
PARENT_TRACE_CONTEXT_KEY: {
4493+
"traceparent": f"00-{self._EXTERNAL_TRACE_ID}-{self._EXTERNAL_SPAN_ID}-01",
4494+
"tracestate": "foo=bar",
4495+
}
4496+
}
4497+
span_ctx = otel_trace.get_current_span(parent_trace_context(conf)).get_span_context()
4498+
assert span_ctx.trace_state.get("foo") == "bar"
4499+
4500+
def test_context_carrier_embeds_external_parent_from_conf(self, dag_maker):
4501+
"""A run with airflow/parent_trace_context rides the external trace_id."""
4502+
with dag_maker("test_tracing_embed_conf"):
4503+
EmptyOperator(task_id="t1")
4504+
dr = dag_maker.create_dagrun(
4505+
conf={PARENT_TRACE_CONTEXT_KEY: f"00-{self._EXTERNAL_TRACE_ID}-{self._EXTERNAL_SPAN_ID}-01"}
4506+
)
4507+
4508+
ctx = TraceContextTextMapPropagator().extract(dr.context_carrier)
4509+
span_ctx = otel_trace.get_current_span(ctx).get_span_context()
4510+
assert format(span_ctx.trace_id, "032x") == self._EXTERNAL_TRACE_ID
4511+
4512+
def test_context_carrier_is_root_without_parent_conf(self, dag_maker):
4513+
"""A run with no parent conf mints its own root trace, not the external one."""
4514+
with dag_maker("test_tracing_root_default"):
4515+
EmptyOperator(task_id="t1")
4516+
dr = dag_maker.create_dagrun()
4517+
4518+
ctx = TraceContextTextMapPropagator().extract(dr.context_carrier)
4519+
span_ctx = otel_trace.get_current_span(ctx).get_span_context()
4520+
assert format(span_ctx.trace_id, "032x") != self._EXTERNAL_TRACE_ID
4521+
4522+
def test_emit_dagrun_span_nests_under_external_parent(self, dag_maker, session):
4523+
"""With the parent conf key set, the dag_run span is a child of the external span."""
4524+
in_mem_exporter = InMemorySpanExporter()
4525+
provider = TracerProvider(id_generator=OverrideableRandomIdGenerator())
4526+
provider.add_span_processor(SimpleSpanProcessor(in_mem_exporter))
4527+
test_tracer = provider.get_tracer("test")
4528+
4529+
with dag_maker("test_tracing_embed_emit", session=session) as dag:
4530+
EmptyOperator(task_id="t1")
4531+
dr = dag_maker.create_dagrun(state=DagRunState.RUNNING)
4532+
dr.dag = dag
4533+
dr.conf = {PARENT_TRACE_CONTEXT_KEY: f"00-{self._EXTERNAL_TRACE_ID}-{self._EXTERNAL_SPAN_ID}-01"}
4534+
# Carrier already rides the external trace with its own child span id, sampled.
4535+
dr.context_carrier = {"traceparent": f"00-{self._EXTERNAL_TRACE_ID}-3333333333333333-01"}
4536+
4537+
with mock.patch("airflow.models.dagrun.tracer", test_tracer):
4538+
dr._emit_dagrun_span(state=DagRunState.SUCCESS)
4539+
4540+
spans = in_mem_exporter.get_finished_spans()
4541+
assert len(spans) == 1
4542+
span = spans[0]
4543+
assert span.context.trace_id == int(self._EXTERNAL_TRACE_ID, 16)
4544+
assert span.context.span_id == int("3333333333333333", 16)
4545+
assert span.parent is not None
4546+
assert span.parent.trace_id == int(self._EXTERNAL_TRACE_ID, 16)
4547+
assert span.parent.span_id == int(self._EXTERNAL_SPAN_ID, 16)
4548+
4549+
def test_emit_dagrun_span_is_root_by_default(self, dag_maker, session):
4550+
"""Without the parent conf key, the dag_run span is a root span (no parent)."""
4551+
in_mem_exporter = InMemorySpanExporter()
4552+
provider = TracerProvider(id_generator=OverrideableRandomIdGenerator())
4553+
provider.add_span_processor(SimpleSpanProcessor(in_mem_exporter))
4554+
test_tracer = provider.get_tracer("test")
4555+
4556+
with dag_maker("test_tracing_root_emit", session=session) as dag:
4557+
EmptyOperator(task_id="t1")
4558+
dr = dag_maker.create_dagrun(state=DagRunState.RUNNING)
4559+
dr.dag = dag
4560+
dr.context_carrier = {"traceparent": f"00-{self._EXTERNAL_TRACE_ID}-3333333333333333-01"}
4561+
4562+
with mock.patch("airflow.models.dagrun.tracer", test_tracer):
4563+
dr._emit_dagrun_span(state=DagRunState.SUCCESS)
4564+
4565+
spans = in_mem_exporter.get_finished_spans()
4566+
assert len(spans) == 1
4567+
assert spans[0].parent is None
4568+
44444569

44454570
class TestDagRunStatsTagsTeamName:
44464571
def test_stats_tags_without_team_name(self, dag_maker):

‎airflow-core/tests/unit/models/test_taskinstance.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4151,6 +4151,26 @@ def test_clear_task_instances_honors_trace_sampled_conf(dag_maker, session, flag
41514151
assert span_ctx.trace_flags.sampled is flag
41524152

41534153

4154+
@pytest.mark.db_test
4155+
def test_clear_task_instances_keeps_external_parent_trace(dag_maker, session):
4156+
"""The regenerated carrier keeps riding the external trace from airflow/parent_trace_context."""
4157+
external_trace_id = "11111111111111111111111111111111"
4158+
with dag_maker("test_clear_parent_trace"):
4159+
EmptyOperator(task_id="t1")
4160+
dag_run = dag_maker.create_dagrun(
4161+
conf={"airflow/parent_trace_context": f"00-{external_trace_id}-2222222222222222-01"}
4162+
)
4163+
ti = dag_run.get_task_instance("t1", session=session)
4164+
ti.state = TaskInstanceState.SUCCESS
4165+
session.flush()
4166+
4167+
clear_task_instances([ti], session)
4168+
4169+
new_ctx = TraceContextTextMapPropagator().extract(dag_run.context_carrier)
4170+
span_ctx = trace.get_current_span(new_ctx).get_span_context()
4171+
assert format(span_ctx.trace_id, "032x") == external_trace_id
4172+
4173+
41544174
@pytest.mark.db_test
41554175
def test_task_instance_repr_does_not_raise_for_deferred_columns(dag_maker, session):
41564176
"""``TaskInstance.__repr__`` must survive *any* deferred column it reads.

‎shared/observability/src/airflow_shared/observability/traces/__init__.py‎

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,11 @@ def generate_trace_id(self):
6060
TASK_SPAN_DETAIL_LEVEL_KEY = "airflow/task_span_detail_level"
6161
DEFAULT_TASK_SPAN_DETAIL_LEVEL = 1
6262
TRACE_SAMPLED_KEY = "airflow/trace_sampled"
63+
PARENT_TRACE_CONTEXT_KEY = "airflow/parent_trace_context"
6364

6465

6566
def new_dagrun_trace_carrier(
66-
task_span_detail_level=None, attributes=None, force_sampled=None
67+
task_span_detail_level=None, attributes=None, force_sampled=None, parent_context=None
6768
) -> dict[str, str]:
6869
"""
6970
Generate a fresh W3C traceparent carrier without creating a recordable span.
@@ -83,9 +84,29 @@ def new_dagrun_trace_carrier(
8384
SAMPLED flag directly (True = always trace this run, False = never) and the
8485
sampler is not consulted. Airflow wires this from the ``airflow/trace_sampled``
8586
run conf key; when None the configured sampler makes the decision.
87+
88+
``parent_context`` optionally embeds the run in an *external* trace: when it
89+
carries a valid span the carrier reuses that trace_id (so the whole run --
90+
dag_run, task_run, worker spans -- lives inside the external trace) and the
91+
sampling decision is made with that parent, so parent-based samplers inherit
92+
the external SAMPLED flag. Airflow wires this from the
93+
``airflow/parent_trace_context`` run conf key; when None the run is a root
94+
trace as before.
8695
"""
96+
parent_span_context = (
97+
trace.get_current_span(context=parent_context).get_span_context() if parent_context else None
98+
)
8799
gen = RandomIdGenerator()
88-
trace_id = gen.generate_trace_id()
100+
if parent_span_context is not None and parent_span_context.is_valid:
101+
# Embed in the external trace: ride its trace_id, inherit its tracestate, and
102+
# let the sampler decide with this parent (so parent-based samplers follow it).
103+
trace_id = parent_span_context.trace_id
104+
parent_trace_state: TraceState | None = parent_span_context.trace_state
105+
sampler_parent_context = parent_context
106+
else:
107+
trace_id = gen.generate_trace_id()
108+
parent_trace_state = None
109+
sampler_parent_context = None
89110

90111
if force_sampled is not None:
91112
sampled = force_sampled
@@ -95,7 +116,7 @@ def new_dagrun_trace_carrier(
95116
sampler = getattr(provider, "sampler", None)
96117
if sampler is not None:
97118
result = sampler.should_sample(
98-
parent_context=None, # root decision
119+
parent_context=sampler_parent_context, # None => root decision; else inherit parent
99120
trace_id=trace_id,
100121
name="dag_run",
101122
attributes=attributes or {},
@@ -109,9 +130,13 @@ def new_dagrun_trace_carrier(
109130
sampled = False
110131
sampler_trace_state = None
111132

112-
# Preserve the detail-level tracestate by merging it onto whatever the
113-
# sampler returned. TraceState is immutable, so update() returns a new one.
114-
trace_state = sampler_trace_state or TraceState()
133+
# Preserve the detail-level tracestate by merging it onto whatever the sampler
134+
# returned, falling back to the external parent's tracestate when embedding
135+
# without a sampler decision. TraceState is immutable, so update() returns a new one.
136+
trace_state = sampler_trace_state
137+
if trace_state is None and parent_trace_state is not None:
138+
trace_state = parent_trace_state
139+
trace_state = trace_state or TraceState()
115140
for key, value in build_trace_state_entries(task_span_detail_level):
116141
trace_state = trace_state.update(key, value)
117142

0 commit comments

Comments
 (0)