Repository navigation
Expand file tree
/
Copy pathdagrun.py
More file actions
3274 lines (2953 loc) · 134 KB
/
Copy pathdagrun.py
File metadata and controls
3274 lines (2953 loc) · 134 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import annotations
import itertools
import logging
import os
import re
from collections import defaultdict
from collections.abc import Callable, Iterable, Iterator, Sequence
from dataclasses import dataclass, field, fields, replace
from datetime import datetime
from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar, cast, overload
from uuid import UUID
import structlog
from opentelemetry import trace
from opentelemetry.context import context
from opentelemetry.trace import StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from sqlalchemy import (
JSON,
Enum,
ForeignKey,
ForeignKeyConstraint,
Index,
Integer,
PrimaryKeyConstraint,
String,
Text,
UniqueConstraint,
Uuid,
and_,
case,
func,
not_,
or_,
text,
tuple_,
update,
)
from sqlalchemy.dialects import postgresql
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.ext.mutable import MutableDict
from sqlalchemy.orm import (
Mapped,
declared_attr,
joinedload,
lazyload,
mapped_column,
relationship,
synonym,
validates,
)
from sqlalchemy.orm.exc import StaleDataError
from sqlalchemy.sql.expression import false, select
from sqlalchemy.sql.functions import coalesce
from airflow._shared.observability.metrics import stats
from airflow._shared.observability.metrics.stats import build_dag_metric_tags
from airflow._shared.observability.traces import (
DAGRUN_PARENT_TRACE_CONTEXT_KEY,
TASK_SPAN_DETAIL_LEVEL_KEY,
TRACE_SAMPLED_KEY,
new_dagrun_trace_carrier,
override_ids,
)
from airflow._shared.timezones import timezone
from airflow.callbacks.callback_requests import DagCallbackRequest, DagRunContext
from airflow.configuration import conf as airflow_conf
from airflow.exceptions import AirflowException, NotMapped, TaskNotFound
from airflow.listeners.listener import get_listener_manager
from airflow.models import Deadline, Log
from airflow.models.backfill import Backfill
from airflow.models.base import Base, StringID
from airflow.models.deadline_alert import DeadlineAlert as DeadlineAlertModel
from airflow.models.dynamic_region import (
LOOP_DECISION_REASON_KEY,
SENTINEL_REGION_ID,
DynamicRegion,
public_region_filter,
)
from airflow.models.taskinstance import TaskInstance as TI, _add_and_prime_mapped_ti
from airflow.models.tasklog import LogTemplate
from airflow.sdk.definitions.context import CallbackMeta, CallbackSource # noqa: SDK001
from airflow.serialization.definitions.deadline import SerializedReferenceModels
from airflow.serialization.definitions.notset import NOTSET, ArgNotSet, is_arg_set
from airflow.ti_deps.dep_context import DepContext
from airflow.ti_deps.dependencies_states import SCHEDULEABLE_STATES
from airflow.utils.helpers import chunks, is_container, prune_dict
from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.retries import retry_db_transaction
from airflow.utils.session import NEW_SESSION, provide_session
from airflow.utils.sqlalchemy import (
ExtendedJSON,
UtcDateTime,
get_dialect_name,
nulls_first,
with_row_locks,
)
from airflow.utils.state import DagRunState, State, TaskInstanceState
from airflow.utils.strings import get_random_string
from airflow.utils.types import DagRunTriggeredByType, DagRunType
if TYPE_CHECKING:
from typing import Literal, TypeAlias
from pydantic import NonNegativeInt
from sqlalchemy.engine import Row, ScalarResult
from sqlalchemy.orm import InstrumentedAttribute, Session
from sqlalchemy.sql.elements import ColumnElement
from sqlalchemy.sql.selectable import Select
from airflow.api_fastapi.execution_api.datamodels.taskinstance import (
DagRun as DRDataModel,
TaskInstance as TIDataModel,
)
from airflow.models.dag_version import DagVersion
from airflow.models.task_coordinates import SchedulingTI
from airflow.sdk import DAG as SDKDAG
from airflow.serialization.definitions.dag import SerializedDAG
from airflow.serialization.definitions.mappedoperator import Operator
from airflow.serialization.definitions.taskgroup import SerializedLoopTaskGroup
from airflow.timetables.base import Timetable
CreatedTasks = TypeVar("CreatedTasks", Iterator["dict[str, Any]"], Iterator[TI])
AttributeValueType: TypeAlias = (
str | bool | int | float | Sequence[str] | Sequence[bool] | Sequence[int] | Sequence[float]
)
RUN_ID_REGEX = r"^(?:manual|scheduled|asset_triggered)__(?:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00)$"
log = structlog.get_logger(__name__)
tracer = trace.get_tracer(__name__)
class InvalidLoopDecision(ValueError):
"""A gate cannot complete successfully with the available decision."""
def validate_loop_decision(stop: bool | None, region_index: int, group: SerializedLoopTaskGroup) -> None:
"""Raise :class:`InvalidLoopDecision` unless ``stop`` is valid for the gate of pass ``region_index``."""
at_limit = region_index >= group.max_iterations
if stop is None:
raise InvalidLoopDecision("Successful loop gate requires a decision")
if not stop and at_limit:
raise InvalidLoopDecision("Loop cannot continue beyond its iteration limit")
if not group.has_until and stop != at_limit:
raise InvalidLoopDecision("Fixed-count loop decision does not match its iteration limit")
@dataclass(frozen=True, slots=True, eq=False)
class FinishedTI:
"""
Immutable view of a task instance in a terminal state.
Keeps the dag_run rescan lightweight and fast: scheduling only reads these columns
off finished task instances, so a fully instrumented TaskInstance per row is
unnecessary. ``task`` is attached from the serialized dag rather than the row.
``eq=False`` keeps identity equality and ``__hash__`` mirrors the key hash, the same as
a TaskInstance.
Anything needing a full TaskInstance must re-fetch it (see ``_ensure_type_task_instance``).
"""
id: UUID
dag_id: str
run_id: str
task_id: str
region_id: UUID
region_index: int
dag_version_id: UUID | None
state: str | None
start_date: datetime | None
end_date: datetime | None
task: Operator | None = field(default=None, kw_only=True)
@property
def map_index(self) -> int:
"""Mirror the ``TaskInstance.map_index`` synonym of ``region_index``."""
return self.region_index
def __hash__(self) -> int:
return hash((self.task_id, self.dag_id, self.run_id, self.region_index))
# Get the necessary columns from the FinishedTI fields so that the class stays
# the single source of truth. For any needed change, add or remove a field there.
# Rows are unpacked into FinishedTI positionally, so the column fields must stay
# first and in declaration order, with ``task`` last.
FINISHED_TI_COLUMNS = tuple(getattr(TI, f.name) for f in fields(FinishedTI) if f.name != "task")
class TISchedulingDecision(NamedTuple):
"""Type of return for DagRun.task_instance_scheduling_decisions."""
tis: list[SchedulingTI]
schedulable_tis: list[TI]
changed_tis: bool
unfinished_tis: list[TI]
finished_tis: list[SchedulingTI]
def _default_run_after(ctx):
params = ctx.get_current_parameters()
return params["data_interval_end"] or params["logical_date"] or timezone.utcnow()
def _creator_note(val):
"""Creator for the ``note`` association proxy."""
if isinstance(val, str):
return DagRunNote(content=val)
if isinstance(val, dict):
return DagRunNote(**val)
return DagRunNote(*val)
def dagrun_trace_attributes(dr) -> dict[str, str]:
"""
Run-identity attributes for a DAG run's trace.
Defined once and used in two places: forwarded to the sampler at carrier
creation (so a custom sampler can differentiate the head-sampling decision by
run kind) and set on the emitted ``dag_run`` span. ``getattr`` guards because
the values may not be populated yet at ``__init__``/clear time.
"""
attributes: dict[str, str] = {}
dag_id = getattr(dr, "dag_id", None)
if dag_id is not None:
attributes["airflow.dag_id"] = str(dag_id)
run_id = getattr(dr, "run_id", None)
if run_id is not None:
attributes["airflow.dag_run.run_id"] = str(run_id)
run_type = getattr(dr, "run_type", None)
if run_type is not None:
attributes["airflow.dag_run.run_type"] = str(run_type)
return attributes
def trace_sampled_override(conf) -> bool | None:
"""
Head-sampling override from the ``airflow/trace_sampled`` run conf key.
Returns the forced SAMPLED flag only when the conf value is an explicit bool
(True = always trace this run, False = never); otherwise None, meaning no
override and the configured sampler decides. Non-bool values are ignored
rather than coerced, so a malformed value never silently flips sampling or
fails run creation.
"""
if not conf:
return None
raw = conf.get(TRACE_SAMPLED_KEY)
if isinstance(raw, bool):
return raw
return None
def parent_trace_context(conf) -> context.Context | None:
"""
External parent trace context from the ``airflow/dagrun_parent_trace_context`` run conf key.
Lets a run be embedded in a trace owned by an external system (an upstream
orchestrator, event pipeline, CI job, another Airflow) instead of being a root
trace. The value is a W3C ``traceparent`` string, or a carrier dict carrying
``traceparent`` (and optionally ``tracestate``); anything else -- including a
malformed traceparent -- is ignored so it can neither silently mis-parent the
run nor fail run creation. Returns an OpenTelemetry ``Context`` when a valid
parent is present, else None (root trace, the default).
"""
if not conf:
return None
match conf.get(DAGRUN_PARENT_TRACE_CONTEXT_KEY):
case str() as traceparent:
carrier = {"traceparent": traceparent}
case {"traceparent": str()} as raw:
# Keep only str members: a non-str tracestate reaches TraceState.from_header
# unvalidated and raises TypeError, which would drop the otherwise-valid parent.
carrier = {k: raw[k] for k in ("traceparent", "tracestate") if isinstance(raw.get(k), str)}
case _:
return None
try:
ctx = TraceContextTextMapPropagator().extract(carrier)
except Exception:
# Never let a malformed conf value fail run creation; fall back to a root trace.
return None
if not trace.get_current_span(ctx).get_span_context().is_valid:
return None
return ctx
class DagRun(Base, LoggingMixin):
"""
Invocation instance of a DAG.
A DAG run can be created by the scheduler (i.e. scheduled runs), or by an
external trigger (i.e. manual runs).
"""
__tablename__ = "dag_run"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
dag_id: Mapped[str] = mapped_column(StringID(), nullable=False)
queued_at: Mapped[datetime | None] = mapped_column(UtcDateTime, nullable=True)
logical_date: Mapped[datetime | None] = mapped_column(UtcDateTime, nullable=True)
start_date: Mapped[datetime | None] = mapped_column(UtcDateTime, nullable=True)
end_date: Mapped[datetime | None] = mapped_column(UtcDateTime, nullable=True)
_state: Mapped[str | None] = mapped_column("state", String(50), default=DagRunState.QUEUED, nullable=True)
run_id: Mapped[str] = mapped_column(StringID(), nullable=False)
creating_job_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
run_type: Mapped[str] = mapped_column(String(50), nullable=False)
triggered_by: Mapped[DagRunTriggeredByType | None] = mapped_column(
Enum(DagRunTriggeredByType, native_enum=False, length=50), nullable=True
) # Airflow component that triggered the run.
triggering_user_name: Mapped[str | None] = mapped_column(
String(512),
nullable=True,
) # The user that triggered the DagRun, if applicable
conf: Mapped[dict[str, Any] | None] = mapped_column(
JSON().with_variant(postgresql.JSONB, "postgresql"), nullable=True
)
# These two must be either both NULL or both datetime.
data_interval_start: Mapped[datetime | None] = mapped_column(UtcDateTime, nullable=True)
data_interval_end: Mapped[datetime | None] = mapped_column(UtcDateTime, nullable=True)
# Earliest time when this DagRun can start running.
run_after: Mapped[datetime] = mapped_column(UtcDateTime, default=_default_run_after, nullable=False)
# When a scheduler last attempted to schedule TIs for this DagRun
last_scheduling_decision: Mapped[datetime | None] = mapped_column(UtcDateTime, nullable=True)
# Foreign key to LogTemplate. DagRun rows created prior to this column's
# existence have this set to NULL. Later rows automatically populate this on
# insert to point to the latest LogTemplate entry.
log_template_id: Mapped[int | None] = mapped_column(
Integer,
ForeignKey("log_template.id", name="task_instance_log_template_id_fkey", ondelete="NO ACTION"),
default=select(func.max(LogTemplate.__table__.c.id)),
nullable=True,
)
# This is nullable because it's too costly to migrate dagruns created prior
# to this column's addition (Airflow 3.2.0). If you want a reasonable
# meaningful non-null value, use ``dr.created_at or dr.run_after``.
created_at: Mapped[datetime | None] = mapped_column(UtcDateTime, nullable=True, default=timezone.utcnow)
updated_at: Mapped[datetime | None] = mapped_column(
UtcDateTime, default=timezone.utcnow, onupdate=timezone.utcnow, nullable=True
)
# Keeps track of the number of times the dagrun had been cleared.
# This number is incremented only when the DagRun is re-Queued,
# when the DagRun is cleared.
clear_number: Mapped[int] = mapped_column(Integer, default=0, nullable=False, server_default="0")
backfill_id: Mapped[int | None] = mapped_column(Integer, ForeignKey("backfill.id"), nullable=True)
"""
The backfill this DagRun is currently associated with.
It's possible this could change if e.g. the dag run is cleared to be rerun, or perhaps re-backfilled.
"""
bundle_version: Mapped[str | None] = mapped_column(StringID(), nullable=True)
scheduled_by_job_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
# Span context carrier, used for context propagation.
context_carrier: Mapped[dict[str, Any] | None] = mapped_column(
MutableDict.as_mutable(ExtendedJSON), nullable=True
)
created_dag_version_id: Mapped[UUID | None] = mapped_column(
Uuid(),
ForeignKey("dag_version.id", name="created_dag_version_id_fkey", ondelete="set null"),
nullable=True,
)
"""The id of the dag version column that was in effect at dag run creation time.
:meta private:
"""
partition_key: Mapped[str | None] = mapped_column(StringID(), nullable=True)
partition_date: Mapped[datetime | None] = mapped_column(UtcDateTime, nullable=True)
# Remove this `if` after upgrading Sphinx-AutoAPI
if not TYPE_CHECKING and "BUILDING_AIRFLOW_DOCS" in os.environ:
dag: SerializedDAG | None
else:
dag: SerializedDAG | None = None
__table_args__ = (
Index("dag_id_state", dag_id, _state),
UniqueConstraint("dag_id", "run_id", name="dag_run_dag_id_run_id_key"),
UniqueConstraint("dag_id", "logical_date", name="dag_run_dag_id_logical_date_key"),
Index("idx_dag_run_dag_id", dag_id),
Index("idx_dag_run_run_after", run_after),
Index("idx_dag_run_dag_id_run_after", dag_id, run_after),
Index("idx_dag_run_created_dag_version_id", created_dag_version_id),
Index(
"idx_dag_run_running_dags",
"state",
"dag_id",
postgresql_where=text("state='running'"),
sqlite_where=text("state='running'"),
),
# since mysql lacks filtered/partial indices, this creates a
# duplicate index on mysql. Not the end of the world
Index(
"idx_dag_run_queued_dags",
"state",
"dag_id",
postgresql_where=text("state='queued'"),
sqlite_where=text("state='queued'"),
),
)
task_instances = relationship(
TI,
back_populates="dag_run",
primaryjoin="and_(DagRun.dag_id == TaskInstance.dag_id, DagRun.run_id == TaskInstance.run_id, TaskInstance.working_set.is_(True))",
cascade="save-update, merge, delete, delete-orphan",
passive_deletes=True,
)
historical_task_instances = relationship(
TI,
primaryjoin="and_(DagRun.dag_id == TaskInstance.dag_id, DagRun.run_id == TaskInstance.run_id, TaskInstance.working_set.is_(None))",
viewonly=True,
)
dag_model = relationship(
"DagModel",
primaryjoin="foreign(DagRun.dag_id) == DagModel.dag_id",
uselist=False,
viewonly=True,
)
dag_run_note = relationship(
"DagRunNote",
back_populates="dag_run",
uselist=False,
cascade="all, delete, delete-orphan",
)
deadlines = relationship(
"Deadline",
back_populates="dagrun",
uselist=True,
cascade="all, delete, delete-orphan",
)
created_dag_version = relationship("DagVersion", uselist=False, passive_deletes=True)
"""
The dag version that was active when the dag run was created, if available.
:meta private:
"""
backfill = relationship(Backfill, uselist=False)
backfill_max_active_runs = association_proxy("backfill", "max_active_runs")
max_active_runs = association_proxy("dag_model", "max_active_runs")
@property
def team_name(self) -> str | None:
"""Name of the team owning this run's Dag, or ``None`` when it is not team-owned."""
# Gate before touching ``dag_model``: single-team deployments must not pay for the load.
if not airflow_conf.getboolean("core", "multi_team"):
return None
return self.dag_model.team_name if self.dag_model else None
note = association_proxy("dag_run_note", "content", creator=_creator_note)
DEFAULT_DAGRUNS_TO_EXAMINE = airflow_conf.getint(
"scheduler",
"max_dagruns_per_loop_to_schedule",
fallback=20,
)
_ti_dag_versions = association_proxy("task_instances", "dag_version")
_historical_dag_versions = association_proxy("historical_task_instances", "dag_version")
# Can be set by manual prefetch, such as attach_dag_versions_to_runs()
# as an alternative to traversing all current and historical attempts. Maps version_id -> DagVersion.
_prefetched_dag_version_ids: dict[UUID, DagVersion] | None = None
def __init__(
self,
dag_id: str | None = None,
run_id: str | None = None,
*,
queued_at: datetime | None | ArgNotSet = NOTSET,
logical_date: datetime | None = None,
run_after: datetime | None = None,
start_date: datetime | None = None,
conf: Any | None = None,
state: DagRunState | None = None,
run_type: str | None = None,
creating_job_id: int | None = None,
data_interval: tuple[datetime, datetime] | None = None,
triggered_by: DagRunTriggeredByType | None = None,
triggering_user_name: str | None = None,
backfill_id: NonNegativeInt | None = None,
bundle_version: str | None = None,
partition_key: str | None = None,
partition_date: datetime | None = None,
note: str | None = None,
):
# For manual runs where logical_date is None, ensure no data_interval is set.
if logical_date is None and data_interval is not None:
raise ValueError("data_interval must be None if logical_date is None")
if data_interval is None:
# Legacy: Only happen for runs created prior to Airflow 2.2.
self.data_interval_start = self.data_interval_end = None
else:
self.data_interval_start, self.data_interval_end = data_interval
self.bundle_version = bundle_version
if dag_id is not None:
self.dag_id = dag_id
if run_id is not None:
self.run_id = run_id
self.logical_date = logical_date
if run_after is not None:
self.run_after = run_after
self.start_date = start_date
self.conf = conf or {}
self.note = note
if state is not None:
self.state = state
if not is_arg_set(queued_at):
self.queued_at = timezone.utcnow() if state == DagRunState.QUEUED else None
elif queued_at is not None:
self.queued_at = queued_at
if run_type is not None:
self.run_type = run_type
self.creating_job_id = creating_job_id
self.backfill_id = backfill_id
self.clear_number = 0
self.triggered_by = triggered_by
self.triggering_user_name = triggering_user_name
self.scheduled_by_job_id = None
self.context_carrier: dict[str, str] = new_dagrun_trace_carrier(
task_span_detail_level=self.conf.get(TASK_SPAN_DETAIL_LEVEL_KEY, None),
attributes=dagrun_trace_attributes(self), # these are for potential use by head sampler
force_sampled=trace_sampled_override(self.conf),
parent_context=parent_trace_context(self.conf),
)
if not isinstance(partition_key, str | None):
raise ValueError(
f"Expected partition_key to be a `str` or `None` but got `{partition_key.__class__.__name__}`"
)
self.partition_key = partition_key
self.partition_date = partition_date
super().__init__()
def __repr__(self):
return (
f"<DagRun {self.dag_id} @ {self.logical_date}: {self.run_id}, state:{self.state}, "
f"queued_at: {self.queued_at}. run_type: {self.run_type}>"
)
@validates("run_id")
def validate_run_id(self, key: str, run_id: str) -> str | None:
if not run_id:
return None
if ".." in run_id and not airflow_conf.getboolean("core", "allow_double_dot_in_ids", fallback=False):
raise ValueError(f"The run_id '{run_id}' must not contain '..' to prevent path traversal")
if re.match(RUN_ID_REGEX, run_id):
return run_id
regex = airflow_conf.get("scheduler", "allowed_run_id_pattern").strip()
if regex and re.match(regex, run_id):
return run_id
raise ValueError(
f"The run_id provided '{run_id}' does not match regex pattern '{regex}' or '{RUN_ID_REGEX}'"
)
@property
def dag_run_data(self) -> DRDataModel:
from airflow.api_fastapi.execution_api.datamodels.taskinstance import DagRun as DRDataModel
return DRDataModel(
dag_id=self.dag_id,
run_id=self.run_id,
logical_date=self.logical_date,
data_interval_start=self.data_interval_start,
data_interval_end=self.data_interval_end,
run_after=self.run_after,
start_date=self.start_date or timezone.utcnow(),
end_date=self.end_date,
run_type=DagRunType(self.run_type),
state=self.state,
conf=self.conf,
consumed_asset_events=[],
partition_key=self.partition_key,
partition_date=self.partition_date,
)
@property
def dag_versions(self) -> list[DagVersion]:
"""Return the DAG versions associated with the TIs of this DagRun."""
# when the dag is in a versioned bundle, we keep the dag version fixed
if self.bundle_version:
return [self.created_dag_version] if self.created_dag_version is not None else []
if self._prefetched_dag_version_ids is not None:
dag_version_objects = list(self._prefetched_dag_version_ids.values())
else:
dag_version_objects = [
dv
for dv in dict.fromkeys(list(self._historical_dag_versions) + list(self._ti_dag_versions))
if dv is not None
]
return sorted(dag_version_objects, key=lambda dv: dv.id)
@property
def version_number(self) -> int | None:
"""Return the DAG version number associated with the latest TI of this DagRun."""
dag_versions = self.dag_versions
if dag_versions:
return dag_versions[-1].version_number
return None
@hybrid_property
def duration(self) -> float | None:
if self.end_date and self.start_date:
return (self.end_date - self.start_date).total_seconds()
return None
@duration.expression # type: ignore[no-redef]
@provide_session
def duration(cls: type[DagRun], *, session: Session = NEW_SESSION) -> ColumnElement[Any]:
dialect_name = get_dialect_name(session)
if dialect_name == "mysql":
return func.timestampdiff(text("SECOND"), cls.start_date, cls.end_date)
if dialect_name == "sqlite":
duration_expr = (func.julianday(cls.end_date) - func.julianday(cls.start_date)) * 86400
else:
duration_expr = func.extract("epoch", cls.end_date - cls.start_date)
when_condition = (
(cls.end_date != None) & (cls.start_date != None), # noqa: E711
duration_expr,
)
return case(when_condition, else_=None)
@provide_session
def check_version_id_exists_in_dr(self, dag_version_id: UUID, *, session: Session = NEW_SESSION):
select_stmt = (
select(TI.dag_version_id)
.where(TI.dag_id == self.dag_id, TI.dag_version_id == dag_version_id, TI.run_id == self.run_id)
.limit(1)
.execution_options(include_all_attempts=True)
)
return session.scalar(select_stmt)
def dag_tags_for_stats(self) -> dict[str, str]:
"""Convert dag tags to metric tags. Tags with ':' become key:value; others are standalone (empty value)."""
if not airflow_conf.getboolean("metrics", "dag_tags_in_metrics", fallback=False):
return {}
try:
# Lazy-loads dag_model.tags if not already loaded. The scheduler hot loop
# (get_running_dag_runs_to_examine) eager-loads these to avoid a per-DagRun query; other,
# low-frequency emission paths fall back to this lazy load. On a detached/expired DagRun
# the load raises — swallow it so metric tagging never breaks the caller.
if not self.dag_model or not self.dag_model.tags:
return {}
return build_dag_metric_tags(tag.name for tag in self.dag_model.tags)
except SQLAlchemyError:
return {}
@property
def stats_tags(self) -> dict[str, str]:
# prune_dict strips falsy values, so merge dag tags after it runs so standalone
# tags (empty value) are preserved for DogStatsD emission.
base = prune_dict(
{
"dag_id": self.dag_id,
# bare value so it serializes as e.g. "scheduled", not "dagruntype.scheduled"
"run_type": getattr(self.run_type, "value", self.run_type),
"team_name": getattr(self, "_team_name", None),
}
)
dag_tags = self.dag_tags_for_stats()
# Built-in keys win on collision; dag tags fill in everything else.
return {**dag_tags, **base}
def get_state(self):
return self._state
def set_state(self, state: DagRunState) -> None:
"""
Change the state of the DagRun.
Changes to attributes are implemented in accordance with the following table
(rows represent old states, columns represent new states):
.. list-table:: State transition matrix
:header-rows: 1
:stub-columns: 1
* -
- QUEUED
- RUNNING
- SUCCESS
- FAILED
* - None
- queued_at = timezone.utcnow()
- if empty: start_date = timezone.utcnow()
end_date = None
- end_date = timezone.utcnow()
- end_date = timezone.utcnow()
* - QUEUED
- queued_at = timezone.utcnow()
- if empty: start_date = timezone.utcnow()
end_date = None
- end_date = timezone.utcnow()
- end_date = timezone.utcnow()
* - RUNNING
- queued_at = timezone.utcnow()
start_date = None
end_date = None
-
- end_date = timezone.utcnow()
- end_date = timezone.utcnow()
* - SUCCESS
- queued_at = timezone.utcnow()
start_date = None
end_date = None
- start_date = timezone.utcnow()
end_date = None
-
-
* - FAILED
- queued_at = timezone.utcnow()
start_date = None
end_date = None
- start_date = timezone.utcnow()
end_date = None
-
-
"""
if state not in State.dag_states:
raise ValueError(f"invalid DagRun state: {state}")
if self._state != state:
if state == DagRunState.QUEUED:
self.queued_at = timezone.utcnow()
self.start_date = None
self.end_date = None
if state == DagRunState.RUNNING:
if self._state in State.finished_dr_states:
self.start_date = timezone.utcnow()
else:
self.start_date = self.start_date or timezone.utcnow()
self.end_date = None
if self._state in State.unfinished_dr_states or self._state is None:
if state in State.finished_dr_states:
self.end_date = timezone.utcnow()
self._state = state
else:
if state == DagRunState.QUEUED:
self.queued_at = timezone.utcnow()
@declared_attr
def state(self):
return synonym("_state", descriptor=property(self.get_state, self.set_state))
@provide_session
def refresh_from_db(self, *, session: Session = NEW_SESSION) -> None:
"""
Reload the current dagrun from the database.
:param session: database session
"""
dr = session.scalars(
select(DagRun).where(DagRun.dag_id == self.dag_id, DagRun.run_id == self.run_id)
).one()
self.id = dr.id
self.state = dr.state
@classmethod
@provide_session
def active_runs_of_dags(
cls,
*,
dag_ids: Iterable[str],
exclude_backfill: bool,
session: Session = NEW_SESSION,
) -> dict[str, int]:
"""
Get the number of active dag runs for each dag.
:meta private:
"""
query = (
select(cls.dag_id, func.count("*"))
.where(cls.dag_id.in_(set(dag_ids)))
.where(cls.state.in_((DagRunState.RUNNING, DagRunState.QUEUED)))
.group_by(cls.dag_id)
)
if exclude_backfill:
query = query.where(cls.run_type != DagRunType.BACKFILL_JOB)
return {dag_id: count for dag_id, count in session.execute(query)}
@classmethod
@retry_db_transaction
def get_running_dag_runs_to_examine(
cls, *, session: Session, eagerly_load_dag_tags: bool
) -> ScalarResult[DagRun]:
"""
Return the next DagRuns that the scheduler should attempt to schedule.
This will return zero or more DagRun rows that are row-level-locked with a "SELECT ... FOR UPDATE"
query, you should ensure that any scheduling decisions are made in a single transaction -- as soon as
the transaction is committed it will be unlocked.
:meta private:
"""
from airflow.models.dag import DagModel
query = (
select(cls)
.with_hint(cls, "USE INDEX (idx_dag_run_running_dags)", dialect_name="mysql")
.where(cls.state == DagRunState.RUNNING)
.join(DagModel, DagModel.dag_id == cls.dag_id)
.where(
DagModel.is_paused == false(),
DagModel.is_stale == false(),
)
.order_by(
nulls_first(cast("ColumnElement[Any]", cls.last_scheduling_decision), session=session),
cls.run_after,
)
.limit(cls.DEFAULT_DAGRUNS_TO_EXAMINE)
)
# When dag tags are emitted as metric tags, eagerly load dag_model.tags so stats_tags does not
# fire a per-DagRun N+1 lazy load in the scheduler loop. The caller owns the feature decision;
# the scheduler passes its cached flag so the loop never reads conf.
if eagerly_load_dag_tags:
query = query.options(joinedload(cls.dag_model).selectinload(DagModel.tags))
query = query.where(DagRun.run_after <= func.now())
result = session.scalars(with_row_locks(query, of=cls, session=session, skip_locked=True)).unique()
return result
@classmethod
@retry_db_transaction
def get_queued_dag_runs_to_set_running(cls, session: Session) -> ScalarResult[DagRun]:
"""
Return the next queued DagRuns that the scheduler should attempt to schedule.
This will return zero or more DagRun rows that are row-level-locked with a "SELECT ... FOR UPDATE"
query, you should ensure that any scheduling decisions are made in a single transaction -- as soon as
the transaction is committed it will be unlocked.
:meta private:
"""
from airflow.models.backfill import Backfill, BackfillDagRun
from airflow.models.dag import DagModel
# For dag runs in the queued state, we check if they have reached the max_active_runs limit
# and if so we drop them
running_drs = (
select(
DagRun.dag_id,
DagRun.backfill_id,
func.count(DagRun.id).label("num_running"),
)
.where(DagRun.state == DagRunState.RUNNING)
.group_by(DagRun.dag_id, DagRun.backfill_id)
.subquery()
)
query = (
select(cls)
.where(cls.state == DagRunState.QUEUED)
.join(
DagModel,
and_(
DagModel.dag_id == cls.dag_id,
DagModel.is_paused == false(),
DagModel.is_stale == false(),
),
)
.join(
BackfillDagRun,
and_(
BackfillDagRun.dag_run_id == DagRun.id,
BackfillDagRun.backfill_id == DagRun.backfill_id,
),
isouter=True,
)
.join(Backfill, isouter=True)
.join(
running_drs,
and_(
running_drs.c.dag_id == DagRun.dag_id,
coalesce(running_drs.c.backfill_id, text("-1"))
== coalesce(DagRun.backfill_id, text("-1")),
),
isouter=True,
)
.where(
# there are two levels of checks for num_running
# the one done in this query verifies that the dag is not maxed out
# it could return many more dag runs than runnable if there is even
# capacity for 1. this could be improved.
coalesce(running_drs.c.num_running, text("0"))
< coalesce(Backfill.max_active_runs, DagModel.max_active_runs),
# don't set paused dag runs as running
not_(coalesce(cast("ColumnElement[bool]", Backfill.is_paused), False)),
)
.order_by(
# ordering by backfill sort ordinal first ensures that backfill dag runs
# have lower priority than all other dag run types (since sort_ordinal >= 1).
# additionally, sorting by sort_ordinal ensures that the backfill
# dag runs are created in the right order when that matters.
# todo: AIP-78 use row_number to avoid starvation; limit the number of returned runs per-dag
nulls_first(cast("ColumnElement[Any]", BackfillDagRun.sort_ordinal), session=session),
nulls_first(cast("ColumnElement[Any]", cls.last_scheduling_decision), session=session),
nulls_first(running_drs.c.num_running, session=session), # many running -> lower priority
cls.run_after,
)
.limit(cls.DEFAULT_DAGRUNS_TO_EXAMINE)
)
query = query.where(DagRun.run_after <= func.now())
return session.scalars(with_row_locks(query, of=cls, session=session, skip_locked=True))
@classmethod
@provide_session
def find(
cls,
dag_id: str | list[str] | None = None,
run_id: Iterable[str] | None = None,
logical_date: datetime | Iterable[datetime] | None = None,
state: DagRunState | None = None,
no_backfills: bool = False,
run_type: DagRunType | None = None,
logical_start_date: datetime | None = None,
logical_end_date: datetime | None = None,
*,
session: Session = NEW_SESSION,
) -> list[DagRun]:
"""
Return a set of dag runs for the given search criteria.
:param dag_id: the dag_id or list of dag_id to find dag runs for
:param run_id: defines the run id for this dag run
:param run_type: type of DagRun
:param logical_date: the logical date
:param state: the state of the dag run
:param no_backfills: return no backfills (True), return all (False).
Defaults to False
:param session: database session
:param logical_start_date: dag run that was executed from this date
:param logical_end_date: dag run that was executed until this date
"""
qry = select(cls)
dag_ids = [dag_id] if isinstance(dag_id, str) else dag_id
if dag_ids:
qry = qry.where(cls.dag_id.in_(dag_ids))
if is_container(run_id):
qry = qry.where(cls.run_id.in_(run_id))
elif run_id is not None:
qry = qry.where(cls.run_id == run_id)
if is_container(logical_date):
qry = qry.where(cls.logical_date.in_(logical_date))
elif logical_date is not None:
qry = qry.where(cls.logical_date == logical_date)
if logical_start_date and logical_end_date:
qry = qry.where(cls.logical_date.between(logical_start_date, logical_end_date))
elif logical_start_date:
qry = qry.where(cls.logical_date >= logical_start_date)
elif logical_end_date:
qry = qry.where(cls.logical_date <= logical_end_date)
if state:
qry = qry.where(cls.state == state)
if run_type:
qry = qry.where(cls.run_type == run_type)
if no_backfills:
qry = qry.where(cls.run_type != DagRunType.BACKFILL_JOB)
return list(session.scalars(qry.order_by(cls.logical_date)).all())
@classmethod
@provide_session
def find_duplicate(cls, dag_id: str, run_id: str, *, session: Session = NEW_SESSION) -> DagRun | None: