Skip to content

Commit 1116f28

Browse files
authored
AIP-65: Add DAG versioning support (#42913)
* AIP-65: Add DAG versioning support This commit introduces versioning for DAGs Changes: - Introduced DagVersion model to handle versioning of DAGs. - Added version_name field to DAG for use in tracking the dagversion by users - Added support for version retrieval in the get_dag_source API endpoint - Modified DAG execution logic to reference dag_version_id instead of the dag_hash to ensure DAG runs are linked to specific versions. Fix tests revert RESTAPI changes * fixup! AIP-65: Add DAG versioning support * fixup! fixup! AIP-65: Add DAG versioning support * fix migration * fix test * more test fixes * update query count * fix static checks * Fix query and add created_at to dag_version table * improve code * Change to using UUID for primary keys * DagCode.bulk_write_code is no longer used * fixup! Change to using UUID for primary keys * fix tests * fixup! fix tests * use uuid for version_name * fixup! use uuid for version_name * use row lock when writing dag version * use row lock when writing dag version * fixup! use row lock when writing dag version * deactivating dag should not remove serialized dags * save version_name as string not uuid * Make dag_version_id unique * fixup! Make dag_version_id unique * Fix tests * Use uuid7 * fix test * fixup! fix test * use binary=False for uuid field to fix sqlite issue * apply suggestions from code review * Remove unnecessary version_name on dagmodel * Fix sqlalchemy 2 warning * Fix conflicts * Apply suggestions from code review Co-authored-by: Jed Cunningham <[email protected]> * fixup! Apply suggestions from code review * fixup! fixup! Apply suggestions from code review * add test for dagversion model and make version_name, number and dag_id unique * Remove commented test as serdag can no longer disappear * Add SQLAlchemy-utils to requirements * mark test_dag_version.py as db_test * make version_name nullable * Apply suggestions from code review * fixup! Apply suggestions from code review * remove file_updater * Use dag_version for creating dagruns instead of dag_version_id * fix conflicts * use if TYPE_CHECKING * Add docstrings to methods * Move getting latest serdags to SerializedDagModel
1 parent da50242 commit 1116f28

46 files changed

Lines changed: 3049 additions & 2366 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎airflow/api/common/trigger_dag.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from airflow.api_internal.internal_api_call import internal_api_call
2626
from airflow.exceptions import DagNotFound, DagRunAlreadyExists
2727
from airflow.models import DagBag, DagModel, DagRun
28+
from airflow.models.dag_version import DagVersion
2829
from airflow.utils import timezone
2930
from airflow.utils.session import NEW_SESSION, provide_session
3031
from airflow.utils.state import DagRunState
@@ -92,14 +93,14 @@ def _trigger_dag(
9293
run_conf = None
9394
if conf:
9495
run_conf = conf if isinstance(conf, dict) else json.loads(conf)
95-
96+
dag_version = DagVersion.get_latest_version(dag.dag_id)
9697
dag_run = dag.create_dagrun(
9798
run_id=run_id,
9899
execution_date=execution_date,
99100
state=DagRunState.QUEUED,
100101
conf=run_conf,
101102
external_trigger=True,
102-
dag_hash=dag_bag.dags_hash.get(dag_id),
103+
dag_version=dag_version,
103104
data_interval=data_interval,
104105
triggered_by=triggered_by,
105106
)

‎airflow/api_connexion/endpoints/dag_run_endpoint.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
from airflow.auth.managers.models.resource_details import DagAccessEntity
6262
from airflow.exceptions import ParamValidationError
6363
from airflow.models import DagModel, DagRun
64+
from airflow.models.dag_version import DagVersion
6465
from airflow.timetables.base import DataInterval
6566
from airflow.utils.airflow_flask_app import get_airflow_app
6667
from airflow.utils.api_migration import mark_fastapi_migration_done
@@ -341,7 +342,7 @@ def post_dag_run(*, dag_id: str, session: Session = NEW_SESSION) -> APIResponse:
341342
)
342343
else:
343344
data_interval = dag.timetable.infer_manual_data_interval(run_after=logical_date)
344-
345+
dag_version = DagVersion.get_latest_version(dag.dag_id)
345346
dag_run = dag.create_dagrun(
346347
run_type=DagRunType.MANUAL,
347348
run_id=run_id,
@@ -350,7 +351,7 @@ def post_dag_run(*, dag_id: str, session: Session = NEW_SESSION) -> APIResponse:
350351
state=DagRunState.QUEUED,
351352
conf=post_body.get("conf"),
352353
external_trigger=True,
353-
dag_hash=get_airflow_app().dag_bag.dags_hash.get(dag_id),
354+
dag_version=dag_version,
354355
session=session,
355356
triggered_by=DagRunTriggeredByType.REST_API,
356357
)

‎airflow/dag_processing/manager.py‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@
5050
from airflow.models.dagwarning import DagWarning
5151
from airflow.models.db_callback_request import DbCallbackRequest
5252
from airflow.models.errors import ParseImportError
53-
from airflow.models.serialized_dag import SerializedDagModel
5453
from airflow.secrets.cache import SecretCache
5554
from airflow.stats import Stats
5655
from airflow.traces.tracer import Trace, add_span
@@ -539,10 +538,6 @@ def deactivate_stale_dags(
539538
if deactivated:
540539
cls.logger().info("Deactivated %i DAGs which are no longer present in file.", deactivated)
541540

542-
for dag_id in to_deactivate:
543-
SerializedDagModel.remove_dag(dag_id)
544-
cls.logger().info("Deleted DAG %s in serialized_dag table", dag_id)
545-
546541
def _run_parsing_loop(self):
547542
# In sync mode we want timeout=None -- wait forever until a message is received
548543
if self._async_mode:
@@ -819,20 +814,10 @@ def _iter_dag_filelocs(fileloc: str) -> Iterator[str]:
819814

820815
dag_filelocs = {full_loc for path in self._file_paths for full_loc in _iter_dag_filelocs(path)}
821816

822-
from airflow.models.dagcode import DagCode
823-
824-
SerializedDagModel.remove_deleted_dags(
825-
alive_dag_filelocs=dag_filelocs,
826-
processor_subdir=self.get_dag_directory(),
827-
)
828817
DagModel.deactivate_deleted_dags(
829818
dag_filelocs,
830819
processor_subdir=self.get_dag_directory(),
831820
)
832-
DagCode.remove_deleted_code(
833-
dag_filelocs,
834-
processor_subdir=self.get_dag_directory(),
835-
)
836821

837822
return True
838823
return False

‎airflow/example_dags/plugins/event_listener.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,9 +164,10 @@ def on_dag_run_running(dag_run: DagRun, msg: str):
164164
"""
165165
print("Dag run in running state")
166166
queued_at = dag_run.queued_at
167-
dag_hash_info = dag_run.dag_hash
168167

169-
print(f"Dag information Queued at: {queued_at} hash info: {dag_hash_info}")
168+
version = dag_run.dag_version.version
169+
170+
print(f"Dag information Queued at: {queued_at} version: {version}")
170171

171172

172173
# [END howto_listen_dagrun_running_task]

‎airflow/jobs/scheduler_job_runner.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,10 @@
5454
)
5555
from airflow.models.backfill import Backfill
5656
from airflow.models.dag import DAG, DagModel
57+
from airflow.models.dag_version import DagVersion
5758
from airflow.models.dagbag import DagBag
5859
from airflow.models.dagrun import DagRun
5960
from airflow.models.dagwarning import DagWarning, DagWarningType
60-
from airflow.models.serialized_dag import SerializedDagModel
6161
from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance
6262
from airflow.stats import Stats
6363
from airflow.ti_deps.dependencies_states import EXECUTION_STATES
@@ -1338,7 +1338,7 @@ def _create_dag_runs(self, dag_models: Collection[DagModel], session: Session) -
13381338
self.log.error("DAG '%s' not found in serialized_dag table", dag_model.dag_id)
13391339
continue
13401340

1341-
dag_hash = self.dagbag.dags_hash.get(dag.dag_id)
1341+
latest_dag_version = DagVersion.get_latest_version(dag.dag_id, session=session)
13421342

13431343
data_interval = dag.get_next_data_interval(dag_model)
13441344
# Explicitly check if the DagRun already exists. This is an edge case
@@ -1358,7 +1358,7 @@ def _create_dag_runs(self, dag_models: Collection[DagModel], session: Session) -
13581358
data_interval=data_interval,
13591359
external_trigger=False,
13601360
session=session,
1361-
dag_hash=dag_hash,
1361+
dag_version=latest_dag_version,
13621362
creating_job_id=self.job.id,
13631363
triggered_by=DagRunTriggeredByType.TIMETABLE,
13641364
)
@@ -1417,7 +1417,7 @@ def _create_dag_runs_asset_triggered(
14171417
)
14181418
continue
14191419

1420-
dag_hash = self.dagbag.dags_hash.get(dag.dag_id)
1420+
latest_dag_version = DagVersion.get_latest_version(dag.dag_id, session=session)
14211421

14221422
# Explicitly check if the DagRun already exists. This is an edge case
14231423
# where a Dag Run is created but `DagModel.next_dagrun` and `DagModel.next_dagrun_create_after`
@@ -1472,7 +1472,7 @@ def _create_dag_runs_asset_triggered(
14721472
state=DagRunState.QUEUED,
14731473
external_trigger=False,
14741474
session=session,
1475-
dag_hash=dag_hash,
1475+
dag_version=latest_dag_version,
14761476
creating_job_id=self.job.id,
14771477
triggered_by=DagRunTriggeredByType.ASSET,
14781478
)
@@ -1750,18 +1750,20 @@ def _verify_integrity_if_dag_changed(self, dag_run: DagRun, session: Session) ->
17501750
17511751
Return True if we determine that DAG still exists.
17521752
"""
1753-
latest_version = SerializedDagModel.get_latest_version_hash(dag_run.dag_id, session=session)
1754-
if dag_run.dag_hash == latest_version:
1753+
latest_dag_version = DagVersion.get_latest_version(dag_run.dag_id, session=session)
1754+
if TYPE_CHECKING:
1755+
assert latest_dag_version
1756+
if dag_run.dag_version_id == latest_dag_version.id:
17551757
self.log.debug("DAG %s not changed structure, skipping dagrun.verify_integrity", dag_run.dag_id)
17561758
return True
17571759

1758-
dag_run.dag_hash = latest_version
1759-
17601760
# Refresh the DAG
17611761
dag_run.dag = self.dagbag.get_dag(dag_id=dag_run.dag_id, session=session)
17621762
if not dag_run.dag:
17631763
return False
17641764

1765+
dag_run.dag_version = latest_dag_version
1766+
17651767
# Verify integrity also takes care of session.flush
17661768
dag_run.verify_integrity(session=session)
17671769
return True
@@ -2041,7 +2043,6 @@ def _cleanup_stale_dags(self, session: Session = NEW_SESSION) -> None:
20412043
20422044
In case one of DagProcessors is stopped (in case there are multiple of them
20432045
for different dag folders), its dags are never marked as inactive.
2044-
Also remove dags from SerializedDag table.
20452046
Executed on schedule only if [scheduler]standalone_dag_processor is True.
20462047
"""
20472048
self.log.debug("Checking dags not parsed within last %s seconds.", self._dag_stale_not_seen_duration)
@@ -2056,7 +2057,6 @@ def _cleanup_stale_dags(self, session: Session = NEW_SESSION) -> None:
20562057
self.log.info("Found (%d) stales dags not parsed after %s.", len(stale_dags), limit_lpt)
20572058
for dag in stale_dags:
20582059
dag.is_active = False
2059-
SerializedDagModel.remove_dag(dag_id=dag.dag_id, session=session)
20602060
session.flush()
20612061

20622062
@provide_session
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
19+
"""
20+
add dag versioning.
21+
22+
Revision ID: 2b47dc6bc8df
23+
Revises: d03e4a635aa3
24+
Create Date: 2024-10-09 05:44:04.670984
25+
26+
"""
27+
28+
from __future__ import annotations
29+
30+
import sqlalchemy as sa
31+
from alembic import op
32+
from sqlalchemy_utils import UUIDType
33+
34+
from airflow.migrations.db_types import StringID
35+
from airflow.models.base import naming_convention
36+
from airflow.utils import timezone
37+
from airflow.utils.sqlalchemy import UtcDateTime
38+
39+
# revision identifiers, used by Alembic.
40+
revision = "2b47dc6bc8df"
41+
down_revision = "d03e4a635aa3"
42+
branch_labels = None
43+
depends_on = None
44+
airflow_version = "3.0.0"
45+
46+
47+
def upgrade():
48+
"""Apply add dag versioning."""
49+
op.create_table(
50+
"dag_version",
51+
sa.Column("id", UUIDType(binary=False), nullable=False),
52+
sa.Column("version_number", sa.Integer(), nullable=False),
53+
sa.Column("version_name", StringID()),
54+
sa.Column("dag_id", StringID(), nullable=False),
55+
sa.Column("created_at", UtcDateTime(), nullable=False, default=timezone.utcnow),
56+
sa.ForeignKeyConstraint(
57+
("dag_id",), ["dag.dag_id"], name=op.f("dag_version_dag_id_fkey"), ondelete="CASCADE"
58+
),
59+
sa.PrimaryKeyConstraint("id", name=op.f("dag_version_pkey")),
60+
sa.UniqueConstraint("dag_id", "version_number", name="dag_id_v_name_v_number_unique_constraint"),
61+
)
62+
with op.batch_alter_table("dag_code", recreate="always", naming_convention=naming_convention) as batch_op:
63+
batch_op.drop_constraint("dag_code_pkey", type_="primary")
64+
batch_op.add_column(
65+
sa.Column("id", UUIDType(binary=False), primary_key=True), insert_before="fileloc_hash"
66+
)
67+
batch_op.create_primary_key("dag_code_pkey", ["id"])
68+
batch_op.add_column(sa.Column("dag_version_id", UUIDType(binary=False), nullable=False))
69+
batch_op.create_foreign_key(
70+
batch_op.f("dag_code_dag_version_id_fkey"),
71+
"dag_version",
72+
["dag_version_id"],
73+
["id"],
74+
ondelete="CASCADE",
75+
)
76+
batch_op.create_unique_constraint("dag_code_dag_version_id_uq", ["dag_version_id"])
77+
78+
with op.batch_alter_table(
79+
"serialized_dag", recreate="always", naming_convention=naming_convention
80+
) as batch_op:
81+
batch_op.drop_constraint("serialized_dag_pkey", type_="primary")
82+
batch_op.add_column(sa.Column("id", UUIDType(binary=False), primary_key=True))
83+
batch_op.drop_index("idx_fileloc_hash")
84+
batch_op.drop_column("fileloc_hash")
85+
batch_op.drop_column("fileloc")
86+
batch_op.create_primary_key("serialized_dag_pkey", ["id"])
87+
batch_op.add_column(sa.Column("dag_version_id", UUIDType(binary=False), nullable=False))
88+
batch_op.create_foreign_key(
89+
batch_op.f("serialized_dag_dag_version_id_fkey"),
90+
"dag_version",
91+
["dag_version_id"],
92+
["id"],
93+
ondelete="CASCADE",
94+
)
95+
batch_op.create_unique_constraint("serialized_dag_dag_version_id_uq", ["dag_version_id"])
96+
97+
with op.batch_alter_table("task_instance", schema=None) as batch_op:
98+
batch_op.add_column(sa.Column("dag_version_id", UUIDType(binary=False)))
99+
batch_op.create_foreign_key(
100+
batch_op.f("task_instance_dag_version_id_fkey"),
101+
"dag_version",
102+
["dag_version_id"],
103+
["id"],
104+
ondelete="CASCADE",
105+
)
106+
107+
with op.batch_alter_table("task_instance_history", schema=None) as batch_op:
108+
batch_op.add_column(sa.Column("dag_version_id", UUIDType(binary=False)))
109+
110+
with op.batch_alter_table("dag_run", schema=None) as batch_op:
111+
batch_op.add_column(sa.Column("dag_version_id", UUIDType(binary=False)))
112+
batch_op.create_foreign_key(
113+
batch_op.f("dag_run_dag_version_id_fkey"),
114+
"dag_version",
115+
["dag_version_id"],
116+
["id"],
117+
ondelete="CASCADE",
118+
)
119+
batch_op.drop_column("dag_hash")
120+
121+
122+
def downgrade():
123+
"""Unapply add dag versioning."""
124+
with op.batch_alter_table("task_instance_history", schema=None) as batch_op:
125+
batch_op.drop_column("dag_version_id")
126+
127+
with op.batch_alter_table("task_instance", schema=None) as batch_op:
128+
batch_op.drop_constraint(batch_op.f("task_instance_dag_version_id_fkey"), type_="foreignkey")
129+
batch_op.drop_column("dag_version_id")
130+
131+
with op.batch_alter_table("dag_code", schema=None) as batch_op:
132+
batch_op.drop_column("id")
133+
batch_op.drop_constraint(batch_op.f("dag_code_dag_version_id_fkey"), type_="foreignkey")
134+
batch_op.drop_column("dag_version_id")
135+
batch_op.create_primary_key("dag_code_pkey", ["fileloc_hash"])
136+
137+
with op.batch_alter_table("serialized_dag", schema=None, naming_convention=naming_convention) as batch_op:
138+
batch_op.drop_column("id")
139+
batch_op.add_column(sa.Column("fileloc", sa.String(length=2000), autoincrement=False, nullable=False))
140+
batch_op.add_column(sa.Column("fileloc_hash", sa.BIGINT(), autoincrement=False, nullable=False))
141+
batch_op.create_index("idx_fileloc_hash", ["fileloc_hash"], unique=False)
142+
batch_op.create_primary_key("serialized_dag_pkey", ["dag_id"])
143+
batch_op.drop_constraint(batch_op.f("serialized_dag_dag_version_id_fkey"), type_="foreignkey")
144+
batch_op.drop_column("dag_version_id")
145+
146+
with op.batch_alter_table("dag_run", schema=None) as batch_op:
147+
batch_op.add_column(sa.Column("dag_hash", sa.String(length=32), autoincrement=False, nullable=True))
148+
batch_op.drop_constraint(batch_op.f("dag_run_dag_version_id_fkey"), type_="foreignkey")
149+
batch_op.drop_column("dag_version_id")
150+
151+
op.drop_table("dag_version")

‎airflow/models/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ def import_all_models():
5757

5858
import airflow.models.asset
5959
import airflow.models.backfill
60+
import airflow.models.dag_version
6061
import airflow.models.dagwarning
6162
import airflow.models.errors
6263
import airflow.models.serialized_dag

‎airflow/models/backfill.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
from airflow.api_connexion.exceptions import NotFound
4444
from airflow.exceptions import AirflowException
4545
from airflow.models.base import Base, StringID
46+
from airflow.models.dag_version import DagVersion
4647
from airflow.settings import json
4748
from airflow.utils import timezone
4849
from airflow.utils.session import create_session
@@ -200,7 +201,7 @@ def _create_backfill_dag_run(
200201
)
201202
)
202203
return
203-
204+
dag_version = DagVersion.get_latest_version(dag.dag_id, session=session)
204205
dr = dag.create_dagrun(
205206
triggered_by=DagRunTriggeredByType.BACKFILL,
206207
execution_date=info.logical_date,
@@ -213,6 +214,7 @@ def _create_backfill_dag_run(
213214
creating_job_id=None,
214215
session=session,
215216
backfill_id=backfill_id,
217+
dag_version=dag_version,
216218
)
217219
session.add(
218220
BackfillDagRun(
@@ -253,7 +255,7 @@ def _create_backfill(
253255
from airflow.models.serialized_dag import SerializedDagModel
254256

255257
with create_session() as session:
256-
serdag = session.get(SerializedDagModel, dag_id)
258+
serdag = session.scalar(SerializedDagModel.latest_item_select_object(dag_id))
257259
if not serdag:
258260
raise NotFound(f"Could not find dag {dag_id}")
259261
# todo: if dag has no schedule, raise

0 commit comments

Comments
 (0)