Skip to content

Commit d9a3478

Browse files
committed
Allow Variable.get to reuse the caller's database session
`MetastoreBackend.get_variable` is decorated with `@provide_session`. Called without a session it goes through `create_session()`, which for a scoped session returns *the caller's own session* and commits it on exit. So any code that reads a Variable while holding a transaction gets that transaction committed underneath it — detaching its objects, or raising `UNEXPECTED COMMIT` under the scheduler's `prohibit_commit` guard, where the error is then swallowed per-backend and surfaces as a missing Variable. `Variable.get` and `Variable.get_variable_from_secrets` now take an optional keyword-only `session`, forwarded only to `MetastoreBackend`. `Variable.update` forwards its own. Affected today, all reached from `_create_dagruns_for_dags` inside the guard: * Deadline Alerts using `VariableInterval` * Custom timetables reading a Variable in `next_dagrun_info` — `next_dagrun` is left NULL, so the Dag is never eligible and never runs, with nothing logged * Dag sync via `update_dags`; `Variable.update`; `Variable.setdefault` Scope: this fixes the core read path only. `airflow.sdk.Variable.get` and `Connection.get_connection_from_secrets` share the defect and are unchanged, so callers going through those are still affected. apache#68917 can drop its duplicated backend walk once this lands. closes: apache#71801
1 parent 4ddb6b4 commit d9a3478

2 files changed

Lines changed: 89 additions & 4 deletions

File tree

‎airflow-core/src/airflow/models/variable.py‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,8 @@ def get(
156156
default_var: Any = __NO_DEFAULT_SENTINEL,
157157
deserialize_json: bool = False,
158158
team_name: str | None = None,
159+
*,
160+
session: Session | None = None,
159161
) -> Any:
160162
"""
161163
Get a value for an Airflow Variable Key.
@@ -164,6 +166,8 @@ def get(
164166
:param default_var: Default value of the Variable if the Variable doesn't exist
165167
:param deserialize_json: Deserialize the value to a Python dict
166168
:param team_name: Team name associated to the task trying to access the variable (if any)
169+
:param session: Existing session to reuse for the metadata database lookup. Callers holding an
170+
open transaction (the scheduler under ``prohibit_commit``, for example) must pass it.
167171
"""
168172
# TODO: This is not the best way of having compat, but it's "better than erroring" for now. This still
169173
# means SQLA etc is loaded, but we can't avoid that unless/until we add import shims as a big
@@ -172,6 +176,11 @@ def get(
172176
# If this is set it means we are in some kind of execution context (Task, Dag Parse or Triggerer perhaps)
173177
# and should use the Task SDK API server path
174178
if hasattr(sys.modules.get("airflow.sdk.execution_time.task_runner"), "SUPERVISOR_COMMS"):
179+
if session is not None:
180+
raise ValueError(
181+
"Variable.get() cannot use a metadata database session from an execution context; "
182+
"reads there go through the Execution API. Use airflow.sdk.Variable.get() instead."
183+
)
175184
warnings.warn(
176185
"Using Variable.get from `airflow.models` is deprecated."
177186
"Please use `get` on Variable from sdk(`airflow.sdk.Variable`) instead",
@@ -192,7 +201,7 @@ def get(
192201
"Multi-team mode is not configured in the Airflow environment but the task trying to access the variable belongs to a team"
193202
)
194203

195-
var_val = Variable.get_variable_from_secrets(key=key, team_name=team_name)
204+
var_val = Variable.get_variable_from_secrets(key=key, team_name=team_name, session=session)
196205
if var_val is None:
197206
if default_var is not cls.__NO_DEFAULT_SENTINEL:
198207
return default_var
@@ -344,7 +353,7 @@ def update(
344353

345354
Variable.check_for_write_conflict(key=key, team_name=team_name)
346355

347-
if Variable.get_variable_from_secrets(key=key, team_name=team_name) is None:
356+
if Variable.get_variable_from_secrets(key=key, team_name=team_name, session=session) is None:
348357
raise KeyError(f"Variable {key} does not exist")
349358

350359
ctx: contextlib.AbstractContextManager
@@ -469,12 +478,18 @@ def check_for_write_conflict(key: str, team_name: str | None = None) -> None:
469478
return None
470479

471480
@staticmethod
472-
def get_variable_from_secrets(key: str, team_name: str | None = None) -> str | None:
481+
def get_variable_from_secrets(
482+
key: str, team_name: str | None = None, *, session: Session | None = None
483+
) -> str | None:
473484
"""
474485
Get Airflow Variable by iterating over all Secret Backends.
475486
476487
:param key: Variable Key
477488
:param team_name: Team name associated to the task trying to access the variable (if any)
489+
:param session: Existing session to reuse for the metadata database lookup. Callers that
490+
already hold a transaction must pass it, otherwise ``MetastoreBackend`` opens the same
491+
scoped session and commits it, which detaches the caller's objects and is rejected
492+
outright under ``prohibit_commit``.
478493
:return: Variable Value
479494
"""
480495
from airflow.sdk import SecretCache
@@ -491,7 +506,16 @@ def get_variable_from_secrets(key: str, team_name: str | None = None) -> str | N
491506
for secrets_backend in ensure_secrets_loaded():
492507
try:
493508
var_val = call_secrets_backend_method(
494-
secrets_backend.get_variable, team_name=team_name, key=key
509+
secrets_backend.get_variable,
510+
team_name=team_name,
511+
key=key,
512+
# Only the metastore backend touches the metadata database, and it is the only
513+
# one whose signature accepts a session.
514+
**(
515+
{"session": session}
516+
if session is not None and isinstance(secrets_backend, MetastoreBackend)
517+
else {}
518+
),
495519
)
496520
if var_val is not None:
497521
break

‎airflow-core/tests/unit/models/test_variable.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
from airflow.sdk import SecretCache
3131
from airflow.secrets import BaseSecretsBackend
3232
from airflow.secrets.metastore import MetastoreBackend
33+
from airflow.utils.sqlalchemy import prohibit_commit
3334

3435
from tests_common.test_utils import db
3536
from tests_common.test_utils.config import conf_vars
@@ -253,6 +254,66 @@ def test_update_forwards_team_name_to_write_conflict_check(self, mock_check, tes
253254
Variable.update(key="key", value="new-value", team_name=testing_team.name, session=session)
254255

255256
assert mock_check.call_args.kwargs["team_name"] == testing_team.name
257+
@mock.patch.object(MetastoreBackend, "get_variable", autospec=True)
258+
@mock.patch("airflow.models.variable.ensure_secrets_loaded")
259+
def test_get_forwards_session_to_metastore_backend(self, mock_ensure_secrets, mock_get_variable, session):
260+
mock_get_variable.return_value = "from_db"
261+
mock_ensure_secrets.return_value = [MetastoreBackend()]
262+
263+
assert Variable.get("some_key", session=session) == "from_db"
264+
assert mock_get_variable.call_args.kwargs["session"] is session
265+
266+
@mock.patch.object(MetastoreBackend, "get_variable", autospec=True)
267+
@mock.patch("airflow.models.variable.ensure_secrets_loaded")
268+
def test_get_without_session_omits_session_kwarg(self, mock_ensure_secrets, mock_get_variable):
269+
mock_get_variable.return_value = "from_db"
270+
mock_ensure_secrets.return_value = [MetastoreBackend()]
271+
272+
assert Variable.get("some_key") == "from_db"
273+
assert "session" not in mock_get_variable.call_args.kwargs
274+
275+
@mock.patch("airflow.models.variable.ensure_secrets_loaded")
276+
def test_get_does_not_forward_session_to_other_backends(self, mock_ensure_secrets, session):
277+
"""Only the metastore backend reads the metadata database, so only it accepts a session."""
278+
mock_backend = mock.Mock()
279+
mock_backend.get_variable.return_value = "from_backend"
280+
mock_backend.__class__.__name__ = "MockSecretsBackend"
281+
mock_ensure_secrets.return_value = [mock_backend]
282+
283+
assert Variable.get("some_key", session=session) == "from_backend"
284+
assert "session" not in mock_backend.get_variable.call_args.kwargs
285+
286+
def test_get_with_session_does_not_commit_under_prohibit_commit(self, session):
287+
"""
288+
A caller holding an open transaction can read a Variable without its session being committed.
289+
290+
Without the session being forwarded, ``MetastoreBackend.get_variable``'s ``provide_session``
291+
takes the same scoped session and commits it, which the guard rejects.
292+
"""
293+
Variable.set(key="interval_key", value="60", session=session)
294+
session.commit()
295+
SecretCache.invalidate_variable("interval_key")
296+
297+
with prohibit_commit(session):
298+
assert Variable.get("interval_key", session=session) == "60"
299+
300+
def test_update_with_session_does_not_commit_under_prohibit_commit(self, session):
301+
"""``update`` verifies existence through the secrets chain, which must reuse the session too."""
302+
Variable.set(key="interval_key", value="60", session=session)
303+
session.commit()
304+
SecretCache.invalidate_variable("interval_key")
305+
306+
with prohibit_commit(session):
307+
Variable.update(key="interval_key", value="120", session=session)
308+
309+
def test_get_rejects_session_in_execution_context(self):
310+
"""Reads from an execution context go via the Execution API, where a session is meaningless."""
311+
task_runner = mock.Mock(SUPERVISOR_COMMS=mock.Mock())
312+
with (
313+
mock.patch.dict("sys.modules", {"airflow.sdk.execution_time.task_runner": task_runner}),
314+
pytest.raises(ValueError, match="cannot use a metadata database session"),
315+
):
316+
Variable.get("some_key", session=mock.Mock())
256317

257318
def test_variable_set_get_round_trip_json(self):
258319
value = {"a": 17, "b": 47}

0 commit comments

Comments
 (0)