Skip to content

Commit cf5f7ad

Browse files
shahar1dabla
authored andcommitted
Detect template-field logic in operator __init__ in prek check (apache#70297)
Template fields are rendered after the constructor runs, so any validation or transformation applied to them in __init__ acts on the un-rendered Jinja expression — the bug class behind apache#69813, where SSHRemoteJobOperator validated a templated remote_base_dir at construction time and cleanup then failed for any custom base. The validate-operators-init hook only recognised top-level assignment statements, so validation calls, conditionals and transformations were invisible to it, and sensors, subscripted AwsBaseOperator[...] bases and aws_template_fields(...)-based classes were not covered at all. Pre-existing violations (83 classes) are exempted via a path::Class ratchet file so the rule is enforced on new code immediately; a fixing PR must remove its entry (stale entries fail the hook). Burn-down is tracked at apache#70296.
1 parent 48dfe62 commit cf5f7ad

3 files changed

Lines changed: 601 additions & 23 deletions

File tree

‎scripts/ci/prek/validate_operators_init.py‎

Lines changed: 304 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -25,54 +25,98 @@
2525

2626
import ast
2727
import sys
28+
from pathlib import Path
2829
from typing import Any
2930

3031
from rich.console import Console
32+
from rich.markup import escape
3133

3234
console = Console(color_system="standard", width=200)
33-
BASE_OPERATOR_CLASS_NAME = "BaseOperator"
35+
# Pre-existing violations exempted from the checks; burn-down tracked at
36+
# https://github.com/apache/airflow/issues/70296
37+
EXEMPTIONS_PATH = Path(__file__).parent / "validate_operators_init_exemptions.txt"
38+
BASE_CLASS_NAME_SUFFIXES = ("BaseOperator", "BaseSensorOperator")
39+
# Helper callables used as template_fields values, mapped to the fields the helper injects
40+
# on top of the explicit arguments. Injected fields are owned and assigned by the base class,
41+
# so only the explicit arguments are fields the defining class must assign itself.
42+
TEMPLATE_FIELD_HELPERS: dict[str, frozenset[str]] = {
43+
"aws_template_fields": frozenset({"aws_conn_id", "region_name", "verify"}),
44+
}
45+
46+
47+
def _resolve_base_name(base: ast.expr) -> str:
48+
"""
49+
Resolve a base-class expression to its plain name.
50+
51+
Unwraps subscripted generics (``AwsBaseOperator[EmrHook]``) and attribute access
52+
(``module.BaseOperator``).
53+
54+
:param base: The base-class expression node.
55+
:return: The resolved name, or an empty string when it cannot be resolved.
56+
"""
57+
if isinstance(base, ast.Subscript):
58+
base = base.value
59+
if isinstance(base, ast.Attribute):
60+
return base.attr
61+
if isinstance(base, ast.Name):
62+
return base.id
63+
return ""
3464

3565

3666
def _is_operator(class_node: ast.ClassDef) -> bool:
3767
"""
38-
Check if a given class node is an operator, based of the string suffix of the base IDs
39-
(ends with "BaseOperator").
68+
Check if a given class node is an operator or sensor, based on the string suffix of the
69+
base IDs (ends with "BaseOperator" or "BaseSensorOperator").
4070
TODO: Enhance this function to work with nested inheritance trees through dynamic imports.
4171
4272
:param class_node: The class node to check.
4373
:return: True if the class definition is of an operator, False otherwise.
4474
"""
45-
for base in class_node.bases:
46-
if isinstance(base, ast.Name) and base.id.endswith(BASE_OPERATOR_CLASS_NAME):
47-
return True
48-
return False
75+
return any(_resolve_base_name(base).endswith(BASE_CLASS_NAME_SUFFIXES) for base in class_node.bases)
76+
77+
78+
def _extract_field_names(value: ast.expr | None) -> list[str] | None:
79+
"""
80+
Extract template-field names from a ``template_fields`` value expression.
81+
82+
Supports a tuple of constants and known helper calls with constant arguments
83+
(e.g. ``aws_template_fields("s3_bucket", "s3_key")``). For helper calls, fields the
84+
helper injects on behalf of the base class are excluded — see ``TEMPLATE_FIELD_HELPERS``.
85+
86+
:param value: The value expression assigned to ``template_fields``.
87+
:return: The extracted field names, or None if the expression shape is not supported.
88+
"""
89+
if isinstance(value, ast.Tuple):
90+
return [str(elt.value) for elt in value.elts if isinstance(elt, ast.Constant)]
91+
if isinstance(value, ast.Call):
92+
injected = TEMPLATE_FIELD_HELPERS.get(_resolve_base_name(value.func))
93+
if injected is not None:
94+
args = [str(arg.value) for arg in value.args if isinstance(arg, ast.Constant)]
95+
return [arg for arg in args if arg not in injected]
96+
return None
4997

5098

5199
def _extract_template_fields(class_node: ast.ClassDef) -> list[str]:
52100
"""
53101
This method takes a class node as input and extracts the template fields from it.
54102
Template fields are identified by an assignment statement where the target is a variable
55-
named "template_fields" and the value is a tuple of constants.
103+
named "template_fields" and the value is a tuple of constants or a known helper call.
56104
57105
:param class_node: The class node representing the class for which template fields need to be extracted.
58106
:return: A list of template fields extracted from the class node.
59107
"""
60108
for class_item in class_node.body:
61109
if isinstance(class_item, ast.Assign):
62110
for target in class_item.targets:
63-
if (
64-
isinstance(target, ast.Name)
65-
and target.id == "template_fields"
66-
and isinstance(class_item.value, ast.Tuple)
67-
):
68-
return [str(elt.value) for elt in class_item.value.elts if isinstance(elt, ast.Constant)]
111+
if isinstance(target, ast.Name) and target.id == "template_fields":
112+
fields = _extract_field_names(class_item.value)
113+
if fields is not None:
114+
return fields
69115
elif isinstance(class_item, ast.AnnAssign):
70-
if (
71-
isinstance(class_item.target, ast.Name)
72-
and class_item.target.id == "template_fields"
73-
and isinstance(class_item.value, ast.Tuple)
74-
):
75-
return [str(elt.value) for elt in class_item.value.elts if isinstance(elt, ast.Constant)]
116+
if isinstance(class_item.target, ast.Name) and class_item.target.id == "template_fields":
117+
fields = _extract_field_names(class_item.value)
118+
if fields is not None:
119+
return fields
76120
return []
77121

78122

@@ -136,7 +180,13 @@ def _handle_constructor_statement(
136180
if isinstance(ctor_stmt.targets[0], ast.Attribute):
137181
for target in ctor_stmt.targets:
138182
if isinstance(target, ast.Attribute) and target.attr in template_fields:
139-
if isinstance(ctor_stmt.value, ast.BoolOp) and isinstance(ctor_stmt.value.op, ast.Or):
183+
if isinstance(ctor_stmt.value, ast.IfExp) and _is_value_preserving_ternary(
184+
ctor_stmt.value, target.attr
185+
):
186+
_handle_assigned_field(
187+
assigned_template_fields, invalid_assignments, target, ctor_stmt.value.body
188+
)
189+
elif isinstance(ctor_stmt.value, ast.BoolOp) and isinstance(ctor_stmt.value.op, ast.Or):
140190
_handle_assigned_field(
141191
assigned_template_fields, invalid_assignments, target, ctor_stmt.value.values[0]
142192
)
@@ -173,6 +223,175 @@ def _handle_assigned_field(
173223
assigned_template_fields.append(target.attr)
174224

175225

226+
def _target_name(target: ast.expr) -> str | None:
227+
"""
228+
Resolve an assignment target to the field name it binds.
229+
230+
:param target: The assignment target node.
231+
:return: The attribute name for ``self.<name>`` targets, the identifier for bare-name
232+
targets, or None for anything else.
233+
"""
234+
if isinstance(target, ast.Attribute) and isinstance(target.value, ast.Name) and target.value.id == "self":
235+
return target.attr
236+
if isinstance(target, ast.Name):
237+
return target.id
238+
return None
239+
240+
241+
def _is_super_init_call(node: ast.Call) -> bool:
242+
"""
243+
Check whether a call node is ``super().__init__(...)``.
244+
245+
:param node: The call node to check.
246+
:return: True if the node calls ``__init__`` on a ``super()`` call.
247+
"""
248+
return (
249+
isinstance(node.func, ast.Attribute)
250+
and node.func.attr == "__init__"
251+
and isinstance(node.func.value, ast.Call)
252+
and isinstance(node.func.value.func, ast.Name)
253+
and node.func.value.func.id == "super"
254+
)
255+
256+
257+
def _is_value_preserving_ternary(value: ast.IfExp, field: str) -> bool:
258+
"""
259+
Check whether a ternary keeps the field value intact when it is set.
260+
261+
Matches ``field if field else <default>`` and ``field if field is not None else <default>``
262+
— both equivalent to the sanctioned ``field or <default>`` defaulting idiom.
263+
264+
:param value: The ternary expression node.
265+
:param field: The template field name.
266+
:return: True if the ternary only substitutes a default for an unset value.
267+
"""
268+
if not (isinstance(value.body, ast.Name) and value.body.id == field):
269+
return False
270+
test = value.test
271+
if isinstance(test, ast.Name) and test.id == field:
272+
return True
273+
return (
274+
isinstance(test, ast.Compare)
275+
and isinstance(test.left, ast.Name)
276+
and test.left.id == field
277+
and len(test.ops) == 1
278+
and isinstance(test.ops[0], (ast.Is, ast.IsNot))
279+
and len(test.comparators) == 1
280+
and isinstance(test.comparators[0], ast.Constant)
281+
and test.comparators[0].value is None
282+
)
283+
284+
285+
def _collect_sanctioned_uses(ctor: ast.FunctionDef, template_fields: list[str]) -> set[int]:
286+
"""
287+
Collect the AST node ids of template-field reads that belong to sanctioned patterns.
288+
289+
Sanctioned patterns are the ones the project documents as safe in a constructor:
290+
``self.field = field``, ``self.field = field or <default>``, the equivalent
291+
value-preserving ternaries, the local rebind ``field = field or <default>``,
292+
tuple assignments pairing names one-to-one, and forwarding via
293+
``super().__init__(field=field)``.
294+
295+
:param ctor: The constructor function node.
296+
:param template_fields: The template fields of the class.
297+
:return: Set of ``id()``s of Name nodes participating in sanctioned patterns.
298+
"""
299+
sanctioned: set[int] = set()
300+
301+
def mark(value: ast.expr | None, field: str) -> None:
302+
if isinstance(value, ast.BoolOp) and isinstance(value.op, ast.Or):
303+
value = value.values[0]
304+
elif isinstance(value, ast.IfExp) and _is_value_preserving_ternary(value, field):
305+
for name_node in ast.walk(value.test):
306+
if isinstance(name_node, ast.Name) and name_node.id == field:
307+
sanctioned.add(id(name_node))
308+
value = value.body
309+
if isinstance(value, ast.Name) and value.id == field:
310+
sanctioned.add(id(value))
311+
312+
for node in ast.walk(ctor):
313+
if isinstance(node, (ast.Assign, ast.AnnAssign)):
314+
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
315+
pairs: list[tuple[ast.expr, ast.expr | None]]
316+
if len(targets) == 1 and isinstance(targets[0], ast.Tuple) and isinstance(node.value, ast.Tuple):
317+
pairs = list(zip(targets[0].elts, node.value.elts))
318+
else:
319+
pairs = [(target, node.value) for target in targets]
320+
for target, value in pairs:
321+
name = _target_name(target)
322+
if name is not None and name in template_fields:
323+
mark(value, name)
324+
elif isinstance(node, ast.Call) and _is_super_init_call(node):
325+
for keyword in node.keywords:
326+
if keyword.arg is not None and keyword.arg in template_fields:
327+
mark(keyword.value, keyword.arg)
328+
return sanctioned
329+
330+
331+
def _check_constructor_field_logic(
332+
class_node: ast.ClassDef, template_fields: list[str], source_lines: list[str]
333+
) -> int:
334+
"""
335+
Check a class's constructor for logic applied to template fields.
336+
337+
Template fields are rendered after the constructor runs, so any read of a template-field
338+
parameter (or ``self.<field>``) outside the sanctioned assignment/forwarding patterns —
339+
validation calls, conditionals, transformations, string interpolation — operates on the
340+
un-rendered Jinja expression and must move to ``execute()``.
341+
342+
:param class_node: The AST node representing the class definition.
343+
:param template_fields: The template fields of the class.
344+
:param source_lines: The source lines of the file, for reporting.
345+
:return: The number of offending source lines found.
346+
"""
347+
ctor = next(
348+
(item for item in class_node.body if isinstance(item, ast.FunctionDef) and item.name == "__init__"),
349+
None,
350+
)
351+
if ctor is None or not template_fields:
352+
return 0
353+
sanctioned = _collect_sanctioned_uses(ctor, template_fields)
354+
args = ctor.args
355+
# Only names bound in the constructor scope can refer to a template field; without this,
356+
# a field named e.g. "json" would false-positive on uses of the stdlib module.
357+
bound_names = {arg.arg for arg in [*args.posonlyargs, *args.args, *args.kwonlyargs]}
358+
bound_names |= {
359+
node.id for node in ast.walk(ctor) if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store)
360+
}
361+
# Parameter defaults evaluate at class-definition scope, where a name that matches a
362+
# template field (e.g. a field named "conf" vs. the configuration module) is not the field.
363+
in_defaults = {
364+
id(node)
365+
for default in [*args.defaults, *args.kw_defaults]
366+
if default is not None
367+
for node in ast.walk(default)
368+
}
369+
370+
findings: dict[int, set[str]] = {}
371+
for node in ast.walk(ctor):
372+
if id(node) in in_defaults:
373+
continue
374+
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load):
375+
if node.id in template_fields and node.id in bound_names and id(node) not in sanctioned:
376+
findings.setdefault(node.lineno, set()).add(node.id)
377+
elif isinstance(node, ast.Attribute) and isinstance(node.ctx, ast.Load):
378+
if isinstance(node.value, ast.Name) and node.value.id == "self" and node.attr in template_fields:
379+
findings.setdefault(node.lineno, set()).add(f"self.{node.attr}")
380+
381+
if findings:
382+
console.print(
383+
f"{class_node.name}'s constructor applies logic to template fields. Template fields "
384+
f"are rendered after the constructor runs, so validation or transformation here acts "
385+
f"on the un-rendered Jinja expression and should move to execute():"
386+
)
387+
for lineno in sorted(findings):
388+
source = source_lines[lineno - 1].strip() if lineno <= len(source_lines) else ""
389+
console.print(
390+
f"[red] line {lineno}: {escape(source)} ({', '.join(sorted(findings[lineno]))})[/red]"
391+
)
392+
return len(findings)
393+
394+
176395
def _check_constructor_template_fields(class_node: ast.ClassDef, template_fields: list[str]) -> int:
177396
"""
178397
This method checks a class's constructor for missing or invalid assignments of template fields.
@@ -220,22 +439,84 @@ def _check_constructor_template_fields(class_node: ast.ClassDef, template_fields
220439
return count
221440

222441

442+
def _load_exemptions() -> dict[str, set[str]]:
443+
"""
444+
Load the exemption list for known violations that predate the constructor-logic check.
445+
446+
Each non-comment line has the form ``<repo-relative-path>::<ClassName>``. Exempted classes
447+
are skipped by all checks; an exempted class with no findings fails as stale so the entry
448+
is removed in the same PR that fixes the class, until the list is empty.
449+
450+
:return: Mapping of repo-relative file path to the exempted class names in that file.
451+
"""
452+
exemptions: dict[str, set[str]] = {}
453+
if not EXEMPTIONS_PATH.exists():
454+
return exemptions
455+
for raw_line in EXEMPTIONS_PATH.read_text().splitlines():
456+
line = raw_line.strip()
457+
if not line or line.startswith("#"):
458+
continue
459+
entry_path, sep, class_name = line.partition("::")
460+
if sep and class_name:
461+
exemptions.setdefault(entry_path, set()).add(class_name)
462+
return exemptions
463+
464+
465+
def _get_exempted_classes(path: str, exemptions: dict[str, set[str]]) -> set[str]:
466+
"""
467+
Find the exempted class names for a checked file.
468+
469+
Exemption entries store repo-relative paths while the hook may receive paths relative
470+
to another working directory, so entries are matched as path suffixes.
471+
472+
:param path: The file path as passed to the script.
473+
:param exemptions: The loaded exemption mapping.
474+
:return: The exempted class names for this file, or an empty set.
475+
"""
476+
resolved = Path(path).resolve().as_posix()
477+
for entry_path, classes in exemptions.items():
478+
if resolved == entry_path or resolved.endswith(f"/{entry_path}"):
479+
return classes
480+
return set()
481+
482+
223483
def main():
224484
"""
225485
Check missing or invalid template fields in constructors of providers' operators.
226486
227487
:return: The total number of errors found.
228488
"""
229489
err = 0
490+
exemptions = _load_exemptions()
230491
for path in sys.argv[1:]:
231492
console.print(f"[yellow]{path}[/yellow]")
232-
tree = ast.parse(open(path).read())
493+
source = open(path).read()
494+
source_lines = source.splitlines()
495+
tree = ast.parse(source)
496+
exempted_classes = _get_exempted_classes(path, exemptions)
497+
exempted_finding_counts: dict[str, int] = {}
233498
for node in ast.walk(tree):
234499
if isinstance(node, ast.ClassDef) and _is_operator(class_node=node):
235500
template_fields = _extract_template_fields(node) or []
501+
if node.name in exempted_classes:
502+
with console.capture():
503+
count = _check_constructor_template_fields(node, template_fields)
504+
count += _check_constructor_field_logic(node, template_fields, source_lines)
505+
exempted_finding_counts[node.name] = count
506+
continue
236507
err += _check_constructor_template_fields(node, template_fields)
508+
err += _check_constructor_field_logic(node, template_fields, source_lines)
509+
for class_name in sorted(exempted_classes):
510+
if not exempted_finding_counts.get(class_name):
511+
err += 1
512+
console.print(
513+
f"[red]Stale exemption for {class_name} — the class has no findings anymore "
514+
f"(or is not detected as an operator); remove its entry from "
515+
f"{EXEMPTIONS_PATH.name}[/red]"
516+
)
237517
return err
238518

239519

240520
if __name__ == "__main__":
241-
sys.exit(main())
521+
# A raw error count wraps at 256 (e.g. 256 findings -> exit code 0), so clamp to 0/1.
522+
sys.exit(1 if main() else 0)

0 commit comments

Comments
 (0)