|
25 | 25 |
|
26 | 26 | import ast |
27 | 27 | import sys |
| 28 | +from pathlib import Path |
28 | 29 | from typing import Any |
29 | 30 |
|
30 | 31 | from rich.console import Console |
| 32 | +from rich.markup import escape |
31 | 33 |
|
32 | 34 | console = Console(color_system="standard", width=200) |
33 | | -BASE_OPERATOR_CLASS_NAME = "BaseOperator" |
| 35 | +# Pre-existing violations exempted from the checks; burn-down tracked at |
| 36 | +# https://github.com/apache/airflow/issues/70296 |
| 37 | +EXEMPTIONS_PATH = Path(__file__).parent / "validate_operators_init_exemptions.txt" |
| 38 | +BASE_CLASS_NAME_SUFFIXES = ("BaseOperator", "BaseSensorOperator") |
| 39 | +# Helper callables used as template_fields values, mapped to the fields the helper injects |
| 40 | +# on top of the explicit arguments. Injected fields are owned and assigned by the base class, |
| 41 | +# so only the explicit arguments are fields the defining class must assign itself. |
| 42 | +TEMPLATE_FIELD_HELPERS: dict[str, frozenset[str]] = { |
| 43 | + "aws_template_fields": frozenset({"aws_conn_id", "region_name", "verify"}), |
| 44 | +} |
| 45 | + |
| 46 | + |
| 47 | +def _resolve_base_name(base: ast.expr) -> str: |
| 48 | + """ |
| 49 | + Resolve a base-class expression to its plain name. |
| 50 | +
|
| 51 | + Unwraps subscripted generics (``AwsBaseOperator[EmrHook]``) and attribute access |
| 52 | + (``module.BaseOperator``). |
| 53 | +
|
| 54 | + :param base: The base-class expression node. |
| 55 | + :return: The resolved name, or an empty string when it cannot be resolved. |
| 56 | + """ |
| 57 | + if isinstance(base, ast.Subscript): |
| 58 | + base = base.value |
| 59 | + if isinstance(base, ast.Attribute): |
| 60 | + return base.attr |
| 61 | + if isinstance(base, ast.Name): |
| 62 | + return base.id |
| 63 | + return "" |
34 | 64 |
|
35 | 65 |
|
36 | 66 | def _is_operator(class_node: ast.ClassDef) -> bool: |
37 | 67 | """ |
38 | | - Check if a given class node is an operator, based of the string suffix of the base IDs |
39 | | - (ends with "BaseOperator"). |
| 68 | + Check if a given class node is an operator or sensor, based on the string suffix of the |
| 69 | + base IDs (ends with "BaseOperator" or "BaseSensorOperator"). |
40 | 70 | TODO: Enhance this function to work with nested inheritance trees through dynamic imports. |
41 | 71 |
|
42 | 72 | :param class_node: The class node to check. |
43 | 73 | :return: True if the class definition is of an operator, False otherwise. |
44 | 74 | """ |
45 | | - for base in class_node.bases: |
46 | | - if isinstance(base, ast.Name) and base.id.endswith(BASE_OPERATOR_CLASS_NAME): |
47 | | - return True |
48 | | - return False |
| 75 | + return any(_resolve_base_name(base).endswith(BASE_CLASS_NAME_SUFFIXES) for base in class_node.bases) |
| 76 | + |
| 77 | + |
| 78 | +def _extract_field_names(value: ast.expr | None) -> list[str] | None: |
| 79 | + """ |
| 80 | + Extract template-field names from a ``template_fields`` value expression. |
| 81 | +
|
| 82 | + Supports a tuple of constants and known helper calls with constant arguments |
| 83 | + (e.g. ``aws_template_fields("s3_bucket", "s3_key")``). For helper calls, fields the |
| 84 | + helper injects on behalf of the base class are excluded — see ``TEMPLATE_FIELD_HELPERS``. |
| 85 | +
|
| 86 | + :param value: The value expression assigned to ``template_fields``. |
| 87 | + :return: The extracted field names, or None if the expression shape is not supported. |
| 88 | + """ |
| 89 | + if isinstance(value, ast.Tuple): |
| 90 | + return [str(elt.value) for elt in value.elts if isinstance(elt, ast.Constant)] |
| 91 | + if isinstance(value, ast.Call): |
| 92 | + injected = TEMPLATE_FIELD_HELPERS.get(_resolve_base_name(value.func)) |
| 93 | + if injected is not None: |
| 94 | + args = [str(arg.value) for arg in value.args if isinstance(arg, ast.Constant)] |
| 95 | + return [arg for arg in args if arg not in injected] |
| 96 | + return None |
49 | 97 |
|
50 | 98 |
|
51 | 99 | def _extract_template_fields(class_node: ast.ClassDef) -> list[str]: |
52 | 100 | """ |
53 | 101 | This method takes a class node as input and extracts the template fields from it. |
54 | 102 | Template fields are identified by an assignment statement where the target is a variable |
55 | | - named "template_fields" and the value is a tuple of constants. |
| 103 | + named "template_fields" and the value is a tuple of constants or a known helper call. |
56 | 104 |
|
57 | 105 | :param class_node: The class node representing the class for which template fields need to be extracted. |
58 | 106 | :return: A list of template fields extracted from the class node. |
59 | 107 | """ |
60 | 108 | for class_item in class_node.body: |
61 | 109 | if isinstance(class_item, ast.Assign): |
62 | 110 | for target in class_item.targets: |
63 | | - if ( |
64 | | - isinstance(target, ast.Name) |
65 | | - and target.id == "template_fields" |
66 | | - and isinstance(class_item.value, ast.Tuple) |
67 | | - ): |
68 | | - return [str(elt.value) for elt in class_item.value.elts if isinstance(elt, ast.Constant)] |
| 111 | + if isinstance(target, ast.Name) and target.id == "template_fields": |
| 112 | + fields = _extract_field_names(class_item.value) |
| 113 | + if fields is not None: |
| 114 | + return fields |
69 | 115 | elif isinstance(class_item, ast.AnnAssign): |
70 | | - if ( |
71 | | - isinstance(class_item.target, ast.Name) |
72 | | - and class_item.target.id == "template_fields" |
73 | | - and isinstance(class_item.value, ast.Tuple) |
74 | | - ): |
75 | | - return [str(elt.value) for elt in class_item.value.elts if isinstance(elt, ast.Constant)] |
| 116 | + if isinstance(class_item.target, ast.Name) and class_item.target.id == "template_fields": |
| 117 | + fields = _extract_field_names(class_item.value) |
| 118 | + if fields is not None: |
| 119 | + return fields |
76 | 120 | return [] |
77 | 121 |
|
78 | 122 |
|
@@ -136,7 +180,13 @@ def _handle_constructor_statement( |
136 | 180 | if isinstance(ctor_stmt.targets[0], ast.Attribute): |
137 | 181 | for target in ctor_stmt.targets: |
138 | 182 | if isinstance(target, ast.Attribute) and target.attr in template_fields: |
139 | | - if isinstance(ctor_stmt.value, ast.BoolOp) and isinstance(ctor_stmt.value.op, ast.Or): |
| 183 | + if isinstance(ctor_stmt.value, ast.IfExp) and _is_value_preserving_ternary( |
| 184 | + ctor_stmt.value, target.attr |
| 185 | + ): |
| 186 | + _handle_assigned_field( |
| 187 | + assigned_template_fields, invalid_assignments, target, ctor_stmt.value.body |
| 188 | + ) |
| 189 | + elif isinstance(ctor_stmt.value, ast.BoolOp) and isinstance(ctor_stmt.value.op, ast.Or): |
140 | 190 | _handle_assigned_field( |
141 | 191 | assigned_template_fields, invalid_assignments, target, ctor_stmt.value.values[0] |
142 | 192 | ) |
@@ -173,6 +223,175 @@ def _handle_assigned_field( |
173 | 223 | assigned_template_fields.append(target.attr) |
174 | 224 |
|
175 | 225 |
|
| 226 | +def _target_name(target: ast.expr) -> str | None: |
| 227 | + """ |
| 228 | + Resolve an assignment target to the field name it binds. |
| 229 | +
|
| 230 | + :param target: The assignment target node. |
| 231 | + :return: The attribute name for ``self.<name>`` targets, the identifier for bare-name |
| 232 | + targets, or None for anything else. |
| 233 | + """ |
| 234 | + if isinstance(target, ast.Attribute) and isinstance(target.value, ast.Name) and target.value.id == "self": |
| 235 | + return target.attr |
| 236 | + if isinstance(target, ast.Name): |
| 237 | + return target.id |
| 238 | + return None |
| 239 | + |
| 240 | + |
| 241 | +def _is_super_init_call(node: ast.Call) -> bool: |
| 242 | + """ |
| 243 | + Check whether a call node is ``super().__init__(...)``. |
| 244 | +
|
| 245 | + :param node: The call node to check. |
| 246 | + :return: True if the node calls ``__init__`` on a ``super()`` call. |
| 247 | + """ |
| 248 | + return ( |
| 249 | + isinstance(node.func, ast.Attribute) |
| 250 | + and node.func.attr == "__init__" |
| 251 | + and isinstance(node.func.value, ast.Call) |
| 252 | + and isinstance(node.func.value.func, ast.Name) |
| 253 | + and node.func.value.func.id == "super" |
| 254 | + ) |
| 255 | + |
| 256 | + |
| 257 | +def _is_value_preserving_ternary(value: ast.IfExp, field: str) -> bool: |
| 258 | + """ |
| 259 | + Check whether a ternary keeps the field value intact when it is set. |
| 260 | +
|
| 261 | + Matches ``field if field else <default>`` and ``field if field is not None else <default>`` |
| 262 | + — both equivalent to the sanctioned ``field or <default>`` defaulting idiom. |
| 263 | +
|
| 264 | + :param value: The ternary expression node. |
| 265 | + :param field: The template field name. |
| 266 | + :return: True if the ternary only substitutes a default for an unset value. |
| 267 | + """ |
| 268 | + if not (isinstance(value.body, ast.Name) and value.body.id == field): |
| 269 | + return False |
| 270 | + test = value.test |
| 271 | + if isinstance(test, ast.Name) and test.id == field: |
| 272 | + return True |
| 273 | + return ( |
| 274 | + isinstance(test, ast.Compare) |
| 275 | + and isinstance(test.left, ast.Name) |
| 276 | + and test.left.id == field |
| 277 | + and len(test.ops) == 1 |
| 278 | + and isinstance(test.ops[0], (ast.Is, ast.IsNot)) |
| 279 | + and len(test.comparators) == 1 |
| 280 | + and isinstance(test.comparators[0], ast.Constant) |
| 281 | + and test.comparators[0].value is None |
| 282 | + ) |
| 283 | + |
| 284 | + |
| 285 | +def _collect_sanctioned_uses(ctor: ast.FunctionDef, template_fields: list[str]) -> set[int]: |
| 286 | + """ |
| 287 | + Collect the AST node ids of template-field reads that belong to sanctioned patterns. |
| 288 | +
|
| 289 | + Sanctioned patterns are the ones the project documents as safe in a constructor: |
| 290 | + ``self.field = field``, ``self.field = field or <default>``, the equivalent |
| 291 | + value-preserving ternaries, the local rebind ``field = field or <default>``, |
| 292 | + tuple assignments pairing names one-to-one, and forwarding via |
| 293 | + ``super().__init__(field=field)``. |
| 294 | +
|
| 295 | + :param ctor: The constructor function node. |
| 296 | + :param template_fields: The template fields of the class. |
| 297 | + :return: Set of ``id()``s of Name nodes participating in sanctioned patterns. |
| 298 | + """ |
| 299 | + sanctioned: set[int] = set() |
| 300 | + |
| 301 | + def mark(value: ast.expr | None, field: str) -> None: |
| 302 | + if isinstance(value, ast.BoolOp) and isinstance(value.op, ast.Or): |
| 303 | + value = value.values[0] |
| 304 | + elif isinstance(value, ast.IfExp) and _is_value_preserving_ternary(value, field): |
| 305 | + for name_node in ast.walk(value.test): |
| 306 | + if isinstance(name_node, ast.Name) and name_node.id == field: |
| 307 | + sanctioned.add(id(name_node)) |
| 308 | + value = value.body |
| 309 | + if isinstance(value, ast.Name) and value.id == field: |
| 310 | + sanctioned.add(id(value)) |
| 311 | + |
| 312 | + for node in ast.walk(ctor): |
| 313 | + if isinstance(node, (ast.Assign, ast.AnnAssign)): |
| 314 | + targets = node.targets if isinstance(node, ast.Assign) else [node.target] |
| 315 | + pairs: list[tuple[ast.expr, ast.expr | None]] |
| 316 | + if len(targets) == 1 and isinstance(targets[0], ast.Tuple) and isinstance(node.value, ast.Tuple): |
| 317 | + pairs = list(zip(targets[0].elts, node.value.elts)) |
| 318 | + else: |
| 319 | + pairs = [(target, node.value) for target in targets] |
| 320 | + for target, value in pairs: |
| 321 | + name = _target_name(target) |
| 322 | + if name is not None and name in template_fields: |
| 323 | + mark(value, name) |
| 324 | + elif isinstance(node, ast.Call) and _is_super_init_call(node): |
| 325 | + for keyword in node.keywords: |
| 326 | + if keyword.arg is not None and keyword.arg in template_fields: |
| 327 | + mark(keyword.value, keyword.arg) |
| 328 | + return sanctioned |
| 329 | + |
| 330 | + |
| 331 | +def _check_constructor_field_logic( |
| 332 | + class_node: ast.ClassDef, template_fields: list[str], source_lines: list[str] |
| 333 | +) -> int: |
| 334 | + """ |
| 335 | + Check a class's constructor for logic applied to template fields. |
| 336 | +
|
| 337 | + Template fields are rendered after the constructor runs, so any read of a template-field |
| 338 | + parameter (or ``self.<field>``) outside the sanctioned assignment/forwarding patterns — |
| 339 | + validation calls, conditionals, transformations, string interpolation — operates on the |
| 340 | + un-rendered Jinja expression and must move to ``execute()``. |
| 341 | +
|
| 342 | + :param class_node: The AST node representing the class definition. |
| 343 | + :param template_fields: The template fields of the class. |
| 344 | + :param source_lines: The source lines of the file, for reporting. |
| 345 | + :return: The number of offending source lines found. |
| 346 | + """ |
| 347 | + ctor = next( |
| 348 | + (item for item in class_node.body if isinstance(item, ast.FunctionDef) and item.name == "__init__"), |
| 349 | + None, |
| 350 | + ) |
| 351 | + if ctor is None or not template_fields: |
| 352 | + return 0 |
| 353 | + sanctioned = _collect_sanctioned_uses(ctor, template_fields) |
| 354 | + args = ctor.args |
| 355 | + # Only names bound in the constructor scope can refer to a template field; without this, |
| 356 | + # a field named e.g. "json" would false-positive on uses of the stdlib module. |
| 357 | + bound_names = {arg.arg for arg in [*args.posonlyargs, *args.args, *args.kwonlyargs]} |
| 358 | + bound_names |= { |
| 359 | + node.id for node in ast.walk(ctor) if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) |
| 360 | + } |
| 361 | + # Parameter defaults evaluate at class-definition scope, where a name that matches a |
| 362 | + # template field (e.g. a field named "conf" vs. the configuration module) is not the field. |
| 363 | + in_defaults = { |
| 364 | + id(node) |
| 365 | + for default in [*args.defaults, *args.kw_defaults] |
| 366 | + if default is not None |
| 367 | + for node in ast.walk(default) |
| 368 | + } |
| 369 | + |
| 370 | + findings: dict[int, set[str]] = {} |
| 371 | + for node in ast.walk(ctor): |
| 372 | + if id(node) in in_defaults: |
| 373 | + continue |
| 374 | + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): |
| 375 | + if node.id in template_fields and node.id in bound_names and id(node) not in sanctioned: |
| 376 | + findings.setdefault(node.lineno, set()).add(node.id) |
| 377 | + elif isinstance(node, ast.Attribute) and isinstance(node.ctx, ast.Load): |
| 378 | + if isinstance(node.value, ast.Name) and node.value.id == "self" and node.attr in template_fields: |
| 379 | + findings.setdefault(node.lineno, set()).add(f"self.{node.attr}") |
| 380 | + |
| 381 | + if findings: |
| 382 | + console.print( |
| 383 | + f"{class_node.name}'s constructor applies logic to template fields. Template fields " |
| 384 | + f"are rendered after the constructor runs, so validation or transformation here acts " |
| 385 | + f"on the un-rendered Jinja expression and should move to execute():" |
| 386 | + ) |
| 387 | + for lineno in sorted(findings): |
| 388 | + source = source_lines[lineno - 1].strip() if lineno <= len(source_lines) else "" |
| 389 | + console.print( |
| 390 | + f"[red] line {lineno}: {escape(source)} ({', '.join(sorted(findings[lineno]))})[/red]" |
| 391 | + ) |
| 392 | + return len(findings) |
| 393 | + |
| 394 | + |
176 | 395 | def _check_constructor_template_fields(class_node: ast.ClassDef, template_fields: list[str]) -> int: |
177 | 396 | """ |
178 | 397 | This method checks a class's constructor for missing or invalid assignments of template fields. |
@@ -220,22 +439,84 @@ def _check_constructor_template_fields(class_node: ast.ClassDef, template_fields |
220 | 439 | return count |
221 | 440 |
|
222 | 441 |
|
| 442 | +def _load_exemptions() -> dict[str, set[str]]: |
| 443 | + """ |
| 444 | + Load the exemption list for known violations that predate the constructor-logic check. |
| 445 | +
|
| 446 | + Each non-comment line has the form ``<repo-relative-path>::<ClassName>``. Exempted classes |
| 447 | + are skipped by all checks; an exempted class with no findings fails as stale so the entry |
| 448 | + is removed in the same PR that fixes the class, until the list is empty. |
| 449 | +
|
| 450 | + :return: Mapping of repo-relative file path to the exempted class names in that file. |
| 451 | + """ |
| 452 | + exemptions: dict[str, set[str]] = {} |
| 453 | + if not EXEMPTIONS_PATH.exists(): |
| 454 | + return exemptions |
| 455 | + for raw_line in EXEMPTIONS_PATH.read_text().splitlines(): |
| 456 | + line = raw_line.strip() |
| 457 | + if not line or line.startswith("#"): |
| 458 | + continue |
| 459 | + entry_path, sep, class_name = line.partition("::") |
| 460 | + if sep and class_name: |
| 461 | + exemptions.setdefault(entry_path, set()).add(class_name) |
| 462 | + return exemptions |
| 463 | + |
| 464 | + |
| 465 | +def _get_exempted_classes(path: str, exemptions: dict[str, set[str]]) -> set[str]: |
| 466 | + """ |
| 467 | + Find the exempted class names for a checked file. |
| 468 | +
|
| 469 | + Exemption entries store repo-relative paths while the hook may receive paths relative |
| 470 | + to another working directory, so entries are matched as path suffixes. |
| 471 | +
|
| 472 | + :param path: The file path as passed to the script. |
| 473 | + :param exemptions: The loaded exemption mapping. |
| 474 | + :return: The exempted class names for this file, or an empty set. |
| 475 | + """ |
| 476 | + resolved = Path(path).resolve().as_posix() |
| 477 | + for entry_path, classes in exemptions.items(): |
| 478 | + if resolved == entry_path or resolved.endswith(f"/{entry_path}"): |
| 479 | + return classes |
| 480 | + return set() |
| 481 | + |
| 482 | + |
223 | 483 | def main(): |
224 | 484 | """ |
225 | 485 | Check missing or invalid template fields in constructors of providers' operators. |
226 | 486 |
|
227 | 487 | :return: The total number of errors found. |
228 | 488 | """ |
229 | 489 | err = 0 |
| 490 | + exemptions = _load_exemptions() |
230 | 491 | for path in sys.argv[1:]: |
231 | 492 | console.print(f"[yellow]{path}[/yellow]") |
232 | | - tree = ast.parse(open(path).read()) |
| 493 | + source = open(path).read() |
| 494 | + source_lines = source.splitlines() |
| 495 | + tree = ast.parse(source) |
| 496 | + exempted_classes = _get_exempted_classes(path, exemptions) |
| 497 | + exempted_finding_counts: dict[str, int] = {} |
233 | 498 | for node in ast.walk(tree): |
234 | 499 | if isinstance(node, ast.ClassDef) and _is_operator(class_node=node): |
235 | 500 | template_fields = _extract_template_fields(node) or [] |
| 501 | + if node.name in exempted_classes: |
| 502 | + with console.capture(): |
| 503 | + count = _check_constructor_template_fields(node, template_fields) |
| 504 | + count += _check_constructor_field_logic(node, template_fields, source_lines) |
| 505 | + exempted_finding_counts[node.name] = count |
| 506 | + continue |
236 | 507 | err += _check_constructor_template_fields(node, template_fields) |
| 508 | + err += _check_constructor_field_logic(node, template_fields, source_lines) |
| 509 | + for class_name in sorted(exempted_classes): |
| 510 | + if not exempted_finding_counts.get(class_name): |
| 511 | + err += 1 |
| 512 | + console.print( |
| 513 | + f"[red]Stale exemption for {class_name} — the class has no findings anymore " |
| 514 | + f"(or is not detected as an operator); remove its entry from " |
| 515 | + f"{EXEMPTIONS_PATH.name}[/red]" |
| 516 | + ) |
237 | 517 | return err |
238 | 518 |
|
239 | 519 |
|
240 | 520 | if __name__ == "__main__": |
241 | | - sys.exit(main()) |
| 521 | + # A raw error count wraps at 256 (e.g. 256 findings -> exit code 0), so clamp to 0/1. |
| 522 | + sys.exit(1 if main() else 0) |
0 commit comments