Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions .lintrunner.toml
Original file line number Diff line number Diff line change
Expand Up @@ -1825,3 +1825,23 @@ command = [
'--',
'@{{PATHSFILE}}'
]

# missing_type_linter reports on regressions in typing
[[linter]]
code = 'MISSING_TYPE_LINTER'
command = [
'python3',
'tools/linter/adapters/missing_type_linter.py',
'--lintrunner',
]
include_patterns = [
'torch/**/*.py',
'torch/**/*.pyi',
]
init_command = [
'python3',
'tools/linter/adapters/pip_init.py',
'--dry-run={{DRYRUN}}',
'pyrefly==0.44.1',
]
is_formatter = false
3 changes: 2 additions & 1 deletion tools/linter/adapters/_linter/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"Block",
"FileLinter",
"is_empty",
"is_public",
"LineWithSets",
"LintResult",
"ParseError",
Expand Down Expand Up @@ -45,4 +46,4 @@ def is_empty(t: TokenInfo) -> bool:
from .block import Block
from .file_linter import FileLinter
from .messages import LintResult
from .python_file import PythonFile
from .python_file import is_public, PythonFile
23 changes: 22 additions & 1 deletion tools/linter/adapters/_linter/file_linter.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
from __future__ import annotations

import json
import shlex
import subprocess
import sys
from abc import abstractmethod
from functools import cached_property
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Any, TYPE_CHECKING
from typing_extensions import Never

from . import ParseError
Expand Down Expand Up @@ -88,6 +90,25 @@ def paths(self) -> list[Path]:
files.append(f)
return sorted(Path(f) for f in files)

def call(self, cmd: str | Sequence[str], check: bool = True, **kwargs: Any) -> str:
"""Run a subprocess and return stdout as a string"""
if self.args.verbose:
print("$", *([cmd] if isinstance(cmd, str) else cmd))

shell = kwargs.get("shell", False)
if shell and not isinstance(cmd, str):
cmd = shlex.join(cmd)
elif not shell and isinstance(cmd, str):
cmd = shlex.split(cmd)
assert shell == isinstance(cmd, str)
p = subprocess.run(cmd, text=True, capture_output=True, **kwargs)

if check:
p.check_returncode()

assert isinstance(p.stdout, str)
return p.stdout

def _lint_file(self, p: Path) -> bool:
if self.args.verbose:
print(p, "Reading", file=sys.stderr)
Expand Down
45 changes: 41 additions & 4 deletions tools/linter/adapters/_linter/python_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from typing import TYPE_CHECKING
from typing_extensions import Self

from . import is_empty, NO_TOKEN, ParseError, ROOT
from . import is_empty, NO_TOKEN, ParseError
from .sets import LineWithSets


Expand All @@ -30,13 +30,16 @@ def __init__(
) -> None:
self.linter_name = linter_name
self._contents = contents
self.path = path.relative_to(ROOT) if path and path.is_absolute() else path
self._path = path

def __repr__(self) -> str:
return f"PythonFile({self._path})"

@cached_property
def contents(self) -> str:
if self._contents is not None:
return self._contents
return self.path.read_text() if self.path else ""
return self.path.read_text() if self._path else ""

@cached_property
def lines(self) -> list[str]:
Expand All @@ -49,8 +52,13 @@ def make(cls, linter_name: str, pc: Path | str | None = None) -> Self:
else:
return cls(linter_name, contents=pc)

@cached_property
def path(self) -> Path:
assert self._path is not None
return self._path

def with_contents(self, contents: str) -> Self:
return self.__class__(self.linter_name, contents=contents, path=self.path)
return self.__class__(self.linter_name, contents=contents, path=self._path)

@cached_property
def omitted(self) -> OmittedLines:
Expand Down Expand Up @@ -158,6 +166,35 @@ def blocks(self) -> list[Block]:

return blocks(self)

@cached_property
def blocks_by_line_number(self) -> dict[int, Block]:
# Lines that don't appear are in the top-level scope
# Later blocks correctly overwrite earlier, parent blocks.
return {i: b for b in self.blocks for i in b.line_range}

def block_name(self, line: int) -> str:
block = self.blocks_by_line_number.get(line)
return block.full_name if block else ""

@cached_property
def is_public(self) -> bool:
return is_public(*self.python_parts)

@cached_property
def python_parts(self) -> tuple[str, ...]:
parts = self.path.with_suffix("").parts
return parts[:-1] if parts[-1] == "__init__" else parts


def is_public(*parts: str) -> bool:
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we fix this function before landing? Otherwise what this linter consider public is not aligned with what other tests consider public.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, direct me to the official definition and I'll use it in a trice!

# TODO: this rule is easy to understand but incomplete.
#
# What is missing is checking `__all__`: see
# https://github.com/pytorch/pytorch/wiki/Public-API-definition-and-documentation

it = (s for p in parts for s in p.split("."))
return not any(i.startswith("_") and not i.startswith("__") for i in it)


class OmittedLines:
"""Read lines textually and find comment lines that end in 'noqa {linter_name}'"""
Expand Down
26 changes: 13 additions & 13 deletions tools/linter/adapters/docstring_linter-grandfather.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,16 +48,16 @@
"def CppFlexAttentionTemplate.modification()": 102
},
"torch/_inductor/codegen/cpp_gemm_template.py": {
"def CppGemmTemplate.get_options()": 255
"def CppGemmTemplate.get_options()": 254
},
"torch/_inductor/codegen/cpp_grouped_gemm_template.py": {
"def CppGroupedGemmTemplate.render()": 157
"def CppGroupedGemmTemplate.render()": 155
},
"torch/_inductor/codegen/cpp_template.py": {
"class CppTemplate": 117
"class CppTemplate": 116
},
"torch/_inductor/codegen/cpp_template_kernel.py": {
"class CppTemplateKernel": 504
"class CppTemplateKernel": 503
},
"torch/_inductor/codegen/cpp_utils.py": {
"def create_epilogue_with_attr()": 164
Expand Down Expand Up @@ -98,13 +98,13 @@
"def SIMDScheduling.generate_node_schedule()": 95
},
"torch/_inductor/codegen/triton.py": {
"class TritonPrinter": 179,
"class TritonScheduling": 413,
"class TritonPrinter": 180,
"class TritonScheduling": 414,
"def TritonScheduling.benchmark_codegened_module()": 88,
"def TritonScheduling.benchmark_combo_kernel()": 93
"def TritonScheduling.benchmark_combo_kernel()": 94
},
"torch/_inductor/codegen/triton_combo_kernel.py": {
"class ComboKernel": 871
"class ComboKernel": 868
},
"torch/_inductor/codegen/wrapper.py": {
"def PythonWrapperCodegen.benchmark_compiled_module()": 96,
Expand Down Expand Up @@ -156,21 +156,21 @@
"def move_reshape_out_of_split_stack()": 109
},
"torch/_inductor/graph.py": {
"class GraphLowering": 2224,
"class GraphLowering": 2223,
"def GraphLowering.call_function()": 119,
"def GraphLowering.create_deferred_runtime_asserts()": 84,
"def GraphLowering.extract_autotune_inputs()": 94,
"def GraphLowering.output()": 92,
"def GraphLowering.placeholder()": 103,
"def GraphLowering.run_node()": 364
"def GraphLowering.run_node()": 363
},
"torch/_inductor/ir.py": {
"class Buffer": 134,
"class Loops": 125,
"class Reduction": 803,
"class Scan": 199,
"class Sort": 151,
"class UserDefinedTritonKernel": 204,
"class UserDefinedTritonKernel": 203,
"class View": 180,
"class WelfordReduction": 220,
"def ExternKernel.process_kernel()": 125,
Expand All @@ -194,7 +194,7 @@
"def avg_pool3d_backward()": 197,
"def cat()": 122,
"def index_put_impl_()": 117,
"def make_pointwise()": 88,
"def make_pointwise()": 87,
"def max_pool2d_with_indices_backward()": 143,
"def scatter_reduce_()": 114,
"def sdpa_constraint()": 135,
Expand All @@ -205,7 +205,7 @@
"def MkldnnRnnLayer.create()": 101
},
"torch/_inductor/mkldnn_lowerings.py": {
"def register_onednn_fusion_ops()": 1207
"def register_onednn_fusion_ops()": 1202
},
"torch/_inductor/mock_cache.py": {
"class PatchCaches": 109
Expand Down
Loading
Loading