Skip to content

Commit b1cda7c

Browse files
john-bodleyjohn-bodley
committed
feature(pre-commit): Keeping up with the Joneses
1 parent f6e769a commit b1cda7c

File tree

448 files changed

+3084
-3305
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

448 files changed

+3084
-3305
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,28 @@
1515
# limitations under the License.
1616
#
1717
repos:
18+
- repo: https://github.com/MarcoGorelli/auto-walrus
19+
rev: v0.2.2
20+
hooks:
21+
- id: auto-walrus
22+
- repo: https://github.com/asottile/pyupgrade
23+
rev: v3.4.0
24+
hooks:
25+
- id: pyupgrade
26+
args:
27+
- --py39-plus
28+
- repo: https://github.com/hadialqattan/pycln
29+
rev: v2.1.2
30+
hooks:
31+
- id: pycln
32+
args:
33+
- --disable-all-dunder-policy
34+
- --exclude=superset/config.py
35+
- --extend-exclude=tests/integration_tests/superset_test_config.*.py
1836
- repo: https://github.com/PyCQA/isort
1937
rev: 5.12.0
2038
hooks:
2139
- id: isort
22-
- repo: https://github.com/MarcoGorelli/auto-walrus
23-
rev: v0.2.2
24-
hooks:
25-
- id: auto-walrus
2640
- repo: https://github.com/pre-commit/mirrors-mypy
2741
rev: v1.3.0
2842
hooks:

‎RELEASING/changelog.py‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@
1717
import os
1818
import re
1919
import sys
20+
from collections.abc import Iterator
2021
from dataclasses import dataclass
21-
from typing import Any, Dict, Iterator, List, Optional, Union
22+
from typing import Any, Optional, Union
2223

2324
import click
2425
from click.core import Context
@@ -67,15 +68,15 @@ class GitChangeLog:
6768
def __init__(
6869
self,
6970
version: str,
70-
logs: List[GitLog],
71+
logs: list[GitLog],
7172
access_token: Optional[str] = None,
7273
risk: Optional[bool] = False,
7374
) -> None:
7475
self._version = version
7576
self._logs = logs
76-
self._pr_logs_with_details: Dict[int, Dict[str, Any]] = {}
77-
self._github_login_cache: Dict[str, Optional[str]] = {}
78-
self._github_prs: Dict[int, Any] = {}
77+
self._pr_logs_with_details: dict[int, dict[str, Any]] = {}
78+
self._github_login_cache: dict[str, Optional[str]] = {}
79+
self._github_prs: dict[int, Any] = {}
7980
self._wait = 10
8081
github_token = access_token or os.environ.get("GITHUB_TOKEN")
8182
self._github = Github(github_token)
@@ -126,7 +127,7 @@ def _has_commit_migrations(self, git_sha: str) -> bool:
126127
"superset/migrations/versions/" in file.filename for file in commit.files
127128
)
128129

129-
def _get_pull_request_details(self, git_log: GitLog) -> Dict[str, Any]:
130+
def _get_pull_request_details(self, git_log: GitLog) -> dict[str, Any]:
130131
pr_number = git_log.pr_number
131132
if pr_number:
132133
detail = self._pr_logs_with_details.get(pr_number)
@@ -156,7 +157,7 @@ def _get_pull_request_details(self, git_log: GitLog) -> Dict[str, Any]:
156157

157158
return detail
158159

159-
def _is_risk_pull_request(self, labels: List[Any]) -> bool:
160+
def _is_risk_pull_request(self, labels: list[Any]) -> bool:
160161
for label in labels:
161162
risk_label = re.match(SUPERSET_RISKY_LABELS, label.name)
162163
if risk_label is not None:
@@ -174,8 +175,8 @@ def _get_changelog_version_head(self) -> str:
174175

175176
def _parse_change_log(
176177
self,
177-
changelog: Dict[str, str],
178-
pr_info: Dict[str, str],
178+
changelog: dict[str, str],
179+
pr_info: dict[str, str],
179180
github_login: str,
180181
) -> None:
181182
formatted_pr = (
@@ -227,7 +228,7 @@ def __repr__(self) -> str:
227228
result += f"**{key}** {changelog[key]}\n"
228229
return result
229230

230-
def __iter__(self) -> Iterator[Dict[str, Any]]:
231+
def __iter__(self) -> Iterator[dict[str, Any]]:
231232
for log in self._logs:
232233
yield {
233234
"pr_number": log.pr_number,
@@ -250,20 +251,20 @@ class GitLogs:
250251

251252
def __init__(self, git_ref: str) -> None:
252253
self._git_ref = git_ref
253-
self._logs: List[GitLog] = []
254+
self._logs: list[GitLog] = []
254255

255256
@property
256257
def git_ref(self) -> str:
257258
return self._git_ref
258259

259260
@property
260-
def logs(self) -> List[GitLog]:
261+
def logs(self) -> list[GitLog]:
261262
return self._logs
262263

263264
def fetch(self) -> None:
264265
self._logs = list(map(self._parse_log, self._git_logs()))[::-1]
265266

266-
def diff(self, git_logs: "GitLogs") -> List[GitLog]:
267+
def diff(self, git_logs: "GitLogs") -> list[GitLog]:
267268
return [log for log in git_logs.logs if log not in self._logs]
268269

269270
def __repr__(self) -> str:
@@ -284,7 +285,7 @@ def _git_checkout(self, git_ref: str) -> None:
284285
print(f"Could not checkout {git_ref}")
285286
sys.exit(1)
286287

287-
def _git_logs(self) -> List[str]:
288+
def _git_logs(self) -> list[str]:
288289
# let's get current git ref so we can revert it back
289290
current_git_ref = self._git_get_current_head()
290291
self._git_checkout(self._git_ref)

‎RELEASING/generate_email.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# See the License for the specific language governing permissions and
1616
# limitations under the License.
1717
#
18-
from typing import Any, Dict, List
18+
from typing import Any
1919

2020
from click.core import Context
2121

@@ -34,7 +34,7 @@
3434
PROJECT_DESCRIPTION = "Apache Superset is a modern, enterprise-ready business intelligence web application"
3535

3636

37-
def string_comma_to_list(message: str) -> List[str]:
37+
def string_comma_to_list(message: str) -> list[str]:
3838
if not message:
3939
return []
4040
return [element.strip() for element in message.split(",")]
@@ -52,15 +52,15 @@ def render_template(template_file: str, **kwargs: Any) -> str:
5252
return template.render(kwargs)
5353

5454

55-
class BaseParameters(object):
55+
class BaseParameters:
5656
def __init__(
5757
self,
5858
version: str,
5959
version_rc: str,
6060
) -> None:
6161
self.version = version
6262
self.version_rc = version_rc
63-
self.template_arguments: Dict[str, Any] = {}
63+
self.template_arguments: dict[str, Any] = {}
6464

6565
def __repr__(self) -> str:
6666
return f"Apache Credentials: {self.version}/{self.version_rc}"

‎docker/pythonpath_dev/superset_config.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
#
2323
import logging
2424
import os
25-
from datetime import timedelta
2625
from typing import Optional
2726

2827
from cachelib.file import FileSystemCache
@@ -42,7 +41,7 @@ def get_env_variable(var_name: str, default: Optional[str] = None) -> str:
4241
error_msg = "The environment variable {} was missing, abort...".format(
4342
var_name
4443
)
45-
raise EnvironmentError(error_msg)
44+
raise OSError(error_msg)
4645

4746

4847
DATABASE_DIALECT = get_env_variable("DATABASE_DIALECT")
@@ -53,7 +52,7 @@ def get_env_variable(var_name: str, default: Optional[str] = None) -> str:
5352
DATABASE_DB = get_env_variable("DATABASE_DB")
5453

5554
# The SQLAlchemy connection string.
56-
SQLALCHEMY_DATABASE_URI = "%s://%s:%s@%s:%s/%s" % (
55+
SQLALCHEMY_DATABASE_URI = "{}://{}:{}@{}:{}/{}".format(
5756
DATABASE_DIALECT,
5857
DATABASE_USER,
5958
DATABASE_PASSWORD,
@@ -80,7 +79,7 @@ def get_env_variable(var_name: str, default: Optional[str] = None) -> str:
8079
DATA_CACHE_CONFIG = CACHE_CONFIG
8180

8281

83-
class CeleryConfig(object):
82+
class CeleryConfig:
8483
broker_url = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
8584
imports = ("superset.sql_lab",)
8685
result_backend = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}"

‎scripts/benchmark_migration.py‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
from inspect import getsource
2424
from pathlib import Path
2525
from types import ModuleType
26-
from typing import Any, Dict, List, Set, Type
26+
from typing import Any
2727

2828
import click
2929
from flask import current_app
@@ -48,12 +48,10 @@ def import_migration_script(filepath: Path) -> ModuleType:
4848
module = importlib.util.module_from_spec(spec)
4949
spec.loader.exec_module(module) # type: ignore
5050
return module
51-
raise Exception(
52-
"No module spec found in location: `{path}`".format(path=str(filepath))
53-
)
51+
raise Exception(f"No module spec found in location: `{str(filepath)}`")
5452

5553

56-
def extract_modified_tables(module: ModuleType) -> Set[str]:
54+
def extract_modified_tables(module: ModuleType) -> set[str]:
5755
"""
5856
Extract the tables being modified by a migration script.
5957
@@ -62,7 +60,7 @@ def extract_modified_tables(module: ModuleType) -> Set[str]:
6260
actually traversing the AST.
6361
"""
6462

65-
tables: Set[str] = set()
63+
tables: set[str] = set()
6664
for function in {"upgrade", "downgrade"}:
6765
source = getsource(getattr(module, function))
6866
tables.update(re.findall(r'alter_table\(\s*"(\w+?)"\s*\)', source, re.DOTALL))
@@ -72,11 +70,11 @@ def extract_modified_tables(module: ModuleType) -> Set[str]:
7270
return tables
7371

7472

75-
def find_models(module: ModuleType) -> List[Type[Model]]:
73+
def find_models(module: ModuleType) -> list[type[Model]]:
7674
"""
7775
Find all models in a migration script.
7876
"""
79-
models: List[Type[Model]] = []
77+
models: list[type[Model]] = []
8078
tables = extract_modified_tables(module)
8179

8280
# add models defined explicitly in the migration script
@@ -123,7 +121,7 @@ def find_models(module: ModuleType) -> List[Type[Model]]:
123121
sorter: TopologicalSorter[Any] = TopologicalSorter()
124122
for model in models:
125123
inspector = inspect(model)
126-
dependent_tables: List[str] = []
124+
dependent_tables: list[str] = []
127125
for column in inspector.columns.values():
128126
for foreign_key in column.foreign_keys:
129127
if foreign_key.column.table.name != model.__tablename__:
@@ -174,30 +172,30 @@ def main(
174172

175173
print("\nIdentifying models used in the migration:")
176174
models = find_models(module)
177-
model_rows: Dict[Type[Model], int] = {}
175+
model_rows: dict[type[Model], int] = {}
178176
for model in models:
179177
rows = session.query(model).count()
180178
print(f"- {model.__name__} ({rows} rows in table {model.__tablename__})")
181179
model_rows[model] = rows
182180
session.close()
183181

184182
print("Benchmarking migration")
185-
results: Dict[str, float] = {}
183+
results: dict[str, float] = {}
186184
start = time.time()
187185
upgrade(revision=revision)
188186
duration = time.time() - start
189187
results["Current"] = duration
190188
print(f"Migration on current DB took: {duration:.2f} seconds")
191189

192190
min_entities = 10
193-
new_models: Dict[Type[Model], List[Model]] = defaultdict(list)
191+
new_models: dict[type[Model], list[Model]] = defaultdict(list)
194192
while min_entities <= limit:
195193
downgrade(revision=down_revision)
196194
print(f"Running with at least {min_entities} entities of each model")
197195
for model in models:
198196
missing = min_entities - model_rows[model]
199197
if missing > 0:
200-
entities: List[Model] = []
198+
entities: list[Model] = []
201199
print(f"- Adding {missing} entities to the {model.__name__} model")
202200
bar = ChargingBar("Processing", max=missing)
203201
try:

‎scripts/cancel_github_workflows.py‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -33,21 +33,21 @@
3333
./cancel_github_workflows.py 1024 --include-last
3434
"""
3535
import os
36-
from typing import Any, Dict, Iterable, Iterator, List, Optional, Union
36+
from collections.abc import Iterable, Iterator
37+
from typing import Any, Literal, Optional, Union
3738

3839
import click
3940
import requests
4041
from click.exceptions import ClickException
4142
from dateutil import parser
42-
from typing_extensions import Literal
4343

4444
github_token = os.environ.get("GITHUB_TOKEN")
4545
github_repo = os.environ.get("GITHUB_REPOSITORY", "apache/superset")
4646

4747

4848
def request(
4949
method: Literal["GET", "POST", "DELETE", "PUT"], endpoint: str, **kwargs: Any
50-
) -> Dict[str, Any]:
50+
) -> dict[str, Any]:
5151
resp = requests.request(
5252
method,
5353
f"https://api.github.com/{endpoint.lstrip('/')}",
@@ -61,8 +61,8 @@ def request(
6161

6262
def list_runs(
6363
repo: str,
64-
params: Optional[Dict[str, str]] = None,
65-
) -> Iterator[Dict[str, Any]]:
64+
params: Optional[dict[str, str]] = None,
65+
) -> Iterator[dict[str, Any]]:
6666
"""List all github workflow runs.
6767
Returns:
6868
An iterator that will iterate through all pages of matching runs."""
@@ -77,16 +77,15 @@ def list_runs(
7777
params={**params, "per_page": 100, "page": page},
7878
)
7979
total_count = result["total_count"]
80-
for item in result["workflow_runs"]:
81-
yield item
80+
yield from result["workflow_runs"]
8281
page += 1
8382

8483

85-
def cancel_run(repo: str, run_id: Union[str, int]) -> Dict[str, Any]:
84+
def cancel_run(repo: str, run_id: Union[str, int]) -> dict[str, Any]:
8685
return request("POST", f"/repos/{repo}/actions/runs/{run_id}/cancel")
8786

8887

89-
def get_pull_request(repo: str, pull_number: Union[str, int]) -> Dict[str, Any]:
88+
def get_pull_request(repo: str, pull_number: Union[str, int]) -> dict[str, Any]:
9089
return request("GET", f"/repos/{repo}/pulls/{pull_number}")
9190

9291

@@ -96,7 +95,7 @@ def get_runs(
9695
user: Optional[str] = None,
9796
statuses: Iterable[str] = ("queued", "in_progress"),
9897
events: Iterable[str] = ("pull_request", "push"),
99-
) -> List[Dict[str, Any]]:
98+
) -> list[dict[str, Any]]:
10099
"""Get workflow runs associated with the given branch"""
101100
return [
102101
item
@@ -108,7 +107,7 @@ def get_runs(
108107
]
109108

110109

111-
def print_commit(commit: Dict[str, Any], branch: str) -> None:
110+
def print_commit(commit: dict[str, Any], branch: str) -> None:
112111
"""Print out commit message for verification"""
113112
indented_message = " \n".join(commit["message"].split("\n"))
114113
date_str = (
@@ -155,7 +154,7 @@ def print_commit(commit: Dict[str, Any], branch: str) -> None:
155154
def cancel_github_workflows(
156155
branch_or_pull: Optional[str],
157156
repo: str,
158-
event: List[str],
157+
event: list[str],
159158
include_last: bool,
160159
include_running: bool,
161160
) -> None:

0 commit comments

Comments
 (0)