Skip to content

Commit d7a436f

Browse files
pkchpkch
committed
Add --lf, -ff
1 parent 5fd33b9 commit d7a436f

2 files changed

Lines changed: 95 additions & 37 deletions

File tree

‎mypy/waiter.py‎

Lines changed: 68 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
This is used for running mypy tests.
44
"""
55

6-
from typing import Dict, List, Optional, Set, Tuple, Any
6+
from typing import Dict, List, Optional, Set, Tuple, Any, Iterable
77

88
import os
99
from multiprocessing import cpu_count
@@ -113,7 +113,8 @@ class Waiter:
113113
LOGSIZE = 50
114114
FULL_LOG_FILENAME = '.runtest_log.json'
115115

116-
def __init__(self, limit: int = 0, *, verbosity: int = 0, xfail: List[str] = []) -> None:
116+
def __init__(self, limit: int = 0, *, verbosity: int = 0, xfail: List[str] = [],
117+
lf: bool = False, ff: bool = False) -> None:
117118
self.verbosity = verbosity
118119
self.queue = [] # type: List[LazySubprocess]
119120
# Index of next task to run in the queue.
@@ -130,12 +131,15 @@ def __init__(self, limit: int = 0, *, verbosity: int = 0, xfail: List[str] = [])
130131
# major mistake to count *all* CPUs on the machine.
131132
limit = len(sched_getaffinity(0))
132133
self.limit = limit
134+
self.lf = lf
135+
self.ff = ff
133136
assert limit > 0
134137
self.xfail = set(xfail)
135138
self._note = None # type: Noter
136139
self.times1 = {} # type: Dict[str, float]
137140
self.times2 = {} # type: Dict[str, float]
138-
self.new_log = defaultdict(dict) # type: Dict[str, Dict[str, Any]]
141+
self.new_log = defaultdict(dict) # type: Dict[str, Dict[str, float]]
142+
self.sequential_tasks = set() # type: Set[str]
139143

140144
def load_log_file(self) -> Optional[List[Dict[str, Dict[str, Any]]]]:
141145
try:
@@ -149,9 +153,13 @@ def load_log_file(self) -> Optional[List[Dict[str, Dict[str, Any]]]]:
149153
test_log = []
150154
return test_log
151155

152-
def add(self, cmd: LazySubprocess) -> int:
156+
def add(self, cmd: LazySubprocess, sequential: bool = False) -> int:
153157
rv = len(self.queue)
158+
if cmd.name in (task.name for task in self.queue):
159+
sys.exit('Duplicate test name: {}'.format(cmd.name))
154160
self.queue.append(cmd)
161+
if sequential:
162+
self.sequential_tasks.add(cmd.name)
155163
return rv
156164

157165
def _start_next(self) -> None:
@@ -187,8 +195,8 @@ def _poll_current(self) -> Tuple[int, int]:
187195
code = cmd.process.poll()
188196
if code is not None:
189197
cmd.end_time = time.perf_counter()
190-
self.new_log[cmd.name]['exit_code'] = code
191-
self.new_log[cmd.name]['runtime'] = cmd.end_time - cmd.start_time
198+
self.new_log['exit_code'][cmd.name] = code
199+
self.new_log['runtime'][cmd.name] = cmd.end_time - cmd.start_time
192200
return pid, code
193201

194202
def _wait_next(self) -> Tuple[List[str], int, int]:
@@ -262,16 +270,43 @@ def run(self) -> int:
262270
self._note = Noter(len(self.queue))
263271
print('SUMMARY %d tasks selected' % len(self.queue))
264272

265-
if self.limit > 1:
266-
logs = self.load_log_file()
267-
if logs:
273+
def avg(lst: Iterable[float]) -> float:
274+
valid_items = [item for item in lst if item is not None]
275+
if not valid_items:
268276
# we don't know how long a new task takes
269277
# better err by putting it in front in case it is slow:
270278
# a fast task in front hurts performance less than a slow task in the back
271-
default = float('inf')
272-
times = {cmd.name: sum(log[cmd.name].get('runtime', default) for log in logs)
273-
/ len(logs) for cmd in self.queue}
274-
self.queue = sorted(self.queue, key=lambda cmd: times[cmd.name], reverse=True)
279+
return float('inf')
280+
else:
281+
return sum(valid_items) / len(valid_items)
282+
283+
logs = self.load_log_file()
284+
if logs:
285+
times = {cmd.name: avg(log['runtime'].get(cmd.name, None) for log in logs)
286+
for cmd in self.queue}
287+
288+
def sort_function(cmd: LazySubprocess) -> Tuple[Any, int, float]:
289+
# longest tasks first
290+
runtime = -times[cmd.name]
291+
# sequential tasks go first by default
292+
sequential = -(cmd.name in self.sequential_tasks)
293+
if self.ff:
294+
# failed tasks first with -ff
295+
exit_code = -logs[-1]['exit_code'].get(cmd.name, 0)
296+
if not exit_code:
297+
# avoid interrupting parallel tasks with sequential in between
298+
# => order: seq failed, parallel failed, parallel passed, seq passed
299+
# => among failed tasks, sequential should go before parallel
300+
# => among successful tasks, sequential should go after parallel
301+
sequential = -sequential
302+
else:
303+
# ignore exit code without -ff
304+
exit_code = 0
305+
return exit_code, sequential, runtime
306+
self.queue = sorted(self.queue, key=sort_function)
307+
if self.lf:
308+
self.queue = [cmd for cmd in self.queue
309+
if logs[-1]['exit_code'].get(cmd.name, 0)]
275310

276311
sys.stdout.flush()
277312
# Failed tasks.
@@ -280,15 +315,35 @@ def run(self) -> int:
280315
total_tests = 0
281316
# Number of failed test cases.
282317
total_failed_tests = 0
318+
running_sequential_task = False
283319
while self.current or self.next < len(self.queue):
284320
while len(self.current) < self.limit and self.next < len(self.queue):
321+
# only start next task if idle, or current and next tasks are both parallel
322+
if running_sequential_task:
323+
break
324+
if self.queue[self.next].name in self.sequential_tasks:
325+
if self.current:
326+
break
327+
else:
328+
running_sequential_task = True
285329
self._start_next()
286330
fails, tests, test_fails = self._wait_next()
331+
running_sequential_task = False
287332
all_failures += fails
288333
total_tests += tests
289334
total_failed_tests += test_fails
290335
if self.verbosity == 0:
291336
self._note.clear()
337+
338+
if self.new_log: # don't append empty log, it will corrupt the cache file
339+
# log only LOGSIZE most recent tests
340+
test_log = (self.load_log_file() + [self.new_log])[:self.LOGSIZE]
341+
try:
342+
with open(self.FULL_LOG_FILENAME, 'w') as fp:
343+
json.dump(test_log, fp, sort_keys=True, indent=4)
344+
except Exception as e:
345+
print('cannot save test log file:', e)
346+
292347
if all_failures:
293348
summary = 'SUMMARY %d/%d tasks and %d/%d tests failed' % (
294349
len(all_failures), len(self.queue), total_failed_tests, total_tests)
@@ -305,16 +360,6 @@ def run(self) -> int:
305360
len(self.queue), total_tests))
306361
print('*** OK ***')
307362
sys.stdout.flush()
308-
309-
if self.limit > 1:
310-
# log only LOGSIZE most recent tests
311-
test_log = (self.load_log_file() + [self.new_log])[:self.LOGSIZE]
312-
try:
313-
with open(self.FULL_LOG_FILENAME, 'w') as fp:
314-
json.dump(test_log, fp, sort_keys=True, indent=4)
315-
except Exception as e:
316-
print('cannot save test log file:', e)
317-
318363
return 0
319364

320365

‎runtests.py‎

Lines changed: 27 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ def get_versions(): # type: () -> typing.List[str]
4242

4343
class Driver:
4444

45-
def __init__(self, whitelist: List[str], blacklist: List[str],
45+
def __init__(self, *, whitelist: List[str], blacklist: List[str],
46+
lf: bool, ff: bool,
4647
arglist: List[str], pyt_arglist: List[str],
4748
verbosity: int, parallel_limit: int,
4849
xfail: List[str], coverage: bool) -> None:
@@ -51,18 +52,13 @@ def __init__(self, whitelist: List[str], blacklist: List[str],
5152
self.arglist = arglist
5253
self.pyt_arglist = pyt_arglist
5354
self.verbosity = verbosity
54-
self.waiter = Waiter(verbosity=verbosity, limit=parallel_limit, xfail=xfail)
55-
self.sequential = Waiter(verbosity=verbosity, limit=1, xfail=xfail)
55+
self.waiter = Waiter(verbosity=verbosity, limit=parallel_limit, xfail=xfail, lf=lf, ff=ff)
5656
self.versions = get_versions()
5757
self.cwd = os.getcwd()
5858
self.mypy = os.path.join(self.cwd, 'scripts', 'mypy')
5959
self.env = dict(os.environ)
6060
self.coverage = coverage
6161

62-
def run(self) -> int:
63-
exit_code = self.sequential.run() & self.waiter.run()
64-
return exit_code
65-
6662
def prepend_path(self, name: str, paths: List[str]) -> None:
6763
old_val = self.env.get(name)
6864
paths = [p for p in paths if isdir(p)]
@@ -115,7 +111,7 @@ def add_pytest(self, name: str, pytest_args: List[str], coverage: bool = False)
115111
else:
116112
args = [sys.executable, '-m', 'pytest'] + pytest_args
117113

118-
self.sequential.add(LazySubprocess(full_name, args, env=self.env))
114+
self.waiter.add(LazySubprocess(full_name, args, env=self.env), sequential=True)
119115

120116
def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None:
121117
name = 'run %s' % name
@@ -168,8 +164,7 @@ def add_flake8(self, cwd: Optional[str] = None) -> None:
168164
self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env))
169165

170166
def list_tasks(self) -> None:
171-
for id, task in enumerate(itertools.chain(self.sequential.queue,
172-
self.waiter.queue)):
167+
for id, task in enumerate(self.waiter.queue):
173168
print('{id}:{task}'.format(id=id, task=task.name))
174169

175170

@@ -184,7 +179,8 @@ def add_basic(driver: Driver) -> None:
184179

185180

186181
def add_selftypecheck(driver: Driver) -> None:
187-
driver.add_mypy_package('package mypy', 'mypy', '--config-file', 'mypy_self_check.ini')
182+
driver.add_mypy_package('package mypy nonstrict optional', 'mypy', '--config-file',
183+
'mypy_self_check.ini')
188184
driver.add_mypy_package('package mypy', 'mypy', '--config-file', 'mypy_strict_optional.ini')
189185

190186

@@ -305,7 +301,8 @@ def add_samples(driver: Driver) -> None:
305301

306302

307303
def usage(status: int) -> None:
308-
print('Usage: %s [-h | -v | -q | [-x] FILTER | -a ARG | -p ARG] ... [-- FILTER ...]'
304+
print('Usage: %s [-h | -v | -q | --lf | --ff | [-x] FILTER | -a ARG | -p ARG]'
305+
'... [-- FILTER ...]'
309306
% sys.argv[0])
310307
print()
311308
print('Run mypy tests. If given no arguments, run all tests.')
@@ -318,6 +315,8 @@ def usage(status: int) -> None:
318315
print('Options:')
319316
print(' -h, --help show this help')
320317
print(' -v, --verbose increase driver verbosity')
318+
print(' --lf rerun only the tests that failed at the last run')
319+
print(' --ff run all tests but run the last failures first')
321320
print(' -q, --quiet decrease driver verbosity')
322321
print(' -jN run N tasks at once (default: one per CPU)')
323322
print(' -a, --argument ARG pass an argument to myunit tasks')
@@ -357,6 +356,8 @@ def main() -> None:
357356
blacklist = [] # type: List[str]
358357
arglist = [] # type: List[str]
359358
pyt_arglist = [] # type: List[str]
359+
lf = False
360+
ff = False
360361
list_only = False
361362
coverage = False
362363

@@ -383,6 +384,12 @@ def main() -> None:
383384
curlist = arglist
384385
elif a == '-p' or a == '--pytest_arg':
385386
curlist = pyt_arglist
387+
# will also pass this option to pytest
388+
elif a == '--lf':
389+
lf = True
390+
# will also pass this option to pytest
391+
elif a == '--ff':
392+
ff = True
386393
elif a == '-l' or a == '--list':
387394
list_only = True
388395
elif a == '-c' or a == '--coverage':
@@ -400,11 +407,17 @@ def main() -> None:
400407
sys.exit('-a must be followed by an argument')
401408
if curlist is pyt_arglist:
402409
sys.exit('-p must be followed by an argument')
410+
if lf and ff:
411+
sys.exit('use either --lf or --ff, not both')
403412
# empty string is a substring of all names
404413
if not whitelist:
405414
whitelist.append('')
415+
if lf:
416+
pyt_arglist.append('--lf')
417+
if ff:
418+
pyt_arglist.append('--ff')
406419

407-
driver = Driver(whitelist=whitelist, blacklist=blacklist,
420+
driver = Driver(whitelist=whitelist, blacklist=blacklist, lf=lf, ff=ff,
408421
arglist=arglist, pyt_arglist=pyt_arglist, verbosity=verbosity,
409422
parallel_limit=parallel_limit, xfail=[], coverage=coverage)
410423

@@ -429,7 +442,7 @@ def main() -> None:
429442
driver.list_tasks()
430443
return
431444

432-
exit_code = driver.run()
445+
exit_code = driver.waiter.run()
433446
t1 = time.perf_counter()
434447
print('total runtime:', t1 - t0, 'sec')
435448

0 commit comments

Comments
 (0)