Skip to content

Commit 9461f69

Browse files
bobholtbobholt
authored
Merge 560facf into 6501418 
2 parents 6501418 + 560facf commit 9461f69

File tree

1 file changed

+62
-22
lines changed

1 file changed

+62
-22
lines changed

‎check_stability.py‎

Lines changed: 62 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
import zipfile
1212
from abc import ABCMeta, abstractmethod
1313
from cStringIO import StringIO as CStringIO
14-
from collections import defaultdict
14+
from collections import defaultdict, OrderedDict
1515
from ConfigParser import RawConfigParser
1616
from io import BytesIO, StringIO
1717

@@ -487,13 +487,44 @@ class LogHandler(reader.LogHandler):
487487
Subclasses reader.LogHandler.
488488
"""
489489
def __init__(self):
490-
self.results = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))
490+
self.results = OrderedDict()
491+
492+
def find_or_create_test(self, data):
493+
test_name = data["test"]
494+
if self.results.get(test_name):
495+
return self.results[test_name]
496+
497+
test = {
498+
"subtests": OrderedDict(),
499+
"status": defaultdict(int)
500+
}
501+
self.results[test_name] = test
502+
return test
503+
504+
def find_or_create_subtest(self, data):
505+
test = self.find_or_create_test(data)
506+
subtest_name = data["subtest"]
507+
508+
if test["subtests"].get(subtest_name):
509+
return test["subtests"][subtest_name]
510+
511+
subtest = {
512+
"status": defaultdict(int),
513+
"messages": set()
514+
}
515+
test["subtests"][subtest_name] = subtest
516+
517+
return subtest
491518

492519
def test_status(self, data):
493-
self.results[data["test"]][data.get("subtest")][data["status"]] += 1
520+
subtest = self.find_or_create_subtest(data)
521+
subtest["status"][data["status"]] += 1
522+
if data.get("message"):
523+
subtest["messages"].add(data["message"])
494524

495525
def test_end(self, data):
496-
self.results[data["test"]][None][data["status"]] += 1
526+
test = self.find_or_create_test(data)
527+
test["status"][data["status"]] += 1
497528

498529

499530
def is_inconsistent(results_dict, iterations):
@@ -511,7 +542,7 @@ def err_string(results_dict, iterations):
511542
rv = ", ".join(rv)
512543
if total_results < iterations:
513544
rv.append("MISSING: %s/%s" % (iterations - total_results, iterations))
514-
if len(results_dict) > 1 or total_results != iterations:
545+
if is_inconsistent(results_dict, iterations):
515546
rv = "**%s**" % rv
516547
return rv
517548

@@ -522,10 +553,12 @@ def process_results(log, iterations):
522553
handler = LogHandler()
523554
reader.handle_log(reader.read(log), handler)
524555
results = handler.results
525-
for test, test_results in results.iteritems():
526-
for subtest, result in test_results.iteritems():
527-
if is_inconsistent(result, iterations):
528-
inconsistent.append((test, subtest, result))
556+
for test_name, test in results.iteritems():
557+
if is_inconsistent(test["status"], iterations):
558+
inconsistent.append((test_name, None, test["status"], None))
559+
for subtest_name, subtest in test["subtests"].iteritems():
560+
if is_inconsistent(subtest["status"], iterations):
561+
inconsistent.append((test_name, subtest_name, subtest["status"], subtest["messages"]))
529562
return results, inconsistent
530563

531564

@@ -572,9 +605,14 @@ def table(headings, data, log):
572605
def write_inconsistent(inconsistent, iterations):
573606
"""Output inconsistent tests to logger.error."""
574607
logger.error("## Unstable results ##\n")
575-
strings = [("`%s`" % markdown_adjust(test), ("`%s`" % markdown_adjust(subtest)) if subtest else "", err_string(results, iterations))
576-
for test, subtest, results in inconsistent]
577-
table(["Test", "Subtest", "Results"], strings, logger.error)
608+
strings = [(
609+
"`%s`" % markdown_adjust(test),
610+
("`%s`" % markdown_adjust(subtest)) if subtest else "",
611+
err_string(results, iterations),
612+
("`%s`" % markdown_adjust(";".join(messages))) if len(messages) else ""
613+
)
614+
for test, subtest, results, messages in inconsistent]
615+
table(["Test", "Subtest", "Results", "Messages"], strings, logger.error)
578616

579617

580618
def write_results(results, iterations, comment_pr):
@@ -592,22 +630,24 @@ def write_results(results, iterations, comment_pr):
592630
"tests" if len(results) > 1
593631
else "test"))
594632

595-
for test, test_results in results.iteritems():
633+
for test_name, test in results.iteritems():
596634
baseurl = "http://w3c-test.org/submissions"
597-
if "https" in os.path.splitext(test)[0].split(".")[1:]:
635+
if "https" in os.path.splitext(test_name)[0].split(".")[1:]:
598636
baseurl = "https://w3c-test.org/submissions"
599637
if pr_number:
600638
logger.info("<details>\n")
601639
logger.info('<summary><a href="%s/%s%s">%s</a></summary>\n\n' %
602-
(baseurl, pr_number, test, test))
640+
(baseurl, pr_number, test_name, test_name))
603641
else:
604-
logger.info("### %s ###" % test)
605-
parent = test_results.pop(None)
606-
strings = [("", err_string(parent, iterations))]
607-
strings.extend(((("`%s`" % markdown_adjust(subtest)) if subtest
608-
else "", err_string(results, iterations))
609-
for subtest, results in test_results.iteritems()))
610-
table(["Subtest", "Results"], strings, logger.info)
642+
logger.info("### %s ###" % test_name)
643+
strings = [("", err_string(test["status"], iterations), "")]
644+
645+
strings.extend(((
646+
("`%s`" % markdown_adjust(subtest_name)) if subtest else "",
647+
err_string(subtest["status"], iterations),
648+
("`%s`" % markdown_adjust(';'.join(subtest["messages"]))) if len(subtest["messages"]) else ""
649+
) for subtest_name, subtest in test["subtests"].items()))
650+
table(["Subtest", "Results", "Messages"], strings, logger.info)
611651
if pr_number:
612652
logger.info("</details>\n")
613653

0 commit comments

Comments
 (0)