diff --git a/.automation_scripts/pytorch-unit-test-scripts/detect_log_failures.py b/.automation_scripts/pytorch-unit-test-scripts/detect_log_failures.py index b563fcf74bc1e..2b456f5954676 100755 --- a/.automation_scripts/pytorch-unit-test-scripts/detect_log_failures.py +++ b/.automation_scripts/pytorch-unit-test-scripts/detect_log_failures.py @@ -39,10 +39,19 @@ RE_INDIVIDUAL_TEST = re.compile( r"(?P\S+\.py::(?P\w+)::(?P\w+))" ) -RE_INDIV_PASSED = re.compile( - r"(?:test/)?(?P\S+\.py)::(?P\w+)::(?P\S+?)\s+PASSED" +# PyTorch's run_test.py prints an authoritative summary at the end of a shard +# listing the exact tests that failed and then passed on rerun-in-new-process, +# e.g. +# The following tests failed and then succeeded when run in a new process +# ['test/inductor/test_compiled_autograd.py::Cls::test_jacfwd', ...] +# We parse this directly instead of guessing from nearby PASSED lines, which +# misattributes flakiness to whatever test happened to pass most recently. +RE_FLAKY_SUMMARY = re.compile( + r"The following tests failed and then succeeded when run in a new process\s*\[(?P[^\]]*)\]" +) +RE_FLAKY_ENTRY = re.compile( + r"['\"](?:test/)?(?P\S+?\.py)::(?P\w+)::(?P[^'\"]+?)['\"]" ) -RE_NEW_PROCESS_SUCCESS = re.compile(r"Test succeeded in new process") CRASH_PATTERNS = [ (re.compile(r"Segmentation fault", re.IGNORECASE), "SEGFAULT"), @@ -69,13 +78,23 @@ def classify_log_file(filename): - """Return (platform, test_config, shard_num) from a log filename like rocm3.txt.""" + """Return (platform, test_config, shard_num) from a log filename like rocm3.txt. + + Commit-vs-commit parity prefixes log files with the short commit SHA + (for example, 09e0c59b_rocm3.txt). In that mode the SHA label is the + platform name used by generate_summary.py, so preserve it here. + """ stem = Path(filename).stem + label = None + m = re.match(r"(?P