diff --git a/examples/different/submissions/slow_accepted/different_slow.py b/examples/different/submissions/slow_accepted/different_slow.py deleted file mode 100644 index 75b93111..00000000 --- a/examples/different/submissions/slow_accepted/different_slow.py +++ /dev/null @@ -1,15 +0,0 @@ -#! /usr/bin/python3 - -import sys - -for line in sys.stdin: - ab = line.split() - a = int(ab[0]) - b = int(ab[1]) - # needless loop just to be slow - x = a - for _ in range(100000000): - x += 1 - diff = abs(a-b) + x - x - - print(diff) diff --git a/problemtools/checks/__init__.py b/problemtools/checks/__init__.py index abbb1c2e..2863d541 100644 --- a/problemtools/checks/__init__.py +++ b/problemtools/checks/__init__.py @@ -1,5 +1,7 @@ from .includes import check_includes +from .submissions import check_submissions __all__ = [ 'check_includes', + 'check_submissions', ] diff --git a/problemtools/checks/submissions.py b/problemtools/checks/submissions.py new file mode 100644 index 00000000..05ed25e6 --- /dev/null +++ b/problemtools/checks/submissions.py @@ -0,0 +1,346 @@ +"""Checks for a problem package's submissions.""" + +from __future__ import annotations + +import math +import os +from collections.abc import Callable +from pathlib import Path +from typing import TYPE_CHECKING + +from ..context import Context +from ..diagnostics import Diagnostics +from ..judge import SubmissionJudge, SubmissionResult +from ..metadata import Metadata +from ..model import LegacyPolicy, Submission, Submissions +from ..run import Program + +if TYPE_CHECKING: + from ..verifyproblem import TestCaseGroup + +# Temporary consts to keep code structure as similar as possible to old code from +# verifyproblem when extracting this to a separate module. +_DIRECTORIES: list[str] = ['accepted', 'partially_accepted', 'wrong_answer', 'run_time_error', 'time_limit_exceeded'] +_DISPLAY_LABEL_BY_DIRECTORY: dict[str, str] = { + 'accepted': 'AC', + 'partially_accepted': 'PAC', + 'wrong_answer': 'WA', + 'run_time_error': 'RTE', + 'time_limit_exceeded': 'TLE', +} + + +def check_submissions( + submissions: Submissions, + metadata: Metadata, + testdata: TestCaseGroup, + output_validator: Program, + custom_grader: Program | None, + tmpdir: str, + context: Context, + set_timelim: Callable[[float], None], + diag: Diagnostics, +) -> None: + """Run all checks on a problem's submissions.""" + _check_has_accepted_submission(submissions, diag) + + policy = submissions.policy + known_submissions = _check_matches_policy(submissions, policy, diag) + + limits = metadata.limits + ac_to_time_limit = limits.time_multipliers.ac_to_time_limit + + fixed_limit: float | None = context.fixed_timelim if context.fixed_timelim is not None else limits.time_limit + lower_bound_runtime: float | None = None # The runtime of the slowest submission used to lower bound the time limit. + + if limits.time_limit is not None and context.fixed_timelim is not None: + diag.warning('There is a fixed time limit in problem.yaml, and you provided one on command line. Using command line.') + + has_testcases = any(tc.matches_filter(context.data_filter) for tc in testdata.get_all_testcases()) + if not has_testcases: + diag.warning('Found no test cases to run on. Did you filter them all out?') + + all_submission_results: list[tuple[Submission, list[SubmissionResult]]] = [] + + for directory in _DIRECTORIES: + label = _DISPLAY_LABEL_BY_DIRECTORY[directory] + runtimes = [] + + for sub in known_submissions: + if sub.directory != directory: + continue + if not context.submission_filter.search(str(sub.path)): + continue + + diag.info(f'Check {label} submission {sub.program}') + + if sub.program.code_size() > 1024 * limits.code: + diag.error( + f'{label} submission {sub.program} has size {sub.program.code_size() / 1024.0:.1f} kiB, ' + f'exceeds code size limit of {limits.code} kiB' + ) + continue + + success, msg = sub.program.compile() + if not success: + diag.error(f'Compile error for {label} submission {sub.program}', additional_info=msg) + continue + + if has_testcases: + timelim, timelim_high = _compute_time_limit(metadata, fixed_limit, lower_bound_runtime) + sub_results = _check_submission( + sub, policy, context, metadata, testdata, output_validator, custom_grader, tmpdir, timelim, timelim_high, diag + ) + runtimes.append(sub_results[-1].runtime) + all_submission_results.append((sub, sub_results)) + + if directory == 'accepted' and has_testcases: + if len(runtimes) > 0: + lower_bound_runtime = max(runtimes) + + if fixed_limit is not None and lower_bound_runtime is not None: + tl_from_subs, _ = _compute_time_limit(metadata, None, lower_bound_runtime) + if lower_bound_runtime * ac_to_time_limit > fixed_limit: + msg = ( + f'Fixed time limit ({_fmt_number(fixed_limit)}) is tighter than the auto-computed limit ' + f'({_fmt_number(tl_from_subs)}) — slowest AC: {_fmt_number(lower_bound_runtime)} x ' + f'multiplier {_fmt_number(ac_to_time_limit)}' + ) + if context.fixed_timelim is not None: # We just warn when the fixed time limit comes from command line + diag.warning(msg) + else: + diag.error(msg) # ... but if it came from problem.yaml, it's an error if bounds aren't kept + + if not math.isclose(fixed_limit, tl_from_subs): + print( + f' Solutions give timelim of {_fmt_number(tl_from_subs)} seconds, but will use provided ' + f'fixed limit of {_fmt_number(fixed_limit)} seconds instead' + ) + + timelim, timelim_margin = _compute_time_limit(metadata, fixed_limit, lower_bound_runtime) + print( + f' Slowest AC runtime: {_fmt_number(lower_bound_runtime)}, setting timelim to {_fmt_number(timelim)} secs, ' + f'safety margin to {_fmt_number(timelim_margin)} secs' + ) + set_timelim(timelim) + + if all_submission_results: + _print_results_table(all_submission_results, testdata, metadata.is_scoring()) + + +def _check_has_accepted_submission(submissions: Submissions, diag: Diagnostics) -> None: + if not any(sub.directory == 'accepted' for sub in submissions.submissions): + diag.error('Require at least one "accepted" submission') + + +def _check_matches_policy(submissions: Submissions, policy: LegacyPolicy, diag: Diagnostics) -> list[Submission]: + """Emit an error for, and exclude, any submission that doesn't match the policy at all + (i.e. sits in an unrecognized directory). Such a submission is never compiled or tested.""" + matched = [] + for sub in submissions.submissions: + if policy.matches(sub): + matched.append(sub) + else: + diag.error(f'Submission {sub.path} does not match any known submissions directory; ignoring it') + return matched + + +def _check_submission( + sub: Submission, + policy: LegacyPolicy, + context: Context, + metadata: Metadata, + testdata: TestCaseGroup, + output_validator: Program, + custom_grader: Program | None, + tmpdir: str, + timelim: float, + timelim_high: float, + diag: Diagnostics, +) -> list[SubmissionResult]: + expected_verdict = policy.expected_verdict(sub) + assert expected_verdict is not None, '_check_submission called on a submission not matching the policy' + partial = sub.directory == 'partially_accepted' + desc = f'{_DISPLAY_LABEL_BY_DIRECTORY[sub.directory]} submission {sub.program}' + + judge = SubmissionJudge( + sub=sub.program, + output_validator=output_validator, + metadata=metadata, + root=testdata, + base_dir=Path(tmpdir), + context=context, + diag=diag, + custom_grader=custom_grader, + ) + if context.executor is not None: + judge.precompute(timelim_high) + results_high = judge.judge(timelim_high) + if not results_high: + diag.fatal('_check_submission called, but found no test cases to run on.') + result_high = results_high[-1] + + results = judge.judge(timelim) + result = results[-1] + + # Check if scores were outside of the range for any groups + if metadata.is_scoring(): + for r in results: + if r.score is not None and r.test_node is not None and r.test_node.is_group: + r.test_node.check_score_in_bounds(sub.program, r.score) + + # Warn if AC (but not PAC) submissions fail on samples. It's not uncommon for sample cases to be + # ignored, so failing on them could be silent otherwise. Skip warning if the result isn't AC - + # then something worse has gone wrong, and we'll error later. + if expected_verdict == 'AC' and not partial and result.verdict == 'AC': + if sample_failure := _find_sample_failure(results): + diag.warning(f'{desc} got {sample_failure.verdict} on sample: {sample_failure}') + + # Warn if a PAC submission would affect time limit, had it been use to compute the time limit. Only do this + # if it gets AC on the computed time limit, otherwise we have other warnings below. + if partial and result.verdict == 'AC': + _warn_pac_too_slow(judge, results, timelim, desc, metadata, diag) + + if result.verdict != result_high.verdict or result.score != result_high.score: + diag.warning( + f'{desc} sensitive to time limit: limit of {timelim} secs -> {result}, limit of {timelim_high} secs -> {result_high}' + ) + + if partial and _fully_accepted(result, testdata, metadata): + diag.warning(f'{desc} was fully accepted: {result}') + elif result.verdict == expected_verdict: + print(f' {desc} OK: {result}') + if ( + not partial + and expected_verdict == 'AC' + and not _fully_accepted(result, testdata, metadata) + and _full_score_finite(testdata, metadata) + ): + # For some heuristic problems, this is expected. Thus, only warn. + diag.warning(f'{desc} did not attain full score (consider moving it to partially_accepted)') + elif result_high.verdict == expected_verdict and not (partial and _fully_accepted(result_high, testdata, metadata)): + print(f' {desc} OK with extra time: {result_high}') + else: + diag.error(f'{desc} got {result}', result_high.additional_info) + + return results + + +def _find_sample_failure(results: list[SubmissionResult]) -> SubmissionResult | None: + for r in results: + if r.verdict != 'AC' and r.test_node is not None and not r.test_node.is_group and r.test_node.is_in_sample_group(): + return r + return None + + +def _warn_pac_too_slow( + judge: SubmissionJudge, results: list[SubmissionResult], timelim: float, desc: str, metadata: Metadata, diag: Diagnostics +) -> None: + """Warn if a PAC submission is slow enough that it would have affected the time limit.""" + runtime_without_affecting_tl = timelim / metadata.limits.time_multipliers.ac_to_time_limit + if judge.judge(runtime_without_affecting_tl)[-1].verdict == 'AC': + return + for t in sorted(r.runtime for r in results if r.runtime > runtime_without_affecting_tl): + if judge.judge(t)[-1].verdict == 'AC': + diag.warning(f'{desc} is slower than all AC submissions. It needs {t:.2f}s to get AC') + return + + +def _get_table_groups(testdata: TestCaseGroup) -> list[TestCaseGroup]: + """Return the groups to show as columns: expand any root child that has subgroups.""" + result = [] + for group in testdata.get_subgroups(): + subgroups = group.get_subgroups() + if subgroups: + result.extend(subgroups) + else: + result.append(group) + return result + + +def _print_results_table( + all_submission_results: list[tuple[Submission, list[SubmissionResult]]], testdata: TestCaseGroup, is_scoring: bool +) -> None: + groups = _get_table_groups(testdata) + + def cell_for_group(results: list[SubmissionResult], group: TestCaseGroup) -> str: + for r in results: + if r.test_node is group: + if r.verdict == 'AC': + if is_scoring and r.score is not None: + score_str = f'{int(r.score)}' if r.score == int(r.score) else f'{r.score:.2f}' + score_part = f'({score_str})' + else: + score_part = '' + return f'AC{score_part}:{r.runtime:.2f}s' + return r.verdict + return '-' + + def cell_for_pts(results: list[SubmissionResult]) -> str: + score = results[-1].score + return f'{score:.0f}' if score is not None else '-' + + def cell_for_time(results: list[SubmissionResult]) -> str: + t = results[-1].runtime + return f'{t:.2f}s' if t >= 0 else '-' + + headers = ['Submission'] + [os.path.basename(g._datadir) for g in groups] + if is_scoring: + headers.append('Pts') + headers.append('Time') + + rows = [] + for sub, results in all_submission_results: + row = [sub.program.name] + for g in groups: + row.append(cell_for_group(results, g)) + if is_scoring: + row.append(cell_for_pts(results)) + row.append(cell_for_time(results)) + rows.append(row) + + widths = [len(h) for h in headers] + for row in rows: + for i, cell in enumerate(row): + widths[i] = max(widths[i], len(cell)) + + print('Submission results:') + indent = ' ' + print(indent + ' '.join(h.ljust(widths[i]) for i, h in enumerate(headers))) + for row in rows: + print(indent + ' '.join(cell.ljust(widths[i]) for i, cell in enumerate(row))) + + +def _compute_time_limit(metadata: Metadata, fixed_limit: float | None, lower_bound_runtime: float | None) -> tuple[float, float]: + if fixed_limit is None and lower_bound_runtime is None: + # 5 minutes is our currently hard coded upper bound for what to allow when we don't know the time limit yet + return 300.0, 300.0 + + limits = metadata.limits + if fixed_limit is not None: + timelim = fixed_limit + else: + assert lower_bound_runtime is not None, 'Assert to keep mypy happy' + exact_timelim = lower_bound_runtime * limits.time_multipliers.ac_to_time_limit + timelim = max(1, math.ceil(exact_timelim / limits.time_resolution)) * limits.time_resolution + + return timelim, timelim * limits.time_multipliers.time_limit_to_tle + + +def _full_score_finite(testdata: TestCaseGroup, metadata: Metadata) -> bool: + min_score, max_score = testdata.get_score_range() + if metadata.legacy_grading.objective == 'min': + return min_score != float('-inf') + else: + return max_score != float('inf') + + +def _fully_accepted(result: SubmissionResult, testdata: TestCaseGroup, metadata: Metadata) -> bool: + min_score, max_score = testdata.get_score_range() + best_score = min_score if metadata.legacy_grading.objective == 'min' else max_score + return result.verdict == 'AC' and (not metadata.is_scoring() or result.score == best_score) + + +def _fmt_number(number: float | None) -> str: + """Format a number with at most 3 decimals, dealing with None.""" + return f'{round(number, 3):g}' if number is not None else '-' diff --git a/problemtools/judge/__init__.py b/problemtools/judge/__init__.py index e338bc62..11421e86 100644 --- a/problemtools/judge/__init__.py +++ b/problemtools/judge/__init__.py @@ -1,9 +1,7 @@ +from ..model import Verdict from .cache import CacheKey from .execute import execute_testcase -from .result import ( - SubmissionResult, - Verdict, -) +from .result import SubmissionResult from .submission_judge import SubmissionJudge from .validate import validate_output diff --git a/problemtools/judge/grade.py b/problemtools/judge/grade.py index 6afe5ed8..ac8795b6 100644 --- a/problemtools/judge/grade.py +++ b/problemtools/judge/grade.py @@ -7,8 +7,9 @@ from typing import cast from ..diagnostics import Diagnostics +from ..model import Verdict from ..run import Program -from .result import SubmissionResult, Verdict +from .result import SubmissionResult _GRADER_OUTPUT_RE = re.compile(r'^((AC)|(WA)|(TLE)|(RTE)|(MLE)|(OLE)|(JE))\s+-?[0-9.]+\s*$') diff --git a/problemtools/judge/result.py b/problemtools/judge/result.py index 3aacc0a4..9024f77e 100644 --- a/problemtools/judge/result.py +++ b/problemtools/judge/result.py @@ -1,12 +1,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING if TYPE_CHECKING: from ..verifyproblem import TestCase, TestCaseGroup -Verdict = Literal['AC', 'TLE', 'OLE', 'MLE', 'RTE', 'WA', 'PAC', 'JE'] - class SubmissionResult: def __init__( diff --git a/problemtools/model/__init__.py b/problemtools/model/__init__.py index 871bee6c..52c04bcd 100644 --- a/problemtools/model/__init__.py +++ b/problemtools/model/__init__.py @@ -1,9 +1,20 @@ +from typing import Literal + +#: A submission's (or testcase's) verdict, e.g. as expected by policy or produced by judging. +Verdict = Literal['AC', 'TLE', 'OLE', 'MLE', 'RTE', 'WA', 'PAC', 'JE'] + from .includes import DEFAULT_LANGUAGE, IncludeFile, Includes, LanguageIncludes, load_includes +from .submissions import LegacyPolicy, Submission, Submissions, load_submissions __all__ = [ 'DEFAULT_LANGUAGE', 'IncludeFile', 'Includes', 'LanguageIncludes', + 'LegacyPolicy', + 'Submission', + 'Submissions', + 'Verdict', 'load_includes', + 'load_submissions', ] diff --git a/problemtools/model/submissions.py b/problemtools/model/submissions.py new file mode 100644 index 00000000..690f4604 --- /dev/null +++ b/problemtools/model/submissions.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from ..languages import Languages +from ..run import Program, find_programs +from . import Verdict +from .includes import Includes + + +@dataclass(frozen=True) +class Submission: + """A single example submission. + + `path` is relative to the submissions directory, e.g. for + submissions/accepted/hello.java, path is accepted/hello.java.""" + + program: Program + path: Path + + def __post_init__(self) -> None: + if len(self.path.parts) != 2: + raise ValueError(f'Submission path must be on the form directory/name, got {self.path}') + + @property + def directory(self) -> str: + """The submission's top-level directory under submissions/.""" + return self.path.parts[0] + + +_VERDICT_BY_DIRECTORY: dict[str, Verdict] = { + 'accepted': 'AC', + 'partially_accepted': 'AC', + 'wrong_answer': 'WA', + 'run_time_error': 'RTE', + 'time_limit_exceeded': 'TLE', +} + + +@dataclass(frozen=True) +class LegacyPolicy: + """The directory-name-based policy for what's expected of a submission, used by problem + formats that predate submissions.yaml: everything is inferred purely from which of the + well-known directories (if any) a submission's path starts with. + + This is a placeholder for the richer (and eventually per-testcase) policy that + submissions.yaml will bring. + """ + + def matches(self, submission: Submission) -> bool: + """Whether this submission is recognized by the policy at all, i.e. sits in a well-known directory.""" + return self.expected_verdict(submission) is not None + + def expected_verdict(self, submission: Submission) -> Verdict | None: + """The expected verdict for this submission, or None if it isn't in a well-known directory.""" + return _VERDICT_BY_DIRECTORY.get(submission.directory) + + def lower_bounds_time_limit(self, submission: Submission) -> bool: + """Whether this submission's runtime should be used to lower-bound the time limit.""" + return submission.directory == 'accepted' + + def expects_full_score(self, submission: Submission) -> bool: + """Whether this submission is expected to achieve full score.""" + return submission.directory == 'accepted' + + +@dataclass(frozen=True) +class Submissions: + """All example submissions for a problem.""" + + submissions: list[Submission] = field(default_factory=list) + policy: LegacyPolicy = field(default_factory=LegacyPolicy) + + +def load_submissions(probdir: Path, language_config: Languages, work_dir: str, includes: Includes) -> Submissions: + subs_root = probdir / 'submissions' + if not subs_root.is_dir(): + return Submissions() + + submissions = [] + for entry in sorted(subs_root.iterdir()): + if entry.is_dir(): + for program in find_programs(str(entry), language_config=language_config, work_dir=work_dir, includes=includes): + submissions.append(Submission(program=program, path=Path(entry.name) / program.name)) + return Submissions(submissions=submissions) diff --git a/problemtools/run/__init__.py b/problemtools/run/__init__.py index 880f3b15..4c066c54 100644 --- a/problemtools/run/__init__.py +++ b/problemtools/run/__init__.py @@ -3,9 +3,9 @@ """ import os +from typing import TYPE_CHECKING from ..languages import Languages -from ..model import Includes from . import rutil from .buildrun import BuildRun from .checktestdata import Checktestdata @@ -16,12 +16,15 @@ from .tools import get_tool_path as get_tool_path from .viva import Viva +if TYPE_CHECKING: + from ..model import Includes + def find_programs( path: str, language_config: Languages, work_dir: str, - includes: Includes = Includes(), # noqa: B008 -- Includes is a frozen dataclass, so this is safe + includes: 'Includes | None' = None, allow_validation_script: bool = False, ) -> list[Program]: """Find all programs in a directory. @@ -67,7 +70,7 @@ def get_program( path: str, language_config: Languages, work_dir: str, - includes: Includes = Includes(), # noqa: B008 -- Includes is a frozen dataclass, so this is safe + includes: 'Includes | None' = None, allow_validation_script: bool = False, ) -> Program | None: """Get a Program object for a program @@ -85,7 +88,7 @@ def get_program( includes: include files to add to the program, resolved per the program's detected language (see - Includes.get_includes_for_language). + Includes.get_includes_for_language). Defaults to no includes. allow_validation_script: if true, also looks for validation scripts in the Checktestdata and VIVA formats. @@ -94,6 +97,12 @@ def get_program( a Program instance, or None if no program was found at the given path. """ + if includes is None: + # Imported lazily (rather than at module scope) since `model` depends on `run` + # (e.g. for `run.find_programs`), so importing it here avoids a circular import. + from ..model import Includes + + includes = Includes() if os.path.isfile(path): if allow_validation_script: diff --git a/problemtools/run/source.py b/problemtools/run/source.py index 7f2ed4c8..d9f112c9 100644 --- a/problemtools/run/source.py +++ b/problemtools/run/source.py @@ -6,20 +6,23 @@ import os import subprocess import tempfile +from typing import TYPE_CHECKING from ..languages import CommandSubstitution, Language -from ..model import LanguageIncludes from . import rutil from .errors import ProgramError from .program import Program +if TYPE_CHECKING: + from ..model import LanguageIncludes + log = logging.getLogger(__name__) class SourceCode(Program): """Class representing a program provided by source code.""" - def __init__(self, path: str, language: Language, work_dir: str, includes: LanguageIncludes) -> None: + def __init__(self, path: str, language: Language, work_dir: str, includes: 'LanguageIncludes') -> None: """Instantiate SourceCode object Args: diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index acee727b..ce21f103 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -7,7 +7,6 @@ import glob import hashlib import logging -import math import os import random import re @@ -32,7 +31,7 @@ from .context import PROBLEM_PARTS, Context from .diagnostics import Diagnostics, LoggingDiagnostics, VerifyError from .formatversion import FormatVersion, get_format_version -from .judge import CacheKey, SubmissionJudge, SubmissionResult, Verdict, validate_output +from .judge import CacheKey, SubmissionResult, validate_output from .version import add_version_arg random.seed(42) @@ -1064,291 +1063,46 @@ def __str__(self) -> str: class Submissions(ProblemPart): - # (verdict, directory, required) - _VERDICTS: Final[list[tuple[Verdict, str, bool]]] = [ - ('AC', 'accepted', True), - ('PAC', 'partially_accepted', False), - ('WA', 'wrong_answer', False), - ('RTE', 'run_time_error', False), - ('TLE', 'time_limit_exceeded', False), - ] + """Seam to integrate a model + checks setup into verifyproblem in a somewhat clean way""" PART_NAME = 'submission' def setup(self) -> None: - self._submissions = {} - srcdir = os.path.join(self.problem.probdir, 'submissions') - for verdict in Submissions._VERDICTS: - acr = verdict[0] - self._submissions[acr] = run.find_programs( - os.path.join(srcdir, verdict[1]), - language_config=self.problem.language_config, - work_dir=self.problem.tmpdir, - includes=self.problem.includes.includes, - ) + self.submissions = model.load_submissions( + Path(self.problem.probdir), self.problem.language_config, self.problem.tmpdir, self.problem.includes.includes + ) def __str__(self) -> str: return 'submissions' - def check_submission( - self, sub: run.Program, context: Context, expected_verdict: Verdict, timelim: float, timelim_high: float - ) -> list[SubmissionResult]: - desc = f'{expected_verdict} submission {sub}' - partial = expected_verdict == 'PAC' - - judge = SubmissionJudge( - sub=sub, - output_validator=self.problem.output_validators.output_validator, - metadata=self.problem.metadata, - root=self.problem.testdata, - base_dir=Path(self.problem.tmpdir), - context=context, - diag=self._diag, - custom_grader=self.problem.graders._grader, - ) - if context.executor is not None: - judge.precompute(timelim_high) - results_high = judge.judge(timelim_high) - if not results_high: - self.fatal('check_submission called, but found no test cases to run on.') - result_high = results_high[-1] - - results = judge.judge(timelim) - result = results[-1] - - # Check if scores were outside of the range for any groups - if self.problem.is_scoring(): - for r in results: - if r.score is not None and isinstance(r.test_node, TestCaseGroup): - r.test_node.check_score_in_bounds(sub, r.score) - - # Warn if AC (but not PAC) submissions fail on samples. It's not uncommon for sample cases to be - # ignored, so failing on them could be silent otherwise. Skip warning if the result isn't AC - - # then something worse has gone wrong, and we'll error later. - if expected_verdict == 'AC' and result.verdict == 'AC': - if sample_failure := self._find_sample_failure(results): - self.warning(f'{desc} got {sample_failure.verdict} on sample: {sample_failure}') - - # Warn if a PAC submission would affect time limit, had it been use to compute the time limit. Only do this - # if it gets AC on the computed time limit, otherwise we have other warnings below. - if partial and result.verdict == 'AC': - self._warn_pac_too_slow(judge, results, timelim, desc) - - if result.verdict != result_high.verdict or result.score != result_high.score: - self.warning( - f'{desc} sensitive to time limit: limit of {timelim} secs -> {result}, limit of {timelim_high} secs -> {result_high}' - ) - - required_verdict: Verdict = 'AC' if partial else expected_verdict - if partial and self.fully_accepted(result): - self.warning(f'{desc} was fully accepted: {result}') - elif result.verdict == required_verdict: - self.msg(f' {desc} OK: {result}') - if not partial and required_verdict == 'AC' and not self.fully_accepted(result) and self.full_score_finite(): - # For some heuristic problems, this is expected. Thus, only warn. - self.warning(f'{desc} did not attain full score (consider moving it to partially_accepted)') - elif result_high.verdict == required_verdict and not (partial and self.fully_accepted(result_high)): - self.msg(f' {desc} OK with extra time: {result_high}') - else: - self.error(f'{desc} got {result}', result_high.additional_info) - - return results - - def _find_sample_failure(self, results: list[SubmissionResult]) -> SubmissionResult | None: - for r in results: - if r.verdict != 'AC' and isinstance(r.test_node, TestCase) and r.test_node.is_in_sample_group(): - return r - return None - - def _warn_pac_too_slow(self, judge: SubmissionJudge, results: list[SubmissionResult], timelim: float, desc: str) -> None: - """Warn if a PAC submission is slow enough that it would have affected the time limit.""" - runtime_without_affecting_tl = timelim / self.problem.metadata.limits.time_multipliers.ac_to_time_limit - if judge.judge(runtime_without_affecting_tl)[-1].verdict == 'AC': - return - for t in sorted(r.runtime for r in results if r.runtime > runtime_without_affecting_tl): - if judge.judge(t)[-1].verdict == 'AC': - self.warning(f'{desc} is slower than all AC submissions. It needs {t:.2f}s to get AC') - return - - def _get_table_groups(self) -> list[TestCaseGroup]: - """Return the groups to show as columns: expand any root child that has subgroups.""" - result = [] - for group in self.problem.testdata.get_subgroups(): - subgroups = group.get_subgroups() - if subgroups: - result.extend(subgroups) - else: - result.append(group) - return result - - def _print_results_table(self, all_submission_results: list[tuple[run.Program, list[SubmissionResult]]]) -> None: - groups = self._get_table_groups() - is_scoring = self.problem.is_scoring() - - def cell_for_group(results: list[SubmissionResult], group: TestCaseGroup) -> str: - for r in results: - if r.test_node is group: - if r.verdict == 'AC': - if is_scoring and r.score is not None: - score_str = f'{int(r.score)}' if r.score == int(r.score) else f'{r.score:.2f}' - score_part = f'({score_str})' - else: - score_part = '' - return f'AC{score_part}:{r.runtime:.2f}s' - return r.verdict - return '-' - - def cell_for_pts(results: list[SubmissionResult]) -> str: - score = results[-1].score - return f'{score:.0f}' if score is not None else '-' - - def cell_for_time(results: list[SubmissionResult]) -> str: - t = results[-1].runtime - return f'{t:.2f}s' if t >= 0 else '-' - - headers = ['Submission'] + [os.path.basename(g._datadir) for g in groups] - if is_scoring: - headers.append('Pts') - headers.append('Time') - - rows = [] - for sub, results in all_submission_results: - row = [sub.name] - for g in groups: - row.append(cell_for_group(results, g)) - if is_scoring: - row.append(cell_for_pts(results)) - row.append(cell_for_time(results)) - rows.append(row) - - widths = [len(h) for h in headers] - for row in rows: - for i, cell in enumerate(row): - widths[i] = max(widths[i], len(cell)) - - self.msg('Submission results:') - indent = ' ' - self.msg(indent + ' '.join(h.ljust(widths[i]) for i, h in enumerate(headers))) - for row in rows: - self.msg(indent + ' '.join(cell.ljust(widths[i]) for i, cell in enumerate(row))) - - def full_score_finite(self) -> bool: - min_score, max_score = self.problem.testdata.get_score_range() - if self.problem.metadata.legacy_grading.objective == 'min': - return min_score != float('-inf') - else: - return max_score != float('inf') - - def fully_accepted(self, result: SubmissionResult) -> bool: - min_score, max_score = self.problem.testdata.get_score_range() - best_score = min_score if self.problem.metadata.legacy_grading.objective == 'min' else max_score - return result.verdict == 'AC' and (not self.problem.is_scoring() or result.score == best_score) - def start_background_work(self, context: Context) -> None: # Send off an early background compile job for each submission and # validator, to avoid a bottleneck step at the start of each test run. self.problem.output_validators.start_background_work(context) - for verdict in Submissions._VERDICTS: - acr = verdict[0] - for sub in self._submissions[acr]: - sub_name = sub.name - if context.submission_filter.search(os.path.join(verdict[1], sub_name)): - context.submit_background_work(lambda s: s.compile(), sub) - - def _compute_time_limit(self, fixed_limit: float | None, lower_bound_runtime: float | None) -> tuple[float, float]: - if fixed_limit is None and lower_bound_runtime is None: - # 5 minutes is our currently hard coded upper bound for what to allow when we don't know the time limit yet - return 300.0, 300.0 - - limits = self.problem.metadata.limits - if fixed_limit is not None: - timelim = fixed_limit - else: - assert lower_bound_runtime is not None, 'Assert to keep mypy happy' - exact_timelim = lower_bound_runtime * limits.time_multipliers.ac_to_time_limit - timelim = max(1, math.ceil(exact_timelim / limits.time_resolution)) * limits.time_resolution - - return timelim, timelim * limits.time_multipliers.time_limit_to_tle + policy = self.submissions.policy + for sub in self.submissions.submissions: + if policy.matches(sub) and context.submission_filter.search(str(sub.path)): + context.submit_background_work(lambda s: s.compile(), sub.program) def check(self, context: Context) -> bool: if self._check_res is not None: return self._check_res self._check_res = True - limits = self.problem.metadata.limits - ac_to_time_limit = limits.time_multipliers.ac_to_time_limit - - fixed_limit: float | None = context.fixed_timelim if context.fixed_timelim is not None else limits.time_limit - lower_bound_runtime: float | None = None # The runtime of the slowest submission used to lower bound the time limit. - - if limits.time_limit is not None and context.fixed_timelim is not None: - self.warning('There is a fixed time limit in problem.yaml, and you provided one on command line. Using command line.') - - has_testcases = any(tc.matches_filter(context.data_filter) for tc in self.problem.testdata.get_all_testcases()) - if not has_testcases: - self.warning('Found no test cases to run on. Did you filter them all out?') - - all_submission_results: list[tuple[run.Program, list[SubmissionResult]]] = [] - - for verdict in Submissions._VERDICTS: - acr = verdict[0] - if verdict[2] and not self._submissions[acr]: - self.error(f'Require at least one "{verdict[1]}" submission') - - runtimes = [] - - for sub in self._submissions[acr]: - sub_name = sub.name - if context.submission_filter.search(os.path.join(verdict[1], sub_name)): - self.info(f'Check {acr} submission {sub}') - - if sub.code_size() > 1024 * limits.code: - self.error( - f'{acr} submission {sub} has size {sub.code_size() / 1024.0:.1f} kiB, exceeds code size limit of {limits.code} kiB' - ) - continue - - success, msg = sub.compile() - if not success: - self.error(f'Compile error for {acr} submission {sub}', additional_info=msg) - continue - - if has_testcases: - timelim, timelim_high = self._compute_time_limit(fixed_limit, lower_bound_runtime) - sub_results = self.check_submission(sub, context, acr, timelim, timelim_high) - runtimes.append(sub_results[-1].runtime) - all_submission_results.append((sub, sub_results)) - - if acr == 'AC' and has_testcases: - if len(runtimes) > 0: - lower_bound_runtime = max(runtimes) - - # Helper function to format numbers with at most 3 decimals and dealing with None - def _f_n(number: float | None) -> str: - return f'{round(number, 3):g}' if number is not None else '-' - - if fixed_limit is not None and lower_bound_runtime is not None: - tl_from_subs, _ = self._compute_time_limit(None, lower_bound_runtime) - if lower_bound_runtime * ac_to_time_limit > fixed_limit: - msg = f'Fixed time limit ({_f_n(fixed_limit)}) is tighter than the auto-computed limit ({_f_n(tl_from_subs)}) — slowest AC: {_f_n(lower_bound_runtime)} x multiplier {_f_n(ac_to_time_limit)}' - if context.fixed_timelim is not None: # We just warn when the fixed time limit comes from command line - self.warning(msg) - else: - self.error(msg) # ... but if it came from problem.yaml, it's an error if bounds aren't kept - - if not math.isclose(fixed_limit, tl_from_subs): - self.msg( - f' Solutions give timelim of {_f_n(tl_from_subs)} seconds, but will use provided fixed limit of {_f_n(fixed_limit)} seconds instead' - ) - - timelim, timelim_margin = self._compute_time_limit(fixed_limit, lower_bound_runtime) - self.msg( - f' Slowest AC runtime: {_f_n(lower_bound_runtime)}, setting timelim to {_f_n(timelim)} secs, safety margin to {_f_n(timelim_margin)} secs' - ) - self.problem._set_timelim(timelim) - - if all_submission_results: - self._print_results_table(all_submission_results) + errors_before = self.errors + checks.check_submissions( + self.submissions, + self.problem.metadata, + self.problem.testdata, + self.problem.output_validators.output_validator, + self.problem.graders._grader, + self.problem.tmpdir, + context, + self.problem._set_timelim, + self._diag, + ) + if self.errors > errors_before: + self._check_res = False return self._check_res