From 434d45f343716f48a1858b7bf0d4b8ef7cfd8dfc Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Thu, 27 Aug 2026 13:23:20 +0200 Subject: [PATCH 1/4] Create model for InputValidators and OutputValidators --- problemtools/model/__init__.py | 6 ++++ problemtools/model/validators.py | 57 ++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 problemtools/model/validators.py diff --git a/problemtools/model/__init__.py b/problemtools/model/__init__.py index 7b1f49d7..8608a8de 100644 --- a/problemtools/model/__init__.py +++ b/problemtools/model/__init__.py @@ -6,21 +6,27 @@ from .includes import DEFAULT_LANGUAGE, IncludeFile, Includes, LanguageIncludes, load_includes from .submissions import LegacyPolicy, Submission, Submissions, load_submissions from .testdata import DEFAULT_CONFIG, SCORING_ONLY_KEYS, TestCase, TestDataGroup, load_testdata +from .validators import DEFAULT_VALIDATOR, InputValidators, OutputValidators, load_input_validators, load_output_validators __all__ = [ 'DEFAULT_CONFIG', 'DEFAULT_LANGUAGE', + 'DEFAULT_VALIDATOR', 'SCORING_ONLY_KEYS', 'IncludeFile', 'Includes', + 'InputValidators', 'LanguageIncludes', 'LegacyPolicy', + 'OutputValidators', 'Submission', 'Submissions', 'TestCase', 'TestDataGroup', 'Verdict', 'load_includes', + 'load_input_validators', + 'load_output_validators', 'load_submissions', 'load_testdata', ] diff --git a/problemtools/model/validators.py b/problemtools/model/validators.py new file mode 100644 index 00000000..460b7019 --- /dev/null +++ b/problemtools/model/validators.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from ..formatversion import FormatVersion +from ..languages import Languages +from ..metadata import Metadata +from ..run import Program, find_programs, get_tool + +#: The problemtools-provided validator used when a problem doesn't ship a custom output validator. +DEFAULT_VALIDATOR = get_tool('default_validator') + + +@dataclass(frozen=True) +class InputValidators: + """A problem's input format validators.""" + + validators: list[Program] = field(default_factory=list) + uses_old_path: bool = False + + +def load_input_validators(probdir: Path, language_config: Languages, work_dir: str) -> InputValidators: + old_path = probdir / 'input_format_validators' + uses_old_path = old_path.is_dir() + validators_path = old_path if uses_old_path else probdir / 'input_validators' + validators = find_programs( + str(validators_path), language_config=language_config, allow_validation_script=True, work_dir=work_dir + ) + return InputValidators(validators=validators, uses_old_path=uses_old_path) + + +@dataclass(frozen=True) +class OutputValidators: + """A problem's output validators: custom validator programs found on disk, if any.""" + + validators: list[Program] = field(default_factory=list) + + def uses_default(self, format: FormatVersion, metadata: Metadata) -> bool: + """Whether the default validator is used, rather than a custom one.""" + if format is FormatVersion.LEGACY: + return metadata.legacy_validation == 'default' + return not self.validators + + def select(self, format: FormatVersion, metadata: Metadata) -> Program | None: + """The output validator that will actually be used, or None if the default validator + is required but not available on this problemtools install.""" + if self.uses_default(format, metadata) or not self.validators: + return DEFAULT_VALIDATOR + return self.validators[0] + + +def load_output_validators(probdir: Path, format: FormatVersion, language_config: Languages, work_dir: str) -> OutputValidators: + validators = find_programs( + str(probdir / format.output_validator_directory), language_config=language_config, work_dir=work_dir + ) + return OutputValidators(validators=validators) From 7969e5a274e2d740b3b73cf1fc6e1eeaf92fd1a5 Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Thu, 27 Aug 2026 13:30:54 +0200 Subject: [PATCH 2/4] Tests for validators (copied over from verifyproblem with minimal changes) --- problemtools/checks/__init__.py | 4 + problemtools/checks/validators.py | 280 ++++++++++++++++++++++++++++++ 2 files changed, 284 insertions(+) create mode 100644 problemtools/checks/validators.py diff --git a/problemtools/checks/__init__.py b/problemtools/checks/__init__.py index 15bc60a5..95be9f24 100644 --- a/problemtools/checks/__init__.py +++ b/problemtools/checks/__init__.py @@ -1,9 +1,13 @@ from .includes import check_includes from .submissions import check_submissions from .testdata import check_testdata +from .validators import check_input_validators, check_output_validators, check_testcase_input __all__ = [ 'check_includes', + 'check_input_validators', + 'check_output_validators', 'check_submissions', + 'check_testcase_input', 'check_testdata', ] diff --git a/problemtools/checks/validators.py b/problemtools/checks/validators.py new file mode 100644 index 00000000..9b645424 --- /dev/null +++ b/problemtools/checks/validators.py @@ -0,0 +1,280 @@ +"""Checks for a problem package's input/output validators.""" + +from __future__ import annotations + +import os +import random +import re +import string +import tempfile +from collections.abc import Callable +from pathlib import Path +from re import Match + +from ..diagnostics import Diagnostics +from ..formatversion import FormatVersion +from ..judge import SubmissionResult, validate_output +from ..metadata import Metadata +from ..model import InputValidators, OutputValidators, TestCase, TestDataGroup +from ..run import ProgramError, SourceCode + +# Junk data. The validator should reject these cases +_JUNK_CASES: list[tuple[str, bytes]] = [ + ('an empty file', b''), + ('a binary file with random bytes', random.Random(42).randbytes(1024)), + ('a text file with the ASCII characters 32 up to 127', bytes(range(32, 127))), + ( + 'a random text file with printable ASCII characters', + bytes(random.Random(42).choices(string.printable.encode('utf8'), k=200)), + ), +] + +# Try to crash the output validator, causing a judge error +_JUNK_CASES_CRASH = [ + ('a file with the number -1', b'-1'), + ('a file with the number 2147483647', b'2147483647'), + ('a file with the number 2147483648', b'2147483648'), + ('a file with the number 9223372036854775808', b'9223372036854775808'), + ('a file with the number 0', b'0'), + ('a file with the number 1', b'1'), + ('a file with the number 1.0', b'1.0'), + ('a file with the string "a"', b'a'), + ('a file with the contents "2\\n-1 1"', b'2\n-1 1'), + ('a file with the contents "2\\n1"', b'2\n1'), + ('a file with the contents "1\\n-1 1"', b'1\n-1 1'), + ('a file with the contents "1\\na"', b'1\na'), + ('a file with the contents "(()"', b'(()'), + ('a file with the contents "1-"', b'1-'), + ('a file with the contents "1/0"', b'1/0'), + ('a file with the contents "2\\n<"', b'2\n<'), + ('a file with the contents "NaN"', b'NaN'), + ('a file with the contents "inf"', b'inf'), + ('a file with the contents "\\x00"', b'\x00'), + ('a file with the contents "\\x80"', b'\x80'), +] + + +def _build_junk_modifier( + desc: str, pattern: str, repl: str | Callable[[Match[str]], str] +) -> tuple[str, Callable[[str], bool], Callable[[str], str]]: + p = re.compile(pattern) + return (desc, lambda text: p.search(text) is not None, lambda text: p.sub(repl, text)) + + +_JUNK_MODIFICATIONS = [ + _build_junk_modifier('spaces added where there already is whitespace', r'\s', lambda m: m.group(0) + ' '), + _build_junk_modifier('spaces added to the end of a line', r'\n', lambda m: m.group(0) + ' '), + _build_junk_modifier('newlines added where there already are newlines', '\n', lambda m: '\n\n'), + _build_junk_modifier('leading zeros added to integers', r'(^|[^.]\b)([0-9]+)\b', r'\g<1>0000000000\g<2>'), + _build_junk_modifier('trailing zeros added to real number decimal portion', r'\.[0-9]+\b', r'\g<0>0000000000'), + ( + 'random junk added to the end of the file', + lambda f: True, + lambda f: f + ''.join(random.choice(string.printable) for _ in range(200)), + ), +] + + +# Temporary helpers to keep code structure as similar as possible to old code from +# verifyproblem when extracting this to a separate module; ProblemAspect still owns the +# "real" versions of these, used by parts not yet extracted (e.g. ProblemStatement, ProblemConfig). +def _warn_directory(format: FormatVersion, probdir: Path, name: str, prop: str, diag: Diagnostics) -> None: + good_dir = getattr(format, prop) + bad_dirs = {getattr(version, prop) for version in FormatVersion} - {good_dir} + for directory in bad_dirs: + if (probdir / directory).exists(): + diag.warning(f'Found directory "{directory}". Version {format} looks for {name} in "{good_dir}"') + + +def _error_in_2023_07(format: FormatVersion, diag: Diagnostics, msg: str, additional_info: str | None = None) -> None: + if format is FormatVersion.LEGACY: + diag.warning(msg, additional_info) + else: + diag.error(msg, additional_info) + + +def check_input_validators(validators: InputValidators, testdata: TestDataGroup, work_dir: str, diag: Diagnostics) -> None: + """Run all checks on a problem's input format validators.""" + if validators.uses_old_path: + diag.warning('input_format_validators is a deprecated name; please use input_validators instead') + + errors_before = diag.errors + if len(validators.validators) == 0: + diag.error('No input format validators found') + + for val in validators.validators: + try: + success, msg = val.compile() + if not success: + diag.error(f'Compile error for {val}', msg) + except ProgramError as e: + diag.error(str(e)) + + # Only sanity check input validators if they all actually compiled + if diag.errors != errors_before: + return + + all_flags: set[str] = set() + + def collect_flags(group: TestDataGroup, flags: set[str]) -> None: + if len(group.get_testcases()) > 0: + flags.add(group.config['input_validator_flags']) + for subgroup in group.get_subgroups(): + collect_flags(subgroup, flags) + + collect_flags(testdata, all_flags) + + fd, file_name = tempfile.mkstemp() + os.close(fd) + for desc, case in _JUNK_CASES: + with open(file_name, 'wb') as f: + f.write(case) + for flags_str in all_flags: + flags = flags_str.split() + for val in validators.validators: + status, _ = val.run(file_name, args=flags, work_dir=work_dir) + if os.WEXITSTATUS(status) != 42: + break + else: + diag.warning(f'No validator rejects {desc} with flags "{" ".join(flags)}"') + + def modified_input_validates(applicable: Callable[[str], bool], modifier: Callable[[str], str]) -> bool: + for testcase in testdata.get_all_testcases(): + try: + with open(testcase.infile) as infile: + infile_data = infile.read() + if not applicable(infile_data): + continue + except UnicodeDecodeError: + continue + + with open(file_name, 'wb') as f: + f.write(modifier(infile_data).encode('utf8')) + + for flags_str in all_flags: + flags = flags_str.split() + for val in validators.validators: + status, _ = val.run(file_name, args=flags, work_dir=work_dir) + if os.WEXITSTATUS(status) != 42: + # expected behavior; validator rejects modified input + return False + + # we found a file we could modify, and all validators + # accepted the modifications + return True + + # no files were modifiable + return False + + for desc, applicable, modifier in _JUNK_MODIFICATIONS: + if modified_input_validates(applicable, modifier): + diag.warning(f'No validator rejects {desc}') + + os.unlink(file_name) + + +def check_testcase_input(validators: InputValidators, testcase: TestCase, work_dir: str, diag: Diagnostics) -> None: + """Run the (already checked) input validators against a single test case's input file.""" + flags = testcase.input_validator_flags + + for val in validators.validators: + # A validator that failed to compile was already reported by check_input_validators; skip it. + success, _ = val.compile() + if not success: + continue + + with tempfile.NamedTemporaryFile() as outfile, tempfile.NamedTemporaryFile() as errfile: + status, _ = val.run(str(testcase.infile), outfile.name, errfile.name, args=flags, work_dir=work_dir) + if not os.WIFEXITED(status): + emsg = f'Input format validator {val} crashed on input {testcase.infile}' + elif os.WEXITSTATUS(status) != 42: + emsg = f'Input format validator {val} did not accept input {testcase.infile}, exit code: {os.WEXITSTATUS(status)}' + else: + continue + validator_stdout = outfile.read().decode('utf-8', 'replace') + validator_stderr = errfile.read().decode('utf-8', 'replace') + validator_output = '\n'.join(out for out in [validator_stdout, validator_stderr] if out) + diag.error(emsg, validator_output) + + +def check_output_validators( + validators: OutputValidators, + format: FormatVersion, + metadata: Metadata, + testdata: TestDataGroup, + probdir: Path, + work_dir: str, + diag: Diagnostics, +) -> None: + """Run all checks on a problem's output validators.""" + _warn_directory(format, probdir, 'output validators', 'output_validator_directory', diag) + + errors_before = diag.errors + + selected = validators.select(format, metadata) + + if len(validators.validators) > 1: + _error_in_2023_07(format, diag, f'Support for multiple output validators has been dropped. will only use {selected}') + + if selected is None: + diag.fatal('Unable to locate default validator') + + safe_output_validator_languages = {'c', 'cpp', 'python3'} + if isinstance(selected, SourceCode) and selected.language.lang_id not in safe_output_validator_languages: + _error_in_2023_07( + format, + diag, + f'Output validator in {selected.language.name}. Only {safe_output_validator_languages} are standardized. ' + 'Check carefully if your CCS supports more (Kattis does not).', + ) + + if validators.uses_default(format, metadata) and validators.validators: + diag.error('There are validator programs but problem.yaml has validation = "default"') + elif not validators.uses_default(format, metadata) and not validators.validators: + diag.fatal('problem.yaml specifies custom validator but no validator programs found') + + try: + success, msg = selected.compile() + if not success: + diag.fatal(f'Compile error for output validator {selected}', msg) + except ProgramError as e: + diag.fatal(f'Compile error for output validator {selected}', str(e)) + + # Only sanity check output validators if they all actually compiled + if diag.errors != errors_before: + return + + def run_junk_case(case_desc: str, junk_content: bytes, testcases: list[TestCase]) -> list[SubmissionResult]: + results = [] + with tempfile.NamedTemporaryFile(mode='wb') as f: + f.write(junk_content) + f.flush() + for testcase in testcases: + result = validate_output( + testcase=testcase, + submission_output=Path(f.name), + output_validator=selected, + metadata=metadata, + base_dir=Path(work_dir), + diag=diag, + ) + results.append(result) + if result.verdict == 'JE': + diag.error(f'{case_desc} as output on test case {testcase} gave {result}') + break + return results + + # Junk cases that the output validator should reject + for desc, junk_case_content in _JUNK_CASES: + results = run_junk_case(desc, junk_case_content, testdata.get_all_testcases()) + rejected = any(result.verdict != 'AC' for result in results) + if not rejected: + diag.warning(f'{desc} gets AC') + + # Malformed cases that a poorly-written output validator might crash on + # Note that these might be valid output, so we only check if it crashes. + # These bugs are rarely dependent on the actual test case, so we just + # run on a few to keep things speedy. + test_cases = testdata.get_all_testcases()[:3] + for desc, junk_case_content in _JUNK_CASES_CRASH: + run_junk_case(desc, junk_case_content, test_cases) From 0875ff5e7c5840e715783da7eb21bac5832a581a Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Thu, 27 Aug 2026 13:31:23 +0200 Subject: [PATCH 3/4] Replace validators in verifyproblem with model + checks --- problemtools/verifyproblem.py | 284 +++++----------------------------- 1 file changed, 35 insertions(+), 249 deletions(-) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 3d7c9c52..ed604600 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -9,15 +9,13 @@ import random import re import shutil -import string import sys import tempfile import traceback import uuid from abc import ABC -from collections.abc import Callable from pathlib import Path -from re import Match, Pattern +from re import Pattern from types import TracebackType from typing import ClassVar, NoReturn, Self @@ -343,188 +341,34 @@ def __str__(self) -> str: return 'attachments' -# Junk data. The validator should reject these cases -_JUNK_CASES: list[tuple[str, bytes]] = [ - ('an empty file', b''), - ('a binary file with random bytes', random.Random(42).randbytes(1024)), - ('a text file with the ASCII characters 32 up to 127', bytes(range(32, 127))), - ( - 'a random text file with printable ASCII characters', - bytes(random.Random(42).choices(string.printable.encode('utf8'), k=200)), - ), -] - -# Try to crash the output validator, causing a judge error -_JUNK_CASES_CRASH = [ - ('a file with the number -1', b'-1'), - ('a file with the number 2147483647', b'2147483647'), - ('a file with the number 2147483648', b'2147483648'), - ('a file with the number 9223372036854775808', b'9223372036854775808'), - ('a file with the number 0', b'0'), - ('a file with the number 1', b'1'), - ('a file with the number 1.0', b'1.0'), - ('a file with the string "a"', b'a'), - ('a file with the contents "2\\n-1 1"', b'2\n-1 1'), - ('a file with the contents "2\\n1"', b'2\n1'), - ('a file with the contents "1\\n-1 1"', b'1\n-1 1'), - ('a file with the contents "1\\na"', b'1\na'), - ('a file with the contents "(()"', b'(()'), - ('a file with the contents "1-"', b'1-'), - ('a file with the contents "1/0"', b'1/0'), - ('a file with the contents "2\\n<"', b'2\n<'), - ('a file with the contents "NaN"', b'NaN'), - ('a file with the contents "inf"', b'inf'), - ('a file with the contents "\\x00"', b'\x00'), - ('a file with the contents "\\x80"', b'\x80'), -] - - -def _build_junk_modifier( - desc: str, pattern: str, repl: str | Callable[[Match[str]], str] -) -> tuple[str, Callable[[str], bool], Callable[[str], str]]: - p = re.compile(pattern) - return (desc, lambda text: p.search(text) is not None, lambda text: p.sub(repl, text)) - - -_JUNK_MODIFICATIONS = [ - _build_junk_modifier('spaces added where there already is whitespace', r'\s', lambda m: m.group(0) + ' '), - _build_junk_modifier('spaces added to the end of a line', r'\n', lambda m: m.group(0) + ' '), - _build_junk_modifier('newlines added where there already are newlines', '\n', lambda m: '\n\n'), - _build_junk_modifier('leading zeros added to integers', r'(^|[^.]\b)([0-9]+)\b', r'\g<1>0000000000\g<2>'), - _build_junk_modifier('trailing zeros added to real number decimal portion', r'\.[0-9]+\b', r'\g<0>0000000000'), - ( - 'random junk added to the end of the file', - lambda f: True, - lambda f: f + ''.join(random.choice(string.printable) for _ in range(200)), - ), -] - - class InputValidators(ProblemPart): + """Seam to integrate a model + checks setup into verifyproblem in a somewhat clean way""" + PART_NAME = 'input_validator' def setup(self) -> None: - input_validators_path = os.path.join(self.problem.probdir, 'input_format_validators') - if os.path.isdir(input_validators_path): - self._uses_old_path = True - else: - self._uses_old_path = False - new_input_validators_path = os.path.join(self.problem.probdir, 'input_validators') - if os.path.isdir(new_input_validators_path): - input_validators_path = new_input_validators_path - self._validators = run.find_programs( - input_validators_path, - language_config=self.problem.language_config, - allow_validation_script=True, - work_dir=self.problem.tmpdir, + self.input_validators = model.load_input_validators( + Path(self.problem.probdir), self.problem.language_config, self.problem.tmpdir ) def __str__(self) -> str: return 'input format validators' def start_background_work(self, context: Context) -> None: - for val in self._validators: + for val in self.input_validators.validators: context.submit_background_work(lambda v: v.compile(), val) - def check(self, context: Context | None) -> bool: + def check(self, context: Context) -> bool: if self._check_res is not None: return self._check_res - if self._uses_old_path: - self.warning('input_format_validators is a deprecated name; please use input_validators instead') self._check_res = True - if len(self._validators) == 0: - self.error('No input format validators found') - - for val in self._validators[:]: - try: - success, msg = val.compile() - if not success: - self.error(f'Compile error for {val}', msg) - self._validators.remove(val) - except run.ProgramError as e: - self.error(str(e)) - - # Only sanity check input validators if they all actually compiled - if self._check_res: - all_flags: set[str] = set() - - def collect_flags(group: model.TestDataGroup, flags: set[str]) -> None: - if len(group.get_testcases()) > 0: - flags.add(group.config['input_validator_flags']) - for subgroup in group.get_subgroups(): - collect_flags(subgroup, flags) - - collect_flags(self.problem.testdata.testdata, all_flags) - - fd, file_name = tempfile.mkstemp() - os.close(fd) - for desc, case in _JUNK_CASES: - with open(file_name, 'wb') as f: - f.write(case) - for flags_str in all_flags: - flags = flags_str.split() - for val in self._validators: - status, _ = val.run(file_name, args=flags, work_dir=self.problem.tmpdir) - if os.WEXITSTATUS(status) != 42: - break - else: - self.warning(f'No validator rejects {desc} with flags "{" ".join(flags)}"') - - def modified_input_validates(applicable: Callable[[str], bool], modifier: Callable[[str], str]) -> bool: - for testcase in self.problem.testdata.testdata.get_all_testcases(): - try: - with open(testcase.infile) as infile: - infile_data = infile.read() - if not applicable(infile_data): - continue - except UnicodeDecodeError: - continue - - with open(file_name, 'wb') as f: - f.write(modifier(infile_data).encode('utf8')) - - for flags_str in all_flags: - flags = flags_str.split() - for val in self._validators: - status, _ = val.run(file_name, args=flags, work_dir=self.problem.tmpdir) - if os.WEXITSTATUS(status) != 42: - # expected behavior; validator rejects modified input - return False - - # we found a file we could modify, and all validators - # accepted the modifications - return True - - # no files were modifiable - return False - - for desc, applicable, modifier in _JUNK_MODIFICATIONS: - if modified_input_validates(applicable, modifier): - self.warning(f'No validator rejects {desc}') - - os.unlink(file_name) - - return self._check_res - - def validate(self, testcase: model.TestCase, diag: Diagnostics) -> None: - flags = testcase.input_validator_flags - # Remove input validators that don't compile, even without -p validators - self.check(None) + errors_before = self.errors + checks.check_input_validators(self.input_validators, self.problem.testdata.testdata, self.problem.tmpdir, self._diag) + if self.errors > errors_before: + self._check_res = False - for val in self._validators: - with tempfile.NamedTemporaryFile() as outfile, tempfile.NamedTemporaryFile() as errfile: - status, _ = val.run(str(testcase.infile), outfile.name, errfile.name, args=flags, work_dir=self.problem.tmpdir) - if not os.WIFEXITED(status): - emsg = f'Input format validator {val} crashed on input {testcase.infile}' - elif os.WEXITSTATUS(status) != 42: - emsg = f'Input format validator {val} did not accept input {testcase.infile}, exit code: {os.WEXITSTATUS(status)}' - else: - continue - validator_stdout = outfile.read().decode('utf-8', 'replace') - validator_stderr = errfile.read().decode('utf-8', 'replace') - validator_output = '\n'.join(out for out in [validator_stdout, validator_stderr] if out) - diag.error(emsg, validator_output) + return self._check_res class Graders(ProblemPart): @@ -561,30 +405,22 @@ def check(self, context: Context) -> bool: class OutputValidators(ProblemPart): - _default_validator = run.get_tool('default_validator') + """Seam to integrate a model + checks setup into verifyproblem in a somewhat clean way""" PART_NAME = 'output_validator' def setup(self) -> None: - self._validators = run.find_programs( - os.path.join(self.problem.probdir, self.problem.format.output_validator_directory), - language_config=self.problem.language_config, - work_dir=self.problem.tmpdir, + self.output_validators = model.load_output_validators( + Path(self.problem.probdir), self.problem.format, self.problem.language_config, self.problem.tmpdir ) self._has_precompiled = False - def uses_default_validator(self) -> bool: - if self.problem.format is FormatVersion.LEGACY: - return self.problem.metadata.legacy_validation == 'default' - return not self._validators - @property def output_validator(self) -> run.Program: - if self.uses_default_validator() or not self._validators: - if self._default_validator is None: - self.fatal('Unable to locate default validator') - return self._default_validator - return self._validators[0] + validator = self.output_validators.select(self.problem.format, self.problem.metadata) + if validator is None: + self.fatal('Unable to locate default validator') + return validator def __str__(self) -> str: return 'output validators' @@ -599,71 +435,18 @@ def check(self, context: Context) -> bool: return self._check_res self._check_res = True - self.warn_directory('output validators', 'output_validator_directory') - - if len(self._validators) > 1: - self.error_in_2023_07( - f'Support for multiple output validators has been dropped. will only use {self.output_validator}' - ) - - safe_output_validator_languages = {'c', 'cpp', 'python3'} - if ( - isinstance(self.output_validator, run.SourceCode) - and self.output_validator.language.lang_id not in safe_output_validator_languages - ): - self.error_in_2023_07( - f'Output validator in {self.output_validator.language.name}. Only {safe_output_validator_languages} are standardized. Check carefully if your CCS supports more (Kattis does not).' - ) - - if self.uses_default_validator() and self._validators: - self.error('There are validator programs but problem.yaml has validation = "default"') - elif not self.uses_default_validator() and not self._validators: - self.fatal('problem.yaml specifies custom validator but no validator programs found') - - try: - success, msg = self.output_validator.compile() - if not success: - self.fatal(f'Compile error for output validator {self.output_validator}', msg) - except run.ProgramError as e: - self.fatal(f'Compile error for output validator {self.output_validator}', str(e)) - - # Only sanity check output validators if they all actually compiled - if self._check_res: - # Sanity check cases that should be rejected by the output validator - def run_junk_case(case_desc: str, junk_content: bytes, testcases: list[model.TestCase]) -> list[SubmissionResult]: - results = [] - with tempfile.NamedTemporaryFile(mode='wb') as f: - f.write(junk_content) - f.flush() - for testcase in testcases: - result = validate_output( - testcase=testcase, - submission_output=Path(f.name), - output_validator=self.output_validator, - metadata=self.problem.metadata, - base_dir=Path(self.problem.tmpdir), - diag=self._diag, - ) - results.append(result) - if result.verdict == 'JE': - self.error(f'{case_desc} as output on test case {testcase} gave {result}') - break - return results - - # Junk cases that the output validator should reject - for desc, junk_case_content in _JUNK_CASES: - results = run_junk_case(desc, junk_case_content, self.problem.testdata.testdata.get_all_testcases()) - rejected = any(result.verdict != 'AC' for result in results) - if not rejected: - self.warning(f'{desc} gets AC') - - # Malformed cases that a poorly-written output validator might crash on - # Note that these might be valid output, so we only check if it crashes. - # These bugs are rarely dependent on the actual test case, so we just - # run on a few to keep things speedy. - test_cases = self.problem.testdata.testdata.get_all_testcases()[:3] - for desc, junk_case_content in _JUNK_CASES_CRASH: - run_junk_case(desc, junk_case_content, test_cases) + errors_before = self.errors + checks.check_output_validators( + self.output_validators, + self.problem.format, + self.problem.metadata, + self.problem.testdata.testdata, + Path(self.problem.probdir), + self.problem.tmpdir, + self._diag, + ) + if self.errors > errors_before: + self._check_res = False return self._check_res @@ -756,6 +539,9 @@ def check(self, context: Context) -> bool: errors_before = self.errors + def validate_input(testcase: model.TestCase, diag: Diagnostics) -> None: + checks.check_testcase_input(self.problem.input_validators.input_validators, testcase, self.problem.tmpdir, diag) + def validate_answer(testcase: model.TestCase, diag: Diagnostics) -> SubmissionResult: return validate_output( testcase=testcase, @@ -773,7 +559,7 @@ def validate_answer(testcase: model.TestCase, diag: Diagnostics) -> SubmissionRe Path(self.problem.probdir), self.problem.graders._grader is not None, Graders._default_grader is not None, - self.problem.input_validators.validate, + validate_input, validate_answer, self._diag, ) From 69cdacf07f830bd166b533102fe5c1e2dd3ae4a8 Mon Sep 17 00:00:00 2001 From: Gunnar Kreitz Date: Thu, 27 Aug 2026 13:46:40 +0200 Subject: [PATCH 4/4] Skip wrapping testcase checking in functions for testcase checks, now that we mave models for validators --- problemtools/checks/__init__.py | 3 +- problemtools/checks/testdata.py | 63 +++++++++++++++++++++++++-------- problemtools/verifyproblem.py | 20 +++-------- 3 files changed, 54 insertions(+), 32 deletions(-) diff --git a/problemtools/checks/__init__.py b/problemtools/checks/__init__.py index 95be9f24..a4ebae73 100644 --- a/problemtools/checks/__init__.py +++ b/problemtools/checks/__init__.py @@ -1,13 +1,12 @@ from .includes import check_includes from .submissions import check_submissions from .testdata import check_testdata -from .validators import check_input_validators, check_output_validators, check_testcase_input +from .validators import check_input_validators, check_output_validators __all__ = [ 'check_includes', 'check_input_validators', 'check_output_validators', 'check_submissions', - 'check_testcase_input', 'check_testdata', ] diff --git a/problemtools/checks/testdata.py b/problemtools/checks/testdata.py index 4d926dbe..be7842ed 100644 --- a/problemtools/checks/testdata.py +++ b/problemtools/checks/testdata.py @@ -6,14 +6,16 @@ import glob import hashlib import os -from collections.abc import Callable from pathlib import Path from ..context import Context from ..diagnostics import Diagnostics, VerifyError -from ..judge import SubmissionResult +from ..formatversion import FormatVersion +from ..judge import validate_output from ..metadata import Metadata -from ..model import DEFAULT_CONFIG, SCORING_ONLY_KEYS, TestCase, TestDataGroup +from ..model import DEFAULT_CONFIG, SCORING_ONLY_KEYS, InputValidators, OutputValidators, TestCase, TestDataGroup +from ..run import Program +from .validators import check_testcase_input def check_testdata( @@ -23,13 +25,28 @@ def check_testdata( probdir: Path, has_custom_grader: bool, has_default_grader: bool, - validate_input: Callable[[TestCase, Diagnostics], None], - validate_answer: Callable[[TestCase, Diagnostics], SubmissionResult], + input_validators: InputValidators, + output_validators: OutputValidators, + format: FormatVersion, + work_dir: str, diag: Diagnostics, ) -> None: """Run all checks on a problem's test data.""" + output_validator = output_validators.select(format, metadata) + if output_validator is None: + diag.fatal('Unable to locate default validator') + _check_group( - testdata, context, metadata, probdir, has_custom_grader, has_default_grader, validate_input, validate_answer, diag + testdata, + context, + metadata, + probdir, + has_custom_grader, + has_default_grader, + input_validators, + output_validator, + work_dir, + diag, ) @@ -40,8 +57,9 @@ def _check_group( probdir: Path, has_custom_grader: bool, has_default_grader: bool, - validate_input: Callable[[TestCase, Diagnostics], None], - validate_answer: Callable[[TestCase, Diagnostics], SubmissionResult], + input_validators: InputValidators, + output_validator: Program, + work_dir: str, diag: Diagnostics, ) -> None: if group.config['grading'] not in ['default', 'custom']: @@ -155,10 +173,19 @@ def _check_group( continue if isinstance(child, TestDataGroup): _check_group( - child, context, metadata, probdir, has_custom_grader, has_default_grader, validate_input, validate_answer, diag + child, + context, + metadata, + probdir, + has_custom_grader, + has_default_grader, + input_validators, + output_validator, + work_dir, + diag, ) else: - _check_testcase(child, metadata, validate_input, validate_answer, diag) + _check_testcase(child, metadata, input_validators, output_validator, work_dir, diag) def _natural_sort_le(a: str, b: str) -> bool: @@ -193,15 +220,16 @@ def parse_num(s: str, i: int) -> tuple[int, int]: def _check_testcase( testcase: TestCase, metadata: Metadata, - validate_input: Callable[[TestCase, Diagnostics], None], - validate_answer: Callable[[TestCase, Diagnostics], SubmissionResult], + input_validators: InputValidators, + output_validator: Program, + work_dir: str, diag: Diagnostics, ) -> None: _check_newlines(testcase.infile, diag) _check_newlines(testcase.ansfile, diag) _check_size_limits(testcase.infile, diag) _check_size_limits(testcase.ansfile, diag) - validate_input(testcase, diag) + check_testcase_input(input_validators, testcase, work_dir, diag) anssize = testcase.ansfile.stat().st_size / 1024.0 / 1024.0 outputlim = metadata.limits.output if anssize > outputlim: @@ -213,7 +241,14 @@ def _check_testcase( f'Answer file ({anssize:.1f} MiB) is within 50% of output limit ({outputlim} MiB), you might want to increase output limit' ) if not metadata.is_interactive() and not metadata.is_multi_pass(): - val_res = validate_answer(testcase, diag) + val_res = validate_output( + testcase=testcase, + submission_output=testcase.ansfile, + output_validator=output_validator, + metadata=metadata, + base_dir=Path(work_dir), + diag=diag, + ) if val_res.verdict != 'AC': if testcase.is_in_sample_group(): diag.error(f'judge answer file got {val_res} on testcase {testcase.path}') diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index ed604600..ad4d5e41 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -25,7 +25,6 @@ from .context import PROBLEM_PARTS, Context from .diagnostics import Diagnostics, LoggingDiagnostics, VerifyError from .formatversion import FormatVersion, get_format_version -from .judge import SubmissionResult, validate_output from .version import add_version_arg random.seed(42) @@ -539,19 +538,6 @@ def check(self, context: Context) -> bool: errors_before = self.errors - def validate_input(testcase: model.TestCase, diag: Diagnostics) -> None: - checks.check_testcase_input(self.problem.input_validators.input_validators, testcase, self.problem.tmpdir, diag) - - def validate_answer(testcase: model.TestCase, diag: Diagnostics) -> SubmissionResult: - return validate_output( - testcase=testcase, - submission_output=testcase.ansfile, - output_validator=self.problem.output_validators.output_validator, - metadata=self.problem.metadata, - base_dir=Path(self.problem.tmpdir), - diag=diag, - ) - checks.check_testdata( self.testdata, context, @@ -559,8 +545,10 @@ def validate_answer(testcase: model.TestCase, diag: Diagnostics) -> SubmissionRe Path(self.problem.probdir), self.problem.graders._grader is not None, Graders._default_grader is not None, - validate_input, - validate_answer, + self.problem.input_validators.input_validators, + self.problem.output_validators.output_validators, + self.problem.format, + self.problem.tmpdir, self._diag, ) if self.errors > errors_before: