diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 8b6a5d89..0c25180a 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -43,6 +43,9 @@ jobs: - name: Run mypy run: | venv/bin/mypy --non-interactive --config-file mypy.ini -p problemtools + - name: Run mypy on tests + run: | + venv/bin/mypy --non-interactive --config-file mypy.ini --exclude '^tests/data/' tests/ packages: # Use a separate job to test debian packaging to speed things up (no need to test this for every python version above) runs-on: ubuntu-latest diff --git a/run_tests.sh b/run_tests.sh index 2cc673b8..dd29bd3c 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -10,3 +10,4 @@ venv/bin/ruff check . venv/bin/ruff format --check . venv/bin/pytest venv/bin/mypy --non-interactive --config-file mypy.ini -p problemtools +venv/bin/mypy --non-interactive --config-file mypy.ini --exclude '^tests/data/' tests/ diff --git a/tests/conftest.py b/tests/conftest.py index 1ea67d38..148ef966 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,22 @@ """Shared pytest fixtures and test doubles.""" +from pathlib import Path + import pytest from problemtools.diagnostics import Diagnostics +def datadir() -> Path: + """Root directory holding static test fixture data.""" + return (Path(__file__).parent / 'data').resolve() + + +def example_directory(problem_name: str) -> Path: + """Path to one of the example problems shipped in the repo's top-level examples/ directory.""" + return (Path(__file__).parent.parent / 'examples' / problem_name).resolve() + + class RecordingDiagnostics(Diagnostics): """A Diagnostics that records messages instead of emitting them, for asserting on in tests.""" diff --git a/tests/config1/broken.yaml b/tests/data/config1/broken.yaml similarity index 100% rename from tests/config1/broken.yaml rename to tests/data/config1/broken.yaml diff --git a/tests/config1/test.yaml b/tests/data/config1/test.yaml similarity index 100% rename from tests/config1/test.yaml rename to tests/data/config1/test.yaml diff --git a/tests/config1/test2.yaml b/tests/data/config1/test2.yaml similarity index 100% rename from tests/config1/test2.yaml rename to tests/data/config1/test2.yaml diff --git a/tests/config2/test2.yaml b/tests/data/config2/test2.yaml similarity index 100% rename from tests/config2/test2.yaml rename to tests/data/config2/test2.yaml diff --git a/tests/languages_examples/py2helper.py2 b/tests/data/languages_examples/py2helper.py2 similarity index 100% rename from tests/languages_examples/py2helper.py2 rename to tests/data/languages_examples/py2helper.py2 diff --git a/tests/languages_examples/py2main.py b/tests/data/languages_examples/py2main.py similarity index 100% rename from tests/languages_examples/py2main.py rename to tests/data/languages_examples/py2main.py diff --git a/tests/languages_examples/py2nosheb.py b/tests/data/languages_examples/py2nosheb.py similarity index 100% rename from tests/languages_examples/py2nosheb.py rename to tests/data/languages_examples/py2nosheb.py diff --git a/tests/languages_examples/src1.zoo b/tests/data/languages_examples/src1.zoo similarity index 100% rename from tests/languages_examples/src1.zoo rename to tests/data/languages_examples/src1.zoo diff --git a/tests/languages_examples/src2.zoo b/tests/data/languages_examples/src2.zoo similarity index 100% rename from tests/languages_examples/src2.zoo rename to tests/data/languages_examples/src2.zoo diff --git a/tests/languages_examples/src3.zpp b/tests/data/languages_examples/src3.zpp similarity index 100% rename from tests/languages_examples/src3.zpp rename to tests/data/languages_examples/src3.zpp diff --git a/tests/problems/footnote/problem.yaml b/tests/data/problems/footnote/problem.yaml similarity index 100% rename from tests/problems/footnote/problem.yaml rename to tests/data/problems/footnote/problem.yaml diff --git a/tests/problems/footnote/statement/problem.en.md b/tests/data/problems/footnote/statement/problem.en.md similarity index 100% rename from tests/problems/footnote/statement/problem.en.md rename to tests/data/problems/footnote/statement/problem.en.md diff --git a/tests/problems/imgrequest/problem.yaml b/tests/data/problems/imgrequest/problem.yaml similarity index 100% rename from tests/problems/imgrequest/problem.yaml rename to tests/data/problems/imgrequest/problem.yaml diff --git a/tests/problems/imgrequest/statement/problem.en.md b/tests/data/problems/imgrequest/statement/problem.en.md similarity index 100% rename from tests/problems/imgrequest/statement/problem.en.md rename to tests/data/problems/imgrequest/statement/problem.en.md diff --git a/tests/problems/imgrequest2/problem.yaml b/tests/data/problems/imgrequest2/problem.yaml similarity index 100% rename from tests/problems/imgrequest2/problem.yaml rename to tests/data/problems/imgrequest2/problem.yaml diff --git a/tests/problems/imgrequest2/statement/problem.en.md b/tests/data/problems/imgrequest2/statement/problem.en.md similarity index 100% rename from tests/problems/imgrequest2/statement/problem.en.md rename to tests/data/problems/imgrequest2/statement/problem.en.md diff --git a/tests/problems/problemnamexss/problem.yaml b/tests/data/problems/problemnamexss/problem.yaml similarity index 100% rename from tests/problems/problemnamexss/problem.yaml rename to tests/data/problems/problemnamexss/problem.yaml diff --git a/tests/problems/problemnamexss/statement/problem.en.md b/tests/data/problems/problemnamexss/statement/problem.en.md similarity index 100% rename from tests/problems/problemnamexss/statement/problem.en.md rename to tests/data/problems/problemnamexss/statement/problem.en.md diff --git a/tests/problems/samplexss/data/sample/1.ans b/tests/data/problems/samplexss/data/sample/1.ans similarity index 100% rename from tests/problems/samplexss/data/sample/1.ans rename to tests/data/problems/samplexss/data/sample/1.ans diff --git a/tests/problems/samplexss/data/sample/1.in b/tests/data/problems/samplexss/data/sample/1.in similarity index 100% rename from tests/problems/samplexss/data/sample/1.in rename to tests/data/problems/samplexss/data/sample/1.in diff --git a/tests/problems/samplexss/data/sample/testdata.yaml b/tests/data/problems/samplexss/data/sample/testdata.yaml similarity index 100% rename from tests/problems/samplexss/data/sample/testdata.yaml rename to tests/data/problems/samplexss/data/sample/testdata.yaml diff --git a/tests/problems/samplexss/data/testdata.yaml b/tests/data/problems/samplexss/data/testdata.yaml similarity index 100% rename from tests/problems/samplexss/data/testdata.yaml rename to tests/data/problems/samplexss/data/testdata.yaml diff --git a/tests/problems/samplexss/problem.yaml b/tests/data/problems/samplexss/problem.yaml similarity index 100% rename from tests/problems/samplexss/problem.yaml rename to tests/data/problems/samplexss/problem.yaml diff --git a/tests/problems/samplexss/statement/problem.en.md b/tests/data/problems/samplexss/statement/problem.en.md similarity index 100% rename from tests/problems/samplexss/statement/problem.en.md rename to tests/data/problems/samplexss/statement/problem.en.md diff --git a/tests/problems/specialcharacterssample/data/sample/1.ans b/tests/data/problems/specialcharacterssample/data/sample/1.ans similarity index 100% rename from tests/problems/specialcharacterssample/data/sample/1.ans rename to tests/data/problems/specialcharacterssample/data/sample/1.ans diff --git a/tests/problems/specialcharacterssample/data/sample/1.in b/tests/data/problems/specialcharacterssample/data/sample/1.in similarity index 100% rename from tests/problems/specialcharacterssample/data/sample/1.in rename to tests/data/problems/specialcharacterssample/data/sample/1.in diff --git a/tests/problems/specialcharacterssample/data/sample/testdata.yaml b/tests/data/problems/specialcharacterssample/data/sample/testdata.yaml similarity index 100% rename from tests/problems/specialcharacterssample/data/sample/testdata.yaml rename to tests/data/problems/specialcharacterssample/data/sample/testdata.yaml diff --git a/tests/problems/specialcharacterssample/data/testdata.yaml b/tests/data/problems/specialcharacterssample/data/testdata.yaml similarity index 100% rename from tests/problems/specialcharacterssample/data/testdata.yaml rename to tests/data/problems/specialcharacterssample/data/testdata.yaml diff --git a/tests/problems/specialcharacterssample/problem.yaml b/tests/data/problems/specialcharacterssample/problem.yaml similarity index 100% rename from tests/problems/specialcharacterssample/problem.yaml rename to tests/data/problems/specialcharacterssample/problem.yaml diff --git a/tests/problems/specialcharacterssample/statement/problem.en.md b/tests/data/problems/specialcharacterssample/statement/problem.en.md similarity index 100% rename from tests/problems/specialcharacterssample/statement/problem.en.md rename to tests/data/problems/specialcharacterssample/statement/problem.en.md diff --git a/tests/problems/statementxss/problem.yaml b/tests/data/problems/statementxss/problem.yaml similarity index 100% rename from tests/problems/statementxss/problem.yaml rename to tests/data/problems/statementxss/problem.yaml diff --git a/tests/problems/statementxss/statement/problem.en.md b/tests/data/problems/statementxss/statement/problem.en.md similarity index 100% rename from tests/problems/statementxss/statement/problem.en.md rename to tests/data/problems/statementxss/statement/problem.en.md diff --git a/tests/problems/twofootnotes/problem.yaml b/tests/data/problems/twofootnotes/problem.yaml similarity index 100% rename from tests/problems/twofootnotes/problem.yaml rename to tests/data/problems/twofootnotes/problem.yaml diff --git a/tests/problems/twofootnotes/statement/problem.en.md b/tests/data/problems/twofootnotes/statement/problem.en.md similarity index 100% rename from tests/problems/twofootnotes/statement/problem.en.md rename to tests/data/problems/twofootnotes/statement/problem.en.md diff --git a/tests/default_validator_tests/run_and_generate_expected.py b/tests/default_validator_tests/run_and_generate_expected.py index 6b4a54ce..716c3d19 100755 --- a/tests/default_validator_tests/run_and_generate_expected.py +++ b/tests/default_validator_tests/run_and_generate_expected.py @@ -21,7 +21,7 @@ from pathlib import Path -def main(): +def main() -> None: """Main function""" parser = argparse.ArgumentParser(description='Run default_validator and generate expected output files.') parser.add_argument('test_dir', type=Path, help='Path to the test directory.') diff --git a/tests/hello/data/secret/hello.ans b/tests/hello/data/secret/hello.ans deleted file mode 100644 index 980a0d5f..00000000 --- a/tests/hello/data/secret/hello.ans +++ /dev/null @@ -1 +0,0 @@ -Hello World! diff --git a/tests/hello/data/secret/hello.in b/tests/hello/data/secret/hello.in deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/hello/input_validators/validate.py b/tests/hello/input_validators/validate.py deleted file mode 100755 index d98eacf9..00000000 --- a/tests/hello/input_validators/validate.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 -import sys -from sys import stdin - -# There shouldn't be any input -assert len(stdin.readline()) == 0 - -sys.exit(42) diff --git a/tests/hello/problem.yaml b/tests/hello/problem.yaml deleted file mode 100644 index c69bfdc4..00000000 --- a/tests/hello/problem.yaml +++ /dev/null @@ -1,9 +0,0 @@ -source: Kattis -license: public domain -name: Hello World! - -# Fix memory limit at 512 MiB. (Note that for most problems, this -# should not be done. It is only done in this case because we include -# a test submission that goes over this limit.) -limits: - memory: 512 diff --git a/tests/hello/problem_statement/problem.en.tex b/tests/hello/problem_statement/problem.en.tex deleted file mode 100644 index 3df9596c..00000000 --- a/tests/hello/problem_statement/problem.en.tex +++ /dev/null @@ -1,9 +0,0 @@ -\problemname{Hello World!} - -\section*{Input} - -There is no input for this problem. - -\section*{Output} - -Output should contain one line, containing the string ``Hello World!''. diff --git a/tests/hello/problem_statement/problem.sv.tex b/tests/hello/problem_statement/problem.sv.tex deleted file mode 100644 index d021c237..00000000 --- a/tests/hello/problem_statement/problem.sv.tex +++ /dev/null @@ -1,10 +0,0 @@ -\problemname{Hej V\"arlden!} % Silly use of LaTeX just to test plainproblemname -%% plainproblemname: Hej Världen! - -\section*{Indata} - -Detta problem har inget indata. - -\section*{Output} - -Utdata ska bestå av en rad, innehållandes strängen ``Hello World!''. diff --git a/tests/hello/submissions/accepted/hello.cc b/tests/hello/submissions/accepted/hello.cc deleted file mode 100644 index 9feeee8e..00000000 --- a/tests/hello/submissions/accepted/hello.cc +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main(void) { - printf("Hello World!\n"); - return 0; -} diff --git a/tests/hello/submissions/accepted/hello.java b/tests/hello/submissions/accepted/hello.java deleted file mode 100644 index 49243411..00000000 --- a/tests/hello/submissions/accepted/hello.java +++ /dev/null @@ -1,5 +0,0 @@ -public class hello { - public static void main(String args[]) { - System.out.println("Hello World!"); - } -} \ No newline at end of file diff --git a/tests/hello/submissions/accepted/hello.kt b/tests/hello/submissions/accepted/hello.kt deleted file mode 100644 index 3dcd543f..00000000 --- a/tests/hello/submissions/accepted/hello.kt +++ /dev/null @@ -1,4 +0,0 @@ -fun main(args: Array) { - val words = if (args.size == 0) arrayOf("Hello", "World!") else args - System.`out`.println(words.joinToString(separator = " ")) -} diff --git a/tests/hello/submissions/accepted/hello.py b/tests/hello/submissions/accepted/hello.py deleted file mode 100644 index 9016c9b6..00000000 --- a/tests/hello/submissions/accepted/hello.py +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env python3 - -print('Hello World!') diff --git a/tests/hello/submissions/accepted/hello_alarm.c b/tests/hello/submissions/accepted/hello_alarm.c deleted file mode 100644 index ba104baa..00000000 --- a/tests/hello/submissions/accepted/hello_alarm.c +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include -#include - -/* Based on the libc manual*/ - -/* This flag controls termination of the main loop. */ -volatile sig_atomic_t keep_going = 1; - -/* The signal handler just clears the flag and re-enables itself. */ -void catch_alarm (int sig) -{ - keep_going = 0; - signal (sig, catch_alarm); -} - -void do_nothing (void) -{ - int i=0; - for (i=0;i<1000;i+=1); -} - -int main (void) -{ - /* Establish a handler for SIGALRM signals. */ - signal (SIGALRM, catch_alarm); - /* Set an alarm to go off in a little while. */ - alarm (1); - /* Check the flag once in a while to see when to quit. */ - while (keep_going) - do_nothing(); - - printf("Hello World!\n"); - return EXIT_SUCCESS; -} diff --git a/tests/hello/submissions/run_time_error/memory_limit.cc b/tests/hello/submissions/run_time_error/memory_limit.cc deleted file mode 100644 index 6d858755..00000000 --- a/tests/hello/submissions/run_time_error/memory_limit.cc +++ /dev/null @@ -1,11 +0,0 @@ -#include -#include - -int main(void) { - char *buf = new char[512*1024*1024]; // 512MB - buf[0] = 0; - for (int i = 1; i < 512*1024*1024; ++i) - buf[i] = 23*buf[i-1]+42; - std::cout << "Hello World!\n" << std::endl; - return 0; -} diff --git a/tests/hello/submissions/wrong_answer/hello.cc b/tests/hello/submissions/wrong_answer/hello.cc deleted file mode 100644 index 80ab5c4f..00000000 --- a/tests/hello/submissions/wrong_answer/hello.cc +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main(void) { - printf("Hello!"); - return 0; -} diff --git a/tests/test_config.py b/tests/test_config.py index c9d60e5d..976dabaa 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,43 +1,44 @@ +from pathlib import Path + import pytest from problemtools import config +from tests.conftest import datadir -def config_paths_mock(): - from pathlib import Path - - return [Path(__file__).parent / 'config1', Path(__file__).parent / 'config2'] +def config_paths_mock() -> list[Path]: + return [datadir() / 'config1', datadir() / 'config2'] -def test_load_basic_config(monkeypatch): +def test_load_basic_config(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(config, '__config_file_paths', config_paths_mock) conf = config.load_config('test.yaml') assert conf == {'prop1': 'hello', 'prop2': 5} -def test_load_updated_config(monkeypatch): +def test_load_updated_config(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(config, '__config_file_paths', config_paths_mock) conf = config.load_config('test2.yaml') assert conf == {'prop1': 'abc', 'prop2': 23, 'prop3': ['hello', 'world']} -def test_load_missing_config(monkeypatch): +def test_load_missing_config(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(config, '__config_file_paths', config_paths_mock) with pytest.raises(config.ConfigError): config.load_config('non_existent_file') -def test_load_broken_config(monkeypatch): +def test_load_broken_config(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(config, '__config_file_paths', config_paths_mock) with pytest.raises(config.ConfigError): config.load_config('broken.yaml') -def test_update_dict(): +def test_update_dict() -> None: update_dict = config.__dict__['__update_dict'] dict1 = {'a': 1, 'b': {'sub1': 1, 'sub2': False}, 'c': 3} diff --git a/tests/test_default_validator.py b/tests/test_default_validator.py index 80e206a9..5500ff9e 100644 --- a/tests/test_default_validator.py +++ b/tests/test_default_validator.py @@ -35,7 +35,7 @@ def validator() -> Path: return VALIDATOR_PATH -def discover_test_cases(): +def discover_test_cases() -> list[Path]: """ Finds and returns a list of all test case directories. A test case directory is expected to start with 'test_'. @@ -46,7 +46,7 @@ def discover_test_cases(): @pytest.mark.parametrize('test_dir', discover_test_cases(), ids=lambda d: d.name) -def test_default_validator(validator: Path, test_dir: Path): +def test_default_validator(validator: Path, test_dir: Path) -> None: """ Runs a single validator test case. The test is parametrized to run for each directory discovered by `discover_test_cases`. @@ -90,4 +90,4 @@ def test_default_validator(validator: Path, test_dir: Path): # If no message is expected, assert that no message was generated. if judgemessage_path.is_file(): actual_message = judgemessage_path.read_bytes() - assert not actual_message, f'A validation message was generated but none was expected: {actual_message}' + assert not actual_message, f'A validation message was generated but none was expected: {actual_message!r}' diff --git a/tests/test_languages.py b/tests/test_languages.py index e0255562..a99eb993 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -6,11 +6,12 @@ import pytest from problemtools import languages +from tests.conftest import datadir class Language_test(TestCase): @staticmethod - def __language_dict(): + def __language_dict() -> dict: return { 'name': 'A Language', 'priority': 100, @@ -21,10 +22,10 @@ def __language_dict(): 'run': '{binary} {memlim}', } - def test_create(self): + def test_create(self) -> None: languages.Language('langid', self.__language_dict()) - def test_update(self): + def test_update(self) -> None: lang = languages.Language('langid', self.__language_dict()) lang.update({'priority': -1}) @@ -51,7 +52,7 @@ def test_update(self): lang.update({'run': 'newrun {mainclass}'}) assert lang.run == 'newrun {mainclass}' - def test_invalid_id(self): + def test_invalid_id(self) -> None: vals = self.__language_dict() with pytest.raises(TypeError): languages.Language(None, vals) # type: ignore @@ -64,25 +65,25 @@ def test_invalid_id(self): with pytest.raises(languages.LanguageConfigError): languages.Language('Capital', vals) - def test_missing_name(self): + def test_missing_name(self) -> None: vals = self.__language_dict() del vals['name'] with pytest.raises(languages.LanguageConfigError): languages.Language('id', vals) - def test_invalid_name(self): + def test_invalid_name(self) -> None: vals = self.__language_dict() vals['name'] = ['A List'] with pytest.raises(languages.LanguageConfigError): languages.Language('id', vals) - def test_missing_priority(self): + def test_missing_priority(self) -> None: vals = self.__language_dict() del vals['priority'] with pytest.raises(languages.LanguageConfigError): languages.Language('id', vals) - def test_invalid_priority(self): + def test_invalid_priority(self) -> None: vals = self.__language_dict() vals['priority'] = 2.3 with pytest.raises(languages.LanguageConfigError): @@ -91,49 +92,49 @@ def test_invalid_priority(self): with pytest.raises(languages.LanguageConfigError): languages.Language('id', vals) - def test_missing_files(self): + def test_missing_files(self) -> None: vals = self.__language_dict() del vals['files'] with pytest.raises(languages.LanguageConfigError): languages.Language('id', vals) - def test_invalid_files(self): + def test_invalid_files(self) -> None: vals = self.__language_dict() vals['files'] = ['*.cc', '*.cpp'] with pytest.raises(languages.LanguageConfigError): languages.Language('id', vals) - def test_without_shebang(self): + def test_without_shebang(self) -> None: vals = self.__language_dict() del vals['shebang'] del vals['shebang_files'] languages.Language('id', vals) - def test_invalid_shebang(self): + def test_invalid_shebang(self) -> None: vals = self.__language_dict() vals['shebang'] = '(Not an RE' with pytest.raises(re.error): languages.Language('id', vals) - def test_shebang_requires_shebang_files(self): + def test_shebang_requires_shebang_files(self) -> None: vals = self.__language_dict() del vals['shebang_files'] with pytest.raises(languages.LanguageConfigError): languages.Language('id', vals) - def test_shebang_files_requires_shebang(self): + def test_shebang_files_requires_shebang(self) -> None: vals = self.__language_dict() del vals['shebang'] with pytest.raises(languages.LanguageConfigError): languages.Language('id', vals) - def test_invalid_shebang_files(self): + def test_invalid_shebang_files(self) -> None: vals = self.__language_dict() vals['shebang_files'] = ['*.foo', '*.bar'] with pytest.raises(languages.LanguageConfigError): languages.Language('id', vals) - def test_get_source_files_does_not_require_readable_files(self): + def test_get_source_files_does_not_require_readable_files(self) -> None: """get_source_files only inspects file names -- unlike get_source_files_for_detection, it must work even for files that don't exist, since it's used on a program whose language is @@ -145,12 +146,12 @@ def test_get_source_files_does_not_require_readable_files(self): with pytest.raises(OSError): lang.get_source_files_for_detection([missing]) - def test_without_compile(self): + def test_without_compile(self) -> None: vals = self.__language_dict() del vals['compile'] languages.Language('id', vals) - def test_invalid_compile(self): + def test_invalid_compile(self) -> None: vals = self.__language_dict() vals['compile'] = ['gcc', '{files}'] with pytest.raises(languages.LanguageConfigError): @@ -159,13 +160,13 @@ def test_invalid_compile(self): with pytest.raises(languages.LanguageConfigError): languages.Language('id', vals) - def test_missing_run(self): + def test_missing_run(self) -> None: vals = self.__language_dict() del vals['run'] with pytest.raises(languages.LanguageConfigError): languages.Language('id', vals) - def test_invalid_run(self): + def test_invalid_run(self) -> None: vals = self.__language_dict() vals['run'] = ['python3', '{mainfile}'] with pytest.raises(languages.LanguageConfigError): @@ -174,7 +175,7 @@ def test_invalid_run(self): with pytest.raises(languages.LanguageConfigError): languages.Language('id', vals) - def test_good_entrypoints(self): + def test_good_entrypoints(self) -> None: vals = self.__language_dict() vals['compile'] = 'echo {binary}' @@ -189,7 +190,7 @@ def test_good_entrypoints(self): vals['run'] = 'echo {mainclass}' languages.Language('id', vals) - def test_bad_entrypoints(self): + def test_bad_entrypoints(self) -> None: vals = self.__language_dict() # Two different entry points @@ -204,8 +205,8 @@ def test_bad_entrypoints(self): languages.Language('id', vals) @staticmethod - def __subs(**overrides): - values = { + def __subs(**overrides: str | int) -> languages.CommandSubstitution: + values: dict[str, str | int] = { 'path': '/tmp/prog', 'files': 'main.foo', 'binary': '/tmp/prog/run', @@ -215,16 +216,16 @@ def __subs(**overrides): 'memlim': 256, } values.update(overrides) - return languages.CommandSubstitution(**values) + return languages.CommandSubstitution(**values) # type: ignore[arg-type] - def test_check_installed_ok(self): + def test_check_installed_ok(self) -> None: # compile is 'echo ...' (found on PATH); run is '{binary} {memlim}', # i.e. the program we just produced, so there's nothing external to # check there. lang = languages.Language('id', self.__language_dict()) assert lang.check_installed() is None - def test_check_installed_does_not_check_produced_entry_point(self): + def test_check_installed_does_not_check_produced_entry_point(self) -> None: # No compile step, and run is just the (self-contained) source file # -- no external program is needed at all. vals = self.__language_dict() @@ -233,7 +234,7 @@ def test_check_installed_does_not_check_produced_entry_point(self): lang = languages.Language('id', vals) assert lang.check_installed() is None - def test_check_installed_missing_compiler(self): + def test_check_installed_missing_compiler(self) -> None: vals = self.__language_dict() vals['compile'] = 'definitely_not_a_real_compiler_xyz {files} {binary}' lang = languages.Language('id', vals) @@ -242,13 +243,13 @@ def test_check_installed_missing_compiler(self): assert 'compiler' in msg assert 'definitely_not_a_real_compiler_xyz' in msg - def test_check_installed_missing_absolute_path(self): + def test_check_installed_missing_absolute_path(self) -> None: vals = self.__language_dict() vals['compile'] = '/nonexistent/path/to/compiler {files} {binary}' lang = languages.Language('id', vals) assert lang.check_installed() is not None - def test_check_installed_missing_runtime(self): + def test_check_installed_missing_runtime(self) -> None: # No compile step -- the runtime must still be checked. vals = self.__language_dict() del vals['compile'] @@ -259,19 +260,19 @@ def test_check_installed_missing_runtime(self): assert 'runtime' in msg assert 'definitely_not_a_real_runtime_xyz' in msg - def test_get_compile_command_none_without_compile_step(self): + def test_get_compile_command_none_without_compile_step(self) -> None: vals = self.__language_dict() del vals['compile'] lang = languages.Language('id', vals) assert lang.get_compile_command(self.__subs()) is None - def test_get_compile_command_resolves_executable(self): + def test_get_compile_command_resolves_executable(self) -> None: lang = languages.Language('id', self.__language_dict()) subs = self.__subs() command = lang.get_compile_command(subs) assert command == [shutil.which('echo'), subs.path, subs.files, subs.binary] - def test_get_run_command_resolves_executable(self): + def test_get_run_command_resolves_executable(self) -> None: vals = self.__language_dict() vals['compile'] = 'echo {mainfile}' vals['run'] = 'echo {mainfile}' @@ -279,14 +280,14 @@ def test_get_run_command_resolves_executable(self): subs = self.__subs() assert lang.get_run_command(subs) == [shutil.which('echo'), subs.mainfile] - def test_get_run_command_leaves_produced_binary_untouched(self): + def test_get_run_command_leaves_produced_binary_untouched(self) -> None: # run is '{binary} {memlim}' in the base dict -- the first token is # the program we just compiled, not something to resolve via PATH. lang = languages.Language('id', self.__language_dict()) subs = self.__subs() assert lang.get_run_command(subs) == [subs.binary, str(subs.memlim)] - def test_get_compile_command_splits_multiple_files(self): + def test_get_compile_command_splits_multiple_files(self) -> None: # {files} is a single space-separated string, but each file must # end up as its own argument rather than one combined string. lang = languages.Language('id', self.__language_dict()) @@ -295,20 +296,20 @@ def test_get_compile_command_splits_multiple_files(self): assert command == [shutil.which('echo'), subs.path, 'main.foo', 'helper1.bar', 'helper2.bar', subs.binary] -__EXAMPLES_PATH = os.path.join(os.path.dirname(__file__), 'languages_examples') +__EXAMPLES_PATH = datadir() / 'languages_examples' -def examples_path(test_file): +def examples_path(test_file: str) -> str: return os.path.join(__EXAMPLES_PATH, test_file) class Languages_test(TestCase): - def test_empty_languages(self): + def test_empty_languages(self) -> None: lang = languages.Languages() assert lang.languages == {} assert lang.detect_language(['foo.cpp', 'foo.c', 'foo.py', 'foo.java']) is None - def test_duplicate_prio(self): + def test_duplicate_prio(self) -> None: lang = languages.Languages() config = { 'c': { @@ -330,7 +331,7 @@ def test_duplicate_prio(self): with pytest.raises(languages.LanguageConfigError): lang.update(config) - def test_invalid_format(self): + def test_invalid_format(self) -> None: lang = languages.Languages() # Dict of strings instead of dict of dict conf1 = {'c': 'C'} @@ -355,16 +356,16 @@ def test_invalid_format(self): with pytest.raises(languages.LanguageConfigError): lang.update(conf1) with pytest.raises(languages.LanguageConfigError): - lang.update(conf2) + lang.update(conf2) # type: ignore[arg-type] with pytest.raises(languages.LanguageConfigError): - lang.update(conf3) + lang.update(conf3) # type: ignore[arg-type] - def test_empty(self): + def test_empty(self) -> None: lang = languages.Languages() lang.update({}) assert lang.languages == {} - def test_zoo(self): + def test_zoo(self) -> None: langs = languages.Languages() zoo = { @@ -383,15 +384,15 @@ def test_zoo(self): langs.update(zoo) lang = langs.detect_language([examples_path(x) for x in ['src1.zoo']]) - assert lang.lang_id == 'zoo' + assert lang is not None and lang.lang_id == 'zoo' lang = langs.detect_language([examples_path(x) for x in ['src2.zoo']]) - assert lang.lang_id == 'zoork' + assert lang is not None and lang.lang_id == 'zoork' lang = langs.detect_language([examples_path(x) for x in ['src2.zoo', 'src3.zpp']]) - assert lang.lang_id == 'zoopp' + assert lang is not None and lang.lang_id == 'zoopp' - def test_shebang_gate_only_affects_detection_not_actual_source_files(self): + def test_shebang_gate_only_affects_detection_not_actual_source_files(self) -> None: """Regression test: once a language has won detection, a helper file matching its "files" glob must not be dropped just because it individually lacks the shebang that broke the tie against another @@ -422,7 +423,7 @@ def test_shebang_gate_only_affects_detection_not_actual_source_files(self): # (unconditional, not gated) = 2. py3 evidence: py2main.py + # py2nosheb.py (both unconditional) = 2. Tie -> priority decides. lang = langs.detect_language(files) - assert lang.lang_id == 'py2' + assert lang is not None and lang.lang_id == 'py2' # All three files -- including the shebang-less helper -- belong # to the now-settled py2 program. diff --git a/tests/test_latex.py b/tests/test_latex.py index 41484a12..94e7a4fd 100644 --- a/tests/test_latex.py +++ b/tests/test_latex.py @@ -3,13 +3,15 @@ from pathlib import Path from problemtools import problem2html, problem2pdf +from problemtools.diagnostics import Diagnostics +from tests.conftest import example_directory -def test_pdf_render_verifyproblem(): +def test_pdf_render_verifyproblem() -> None: # Same options as in verifyproblem options = problem2pdf.get_parser().parse_args(['']) - problem_path = Path(__file__).parent / '..' / 'examples' / 'guess' - options.problem = str(problem_path.resolve()) + problem_path = example_directory('guess') + options.problem = str(problem_path) options.language = 'en' options.nopdf = True options.quiet = True @@ -17,26 +19,26 @@ def test_pdf_render_verifyproblem(): assert False, 'PDF conversion failed' -def test_pdf_render_problem2pdf(): +def test_pdf_render_problem2pdf() -> None: # Same options as typical problem2pdf usage with tempfile.TemporaryDirectory() as temp_dir: - problem_path = Path(__file__).parent / '..' / 'examples' / 'guess' + problem_path = example_directory('guess') temp_filename = Path(temp_dir) / 'guess.pdf' - options = problem2pdf.get_parser().parse_args(['-o', str(temp_filename), '-l', 'en', '-q', str(problem_path.resolve())]) + options = problem2pdf.get_parser().parse_args(['-o', str(temp_filename), '-l', 'en', '-q', str(problem_path)]) if not problem2pdf.convert(options): assert False, 'PDF conversion failed' with open(temp_filename, 'rb') as temp_file: assert temp_file.read(5) == b'%PDF-', 'Output header does not look like a PDF.' -def test_html_render_different(diag): +def test_html_render_different(diag: Diagnostics) -> None: # Same options as typical problem2html usage with tempfile.TemporaryDirectory() as temp_dir: - problem_path = Path(__file__).parent / '..' / 'examples' / 'different' - temp_dir = Path(temp_dir) / 'different_html' - options = problem2html.get_parser().parse_args(['-d', str(temp_dir), '-l', 'en', '-q', str(problem_path.resolve())]) + problem_path = example_directory('different') + out_dir = Path(temp_dir) / 'different_html' + options = problem2html.get_parser().parse_args(['-d', str(out_dir), '-l', 'en', '-q', str(problem_path)]) problem2html.convert(options, diag) - with open(temp_dir / 'index.html', 'r') as temp_file: + with open(out_dir / 'index.html', 'r') as temp_file: full_html = temp_file.read() assert re.search('', full_html) assert re.search('A Different Problem', full_html) @@ -45,14 +47,14 @@ def test_html_render_different(diag): assert re.search('71293781758123 72784', full_html) # part of sample -def test_html_render_guess(diag): +def test_html_render_guess(diag: Diagnostics) -> None: # Same options as typical problem2html usage with tempfile.TemporaryDirectory() as temp_dir: - problem_path = Path(__file__).parent / '..' / 'examples' / 'guess' - temp_dir = Path(temp_dir) / 'guess_html' - options = problem2html.get_parser().parse_args(['-d', str(temp_dir), '-l', 'en', '-q', str(problem_path.resolve())]) + problem_path = example_directory('guess') + out_dir = Path(temp_dir) / 'guess_html' + options = problem2html.get_parser().parse_args(['-d', str(out_dir), '-l', 'en', '-q', str(problem_path)]) problem2html.convert(options, diag) - with open(temp_dir / 'index.html', 'r') as temp_file: + with open(out_dir / 'index.html', 'r') as temp_file: full_html = temp_file.read() assert re.search('', full_html) assert re.search('Guess the Number', full_html) diff --git a/tests/test_markdown.py b/tests/test_markdown.py index 71e48521..d795e32b 100644 --- a/tests/test_markdown.py +++ b/tests/test_markdown.py @@ -1,22 +1,22 @@ -from pathlib import Path - import pytest +from problemtools.diagnostics import Diagnostics from problemtools.statement_util import find_footnotes +from tests.conftest import datadir from tests.test_xss import render, renderpdf # TODO: add when guess is updated to 2023-07 # def test_pdf_render(): # with tempfile.TemporaryDirectory() as temp_dir: -# problem_path = Path(__file__).parent / '..' / 'examples' / 'guess' +# problem_path = example_directory('guess') # args, _unknown = problem2pdf.get_parser().parse_known_args( # ['--problem', str(problem_path.resolve()), '-l', 'sv', '--dest-dir', str(temp_dir)] # ) # problem2pdf.convert(args) -def test_sample_escaping(diag): - problem_path = Path(__file__).parent / 'problems' / 'specialcharacterssample' +def test_sample_escaping(diag: Diagnostics) -> None: + problem_path = datadir() / 'problems' / 'specialcharacterssample' html = render(problem_path, diag) all_printable = r"""0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" # We escape &, < and > @@ -26,33 +26,33 @@ def test_sample_escaping(diag): assert all_printable in html -def test_footnotes(diag): +def test_footnotes(diag: Diagnostics) -> None: # We always want footnotes to be at the bottom # When we insert samples, we need to insert them right above the first footnote # To do this, we search for a string (very fragile) - problem_path = Path(__file__).parent / 'problems' / 'footnote' + problem_path = datadir() / 'problems' / 'footnote' html = render(problem_path, diag) assert find_footnotes(html) is not None - problem_path = Path(__file__).parent / 'problems' / 'twofootnotes' + problem_path = datadir() / 'problems' / 'twofootnotes' html = render(problem_path, diag) assert find_footnotes(html) is not None -def test_footnotes_href(diag): +def test_footnotes_href(diag: Diagnostics) -> None: # We use allowlist-based id values for footnotes. Ensure they have not changed - problem_path = Path(__file__).parent / 'problems' / 'footnote' + problem_path = datadir() / 'problems' / 'footnote' html = render(problem_path, diag) assert 'fn1' in html and 'fnref1' in html -def test_invalid_image_throws(diag): +def test_invalid_image_throws(diag: Diagnostics) -> None: # If images can point to img that doesn't exist, it's arbitrary web request for problem in ('imgrequest', 'imgrequest2'): - problem_path = Path(__file__).parent / 'problems' / problem + problem_path = datadir() / 'problems' / problem with pytest.raises(ValueError): render(problem_path, diag) # Pandoc won't make a web request for imgrequest2 with pytest.raises(ValueError): - renderpdf(Path(__file__).parent / 'problems' / 'imgrequest') + renderpdf(datadir() / 'problems' / 'imgrequest') diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 70a90e6c..7c12f70b 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -1,15 +1,16 @@ -from pathlib import Path +from typing import Any import pytest from pydantic import ValidationError from problemtools import metadata from problemtools.formatversion import FormatVersion +from tests.conftest import example_directory # A few quick tests of config parsing. pytest structure isn't great here, so code gets repetitive, but I wanted *something* basic in place at least. -def test_parse_empty_legacy(): +def test_parse_empty_legacy() -> None: m = metadata.parse_metadata(FormatVersion.LEGACY, {}) # Just check off a few random things assert not m.name @@ -17,18 +18,18 @@ def test_parse_empty_legacy(): assert not m.credits.authors -def test_parse_legacy_with_problem_names(): +def test_parse_legacy_with_problem_names() -> None: m = metadata.parse_metadata(FormatVersion.LEGACY, {}, {'en': 'Hello World!'}) assert m.name['en'] == 'Hello World!' -def test_parse_empty_2023_fails(): +def test_parse_empty_2023_fails() -> None: with pytest.raises(ValidationError): metadata.parse_metadata(FormatVersion.V_2023_07, {}, {'en': 'Hello World!'}) @pytest.fixture -def minimal_2023_conf(): +def minimal_2023_conf() -> dict[str, Any]: return { 'problem_format_version': '2023-07-draft', 'uuid': '46fa942f-44c3-46c0-8ddc-22e02d2e5d2b', @@ -36,21 +37,21 @@ def minimal_2023_conf(): } -def test_parse_minimal_2023(minimal_2023_conf): +def test_parse_minimal_2023(minimal_2023_conf: dict[str, Any]) -> None: m = metadata.parse_metadata(FormatVersion.V_2023_07, minimal_2023_conf, {'en': 'Hello World!'}) assert m.name['en'] == 'Hello World!' assert not m.source assert not m.credits.authors -def test_parse_typo_fails(minimal_2023_conf): +def test_parse_typo_fails(minimal_2023_conf: dict[str, Any]) -> None: c = minimal_2023_conf c['limits'] = {'typo': 1} with pytest.raises(ValidationError): metadata.parse_metadata(FormatVersion.V_2023_07, c, {'en': 'Hello World!'}) -def test_parse_single_author_2023(minimal_2023_conf): +def test_parse_single_author_2023(minimal_2023_conf: dict[str, Any]) -> None: c = minimal_2023_conf c['credits'] = ' \t Authy McAuth \t \t\t ' # Add some extra whitespace to check that we strip m = metadata.parse_metadata(FormatVersion.V_2023_07, c, {'en': 'Hello World!'}) @@ -59,7 +60,7 @@ def test_parse_single_author_2023(minimal_2023_conf): assert m.credits.authors[0].email == 'authy@mcauth.example' -def test_parse_single_source_2023(minimal_2023_conf): +def test_parse_single_source_2023(minimal_2023_conf: dict[str, Any]) -> None: c = minimal_2023_conf c['source'] = 'NWERC 2024' m = metadata.parse_metadata(FormatVersion.V_2023_07, c, {'en': 'Hello World!'}) @@ -68,7 +69,7 @@ def test_parse_single_source_2023(minimal_2023_conf): assert m.source[0].url is None -def test_parse_multi_source(minimal_2023_conf): +def test_parse_multi_source(minimal_2023_conf: dict[str, Any]) -> None: c = minimal_2023_conf c['source'] = [ {'name': 'NWERC 2024', 'url': 'https://2024.nwerc.example/contest'}, @@ -85,7 +86,7 @@ def test_parse_multi_source(minimal_2023_conf): assert m.source[2].url is None -def test_parse_complex_type(minimal_2023_conf): +def test_parse_complex_type(minimal_2023_conf: dict[str, Any]) -> None: c = minimal_2023_conf c['type'] = ['scoring', 'multi-pass', 'interactive'] m = metadata.parse_metadata(FormatVersion.V_2023_07, c, {'en': 'Hello World!'}) @@ -100,8 +101,8 @@ def test_parse_complex_type(minimal_2023_conf): assert not m.is_submit_answer() -def test_load_hello(): - m = metadata.load_metadata(Path(__file__).parent / 'hello') +def test_load_hello() -> None: + m = metadata.load_metadata(example_directory('hello')) assert m.name['en'] == 'Hello World!' assert m.name['sv'] == 'Hej Världen!' assert len(m.source) == 1 diff --git a/tests/test_output_validator.py b/tests/test_output_validator.py index 3562eb7f..17140d4d 100644 --- a/tests/test_output_validator.py +++ b/tests/test_output_validator.py @@ -6,7 +6,7 @@ from problemtools.judge.validate import _get_feedback -def test_output_validator_feedback(): +def test_output_validator_feedback() -> None: r = random.Random(0) with tempfile.TemporaryDirectory() as directory: feedback = pathlib.Path(directory) / 'feedback.txt' @@ -16,7 +16,7 @@ def test_output_validator_feedback(): assert data is not None and text in data -def test_output_validator_feedback_non_unicode(): +def test_output_validator_feedback_non_unicode() -> None: r = random.Random(0) with tempfile.TemporaryDirectory() as directory: feedback = pathlib.Path(directory) / 'feedback.txt' diff --git a/tests/test_run_limit.py b/tests/test_run_limit.py index 73985b7d..e99a019e 100644 --- a/tests/test_run_limit.py +++ b/tests/test_run_limit.py @@ -5,7 +5,7 @@ class Limit_test(TestCase): - def test_less(self): + def test_less(self) -> None: less = limit.__dict__['__limit_less'] assert less(42, 42) assert not less(42, 41) diff --git a/tests/test_score_range.py b/tests/test_score_range.py index a3e832e5..28ad4da2 100644 --- a/tests/test_score_range.py +++ b/tests/test_score_range.py @@ -3,6 +3,7 @@ from problemtools.checks.testdata import _check_score_range from problemtools.model.paths import abspath, relpath from problemtools.model.testdata import TestCase, TestDataGroup +from tests.conftest import RecordingDiagnostics # Not test classes -- just named that way by the model. Tell pytest not to collect them. TestCase.__test__ = False # type: ignore[attr-defined] @@ -49,7 +50,7 @@ def make_group( ) -def test_group_sum_is_default_aggregator(diag): +def test_group_sum_is_default_aggregator(diag: RecordingDiagnostics) -> None: group = make_group( 'g', [make_testcase('a'), make_testcase('b'), make_testcase('c')], range_='0 30', accept_score=10, reject_score=0 ) @@ -57,12 +58,12 @@ def test_group_sum_is_default_aggregator(diag): assert diag.messages == [] -def test_group_avg_aggregator(diag): +def test_group_avg_aggregator(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], grader_flags='avg', accept_score=10, reject_score=0) assert _check_score_range(group, False, diag) == (0, 10) -def test_avg_aggregator_with_on_reject_break_widens_range(diag): +def test_avg_aggregator_with_on_reject_break_widens_range(diag: RecordingDiagnostics) -> None: # subA (graded first) has a much wider range than subB. With on_reject: break, a non-AC subA # stops grading before subB is ever run, so the group's score is just subA's -- as high as # 100 -- even though grading both would pull the maximum average down to 55. @@ -73,7 +74,7 @@ def test_avg_aggregator_with_on_reject_break_widens_range(diag): assert diag.messages == [] -def test_avg_aggregator_with_on_reject_continue_is_tighter(diag): +def test_avg_aggregator_with_on_reject_continue_is_tighter(diag: RecordingDiagnostics) -> None: # Same as above, but without break: subB is always graded too, so avg is always over both. subA = make_group('g.subA', [make_testcase('a')], range_='0 100', accept_score=100, reject_score=0) subB = make_group('g.subB', [make_testcase('b')], range_='0 10', accept_score=10, reject_score=0) @@ -82,18 +83,18 @@ def test_avg_aggregator_with_on_reject_continue_is_tighter(diag): assert diag.messages == [] -def test_reject_above_accept(diag): +def test_reject_above_accept(diag: RecordingDiagnostics) -> None: # Corner case: check we don't end up with a broken range if reject_score is above accept_score group = make_group('g', [make_testcase('a')], accept_score=0, reject_score=5) assert _check_score_range(group, False, diag) == (0, 5) -def test_custom_scoring_possible_is_unbounded(diag): +def test_custom_scoring_possible_is_unbounded(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], accept_score=10, reject_score=0) assert _check_score_range(group, True, diag) == (-INF, INF) -def test_group_min_aggregator(diag): +def test_group_min_aggregator(diag: RecordingDiagnostics) -> None: group = make_group( 'g', [make_testcase('a'), make_testcase('b')], @@ -103,27 +104,27 @@ def test_group_min_aggregator(diag): assert _check_score_range(group, False, diag) == (0, 1) -def test_group_max_aggregator(diag): +def test_group_max_aggregator(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a'), make_testcase('b')], grader_flags='max') assert _check_score_range(group, False, diag) == (0, 1) -def test_last_aggregator_flag_wins(diag): +def test_last_aggregator_flag_wins(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], grader_flags='max min avg', accept_score=10, reject_score=0) assert _check_score_range(group, False, diag) == (0, 10) -def test_custom_grading_is_unbounded(diag): +def test_custom_grading_is_unbounded(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], grading='custom') assert _check_score_range(group, False, diag) == (-INF, INF) -def test_empty_group_scores_zero(diag): +def test_empty_group_scores_zero(diag: RecordingDiagnostics) -> None: group = make_group('g', []) assert _check_score_range(group, False, diag) == (0, 0) -def test_nested_groups_compose(diag): +def test_nested_groups_compose(diag: RecordingDiagnostics) -> None: subtask1 = make_group( 'g.subtask1', [make_testcase('a'), make_testcase('b')], @@ -138,14 +139,14 @@ def test_nested_groups_compose(diag): assert diag.messages == [] -def test_ignore_sample_at_root_skips_sample_group(diag): +def test_ignore_sample_at_root_skips_sample_group(diag: RecordingDiagnostics) -> None: sample = make_group('sample', [make_testcase('s')], accept_score=1000, reject_score=0) secret = make_group('secret', [make_testcase('a')], accept_score=100, reject_score=0) root = make_group('data', [sample, secret], grader_flags='ignore_sample', is_root=True) assert _check_score_range(root, False, diag) == (0, 100) -def test_ignore_sample_is_a_no_op_below_root(diag): +def test_ignore_sample_is_a_no_op_below_root(diag: RecordingDiagnostics) -> None: sample = make_group('sample', [make_testcase('s')], accept_score=1000, reject_score=0) secret = make_group('secret', [make_testcase('a')], accept_score=100, reject_score=0) # Misconfigured (checks._check_group flags this separately). We just aggregate all children @@ -156,13 +157,13 @@ def test_ignore_sample_is_a_no_op_below_root(diag): # --- Declared range vs. what can be inferred --- -def test_no_warning_when_declared_matches_computed(diag): +def test_no_warning_when_declared_matches_computed(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], range_='0 10', accept_score=10, reject_score=0) assert _check_score_range(group, False, diag) == (0, 10) assert diag.messages == [] -def test_looser_declared_range_warns_and_suggests_tightening(diag): +def test_looser_declared_range_warns_and_suggests_tightening(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], range_='0 100', accept_score=10, reject_score=0) assert _check_score_range(group, False, diag) == (0, 10) assert diag.messages == [ @@ -173,7 +174,7 @@ def test_looser_declared_range_warns_and_suggests_tightening(diag): ] -def test_no_declared_range_warns_and_suggests_one(diag): +def test_no_declared_range_warns_and_suggests_one(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], accept_score=10, reject_score=0) # range left at the default assert _check_score_range(group, False, diag) == (0, 10) assert diag.messages == [ @@ -187,7 +188,7 @@ def test_no_declared_range_warns_and_suggests_one(diag): ] -def test_narrower_declared_range_is_trusted_without_warning(diag): +def test_narrower_declared_range_is_trusted_without_warning(diag: RecordingDiagnostics) -> None: # A "bad guarantee": the group's own children can clearly reach 100, but the author declared a # narrower range. We don't warn -- that's a promise checked elsewhere (checks.submissions) -- # but we do trust it for the returned (propagated) value. @@ -196,7 +197,7 @@ def test_narrower_declared_range_is_trusted_without_warning(diag): assert diag.messages == [] -def test_narrower_declared_range_propagates_to_parent(diag): +def test_narrower_declared_range_propagates_to_parent(diag: RecordingDiagnostics) -> None: # The parent's aggregate must reflect the child's effective (trusted) range, not its raw # aggregate -- so a narrow declaration deep in the tree is reflected in ancestors' results too. bad_child = make_group('g.secret', [make_testcase('a')], range_='0 10', accept_score=100, reject_score=0) @@ -214,7 +215,7 @@ def test_narrower_declared_range_propagates_to_parent(diag): ] -def test_disjoint_declared_range_warns_distinctly(diag): +def test_disjoint_declared_range_warns_distinctly(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], range_='0 10', accept_score=100, reject_score=50) assert _check_score_range(group, False, diag) == (0, 10) assert diag.messages == [ @@ -225,7 +226,7 @@ def test_disjoint_declared_range_warns_distinctly(diag): ] -def test_root_with_unbounded_aggregate_and_no_declared_range_still_warns(diag): +def test_root_with_unbounded_aggregate_and_no_declared_range_still_warns(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], grading='custom', is_root=True) assert _check_score_range(group, False, diag) == (-INF, INF) assert diag.messages == [ @@ -240,13 +241,13 @@ def test_root_with_unbounded_aggregate_and_no_declared_range_still_warns(diag): ] -def test_non_root_with_unbounded_aggregate_and_no_declared_range_is_silent(diag): +def test_non_root_with_unbounded_aggregate_and_no_declared_range_is_silent(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], grading='custom', is_root=False) assert _check_score_range(group, False, diag) == (-INF, INF) assert diag.messages == [] -def test_root_with_negative_minimum_and_no_other_issue_warns(diag): +def test_root_with_negative_minimum_and_no_other_issue_warns(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], accept_score=10, reject_score=-5, range_='-5 10', is_root=True) assert _check_score_range(group, False, diag) == (-5, 10) assert diag.messages == [ @@ -260,14 +261,14 @@ def test_root_with_negative_minimum_and_no_other_issue_warns(diag): ] -def test_negative_minimum_below_root_is_not_flagged(diag): +def test_negative_minimum_below_root_is_not_flagged(diag: RecordingDiagnostics) -> None: # negative scores for non-root test groups is a bit weird, but IMHO not weird enough to warn about group = make_group('g', [make_testcase('a')], accept_score=10, reject_score=-5, range_='-5 10', is_root=False) assert _check_score_range(group, False, diag) == (-5, 10) assert diag.messages == [] -def test_negative_minimum_at_root_is_suppressed_by_other_warnings(diag): +def test_negative_minimum_at_root_is_suppressed_by_other_warnings(diag: RecordingDiagnostics) -> None: # Warning about a negative range is low priority. If we can instead suggest to narrow the range, # that's typically a better warning. group = make_group('g', [make_testcase('a')], accept_score=10, reject_score=0, range_='-10 200', is_root=True) @@ -276,14 +277,14 @@ def test_negative_minimum_at_root_is_suppressed_by_other_warnings(diag): assert 'looser than the computed range' in diag.messages[0][1] -def test_invalid_range_format_errors_and_falls_back_to_aggregate(diag): +def test_invalid_range_format_errors_and_falls_back_to_aggregate(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], range_='not a range', accept_score=10, reject_score=0) assert _check_score_range(group, False, diag) == (0, 10) assert diag.errors == 1 assert "Invalid format 'not a range'" in diag.messages[0][1] -def test_min_greater_than_max_errors_and_falls_back_to_aggregate(diag): +def test_min_greater_than_max_errors_and_falls_back_to_aggregate(diag: RecordingDiagnostics) -> None: group = make_group('g', [make_testcase('a')], range_='10 0', accept_score=10, reject_score=0) assert _check_score_range(group, False, diag) == (0, 10) assert diag.errors == 1 diff --git a/tests/test_verify_hello.py b/tests/test_verify_hello.py index 95a48935..976c7d4a 100644 --- a/tests/test_verify_hello.py +++ b/tests/test_verify_hello.py @@ -1,18 +1,11 @@ -import logging -import pathlib - from problemtools import checks, model -from problemtools.diagnostics import LoggingDiagnostics - +from problemtools.diagnostics import Diagnostics +from tests.conftest import example_directory -def _make_diag(shortname: str) -> LoggingDiagnostics: - return LoggingDiagnostics.create(shortname, log_level=logging.WARNING) +def test_load_hello(diag: Diagnostics) -> None: + probdir = example_directory('hello') -def test_load_hello(): - probdir = (pathlib.Path(__file__).parent / 'hello').resolve() - - diag = _make_diag('hello') problem = model.load_problem(probdir, diag) assert problem.shortname == 'hello' @@ -27,9 +20,13 @@ def test_load_hello(): assert not problem.metadata.is_multi_pass() assert not problem.metadata.is_submit_answer() + assert len(problem.testdata.get_all_testcases()) == 1, 'Hello should have exactly 1 test case' + assert problem.graders.grader is None, 'Hello uses the default grader' + assert problem.output_validators.uses_default(problem.format_version, problem.metadata), 'Hello uses the default validator' + -def test_load_twice(): - probdir = (pathlib.Path(__file__).parent / 'hello').resolve() +def test_load_twice(diag: Diagnostics) -> None: + probdir = example_directory('hello') - model.load_problem(probdir, _make_diag('hello')) - model.load_problem(probdir, _make_diag('hello')) + model.load_problem(probdir, diag) + model.load_problem(probdir, diag) diff --git a/tests/test_xss.py b/tests/test_xss.py index 597ab468..5a6ce556 100644 --- a/tests/test_xss.py +++ b/tests/test_xss.py @@ -3,9 +3,11 @@ from pathlib import Path from problemtools import problem2html, problem2pdf +from problemtools.diagnostics import Diagnostics +from tests.conftest import datadir -def render(problem_path, diag): +def render(problem_path: Path, diag: Diagnostics) -> str: with tempfile.TemporaryDirectory() as temp_dir: args, _unknown = problem2html.get_parser().parse_known_args( ['--problem', str(problem_path.resolve()), '--dest-dir', str(temp_dir)] @@ -16,26 +18,26 @@ def render(problem_path, diag): return html -def renderpdf(problem_path): +def renderpdf(problem_path: Path) -> None: with tempfile.TemporaryDirectory() as temp_dir: outpath = os.path.join(temp_dir, 'out.pdf') args, _unknown = problem2pdf.get_parser().parse_known_args(['--problem', str(problem_path.resolve()), '--o', outpath]) problem2pdf.convert(args) -def test_no_xss_statement(diag): - problem_path = Path(__file__).parent / 'problems' / 'statementxss' +def test_no_xss_statement(diag: Diagnostics) -> None: + problem_path = datadir() / 'problems' / 'statementxss' html = render(problem_path, diag) assert 'alert' not in html -def test_no_xss_problemname(diag): - problem_path = Path(__file__).parent / 'problems' / 'problemnamexss' +def test_no_xss_problemname(diag: Diagnostics) -> None: + problem_path = datadir() / 'problems' / 'problemnamexss' html = render(problem_path, diag) assert '