diff --git a/.github/ISSUE_TEMPLATE/FeatureRequestTemplate.md b/.github/ISSUE_TEMPLATE/FeatureRequestTemplate.md new file mode 100644 index 0000000..fc9ea7f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/FeatureRequestTemplate.md @@ -0,0 +1,49 @@ +--- +name: Feature Request +about: Suggest a new feature for this project +title: '[FEATURE] ' +labels: enhancement +assignees: '' + +--- + +## Feature Description + + + +## Value Statement + + +**As a** [type of user] +**I want** [goal/desire] +**So that** [benefit/value] + + + + +## Acceptance Criteria + + +- [ ] +- [ ] +- [ ] + + + + + diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index e5e4889..9403646 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -17,11 +17,11 @@ jobs: python: '3.13' tox_env: 'docs' os: 'ubuntu-latest' - - name: 'py311 (ubuntu)' - python: '3.11' - python_arch: 'x64' - tox_env: 'py311' - os: 'ubuntu-latest' +# - name: 'py311 (ubuntu)' +# python: '3.11' +# python_arch: 'x64' +# tox_env: 'py311' +# os: 'ubuntu-latest' - name: 'py312 (ubuntu)' python: '3.12' python_arch: 'x64' @@ -32,16 +32,16 @@ jobs: python_arch: 'x64' tox_env: 'py313' os: 'ubuntu-latest' - - name: 'pypy310 (ubuntu)' - python: 'pypy-3.10' - python_arch: 'x64' - tox_env: 'pypy310' - os: 'ubuntu-latest' - - name: 'pypy311 (ubuntu)' - python: 'pypy-3.11' - python_arch: 'x64' - tox_env: 'pypy311' - os: 'ubuntu-latest' +# - name: 'pypy310 (ubuntu)' +# python: 'pypy-3.10' +# python_arch: 'x64' +# tox_env: 'pypy310' +# os: 'ubuntu-latest' +# - name: 'pypy311 (ubuntu)' +# python: 'pypy-3.11' +# python_arch: 'x64' +# tox_env: 'pypy311' +# os: 'ubuntu-latest' steps: - uses: actions/checkout@v5 with: diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 9cb9f57..227871e 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -5,3 +5,4 @@ Reference :glob: agh* + pytest_plugin* diff --git a/docs/reference/pytest_plugin.rst b/docs/reference/pytest_plugin.rst new file mode 100644 index 0000000..7b6c6d0 --- /dev/null +++ b/docs/reference/pytest_plugin.rst @@ -0,0 +1,11 @@ +agh.pytest_plugin +=== + +.. testsetup:: + + from agh.pytest_plugin import * + +.. automodule:: agh.pytest_plugin + :members: + :undoc-members: + :show-inheritance: diff --git a/pyproject.toml b/pyproject.toml index 1154749..d274f51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,13 +22,13 @@ classifiers = [ "Intended Audience :: Developers", "Operating System :: Unix", "Operating System :: POSIX", - "Operating System :: Microsoft :: Windows", + # "Operating System :: Microsoft :: Windows", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", # "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", + # "Programming Language :: Python :: 3.10", + # "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", @@ -42,11 +42,12 @@ classifiers = [ keywords = [ # eg: "keyword1", "keyword2", "keyword3", ] -requires-python = ">=3.9" +requires-python = ">=3.12" dependencies = [ "pytest>=8.4.2", "pytest-shell-utilities>=1.9.7", "rich>=14.2.0", + "ipython>=9.6.0", "argcomplete", "rich-argparse", ] diff --git a/src/agh/agh_data.py b/src/agh/agh_data.py index 7b7889e..419c526 100644 --- a/src/agh/agh_data.py +++ b/src/agh/agh_data.py @@ -2,6 +2,7 @@ import json import os import pathlib +from collections.abc import Callable from collections.abc import Generator from dataclasses import asdict from dataclasses import dataclass @@ -12,15 +13,24 @@ from pathlib import Path from string import Template from typing import Any +from typing import Literal from typing import Self from typing import get_args from typing import get_origin import agh.anonymizer as anonymizer +DEFAULT_MAX_OUT_FILE_SIZE = 20 * 1024 + +META_INTERNAL_SUB_OUTPUT = "OUTPUT_INFO" + META_INTERNAL_SUB_KEY = "SUBMISSION" META_AGH_INTERNAL_KEY = "AGH_INTERNAL" +META_INTERNAL_SUB_KEYS = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY] +META_INTERNAL_SUB_OUTPUT_COMPLETE = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "COMPLETED_OUTPUT"] +META_INTERNAL_SUB_OUTPUT_GRADED = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "GRADED"] +META_INTERNAL_SUB_OUTPUT_NON_ANON = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "NON_ANON"] _USER_DEFAULTS_FILE = Path.home() / ".config" / "agh" / ".agh_user_defaults.json" @@ -161,14 +171,24 @@ def sectionAttr(self) -> str: return " {.unlisted .unnumbered}" return "" - def asQmdSection(self, heading_level: int) -> str: + def asQmdSection(self, heading_level: int, max_file_size: int = DEFAULT_MAX_OUT_FILE_SIZE) -> str: if not self.include_in_output: return "" - if not self.path.exists(): - self.path.touch() + # if not self.path.exists(): + # self.path.touch() + # Check for excessive length. hdr_txt = "#" * heading_level + " " + self.title + self.sectionAttr - return f"\n\n{self.description}\n\n{hdr_txt}\n\n```{{.{self.type}}}\n{{{{< include {self.path} >}}}}\n```\n\n" + ret_val = f"\n\n{hdr_txt}\n\n{self.description}\n\n" + if self.path.exists() and self.path.stat().st_size > max_file_size: + with self.path.open("rt", encoding="ascii", errors="backslashreplace") as f: + ret_val += ( + f"**[File too large! Contents truncated to {max_file_size} bytes.]{{.mark}}**\n\n```{{." + f"{self.type}}}\n{f.read(max_file_size)}\n```\n\n" + ) + return ret_val + + return f"{ret_val}```{{.{self.type}}}\n{{{{< include {self.path} >}}}}\n```\n\n" # Mark all fields as keyword-only so that we can load directly from JSON. @@ -181,10 +201,14 @@ class OutputSectionData(SubmissionFileData): included_sections: list["OutputSectionData"] = field(default_factory=list) only_output_if_data: bool = False post_script: str = "" + _errors: list[dict[Literal["title", "message"], str]] = field(default_factory=list) + _warnings: list[dict[Literal["title", "message"], str]] = field(default_factory=list) @property def hasData(self): - if len(self.included_sections) > 0 or len(self.included_files) > 0: + if len(self._errors) > 0 or len(self._warnings) > 0: + return True + elif len(self.included_sections) > 0 or len(self.included_files) > 0: return True in [cur_sub_sec.hasData for cur_sub_sec in self.included_sections] or True in [ (cur_inc_file.path.exists() and cur_inc_file.path.stat().st_size > 1) for cur_inc_file in self.included_files ] @@ -198,9 +222,35 @@ def asQmdSection(self) -> str: return "" inc_file_txt = "\n\n".join([cur_inc_file.asQmdSection(self.heading_level + 1) for cur_inc_file in self.included_files]) inc_sec_txt = "\n\n".join([cur_inc_sec.asQmdSection() for cur_inc_sec in self.included_sections]) - pre = "#" * self.heading_level + f" {self.title} {self.sectionAttr}\n\n{self.description}\n\n{self.text}" + + warnings = "" + for warn in self._warnings: + warnings += f"\n\n+ **{warn['title']}:** {warn['message']}" + if warnings != "": + warnings = f'\n\n::: {{.callout-note title="To Consider"}}\n\n{warnings}\n\n:::\n\n' + + errors = "" + for err in self._errors: + errors += f"\n\n+ **{err['title']}:** {err['message']}" + if errors != "": + errors = f'\n\n::: {{.callout-important title="Errors"}}\n\n{errors}\n\n:::\n\n' + + pre = "#" * self.heading_level + (f" {self.title} {self.sectionAttr}\n\n{self.description}\n\n{self.text}{errors}{warnings}") return f"{pre}\n\n{inc_file_txt}\n\n{inc_sec_txt}\n\n{self.post_script}" + def addSection(self, section: "OutputSectionData") -> Self: + self.included_sections.append(section) + section.heading_level = self.heading_level + 1 + return self + + def addWarning(self, warn_title: str, warn_msg: str) -> Self: + self._errors.append({"title": warn_title, "message": warn_msg}) + return self + + def addError(self, error_title: str, error_msg: str) -> Self: + self._errors.append({"title": error_title, "message": error_msg}) + return self + @dataclass(kw_only=True) class MetaDataclassJson(DataclassJson): @@ -226,7 +276,7 @@ def _getMetadata(self, metadata_key: str, default: Any = None) -> dict[str, Any] return default return cur_metadata - def getMetadata(self, *args, default: Any = None) -> dict[str, Any]: + def getMetadata(self, *args, default: Any = None) -> dict[str, Any] | Any: """Returns the metadata associated with the assignment. If the key is not present in the metadata, it will return the default value. @@ -627,20 +677,27 @@ def name(self, value): @property def Submissions(self): for submission_file in self._directory.rglob(Submission.SUBMISSION_FILE_NAME): - yield Submission.load(submission_file) + try: + yield Submission.load(submission_file) + except Exception as e: + print(f"Error loading submission file {submission_file}: {e}") + continue class LinkProto(IntEnum): # Raise an error if the file exists. RAISE_ERROR = 0 # Ignore an error and continue with the rest of the links if the file exists (don't do anything). - IGNORE_ERROR = 1 + SKIP_FILE = 1 # Overwrite the existing file if it already exists. LINK_OVERWRITE = 2 def PostProcessSubmission( - self, submission_file: "pathlib.Path|Submission", exists_protocol: LinkProto = LinkProto.RAISE_ERROR + self, + submission_file: "pathlib.Path|Submission", + exists_protocol: LinkProto = LinkProto.RAISE_ERROR, + warning_callback: Callable[[str], None | Any] | None = None, ) -> "Submission": - ret_val = submission_file + ret_val: Submission = submission_file if isinstance(submission_file, pathlib.Path): ret_val = Submission.load(filepath=submission_file) @@ -653,7 +710,10 @@ def all_linked_files() -> Generator[Path]: yield link_item for link_item in self._optional_files.values(): if link_item.copy_to_sub_if_missing and not (ret_val.evaluation_directory / link_item.path.name).exists(): - yield link_item.path + ret_path = link_item.path + if not ret_path.exists(): + ret_path = self.link_template_dir / link_item.path.name + yield ret_path for link_item in all_linked_files(): link_tgt = ret_val.evaluation_directory / link_item.name @@ -663,55 +723,88 @@ def all_linked_files() -> Generator[Path]: link_item = link_item.readlink() # Depending on the protocol handle if there is already an existing link. - if link_tgt.exists(): + if link_tgt.exists(follow_symlinks=False): # Handle if the link exists but is pointing to the correct target already - continue. if link_tgt.is_symlink() and link_tgt.readlink() == link_item: - continue - - match exists_protocol: - case self.LinkProto.RAISE_ERROR: - raise FileExistsError(link_tgt) - case self.LinkProto.IGNORE_ERROR: - continue - case self.LinkProto.LINK_OVERWRITE: - link_tgt.unlink() - case _: - raise NotImplementedError("New existing link protocol added, but code not added") + link_tgt.unlink(missing_ok=True) + else: + match exists_protocol: + case self.LinkProto.RAISE_ERROR: + raise FileExistsError(link_tgt) + case self.LinkProto.SKIP_FILE: + continue + case self.LinkProto.LINK_OVERWRITE: + link_tgt.unlink(missing_ok=True) + case _: + raise NotImplementedError("New existing link protocol added, but code not added") link_tgt.symlink_to(link_item.absolute(), target_is_directory=link_item.is_dir()) # Now start linking the output stuff together. + return self.postProcessSubmissionRender(ret_val, warning_callback=warning_callback) + + def postProcessSubmissionRender( + self, submission: "Submission", warning_callback: Callable[[str], None | Any] | None = None + ) -> "Submission": for output_file in self._options.output_files: - output_in_sub_dir = ret_val.evaluation_directory / output_file - output_file = self.complete_eval_dir / (ret_val.evaluation_directory.name + output_in_sub_dir.suffix) - output_not_anon = self.d2l_named_dir / (ret_val.original_name + output_in_sub_dir.suffix) + output_in_sub_dir = submission.evaluation_directory / output_file + output_file = self.complete_eval_dir / (submission.evaluation_directory.name + output_in_sub_dir.suffix) + submission.setMetadata(*META_INTERNAL_SUB_OUTPUT_COMPLETE, value=str(output_file)) + output_not_anon = self.d2l_named_dir / (submission.original_name + output_in_sub_dir.suffix) + submission.setMetadata(*META_INTERNAL_SUB_OUTPUT_NON_ANON, value=str(output_not_anon)) output_graded = self.graded_output_dir / output_file.name + submission.setMetadata(*META_INTERNAL_SUB_OUTPUT_GRADED, value=str(output_graded)) try: + # Copy output_in_sub_dir to output_graded if needed + if (not output_graded.exists()) or ( + output_graded.exists() and output_graded.stat().st_ctime_ns == output_graded.stat().st_mtime_ns + ): + output_graded.parent.mkdir(parents=True, exist_ok=True) + + # This is to ensure that ctime == mtime! + output_graded.unlink(missing_ok=True) + + output_graded.write_bytes(output_in_sub_dir.read_bytes()) + elif warning_callback is not None: + warning_callback( + f'The graded output file "{output_graded}" already exists and appears modified. ' + f"{output_graded} will not be overwritten." + ) + # stats = output_graded.stat() + # warning_callback(f'The graded output file info a:{stats.st_atime_ns}, c: {stats.st_ctime_ns}, + # m: {stats.st_mtime_ns}.') + output_file.unlink(missing_ok=True) output_not_anon.unlink(missing_ok=True) except FileNotFoundError: pass + # Create symbolic links output_file.symlink_to( output_in_sub_dir.relative_to(output_file.parent, walk_up=True), target_is_directory=output_file.is_dir() ) + # output_file.chmod(0o444) + output_not_anon.symlink_to( output_graded.relative_to(output_not_anon.parent, walk_up=True), target_is_directory=output_not_anon.is_dir() ) - return ret_val + return submission - def AddSubmission(self, submission_file: pathlib.Path, override_anon: bool | None = None) -> "Submission": + def AddSubmission( + self, submission_file: pathlib.Path, override_anon: bool | None = None, warning_callback: Callable[[str], None | Any] | None = None + ) -> "Submission": """Add a new submission to the assignment. :param submission_file: The path to the submission file to add. :param override_anon: If none then abide by the default for the assignment. If True then make it anonymous even if assignment is default non-anonymous. If False then make it non-anonymous even if assignment is default anonymous. + :param warning_callback: A callback function to be called when a warning is encountered. :return: The new submission. :rtype: "Submission" """ ret_val = Submission.new(self, submission_file=submission_file, override_anon=override_anon) ret_val.save() - return self.PostProcessSubmission(ret_val, exists_protocol=self.LinkProto.RAISE_ERROR) + return self.PostProcessSubmission(ret_val, exists_protocol=self.LinkProto.RAISE_ERROR, warning_callback=warning_callback) # def __pytest_cmd(self): # return "pytest ./ -p shell-utilities -p agh" @@ -991,7 +1084,8 @@ def new(cls, assignment: Assignment, submission_file: pathlib.Path, override_ano base_file_name = file_name base_file_name_set = True case _: - anon_name = submission_file.stem + anon_name = submission_file.with_suffix("").name + anon_name = anon_name.replace(" ", "_") evaluation_directory = assignment.eval_dir / anon_name evaluation_directory.mkdir(exist_ok=True, parents=True) @@ -1081,23 +1175,54 @@ def name(self): return self.anon_name @property - def main_output_file(self) -> Path | None: - """Check if the submission has output files.""" + def main_output_files(self) -> list[Path | None]: + """Return the submission's output files. + :return: A list of output files as follows: + ``[, + , , + ]`` + """ assign = Assignment.load() + rendered = None for output_file in assign._options.output_files: output_file = self.evaluation_directory / output_file - if not output_file.exists(): - return None - else: - return output_file - return None + if output_file.exists(): + rendered = output_file + + main = self.getMetadata(*META_INTERNAL_SUB_OUTPUT_COMPLETE) + graded = self.getMetadata(*META_INTERNAL_SUB_OUTPUT_GRADED) + non_anon: None | str | Path = self.getMetadata(*META_INTERNAL_SUB_OUTPUT_NON_ANON) + + if main is not None: + main = Path(main) + if graded is not None: + graded = Path(graded) + if non_anon is not None: + non_anon = Path(non_anon) + + return [rendered, main, graded, non_anon] + + # The next three methods have to do with getting, setting, and clearing errors or warnings. + def _getErrWarnList(self, type: Literal["errors", "warnings"]) -> list[str]: + return list(self.getMetadata(*META_INTERNAL_SUB_KEYS, type, default={}).values()) + + def _setErrWarnItem(self, type: Literal["errors", "warnings"], key: str, txt_or_markdown: str) -> Self: + keys = list(META_INTERNAL_SUB_KEYS) + keys.append(type) + keys.append(key) + return self.setMetadata(*keys, value=txt_or_markdown) + + def _delErrWarnItem(self, type: Literal["errors", "warnings"], key: str) -> Self: + exist_md_dict: dict[Any, Any] = self.getMetadata(*META_INTERNAL_SUB_KEYS, type, default={}) + exist_md_dict.pop(key, None) + return self.setMetadata(*META_INTERNAL_SUB_KEYS, type, value=exist_md_dict) @property def errors(self) -> None | list[str]: """Check if the submission has errors. These are NOT testing errors, but anything preventing the submission from being tested. """ - errors: list[str] = self.getMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "errors", default=[]) + errors: list[str] = self._getErrWarnList("errors") if not self.as_submitted_dir.exists(): errors.append(f"Submission directory '{self.as_submitted_dir.absolute()}' does not exist.") return errors @@ -1111,27 +1236,32 @@ def errors(self) -> None | list[str]: return None - def addError(self, txt_or_markdown: str) -> "Submission": + def addError(self, key: str, txt_or_markdown: str) -> "Submission": """Add an error to the submission. These are NOT testing errors, but anything preventing the submission from being tested. """ # DON'T USE the property above. It adds transient errors. Just get the metadata and add to it. - errors: list[str] = self.getMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "errors", default=[]) - errors.append(txt_or_markdown) - return self.setMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "errors", value=errors) + return self._setErrWarnItem("errors", key, txt_or_markdown) + # errors: list[str] = self.getMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "errors", default=[]) + # errors.append(txt_or_markdown) + # return self.setMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "errors", value=errors) + + def delError(self, key: str) -> "Submission": + return self._delErrWarnItem("errors", key) @property def warnings(self) -> None | list[str]: """Check if the submission has warnings. These are NOT testing warnings, but anything possibly preventing the submission from being tested. """ - return self.getMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "warnings") + return self._getErrWarnList("warnings") - def addWarning(self, txt_or_markdown: str) -> "Submission": + def addWarning(self, key: str, txt_or_markdown: str) -> "Submission": """Add a warning to the submission. These are NOT testing warnings, but anything possibly preventing the submission from being tested. """ - warnings = self.warnings - warnings.append(txt_or_markdown) - return self.setMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "warnings", value=warnings) + return self._setErrWarnItem("warnings", key, txt_or_markdown) + + def delWarning(self, key: str) -> "Submission": + return self._delErrWarnItem("warnings", key) diff --git a/src/agh/cli.py b/src/agh/cli.py index b442211..9c9fea0 100644 --- a/src/agh/cli.py +++ b/src/agh/cli.py @@ -22,9 +22,11 @@ import functools import re import sys +from asyncio import CancelledError from dataclasses import dataclass from dataclasses import field from pathlib import Path +from typing import Literal from urllib import parse import argcomplete @@ -92,7 +94,7 @@ def __call__(self, parser, namespace, values, option_string=None): for name, cur_sub_parser in all_sub_parsers: # console.print(Panel(cur_sub_parser.format_help(),title=f"[bold]Subcommand: {name}", expand=False, # style="b", )) - console.rule(('>' * 5) + f" [b red]{name}[/] ", align='left') + console.rule((">" * 5) + f" [b red]{name}[/] ", align="left") cur_sub_parser.print_help() parser.exit(0) @@ -121,8 +123,7 @@ def SubFileCompleter(property: str, prefix: str, **kwargs): def submissionCompleter(*args, **kwargs): # return ['Bob','Tom'] try: - ret_val = [str(subm.evaluation_directory.resolve().relative_to(Path.cwd(), walk_up=True)) for subm in - Assignment.load().Submissions] + ret_val = [str(subm.evaluation_directory.resolve().relative_to(Path.cwd(), walk_up=True)) for subm in Assignment.load().Submissions] return ret_val except Exception as e: argcomplete.warn("No assignment found. Cannot complete submission files.") @@ -133,11 +134,28 @@ def submissionCompleter(*args, **kwargs): # console.log(SubFileCompleter("unprocessed_dir", '')) parser = MyArgParser( - description="agh --- Assignment Grading Helper", prog="agh", formatter_class=RichHelpFormatter, - conflict_handler="resolve" + description="agh --- Assignment Grading Helper", prog="agh", formatter_class=RichHelpFormatter, conflict_handler="resolve" ) parser.add_argument("--version", action="version", version=__version__) parser.add_argument("-H", "--full-help", action=FullHelp, help="Show full (all options) help") +parser.add_argument( + "-D", + "--debug-core-files", + action="store_true", + dest="debug_core_files", + help="This instructs the OS to store core files such that debugging information can be gleaned " + "from crashes. [b red u]THIS MUST BE CALLED WITH ROOT PERMISSIONS.[/b red u]Ex: `sudo agh -D`", + default=False, +) +parser.add_argument( + "--restore-default-core-location", + action="store_true", + dest="restore_default_core_location", + help="This restores the default core location after calling `sudo agh -D`. [b red u]THIS MUST BE " + "CALLED WITH ROOT PERMISSIONS.[/b red u]Ex: `sudo agh --restore-default-core-location`", + default=False, +) + subparsers = parser.add_subparsers(dest="command", help="Assignment/Submission/etc. commands") @@ -155,22 +173,16 @@ def submissionCompleter(*args, **kwargs): ################################################################################ ################################################################################ -assignment_sub_parser = subparsers.add_parser("assignment", help="Assignment commands", - formatter_class=RichHelpFormatter) +assignment_sub_parser = subparsers.add_parser("assignment", help="Assignment commands", formatter_class=RichHelpFormatter) assignment_subparsers = assignment_sub_parser.add_subparsers(dest="assignment_command", help="Assignment commands") # assignment > new assignment command -assignment_new_parser = assignment_subparsers.add_parser("new", help="Create new assignment", - formatter_class=RichHelpFormatter) -assignment_new_parser.add_argument("name", help="Assignment name", type=str).completer = lambda **kwargs: [ - f"Assignment {Path.cwd().name}"] -assignment_new_parser.add_argument("course", help="Course code", type=str).completer = lambda **kwargs: [ - f"CSCI-{Path.cwd().parent.name}"] -assignment_new_parser.add_argument("term", help="Term", - choices=["Fall", "Spring", "Maymester", "Summer I", "Summer II"], type=str) +assignment_new_parser = assignment_subparsers.add_parser("new", help="Create new assignment", formatter_class=RichHelpFormatter) +assignment_new_parser.add_argument("name", help="Assignment name", type=str).completer = lambda **kwargs: [f"Assignment {Path.cwd().name}"] +assignment_new_parser.add_argument("course", help="Course code", type=str).completer = lambda **kwargs: [f"CSCI-{Path.cwd().parent.name}"] +assignment_new_parser.add_argument("term", help="Term", choices=["Fall", "Spring", "Maymester", "Summer I", "Summer II"], type=str) assignment_new_parser.add_argument("-y", "--year", help="Year", type=int, default=cur_date.year) -assignment_new_parser.add_argument("-a", "--anon", help="Anonymize names", action=argparse.BooleanOptionalAction, - default=True) +assignment_new_parser.add_argument("-a", "--anon", help="Anonymize names", action=argparse.BooleanOptionalAction, default=True) # assignment > info command assignment_info_parser = assignment_subparsers.add_parser("info", help="Show assignment info") @@ -179,8 +191,7 @@ def submissionCompleter(*args, **kwargs): # Add required files command assign_add_required_parser = assignment_subparsers.add_parser("add-required", help="Add required files") assign_add_required_parser.add_argument("files", nargs="+", help="Required file names", type=Path) -assign_add_required_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda \ - **kwargs: [ +assign_add_required_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda **kwargs: [ "txt", "py", "c", @@ -189,8 +200,7 @@ def submissionCompleter(*args, **kwargs): "default", "make", ] -assign_add_required_parser.add_argument("-d", "--description", help="Description of the required files", type=str, - default="") +assign_add_required_parser.add_argument("-d", "--description", help="Description of the required files", type=str, default="") assign_add_required_parser.add_argument("-t", "--title", help="Title of the required files", type=str, default="") assign_add_required_parser.add_argument( "-i", "--include-in-output", help="Include in output", action=argparse.BooleanOptionalAction, default=True @@ -199,8 +209,7 @@ def submissionCompleter(*args, **kwargs): # Add optional files command assign_add_optional_parser = assignment_subparsers.add_parser("add-optional", help="Add optional files") assign_add_optional_parser.add_argument("files", nargs="+", help="Optional file names") -assign_add_optional_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda \ - **kwargs: [ +assign_add_optional_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda **kwargs: [ "txt", "py", "c", @@ -209,8 +218,7 @@ def submissionCompleter(*args, **kwargs): "make", "default", ] -assign_add_optional_parser.add_argument("-d", "--description", help="Description of the optional files", type=str, - default="") +assign_add_optional_parser.add_argument("-d", "--description", help="Description of the optional files", type=str, default="") assign_add_optional_parser.add_argument("-t", "--title", help="Title of the optional files", type=str, default="") assign_add_optional_parser.add_argument( "-i", "--include-in-output", help="Include in output", action=argparse.BooleanOptionalAction, default=True @@ -226,23 +234,31 @@ def submissionCompleter(*args, **kwargs): # submission > add command sub_add_subparser = sub_subparsers.add_parser("add", help="Add a submission file.") -sub_add_subparser.add_argument("files", nargs="+", help="Submission files to add", - type=Path).completer = functools.partial( +sub_add_subparser.add_argument("files", nargs="+", help="Submission files to add", type=Path).completer = functools.partial( SubFileCompleter, "unprocessed_dir" ) -sub_add_subparser.add_argument("-a", "--anonymous", dest="override_anon", action="store_true", - help="Override the assignment default and make this submission anonymous.", - default=None) -sub_add_subparser.add_argument("-n", "--non-anonymous", dest="override_anon", action="store_false", - help="Override the assignment default and make this submission non-anonymous.", - default=None) +sub_add_subparser.add_argument( + "-a", + "--anonymous", + dest="override_anon", + action="store_true", + help="Override the assignment default and make this submission anonymous.", + default=None, +) +sub_add_subparser.add_argument( + "-n", + "--non-anonymous", + dest="override_anon", + action="store_false", + help="Override the assignment default and make this submission non-anonymous.", + default=None, +) # submission > fix command sub_fix_subparser = sub_subparsers.add_parser( "fix", help="Fix a submission. Try this if you accidentally deleted something. This may re-create links etc." ) -sub_fix_subparser.add_argument("submissions", nargs="+", help="Submissions to fix", - type=str).completer = submissionCompleter +sub_fix_subparser.add_argument("submissions", nargs="+", help="Submissions to fix", type=str).completer = submissionCompleter ################################################################################ ################################################################################ @@ -252,51 +268,75 @@ def submissionCompleter(*args, **kwargs): # Add run command run_parser = subparsers.add_parser("run", help="Run submission files. This executes build, test, and render.") run_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter run_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add test command -test_parser = subparsers.add_parser("test", - help="Test submission files. This just runs the tests for the given submissions.") +test_parser = subparsers.add_parser("test", help="Test submission files. This just runs the tests for the given submissions.") test_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter test_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add build command build_parser = subparsers.add_parser("build", help="Build submission files") build_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter build_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add render command render_parser = subparsers.add_parser("render", help="Render submission files") render_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter -render_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", - default=False) +render_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) argcomplete.autocomplete(parser) -def display_assignment_info(cli_args: argparse.Namespace): +def printableLinkWithIcon( + link: Path, icon: str | Literal[":file_folder:", ":notebook:"] = ":file_folder:", link_text: str | None = None +) -> str: + """This takes a path and returns a string that can be printed to the terminal as a link. + :param link: Path to the file or directory. If the path does not exist, a red "x" is used as the icon. + :param icon: The icon to use as part of the link. + :param link_text: The text to use for the link. If None, the link text is the path relative to the root directory. + :return: A string that can be printed to the terminal as a link. + """ + if link is None and link_text is None: + raise ValueError("The link and link_text cannot both be None.") + if link is None: + link = "" + icon = ":x:" + else: + if not link.exists(): + icon = ":x:" + link = link.resolve() + + if link_text is None: + link_text = str(link) + + return f"[link file://{link}]{icon}{link_text}[/]" + + +def displayAssignmentInfo(cli_args: argparse.Namespace): assignment = Assignment.load() console.print(f'[label]Assignment "{assignment.name}"') - console.print( - f"[label]Course:[/] {assignment.course}, [label]Term:[/] {assignment._grade_period}, [label]Year:[/] " - f"{assignment.year}") + console.print(f"[label]Course:[/] {assignment.course}, [label]Term:[/] {assignment._grade_period}, [label]Year:[/] {assignment.year}") submissions = list(assignment.Submissions) console.print(f"[label]Submissions:[/] {len(submissions)}") + console.print( + f"[label]Directories:[/] " + f"{printableLinkWithIcon(assignment.root_directory, link_text='Root')} :diamonds: " + f"{printableLinkWithIcon(assignment.eval_dir, link_text='Evaluations')} :diamonds: " + f"{printableLinkWithIcon(assignment.graded_output_dir, link_text='Output for Grading')} :diamonds: " + f"{printableLinkWithIcon(assignment.tests_dir, link_text='Evaluation Tests')} :diamonds: " + f"{printableLinkWithIcon(assignment.link_template_dir, link_text='Templates (files linked into each evaluation)')}" + ) - files_table = rich.table.Table(title="[label][req]Required[/req]/[opt]Optional[/opt] Files", expand=True, - show_lines=True) + files_table = rich.table.Table(title="[label][req]Required[/req]/[opt]Optional[/opt] Files", expand=True, show_lines=True) files_table.add_column("Link", justify="center") files_table.add_column("Output", justify="center") files_table.add_column("Name", justify="left") @@ -327,39 +367,45 @@ def add_submission_file(required_file: SubmissionFileData, style: str = ""): console.log(assignment, submissions) -def display_submission_info(cli_args: argparse.Namespace, assignment: Assignment): - # todo: "[error] Add links to the editable output files." - console.print("[error] Add links to the editable output files.") - console.rule("[bold dark_green]Submission Info") - submission_table = rich.table.Table(title="Submissions") +def displaySubmissionInfo(cli_args: argparse.Namespace, assignment: Assignment): + submission_table = rich.table.Table(title="Submissions", expand=True) submission_table.add_column("Err", justify="center") submission_table.add_column("Warn", justify="center") submission_table.add_column("Output", justify="center") submission_table.add_column("Name", justify="left") + submission_table.add_column("Grading Output", justify="center") for submission in assignment.Submissions: + # For warnings and errors build 'links' that are just text so that there is + # information when the user hovers over the link in the terminal. warnings = submission.warnings has_warnings = warnings is not None and len(warnings) > 0 if warnings: warning_str = "\n".join(warnings) - warnings = f"[link {parse.quote(warning_str)}] :exclamation: [/]" + warnings = f"[link {parse.quote(warning_str)}]:exclamation:[/]" else: - warnings = ":thumbsup:" + warnings = ":white_check_mark:" errors = submission.errors has_errors = errors is not None and len(errors) > 0 if errors: errors_str = "\n".join(errors) - errors = f"[link {parse.quote(errors_str)}] :x: [/]" + errors = f"[link {parse.quote(errors_str)}]:x:[/]" else: - errors = ":+1:" + errors = ":white_check_mark:" - output = submission.main_output_file - has_output = output is not None - if output: - output = f"[link file://{output}] :notebook: [/]" + output, main, graded, _ = submission.main_output_files + has_output = main is not None and main.exists() + if isinstance(main, Path) and main.exists(): + output = f"[link file://{main.absolute()}] :notebook: [/]" else: - output = ":-1:" + output = ":x:" # + str(main) + + # has_graded_output = graded is not None and graded.exists() + if isinstance(graded, Path) and graded.exists(): + graded_output = f"[link file://{graded.absolute()}] :notebook: [/]" + else: + graded_output = ":x:" # + str(graded) sub_color = "green" match (has_errors, has_warnings, has_output): @@ -371,12 +417,18 @@ def display_submission_info(cli_args: argparse.Namespace, assignment: Assignment sub_color = "yellow" case (False, False, True): sub_color = "green" - submission_table.add_row(errors, warnings, output, f"[bold {sub_color}] {submission.name} [/]") + submission_table.add_row( + errors, + warnings, + output, + f"[bold {sub_color}]{printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)}[/]", + graded_output, + ) console.print(submission_table) -def get_current_assignment() -> Assignment: +def getCurrentAssignment() -> Assignment: try: ret_val = Assignment.load() return ret_val @@ -396,17 +448,22 @@ def handleAssignmentCmd(cli_args: argparse.Namespace): pass with console.status("Creating assignment", spinner="dots"): console.print(f'[bold green]Creating assignment "{cli_args.name}"') - new_assignment = Assignment(_name=cli_args.name, _course=cli_args.course, _grade_period=cli_args.term, - _year=cli_args.year, _options=GraderOptions(anonymize_names=cli_args.anon)) + new_assignment = Assignment( + _name=cli_args.name, + _course=cli_args.course, + _grade_period=cli_args.term, + _year=cli_args.year, + _options=GraderOptions(anonymize_names=cli_args.anon), + ) console.print(f"[bold green]saving {new_assignment.name}.") new_assignment.save() console.print("[bold green]Creating directories.") new_assignment.createMissingDirectories() console.print(f'[bold green]Assignment "{new_assignment.name}" created successfully.') case "info": - display_assignment_info(cli_args) + displayAssignmentInfo(cli_args) case "add-required" | "add-optional": - assignment = get_current_assignment() + assignment = getCurrentAssignment() if len(cli_args.files) > 1 and (len(cli_args.title) or len(cli_args.description)): console.log("[error]Cannot specify a title or description when adding multiple files.") @@ -420,8 +477,15 @@ def handleAssignmentCmd(cli_args: argparse.Namespace): # For each file call the method curried from above. for cur_file in cli_args.files: - method(SubmissionFileData(path=Path(cur_file), title=cli_args.title, description=cli_args.description, - include_in_output=cli_args.include_in_output, type=cli_args.type)) + method( + SubmissionFileData( + path=Path(cur_file), + title=cli_args.title, + description=cli_args.description, + include_in_output=cli_args.include_in_output, + type=cli_args.type, + ) + ) console.print(f"[bold green]Added file '{cur_file}'") assignment.save() case _: @@ -436,17 +500,24 @@ def handleSubmissionCmd(cli_args: argparse.Namespace): case "add": with console.status(f"Adding submission{'s' if len(cli_args.files) > 1 else ''}...", spinner="dots"): console.print("Loading assignment.") - assignment = get_current_assignment() + assignment = getCurrentAssignment() console.print(f"Adding {len(cli_args.files)} submissions.") # console.log(cli_args, style="error") for cur_file in cli_args.files: console.print(f"Adding {cur_file}") - assignment.AddSubmission(cur_file, override_anon=cli_args.override_anon).save() + try: + assignment.AddSubmission( + cur_file, + override_anon=cli_args.override_anon, + warning_callback=lambda warn: console.print(warn, style="warning"), + ).save() + except Exception as e: + console.print(f"[error]Error adding submission '{cur_file}': {e}") assignment.save() case "fix": with console.status("Fixing submissions...", spinner="dots"): console.print("Loading assignment.") - assignment = get_current_assignment() + assignment = getCurrentAssignment() console.print(f"Fixing {len(cli_args.submissions)} submissions.") for cur_file in cli_args.submissions: console.print(f"Fixing {cur_file}") @@ -455,7 +526,11 @@ def handleSubmissionCmd(cli_args: argparse.Namespace): console.print(f"[error]No submission directory found for {cur_file}.") cur_subm = Submission.load(cur_subm_dir) cur_subm.fix(assignment=assignment) - assignment.PostProcessSubmission(cur_subm).save() + assignment.PostProcessSubmission( + cur_subm, + exists_protocol=assignment.LinkProto.SKIP_FILE, + warning_callback=lambda warn: console.print(warn, style="warning"), + ).save() case _: console.log(cli_args, style="error") @@ -470,9 +545,13 @@ class RunOutputInfo(DataclassJson): return_code: int | None = None +def verbose_print(cli_args: argparse.Namespace, *args, **kwargs) -> None: + if cli_args.verbose: + console.print(*args, **kwargs) + + async def parse_pytest_output( - assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, - task_id + assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, task_id ): output_info = RunOutputInfo() @@ -491,14 +570,29 @@ async def error_collector(): break line = line.decode().strip() output_info.output.append(line) - if "collecting ..." in line: + if "collecting ..." in line and "selected" in line: + match = re.search(r"collected \d+ items / \d+ deselected / (\d+) selected", line) + if match: + output_info.collected = int(match.group(1)) + progress.update( + task_id, + total=output_info.collected, + name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name), + ) + elif "collecting ..." in line: match = re.search(r"collected (\d+) items", line) if match: output_info.collected = int(match.group(1)) - progress.update(task_id, total=output_info.collected) + progress.update( + task_id, + total=output_info.collected, + name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name), + ) elif " PASSED " in line or " FAILED " in line or " SKIPPED " in line: - progress.update(task_id, advance=1) - progress.update(task_id, description=line) + progress.update(task_id, advance=1, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) + elif line == "": + continue + progress.update(task_id, description=line, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) await proc.wait() await err_coll_task @@ -511,7 +605,11 @@ async def error_collector(): async def run_pytest( - assignment: Assignment, submission: Submission, progress: rich.progress.Progress, extra_pytest_args: str = "" + assignment: Assignment, + submission: Submission, + progress: rich.progress.Progress, + cli_args: argparse.Namespace, + extra_pytest_args: str = "", ) -> tuple[Submission, bool]: """Run pytest on the given submission. @@ -529,31 +627,42 @@ async def run_pytest( # This resolves that path relative to the submission directory. tests_path = submission.evaluation_directory / assignment.tests_dir.name if not tests_path.exists(): - task_id = progress.add_task("Testing...", total=1) + task_id = progress.add_task( + "Testing...", total=1, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name) + ) progress.update( task_id, advance=1, completed=True, - description=f"[error]Tests directory '{tests_path.absolute()}'not found. Perhaps run fix on " - f"{submission.name} first?", + description=f"[error]Tests directory '{tests_path.absolute()}'not found. Perhaps run fix on {submission.name} first?", + name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name), ) return submission, False + cmd_str: str = f"pytest -v -p agh-pytest-plugin --agh {extra_pytest_args} {tests_path.absolute()}/*" + # verbose_print(cli_args, 'Running pytest...', cmd_str) # Setup the progress bar. - task_id = progress.add_task(f"Testing {tests_path.absolute()}...", total=None) - - # Run pytest. - proc = await asyncio.create_subprocess_shell( - f"pytest -v -p agh-pytest-plugin {extra_pytest_args} {tests_path.absolute()}/*", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, + task_id = progress.add_task( + f"Testing {tests_path.absolute()}...", + total=None, + name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name), ) - return_code = await parse_pytest_output(assignment, submission, proc, progress, task_id) + + try: + # Run pytest. + proc = await asyncio.create_subprocess_shell( + cmd_str, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=assignment.eval_dir.absolute(), + ) + return_code = await parse_pytest_output(assignment, submission, proc, progress, task_id) + except CancelledError: + return_code = -1 return submission, return_code == 0 -async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment: Assignment, - extra_pytest_args: str = ""): +async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment: Assignment, extra_pytest_args: str = ""): """This function asynchronously runs pytest on all submissions specified. It provides a progress bar for each submission to indicate the progress of the tests. @@ -566,10 +675,12 @@ async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment # If there are no submissions specified, run on all submissions. if cli_args.submissions is None: cli_args.submissions = list(assignment.Submissions) + cli_args.submissions.sort(key=lambda x: x.name) else: # The CLI submissions provide the directory for the submission. We convert # to the submission objects and report any that don't exist. submission_list = [] + cli_args.submissions.sort() for submission in cli_args.submissions: try: submission_list.append(Submission.load(submission)) @@ -589,27 +700,32 @@ async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment with rich.progress.Progress( rich.progress.SpinnerColumn(spinner_name="dots"), + rich.progress.TextColumn("{task.fields[name]}", style="label", justify="center"), *rich.progress.Progress.get_default_columns(), - rich.progress.TimeRemainingColumn(), ) as progress: - tasks = [run_pytest(assignment, submission, progress, extra_pytest_args=extra_pytest_args) for submission in - cli_args.submissions] - results = await asyncio.gather(*tasks) - + try: + tasks = [ + run_pytest(assignment, submission, progress, cli_args, extra_pytest_args=extra_pytest_args) + for submission in cli_args.submissions + ] + results = await asyncio.gather(*tasks) + except KeyboardInterrupt: + # sys.exit(0) + pass for submission, success in results: if success: console.print(f"[green]Tests passed for {submission.name}[/green]") else: console.print(f"[red]Tests failed for {submission.name}[/red]") - if cli_args.verbose: - console.print("[bold label]Output:[/]") - for line in assignment.getMetadata(META_KEY_RUN_OUTPUT, submission.name, "output", default=[]): - console.print(line) - console.print("[bold label]Errors:[/]") - err_lines = assignment.getMetadata(META_KEY_RUN_OUTPUT, submission.name, "error", default=[]) - if err_lines: - for line in err_lines: - console.print(line, style="error") + if cli_args.verbose: + console.print("[bold label]Output:[/]") + for line in assignment.getMetadata(META_KEY_RUN_OUTPUT, submission.name, "output", default=[]): + console.print(line) + console.print("[bold label]Errors:[/]") + err_lines = assignment.getMetadata(META_KEY_RUN_OUTPUT, submission.name, "error", default=[]) + if err_lines: + for line in err_lines: + console.print(line, style="error") def run(args=None): @@ -621,33 +737,76 @@ def run(args=None): args = ["status"] cli_args = parser.parse_args(args=args) console.rule(f"[b i]agh[/] - Assignment Grading Helper - Version: [b i]{__version__}") + + # Core file Handling. + prior_pattern_path = getCurrentAssignment().root_directory / ".prior_core_pattern.txt" + core_pattern_path = Path("/proc/sys/kernel/core_pattern") + if (cli_args.debug_core_files or cli_args.restore_default_core_location) and cli_args.command is not None: + console.print("[error]The debug core files option can only be used without any other command.") + exit(1) + elif cli_args.debug_core_files: + # Handle setting up the system to store the core dump where testing scripts will pick it up. + target_core_loc = "aghAssignmentCoreDump.core" + prior_loc = core_pattern_path.read_text() + if target_core_loc in prior_loc: + console.print("[error]Core dumps already enabled.") + exit(0) + else: + prior_pattern_path.write_text(prior_loc) + + try: + core_pattern_path.write_text(target_core_loc) + except Exception as e: + console.print(f"[error]Error setting core dump location: {e}") + exit(1) + console.print(f"[bold green]Core dump location set to '{target_core_loc}'") + console.print("[bold green]To restore the default core dump location, run 'agh debug-core-files --restore'") + exit(0) + elif cli_args.restore_default_core_location: + if not prior_pattern_path.exists(): + console.print("[error]No prior core dump location found.") + exit(1) + else: + prior_loc = prior_pattern_path.read_text() + try: + core_pattern_path.write_text(prior_loc) + except Exception as e: + console.print(f"[error]Error setting core dump location: {e}") + exit(1) + console.print(f"[bold green]Core dump location restored to '{prior_loc}'") + exit(0) + + # Command handling. match cli_args.command: case "status": - assignment = get_current_assignment() - display_assignment_info(cli_args) - display_submission_info(cli_args, assignment) + assignment = getCurrentAssignment() + displayAssignmentInfo(cli_args) + displaySubmissionInfo(cli_args, assignment) case "assignment": handleAssignmentCmd(cli_args) case "submission": handleSubmissionCmd(cli_args) case "run": - assignment = get_current_assignment() - asyncio.run(execute_pytest_on_submissions(cli_args, assignment)) + assignment = getCurrentAssignment() + try: + asyncio.run(execute_pytest_on_submissions(cli_args, assignment)) + except KeyboardInterrupt: + pass case "test": - assignment = get_current_assignment() - asyncio.run( - execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) + assignment = getCurrentAssignment() + asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) case "build": - assignment = get_current_assignment() + assignment = getCurrentAssignment() asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "build"')) case "render": - assignment = get_current_assignment() + assignment = getCurrentAssignment() asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "render"')) case _: console.log(cli_args, style="error") # print(start(args)) parser.exit(0) + # todo: create custom url scheme so I can run things from links in the output. # To create a custom URL scheme in Ubuntu that executes a command in the terminal, you need to define a desktop # entry for the scheme and create a script to handle the URL. diff --git a/src/agh/pytest_plugin.py b/src/agh/pytest_plugin.py index 6c9a7ff..30f8f44 100644 --- a/src/agh/pytest_plugin.py +++ b/src/agh/pytest_plugin.py @@ -1,7 +1,20 @@ +import os +import signal +from collections.abc import Callable +from pathlib import Path + import pytest +from pytestshellutils.shell import ProcessResult +from pytestshellutils.shell import ScriptSubprocess from .agh_data import Assignment +from .agh_data import OutputSectionData from .agh_data import Submission +from .agh_data import SubmissionFileData + +TEST_MD_KEY = "TEST_INFO" + +CORE_DUMP_FILE_NAME = "aghAssignmentCoreDump.core" class AghPtPlugin: @@ -18,21 +31,6 @@ def pytest_terminal_summary(self, terminalreporter, exitstatus, config): # terminalreporter.write_line("JSON report saved to rich_parallel_report.json") -@pytest.fixture -def agh_test_plugin(request): - return "bob" - - -@pytest.fixture -def json_report_data(request): - def add_data(key, value): - nodeid = request.node.nodeid - report_data = request.config._json_report_data.setdefault(nodeid, {}) - report_data[key] = value - - return add_data - - def pytest_addoption(parser): parser.addoption("--agh", action="store_true", help="Enable AGH, assignment grading helper, extensions.") @@ -47,72 +45,373 @@ def pytest_configure(config): @pytest.fixture def agh_submission(request): - return Submission.load(request.path) + return Submission.load(request.path.parent) @pytest.fixture def agh_assignment(request): print(request.path) - return Assignment.load(request.path) + return Assignment.load(request.path.parent) + + +@pytest.fixture +def resultsDir(agh_submission) -> Path: + ret_val = agh_submission.evaluation_directory / "results" + ret_val.mkdir(parents=True, exist_ok=True) + return ret_val def register_render_env_var(env_var_name: str, env_var_value, cache: pytest.Cache): env_vars = cache.get("agh_render_env_vars", set()) env_vars.add(env_var_name) - cache.set("agh_render_env_vars", env_vars) + cache.set("agh_render_env_vars", list(env_vars)) cache.set(env_var_name, env_var_value) +def storeRunOutErr(tgt_name: str, res, resultsDir): + stdout_file = resultsDir / f"{tgt_name}.stdout" + stdout_file.write_text(res.stdout) + stderr_file = resultsDir / f"{tgt_name}.stderr" + stderr_file.write_text(res.stderr) + + +evaluationDataOS = OutputSectionData(path=Path("eval_data_section.md"), title="Evaluation Data", heading_level=1) +instructor_out_data = OutputSectionData(path=Path("instructor_data_section.md"), title="Instructor Data", heading_level=1) + + +def _make_sections(resultsDir: Path, agh_assignment: Assignment, agh_submission: Submission): + global evaluationDataOS, instructor_out_data + + orig_cwd = Path.cwd() + os.chdir(agh_submission.evaluation_directory) + try: + yourCodeOS = OutputSectionData(path=Path("code_section.md"), title="Your Code", heading_level=1) + + instructor_out_data.path = resultsDir / instructor_out_data.path.name + instructor_out_data.path.write_text(instructor_out_data.asQmdSection()) + + # todo: handle loading from file. + # for cur_section in [evaluationDataOS, yourCodeOS, instructor_out_data]: + # if not cur_section.hasData: + # # Try loading it from the filesystem. + # cur_section.path = resultsDir / (cur_section.path.with_suffix('.json')) + # if cur_section.path.exists(): + + evalOut = resultsDir / evaluationDataOS.path + evalOut.write_text(evaluationDataOS.asQmdSection()) + + codeOut = resultsDir / yourCodeOS.path + for s_file in agh_assignment.required_files.values(): + if s_file.include_in_output: + tgt = agh_submission.as_submitted_dir.absolute() / s_file.path.name + + # Create a submission file that points to my submission. + my_sub_src_file = SubmissionFileData(**s_file.asdict()) + my_sub_src_file.path = tgt.relative_to(agh_submission.evaluation_directory) + tgt = agh_submission.evaluation_directory / my_sub_src_file.path + if not tgt.exists(): + tgt.touch() + yourCodeOS.included_files.append(my_sub_src_file) + codeOut.write_text(yourCodeOS.asQmdSection()) + finally: + os.chdir(orig_cwd) + + @pytest.fixture -@pytest.mark.build -def agh_build_makefile(agh_submission, shell, cache, request): +def agh_build_makefile(agh_submission, shell, cache, request, resultsDir) -> Callable[[str], str]: request.applymarker(pytest.mark.build) - def build(target: str | None = None): + def build(target: str | None = None, include_build_in_eval: bool = True): # Check to see if this is the first time we're building this submission. first_build = False - if not cache.get("agh_build_makefile", False): + if agh_submission.getMetadata(TEST_MD_KEY, "initial_build_success", default=None) is None: first_build = True + agh_submission.setMetadata(TEST_MD_KEY, "initial_build_success", value=False) # Build the submission. cmd = ["make"] if target is not None: cmd.append(target) - res = shell.run(cmd, shell=True, cwd=agh_submission.evaluation_directory, env={"AGH_BUILD_TESTING": 1}) + res = shell.run(*cmd, shell=True, cwd=agh_submission.evaluation_directory, env={"AGH_BUILD_TESTING": "1"}) # Update permanent cache state for initial build ok. if first_build: - cache.set("agh_build_makefile", True) - build_ok_key = "agh_build_makefile_ok" - register_render_env_var(build_ok_key, res.returncode == 0, cache) + agh_submission.setMetadata(TEST_MD_KEY, "initial_build_success", value=res.returncode == 0) + + build_out_section = OutputSectionData(path=Path("build_data.md"), title="Build Output") + if include_build_in_eval: + evaluationDataOS.addSection(build_out_section) + stdout_file = resultsDir / f"{target if target else ''}build.stdout" + stdout_file.parent.mkdir(exist_ok=True) + stdout_file.write_text(res.stdout) + build_out_section.included_files.append( + SubmissionFileData(path=stdout_file.relative_to(agh_submission.evaluation_directory), title="Build Stdout Output") + ) + stderr_file = resultsDir / f"{target if target else ''}build.stderr" + stderr_file.write_text(res.stderr) + if len(res.stderr) > 0: + build_out_section.included_files.append( + SubmissionFileData(path=stdout_file.relative_to(agh_submission.evaluation_directory), title="Build Stderr Output") + ) + return res - yield build + return build - if build.res != 0: - print(f"Build failed for {agh_submission.submission_file}") + +@pytest.fixture +def _core_file_saved(agh_submission): + core_path = Path("/proc/sys/kernel/core_pattern") + path_good = core_path.exists() and "apport" not in core_path.read_text().strip() + if path_good: + agh_submission.delWarning("core_file_saved") + else: + agh_submission.addWarning("core_file_saved", "Core file pattern will not allow debugging information to be captured.") + return path_good @pytest.fixture -def agh_env_vars(cache): - environ = {} - for env_var_name in cache.get("agh_render_env_vars", set()): - environ[env_var_name] = cache.get(env_var_name, "") - return environ +def agh_run_executable( + agh_submission, shell: ScriptSubprocess, resultsDir, _core_file_saved +) -> Callable[..., tuple[ProcessResult, OutputSectionData]]: + def run_executable( + command: str, + test_key: str, + test_exe_file: Path, + timeout_sec: int = 25, + kill_timeout_sec: int = 50, + parent_section: OutputSectionData | None = None, + handle_core_dump: bool = True, + handle_timeout: bool = True, + **kwargs, + ) -> tuple[ProcessResult, OutputSectionData]: + """Run an executable and return the results. + .. important:: + + You must finish setting up the returned output section with a title etc. + """ + + # Build up the shell command line based on options selected by the grader. + shell_cmd_line = command + if handle_timeout: + shell_cmd_line = f"timeout -vk {kill_timeout_sec} -s SIGXCPU {timeout_sec} " + shell_cmd_line + if handle_core_dump: + shell_cmd_line = "ulimit -c unlimited && " + shell_cmd_line + + result = shell.run(shell_cmd_line, shell=True, cwd=agh_submission.evaluation_directory, **kwargs) + + if parent_section is None: + parent_section = evaluationDataOS + + current_out_section = OutputSectionData(path=Path(resultsDir / f"{test_key}_section.md")) + parent_section.addSection(current_out_section) + + std_out_file = resultsDir / f"{test_key}.stdout" + current_out_section.included_files.append( + SubmissionFileData( + path=std_out_file.relative_to(agh_submission.evaluation_directory), + title="Standard Output", + description="This is the standard output from running your code.", + type="default", + ) + ) + std_out_file.parent.mkdir(exist_ok=True) + std_out_file.write_text(result.stdout, encoding="ascii", errors="backslashreplace") + + if len(result.stderr) > 0: + std_err_file = resultsDir / f"{test_key}.stderr" + current_out_section.included_files.append( + SubmissionFileData( + path=std_err_file.relative_to(agh_submission.evaluation_directory), + title="Standard Error", + description="This is the standard error from running your code.", + type="default", + ) + ) + std_err_file.write_text(result.stderr, encoding="ascii", errors="backslashreplace") + + # Handle core dumps. + core_dump_file = agh_submission.evaluation_directory / CORE_DUMP_FILE_NAME + if core_dump_file.exists(): + # Run gdb on the core dump + result_debug = shell.run( + f'gdb. / {test_exe_file} {core_dump_file.name} --eval-command "thread apply all bt full" --batch', + shell=True, + cwd=agh_submission.evaluation_directory, + ) + core_dump_file.unlink() + + if len(result_debug.stdout) > 0: + # There is data add to the eval section. + debug_output_file = resultsDir / (test_key + ".backtrace") + debug_output_file.write_text(result_debug.stdout) + current_out_section.included_files.append( + SubmissionFileData( + path=debug_output_file.relative_to(agh_submission.evaluation_directory), + title="Backtrace from Debug", + type="default", + description="Your code had an error that caused it to crash. This is the debugging backtrace from that crash.", + ) + ) + else: + # todo: Handle this better. + current_out_section.text += "\n\n**Warning:** no backtrace data available from core file!" + + err_code = result.returncode + if err_code: + # current_out_section.text += f"\n\n**Warning:** Exe exited with error code: {err_code}!" + if 124 <= err_code <= 128: + current_out_section.addWarning("Timeout", f"Your executable took too long to run and had to be terminated: {err_code}!") + elif err_code > 128: + sig_name = "" + try: + sig_name = signal.strsignal(err_code - 128) + except ValueError: + pass + + agh_submission.setMetadata(TEST_MD_KEY, "EXE_FAULT", value=True) + current_out_section.addError( + "Crash Likely", f"Exit Code: {err_code}\n\nExe exited with signal {sig_name}: {err_code - 128}" + ) + # if err_code == 23: # Leak sanitizer exitcode. + # threadWarn = EvalFile(testStdOut.with_suffix('.md'), '', '', just_text=True, unlisted=unlisted) + # curEFiles.insert(0, threadWarn) + # threadWarn.file.write_text( + # f'\n\n::: {{.callout-important title="EXE Issue Detected"}}\n\n**Exit Code: {err_code}**\n\n| ' + # f'{testStdErr.read_text().replace(str(cDir.absolute()), ".").replace("\n", "\n| ")}\n\n::> + # # with myWarnFile.open('a') as infoFile: + # # infoFile.write(f'\n - [ ] Memory Checked.\n\n') + return (result, current_out_section) + + return run_executable + + +# todo: This is a good idea, but I feel that I need to get it working straightforwardly first. +# +# class MetaSingleton(type): +# _instances: ClassVar[dict] = {} +# +# def __call__(cls, *args, **kwargs): +# if cls not in cls._instances: +# cls._instances[cls] = super(MetaSingleton, cls).__call__(*args, **kwargs) # noqa: +# UP008 +# return cls._instances[cls] +# +# +# class AdvShell(metaclass=MetaSingleton): +# """This class is a singleton that manages the invocation of an executable. **It must be +# subclassed.** +# +# This class is designed to allow for testing independent preconditions/postconditions +# on the execution of a command. By subclassing this class the executable is +# invoked a single time, but the preconditions/postconditions can be tested against the +# output independently of the executable. +# +# .. topic:: Example +# +# .. code-block:: python +# +# class TestSimple(AdvShell): +# def __init__(self): +# self.results = self.run(...) +# +# def test_success(self): +# assert self.results.returncode == 0, "Your executable exited with a +# non-zero return code." +# +# def test_out_length(self): +# assert len(self.results.stdout) > 10, "Your executable did not capture +# enough of the test data." +# +# def test_err_length(self): +# assert len(self.results.stderr) < 2, "Your executable was not clean in +# stderr." +# +# ... +# +# class TestBadCLI(AdvShell): +# def __init__(self): +# self.results = self.run() +# +# def test_returncode(self): +# assert self.results.returncode != 0, "Your executable did not fail with a +# non-zero return code." +# +# def test_err_length(self): +# assert len(self.results.stderr) > 0, "Your executable did not output any +# error messages." +# +# def test_out_length(self): +# assert len(self.results.stdout) == 0, "Your executable may have printed the +# help message to stdout instead of stderr." +# ... +# +# +# For each of the classes above, the self.run would only be invoked once each. +# The tests would be run against the results of the single invocation but allow the +# instructor +# to indicate multiple issues within that invocation in a **pytest** appropriate manner. +# +# .. warning:: +# THIS CLASS SHOULD NOT BE INSTANTIATED DIRECTLY, subclass it instead. +# +# """ +# +# def __init__(self): +# if type(self) is AdvShell: +# raise TypeError("AdvShell class cannot be instantiated directly") +# +# def run(self, *args, **kwargs): +# """This method runs all the checks for the executable: Error code != 0, OS captured +# faults, -fsanitize=leak errors, -fsanitize=thread errors.""" + +""" +INPUT + command string + +OUTPUT + command is run once each test class + output of one run is available in each function + +""" @pytest.fixture -@pytest.mark.render -def agh_render_quarto(agh_submission, shell, agh_env_vars, request): +def agh_render_output( + agh_submission: Submission, + shell, + request: pytest.FixtureRequest, + resultsDir: Path, + agh_assignment: Assignment, +): request.applymarker(pytest.mark.render) - def render(target: str | None = None, *args): + def render(target: str | None = agh_assignment._options.output_template_name, *args: str): + _make_sections(resultsDir, agh_assignment, agh_submission) cmd = ["quarto", "render"] if target is not None: cmd.append(target) if len(args) > 0: cmd.extend(args) - res = shell.run(cmd, shell=True, cwd=agh_submission.evaluation_directory, env=agh_env_vars) - return res + + # Clear all render specific errors and warnings + agh_submission.delWarning("render warning").delError("render error").delWarning("render issue").save() + + try: + cmd_str = " ".join(cmd) + print(f"Executing quarto: {cmd_str}") + res = shell.run(cmd_str, shell=True, cwd=agh_submission.evaluation_directory) + (resultsDir / ".render.stdout.txt").write_text(f"{cmd_str}\n" + res.stdout) + (resultsDir / ".render.stderr.txt").write_text(res.stderr) + if res.returncode != 0: + agh_submission.addWarning("render issue", f"Render '{cmd_str}' failed with return code {res.returncode}.").save() + raise RuntimeError(f"quarto failed with return code {res.returncode}") + agh_assignment.postProcessSubmissionRender( + agh_submission, warning_callback=lambda warn: agh_submission.addWarning("render warning", warn) + ).save() + return res + except Exception as e: + agh_submission.addError("render error", f"Quarto failed with error {e}.").save() + request.raiseerror(f"Error rendering {agh_submission.submission_file}: {e}") return render diff --git a/tests/test_assignment.py b/tests/test_assignment.py index 9286e35..690e1ba 100644 --- a/tests/test_assignment.py +++ b/tests/test_assignment.py @@ -23,7 +23,7 @@ def test_defaults(self): # Defaults self.assertEqual(a._name, "assignment") self.assertEqual(a._year, 2025) - self.assertEqual(a._grade_period, "Fall") + # self.assertEqual(a._grade_period, "Fall") This now switches based on the time of year. self.assertEqual(a._course, "CSCI-340") # self.assertEqual(a._submission_files, []) self.assertEqual(a._required_files, {}) @@ -345,7 +345,7 @@ def test_pps_links_test_files(temp_assignment, temp_submission_file): # def test_postprocesssubmission_ignores_existing_link(temp_assignment, temp_submission_file, tmp_path): -# """Test that PostProcessSubmission ignores existing link when protocol is IGNORE_ERROR.""" +# """Test that PostProcessSubmission ignores existing link when protocol is SKIP_FILE.""" # conflict_file = tmp_path / "tests" # conflict_file.mkdir() # @@ -353,9 +353,9 @@ def test_pps_links_test_files(temp_assignment, temp_submission_file): # (new_submission.evaluation_directory / "tests").symlink_to(conflict_file) # # try: -# temp_assignment.PostProcessSubmission(temp_submission_file, exists_protocol=LinkProto.IGNORE_ERROR) +# temp_assignment.PostProcessSubmission(temp_submission_file, exists_protocol=LinkProto.SKIP_FILE) # except FileExistsError: -# pytest.fail("PostProcessSubmission should not raise FileExistsError when IGNORE_ERROR is used.") +# pytest.fail("PostProcessSubmission should not raise FileExistsError when SKIP_FILE is used.") # def test_postprocesssubmission_overwrites_existing_link(temp_assignment, temp_submission_file, tmp_path): diff --git a/tox.ini b/tox.ini index c1ea9cb..17048db 100644 --- a/tox.ini +++ b/tox.ini @@ -14,10 +14,11 @@ envlist = clean, check, docs, - {3.13,py311,py312,py313,pypy310,pypy311}, + {3.13,py312,py313,pypy312,pypy313}, report ignore_basepython_conflict = true +; {3.13,py311,py312,py313,pypy310,pypy311}, ;{py39,py310,py311,py312,py313,pypy39,pypy310,pypy311}, [testenv]