From 690d083a836954bff6e4f21a76a91f2e47b83c45 Mon Sep 17 00:00:00 2001 From: Tuck Williamson Date: Fri, 17 Oct 2025 11:10:20 -0400 Subject: [PATCH 01/10] Improved function naming for consistency. Enhanced metadata handling, added verbose print option, and updated graded/non-anon output management. Introduced error/warning handling mechanisms for submissions. Refined pytest integration and post-processing. Added support for results directory in pytest operations. Updated dependencies. --- pyproject.toml | 1 + src/agh/agh_data.py | 131 +++++++++++++++++++++++++++++---------- src/agh/cli.py | 106 ++++++++++++++++++------------- src/agh/pytest_plugin.py | 113 ++++++++++++++++++++++++--------- 4 files changed, 246 insertions(+), 105 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1154749..8cc0d70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dependencies = [ "pytest>=8.4.2", "pytest-shell-utilities>=1.9.7", "rich>=14.2.0", + "ipython>=9.6.0", "argcomplete", "rich-argparse", ] diff --git a/src/agh/agh_data.py b/src/agh/agh_data.py index 7b7889e..ac9ed93 100644 --- a/src/agh/agh_data.py +++ b/src/agh/agh_data.py @@ -11,16 +11,25 @@ from enum import IntEnum from pathlib import Path from string import Template -from typing import Any +from typing import Any, Callable, LiteralString, Literal from typing import Self from typing import get_args from typing import get_origin import agh.anonymizer as anonymizer +META_INTERNAL_SUB_OUTPUT = "OUTPUT_INFO" + META_INTERNAL_SUB_KEY = "SUBMISSION" META_AGH_INTERNAL_KEY = "AGH_INTERNAL" +META_INTERNAL_SUB_KEYS = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY] +META_INTERNAL_SUB_OUTPUT_COMPLETE = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, + 'COMPLETED_OUTPUT'] +META_INTERNAL_SUB_OUTPUT_GRADED = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, + 'GRADED'] +META_INTERNAL_SUB_OUTPUT_NON_ANON = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, + 'NON_ANON'] _USER_DEFAULTS_FILE = Path.home() / ".config" / "agh" / ".agh_user_defaults.json" @@ -164,8 +173,8 @@ def sectionAttr(self) -> str: def asQmdSection(self, heading_level: int) -> str: if not self.include_in_output: return "" - if not self.path.exists(): - self.path.touch() + # if not self.path.exists(): + # self.path.touch() hdr_txt = "#" * heading_level + " " + self.title + self.sectionAttr return f"\n\n{self.description}\n\n{hdr_txt}\n\n```{{.{self.type}}}\n{{{{< include {self.path} >}}}}\n```\n\n" @@ -226,7 +235,7 @@ def _getMetadata(self, metadata_key: str, default: Any = None) -> dict[str, Any] return default return cur_metadata - def getMetadata(self, *args, default: Any = None) -> dict[str, Any]: + def getMetadata(self, *args, default: Any = None) -> dict[str, Any]|Any: """Returns the metadata associated with the assignment. If the key is not present in the metadata, it will return the default value. @@ -637,10 +646,10 @@ class LinkProto(IntEnum): # Overwrite the existing file if it already exists. LINK_OVERWRITE = 2 - def PostProcessSubmission( - self, submission_file: "pathlib.Path|Submission", exists_protocol: LinkProto = LinkProto.RAISE_ERROR - ) -> "Submission": - ret_val = submission_file + def PostProcessSubmission(self, submission_file: "pathlib.Path|Submission", + exists_protocol: LinkProto = LinkProto.RAISE_ERROR, + warning_callback: Callable[[str],None] | None = None) -> "Submission": + ret_val:Submission = submission_file if isinstance(submission_file, pathlib.Path): ret_val = Submission.load(filepath=submission_file) @@ -681,37 +690,61 @@ def all_linked_files() -> Generator[Path]: link_tgt.symlink_to(link_item.absolute(), target_is_directory=link_item.is_dir()) # Now start linking the output stuff together. + return self.postProcessSubmissionRender(ret_val, warning_callback=warning_callback) + + def postProcessSubmissionRender(self, submission: "Submission", warning_callback: Callable[[str],None] | None = None) -> "Submission": for output_file in self._options.output_files: - output_in_sub_dir = ret_val.evaluation_directory / output_file - output_file = self.complete_eval_dir / (ret_val.evaluation_directory.name + output_in_sub_dir.suffix) - output_not_anon = self.d2l_named_dir / (ret_val.original_name + output_in_sub_dir.suffix) + output_in_sub_dir = submission.evaluation_directory / output_file + output_file = self.complete_eval_dir / (submission.evaluation_directory.name + output_in_sub_dir.suffix) + submission.setMetadata(*META_INTERNAL_SUB_OUTPUT_COMPLETE, value=str(output_file)) + output_not_anon = self.d2l_named_dir / (submission.original_name + output_in_sub_dir.suffix) + submission.setMetadata(*META_INTERNAL_SUB_OUTPUT_NON_ANON, value=str(output_not_anon)) output_graded = self.graded_output_dir / output_file.name + submission.setMetadata(*META_INTERNAL_SUB_OUTPUT_GRADED, value=str(output_graded)) try: + # Copy output_in_sub_dir to output_graded if needed + if (not output_graded.exists()) or (output_graded.exists() and output_graded.stat().st_ctime_ns == output_graded.stat().st_mtime_ns): + output_graded.parent.mkdir(parents=True, exist_ok=True) + + # This is to ensure that ctime == mtime! + output_graded.unlink(missing_ok=True) + + output_graded.write_bytes(output_in_sub_dir.read_bytes()) + elif warning_callback is not None: + warning_callback(f'The graded output file "{output_graded}" already exists and appears modified. {output_graded} will not be overwritten.') + # stats = output_graded.stat() + # warning_callback(f'The graded output file info a:{stats.st_atime_ns}, c: {stats.st_ctime_ns}, m: {stats.st_mtime_ns}.') + output_file.unlink(missing_ok=True) output_not_anon.unlink(missing_ok=True) except FileNotFoundError: pass + # Create symbolic links output_file.symlink_to( output_in_sub_dir.relative_to(output_file.parent, walk_up=True), target_is_directory=output_file.is_dir() ) + output_file.chmod(0o444) + output_not_anon.symlink_to( output_graded.relative_to(output_not_anon.parent, walk_up=True), target_is_directory=output_not_anon.is_dir() ) - return ret_val + return submission - def AddSubmission(self, submission_file: pathlib.Path, override_anon: bool | None = None) -> "Submission": + def AddSubmission(self, submission_file: pathlib.Path, override_anon: bool | None = None, + warning_callback: Callable[[str],None] | None = None) -> "Submission": """Add a new submission to the assignment. :param submission_file: The path to the submission file to add. :param override_anon: If none then abide by the default for the assignment. If True then make it anonymous even if assignment is default non-anonymous. If False then make it non-anonymous even if assignment is default anonymous. + :param warning_callback: A callback function to be called when a warning is encountered. :return: The new submission. :rtype: "Submission" """ ret_val = Submission.new(self, submission_file=submission_file, override_anon=override_anon) ret_val.save() - return self.PostProcessSubmission(ret_val, exists_protocol=self.LinkProto.RAISE_ERROR) + return self.PostProcessSubmission(ret_val, exists_protocol=self.LinkProto.RAISE_ERROR, warning_callback=warning_callback) # def __pytest_cmd(self): # return "pytest ./ -p shell-utilities -p agh" @@ -991,7 +1024,7 @@ def new(cls, assignment: Assignment, submission_file: pathlib.Path, override_ano base_file_name = file_name base_file_name_set = True case _: - anon_name = submission_file.stem + anon_name = submission_file.with_suffix("").name evaluation_directory = assignment.eval_dir / anon_name evaluation_directory.mkdir(exist_ok=True, parents=True) @@ -1081,23 +1114,53 @@ def name(self): return self.anon_name @property - def main_output_file(self) -> Path | None: - """Check if the submission has output files.""" + def main_output_files(self) -> list[Path | None ]: + """Return the submission's output files. + :return: A list of output files as follows: + ``[, + , , ]`` + """ assign = Assignment.load() + rendered = None for output_file in assign._options.output_files: output_file = self.evaluation_directory / output_file - if not output_file.exists(): - return None - else: - return output_file - return None + if output_file.exists(): + rendered = output_file + + main = self.getMetadata(*META_INTERNAL_SUB_OUTPUT_COMPLETE) + graded = self.getMetadata(*META_INTERNAL_SUB_OUTPUT_GRADED) + non_anon :None|str|Path= self.getMetadata(*META_INTERNAL_SUB_OUTPUT_NON_ANON) + + if main is not None: + main = Path(main) + if graded is not None: + graded = Path(graded) + if non_anon is not None: + non_anon = Path(non_anon) + + return [rendered, main, graded, non_anon] + + # The next three methods have to do with getting, setting, and clearing errors or warnings. + def _getErrWarnList(self, type: Literal["errors", "warnings"]) -> list[str]: + return list(self.getMetadata(*META_INTERNAL_SUB_KEYS, type, default={}).values()) + + def _setErrWarnItem(self, type: Literal["errors", "warnings"], key: str, txt_or_markdown:str) -> Self: + keys = list(META_INTERNAL_SUB_KEYS) + keys.append(type) + keys.append(key) + return self.setMetadata(*keys, value=txt_or_markdown) + + def _delErrWarnItem(self, type: Literal["errors", "warnings"], key: str) -> Self: + exist_md_dict:dict[Any, Any] = self.getMetadata(*META_INTERNAL_SUB_KEYS, type, default={}) + exist_md_dict.pop(key, None) + return self.setMetadata(*META_INTERNAL_SUB_KEYS, type, value=exist_md_dict) @property def errors(self) -> None | list[str]: """Check if the submission has errors. These are NOT testing errors, but anything preventing the submission from being tested. """ - errors: list[str] = self.getMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "errors", default=[]) + errors: list[str] = self._getErrWarnList("errors") if not self.as_submitted_dir.exists(): errors.append(f"Submission directory '{self.as_submitted_dir.absolute()}' does not exist.") return errors @@ -1111,27 +1174,31 @@ def errors(self) -> None | list[str]: return None - def addError(self, txt_or_markdown: str) -> "Submission": + def addError(self, key:str, txt_or_markdown: str) -> "Submission": """Add an error to the submission. These are NOT testing errors, but anything preventing the submission from being tested. """ # DON'T USE the property above. It adds transient errors. Just get the metadata and add to it. - errors: list[str] = self.getMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "errors", default=[]) - errors.append(txt_or_markdown) - return self.setMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "errors", value=errors) + return self._setErrWarnItem("errors", key, txt_or_markdown) + # errors: list[str] = self.getMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "errors", default=[]) + # errors.append(txt_or_markdown) + # return self.setMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "errors", value=errors) + def delError(self, key:str) -> "Submission": + return self._delErrWarnItem("errors", key) @property def warnings(self) -> None | list[str]: """Check if the submission has warnings. These are NOT testing warnings, but anything possibly preventing the submission from being tested. """ - return self.getMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "warnings") + return self._getErrWarnList("warnings") - def addWarning(self, txt_or_markdown: str) -> "Submission": + def addWarning(self, key:str, txt_or_markdown: str) -> "Submission": """Add a warning to the submission. These are NOT testing warnings, but anything possibly preventing the submission from being tested. """ - warnings = self.warnings - warnings.append(txt_or_markdown) - return self.setMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "warnings", value=warnings) + return self._setErrWarnItem("warnings", key, txt_or_markdown) + + def delWarning(self, key:str) -> "Submission": + return self._delErrWarnItem("warnings", key) diff --git a/src/agh/cli.py b/src/agh/cli.py index b442211..83734cc 100644 --- a/src/agh/cli.py +++ b/src/agh/cli.py @@ -286,7 +286,7 @@ def submissionCompleter(*args, **kwargs): argcomplete.autocomplete(parser) -def display_assignment_info(cli_args: argparse.Namespace): +def displayAssignmentInfo(cli_args: argparse.Namespace): assignment = Assignment.load() console.print(f'[label]Assignment "{assignment.name}"') console.print( @@ -327,39 +327,46 @@ def add_submission_file(required_file: SubmissionFileData, style: str = ""): console.log(assignment, submissions) -def display_submission_info(cli_args: argparse.Namespace, assignment: Assignment): - # todo: "[error] Add links to the editable output files." - console.print("[error] Add links to the editable output files.") - console.rule("[bold dark_green]Submission Info") - submission_table = rich.table.Table(title="Submissions") +def displaySubmissionInfo(cli_args: argparse.Namespace, assignment: Assignment): + submission_table = rich.table.Table(title="Submissions", expand=True) submission_table.add_column("Err", justify="center") submission_table.add_column("Warn", justify="center") submission_table.add_column("Output", justify="center") submission_table.add_column("Name", justify="left") + submission_table.add_column("Grading Output", justify="center") for submission in assignment.Submissions: + + # For warnings and errors build 'links' that are just text so that there is + # information when the user hovers over the link in the terminal. warnings = submission.warnings has_warnings = warnings is not None and len(warnings) > 0 if warnings: warning_str = "\n".join(warnings) - warnings = f"[link {parse.quote(warning_str)}] :exclamation: [/]" + warnings = f"[link {parse.quote(warning_str)}]:exclamation:[/]" else: - warnings = ":thumbsup:" + warnings = ":white_check_mark:" errors = submission.errors has_errors = errors is not None and len(errors) > 0 if errors: errors_str = "\n".join(errors) - errors = f"[link {parse.quote(errors_str)}] :x: [/]" + errors = f"[link {parse.quote(errors_str)}]:x:[/]" + else: + errors = ":white_check_mark:" + + output, main, graded, non_anon = submission.main_output_files + has_output = main is not None and main.exists() + if isinstance(main, Path) and main.exists(): + output = f"[link file://{main.absolute()}] :notebook: [/]" else: - errors = ":+1:" + output = ":x:" # + str(main) - output = submission.main_output_file - has_output = output is not None - if output: - output = f"[link file://{output}] :notebook: [/]" + # has_graded_output = graded is not None and graded.exists() + if isinstance(graded, Path) and graded.exists(): + graded_output = f"[link file://{graded.absolute()}] :notebook: [/]" else: - output = ":-1:" + graded_output = ":x:" #+ str(graded) sub_color = "green" match (has_errors, has_warnings, has_output): @@ -371,12 +378,12 @@ def display_submission_info(cli_args: argparse.Namespace, assignment: Assignment sub_color = "yellow" case (False, False, True): sub_color = "green" - submission_table.add_row(errors, warnings, output, f"[bold {sub_color}] {submission.name} [/]") + submission_table.add_row(errors, warnings, output, f"[bold {sub_color}] {submission.name} [/]", graded_output) console.print(submission_table) -def get_current_assignment() -> Assignment: +def getCurrentAssignment() -> Assignment: try: ret_val = Assignment.load() return ret_val @@ -404,9 +411,9 @@ def handleAssignmentCmd(cli_args: argparse.Namespace): new_assignment.createMissingDirectories() console.print(f'[bold green]Assignment "{new_assignment.name}" created successfully.') case "info": - display_assignment_info(cli_args) + displayAssignmentInfo(cli_args) case "add-required" | "add-optional": - assignment = get_current_assignment() + assignment = getCurrentAssignment() if len(cli_args.files) > 1 and (len(cli_args.title) or len(cli_args.description)): console.log("[error]Cannot specify a title or description when adding multiple files.") @@ -436,17 +443,20 @@ def handleSubmissionCmd(cli_args: argparse.Namespace): case "add": with console.status(f"Adding submission{'s' if len(cli_args.files) > 1 else ''}...", spinner="dots"): console.print("Loading assignment.") - assignment = get_current_assignment() + assignment = getCurrentAssignment() console.print(f"Adding {len(cli_args.files)} submissions.") # console.log(cli_args, style="error") for cur_file in cli_args.files: console.print(f"Adding {cur_file}") - assignment.AddSubmission(cur_file, override_anon=cli_args.override_anon).save() + try: + assignment.AddSubmission(cur_file, override_anon=cli_args.override_anon, warning_callback=lambda warn: console.print(warn, style="warning")).save() + except Exception as e: + console.print(f"[error]Error adding submission '{cur_file}': {e}") assignment.save() case "fix": with console.status("Fixing submissions...", spinner="dots"): console.print("Loading assignment.") - assignment = get_current_assignment() + assignment = getCurrentAssignment() console.print(f"Fixing {len(cli_args.submissions)} submissions.") for cur_file in cli_args.submissions: console.print(f"Fixing {cur_file}") @@ -455,7 +465,7 @@ def handleSubmissionCmd(cli_args: argparse.Namespace): console.print(f"[error]No submission directory found for {cur_file}.") cur_subm = Submission.load(cur_subm_dir) cur_subm.fix(assignment=assignment) - assignment.PostProcessSubmission(cur_subm).save() + assignment.PostProcessSubmission(cur_subm, warning_callback=lambda warn: console.print(warn, style="warning")).save() case _: console.log(cli_args, style="error") @@ -469,6 +479,9 @@ class RunOutputInfo(DataclassJson): collected: int | None = None return_code: int | None = None +def verbose_print(cli_args:argparse.Namespace, *args, **kwargs) -> None: + if cli_args.verbose: + console.print(*args, **kwargs) async def parse_pytest_output( assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, @@ -491,7 +504,12 @@ async def error_collector(): break line = line.decode().strip() output_info.output.append(line) - if "collecting ..." in line: + if "collecting ..." in line and 'selected' in line: + match = re.search(r"collected \d+ items / \d+ deselected / (\d+) selected", line) + if match: + output_info.collected = int(match.group(1)) + progress.update(task_id, total=output_info.collected) + elif "collecting ..." in line: match = re.search(r"collected (\d+) items", line) if match: output_info.collected = int(match.group(1)) @@ -511,7 +529,7 @@ async def error_collector(): async def run_pytest( - assignment: Assignment, submission: Submission, progress: rich.progress.Progress, extra_pytest_args: str = "" + assignment: Assignment, submission: Submission, progress: rich.progress.Progress, cli_args:argparse.Namespace, extra_pytest_args: str = "" ) -> tuple[Submission, bool]: """Run pytest on the given submission. @@ -539,12 +557,14 @@ async def run_pytest( ) return submission, False + cmd_str : str = f"pytest -v -p agh-pytest-plugin --agh {extra_pytest_args} {tests_path.absolute()}/*" + # verbose_print(cli_args, 'Running pytest...', cmd_str) # Setup the progress bar. task_id = progress.add_task(f"Testing {tests_path.absolute()}...", total=None) # Run pytest. proc = await asyncio.create_subprocess_shell( - f"pytest -v -p agh-pytest-plugin {extra_pytest_args} {tests_path.absolute()}/*", + cmd_str, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) @@ -592,7 +612,7 @@ async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment *rich.progress.Progress.get_default_columns(), rich.progress.TimeRemainingColumn(), ) as progress: - tasks = [run_pytest(assignment, submission, progress, extra_pytest_args=extra_pytest_args) for submission in + tasks = [run_pytest(assignment, submission, progress, cli_args, extra_pytest_args=extra_pytest_args) for submission in cli_args.submissions] results = await asyncio.gather(*tasks) @@ -601,15 +621,15 @@ async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment console.print(f"[green]Tests passed for {submission.name}[/green]") else: console.print(f"[red]Tests failed for {submission.name}[/red]") - if cli_args.verbose: - console.print("[bold label]Output:[/]") - for line in assignment.getMetadata(META_KEY_RUN_OUTPUT, submission.name, "output", default=[]): - console.print(line) - console.print("[bold label]Errors:[/]") - err_lines = assignment.getMetadata(META_KEY_RUN_OUTPUT, submission.name, "error", default=[]) - if err_lines: - for line in err_lines: - console.print(line, style="error") + if cli_args.verbose: + console.print("[bold label]Output:[/]") + for line in assignment.getMetadata(META_KEY_RUN_OUTPUT, submission.name, "output", default=[]): + console.print(line) + console.print("[bold label]Errors:[/]") + err_lines = assignment.getMetadata(META_KEY_RUN_OUTPUT, submission.name, "error", default=[]) + if err_lines: + for line in err_lines: + console.print(line, style="error") def run(args=None): @@ -623,25 +643,25 @@ def run(args=None): console.rule(f"[b i]agh[/] - Assignment Grading Helper - Version: [b i]{__version__}") match cli_args.command: case "status": - assignment = get_current_assignment() - display_assignment_info(cli_args) - display_submission_info(cli_args, assignment) + assignment = getCurrentAssignment() + displayAssignmentInfo(cli_args) + displaySubmissionInfo(cli_args, assignment) case "assignment": handleAssignmentCmd(cli_args) case "submission": handleSubmissionCmd(cli_args) case "run": - assignment = get_current_assignment() + assignment = getCurrentAssignment() asyncio.run(execute_pytest_on_submissions(cli_args, assignment)) case "test": - assignment = get_current_assignment() + assignment = getCurrentAssignment() asyncio.run( execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) case "build": - assignment = get_current_assignment() + assignment = getCurrentAssignment() asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "build"')) case "render": - assignment = get_current_assignment() + assignment = getCurrentAssignment() asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "render"')) case _: console.log(cli_args, style="error") diff --git a/src/agh/pytest_plugin.py b/src/agh/pytest_plugin.py index 6c9a7ff..34d0d58 100644 --- a/src/agh/pytest_plugin.py +++ b/src/agh/pytest_plugin.py @@ -1,8 +1,12 @@ +import subprocess + import pytest from .agh_data import Assignment from .agh_data import Submission - +from .agh_data import OutputSectionData +from .agh_data import SubmissionFileData +from pathlib import Path class AghPtPlugin: def __init__(self, config): @@ -18,21 +22,6 @@ def pytest_terminal_summary(self, terminalreporter, exitstatus, config): # terminalreporter.write_line("JSON report saved to rich_parallel_report.json") -@pytest.fixture -def agh_test_plugin(request): - return "bob" - - -@pytest.fixture -def json_report_data(request): - def add_data(key, value): - nodeid = request.node.nodeid - report_data = request.config._json_report_data.setdefault(nodeid, {}) - report_data[key] = value - - return add_data - - def pytest_addoption(parser): parser.addoption("--agh", action="store_true", help="Enable AGH, assignment grading helper, extensions.") @@ -47,25 +36,63 @@ def pytest_configure(config): @pytest.fixture def agh_submission(request): - return Submission.load(request.path) + return Submission.load(request.path.parent) @pytest.fixture def agh_assignment(request): print(request.path) - return Assignment.load(request.path) + return Assignment.load(request.path.parent) + + +@pytest.fixture +def resultsDir(agh_submission) -> Path: + ret_val = agh_submission.evaluation_directory / "results" + ret_val.mkdir(parents=True, exist_ok=True) + return (ret_val) def register_render_env_var(env_var_name: str, env_var_value, cache: pytest.Cache): env_vars = cache.get("agh_render_env_vars", set()) env_vars.add(env_var_name) - cache.set("agh_render_env_vars", env_vars) + cache.set("agh_render_env_vars", list(env_vars)) cache.set(env_var_name, env_var_value) +def storeRunOutErr(tgt_name: str, res, resultsDir): + stdout_file = (resultsDir / f'{tgt_name}.stdout') + stdout_file.write_text(res.stdout) + stderr_file = (resultsDir / f'{tgt_name}.stderr') + stderr_file.write_text(res.stderr) + + +evaluationDataOS = OutputSectionData(path=Path('eval_data_section.md'), title="Evaluation Data", + heading_level=1) +yourCodeOS = OutputSectionData(path=Path('code_section.md'), title="Your Code", heading_level=1) + + +def _make_sections(resultsDir: Path, agh_assignment: Assignment, agh_submission: Submission): + global evaluationDataOS, yourCodeOS + evalOut = resultsDir / evaluationDataOS.path + evalOut.write_text(evaluationDataOS.asQmdSection()) + + codeOut = resultsDir / yourCodeOS.path + for s_file in agh_assignment.required_files.values(): + if s_file.include_in_output: + tgt = agh_submission.as_submitted_dir.absolute() / s_file.path.name + + # Create a submission file that points to my submission. + my_sub_src_file = SubmissionFileData(**s_file.asdict()) + my_sub_src_file.path = tgt.relative_to(agh_submission.evaluation_directory) + tgt = agh_submission.evaluation_directory / my_sub_src_file.path + if not tgt.exists(): + tgt.touch() + yourCodeOS.included_files.append(my_sub_src_file) + codeOut.write_text(yourCodeOS.asQmdSection()) + + @pytest.fixture -@pytest.mark.build -def agh_build_makefile(agh_submission, shell, cache, request): +def agh_build_makefile(agh_submission, shell, cache, request, resultsDir): request.applymarker(pytest.mark.build) def build(target: str | None = None): @@ -78,41 +105,67 @@ def build(target: str | None = None): cmd = ["make"] if target is not None: cmd.append(target) - res = shell.run(cmd, shell=True, cwd=agh_submission.evaluation_directory, env={"AGH_BUILD_TESTING": 1}) + res = shell.run(*cmd, shell=True, cwd=agh_submission.evaluation_directory, env={"AGH_BUILD_TESTING": '1'}) # Update permanent cache state for initial build ok. if first_build: cache.set("agh_build_makefile", True) build_ok_key = "agh_build_makefile_ok" register_render_env_var(build_ok_key, res.returncode == 0, cache) + + stdout_file = (resultsDir / f'{target if target else ""}build.stdout') + stdout_file.parent.mkdir(exist_ok=True) + stdout_file.write_text(res.stdout) + stderr_file = (resultsDir / f'{target if target else ""}build.stderr') + stderr_file.write_text(res.stderr) + return res yield build - if build.res != 0: - print(f"Build failed for {agh_submission.submission_file}") + # if build.res != 0: + # print(f"Build failed for {agh_submission.submission_file}") @pytest.fixture def agh_env_vars(cache): environ = {} for env_var_name in cache.get("agh_render_env_vars", set()): - environ[env_var_name] = cache.get(env_var_name, "") + environ[env_var_name] = str(cache.get(env_var_name, "")) return environ @pytest.fixture -@pytest.mark.render -def agh_render_quarto(agh_submission, shell, agh_env_vars, request): +def agh_render_output(agh_submission: Submission, shell, agh_env_vars: dict[str, str], request: pytest.FixtureRequest, + resultsDir: Path, agh_assignment: Assignment): request.applymarker(pytest.mark.render) - def render(target: str | None = None, *args): + def render(target: str | None = agh_assignment._options.output_template_name, *args: str): + _make_sections(resultsDir, agh_assignment, agh_submission) cmd = ["quarto", "render"] if target is not None: cmd.append(target) if len(args) > 0: cmd.extend(args) - res = shell.run(cmd, shell=True, cwd=agh_submission.evaluation_directory, env=agh_env_vars) - return res + + #Clear all render specific errors and warnings + agh_submission.delWarning('render warning').delError('render error').delWarning('render issue').save() + + try: + cmd_str = ' '.join(cmd) + print(f'Executing quarto: {cmd_str}') + res = shell.run(cmd_str, shell=True, cwd=agh_submission.evaluation_directory) + (resultsDir/ '.render.stdout.txt').write_text(f"{cmd_str}\n" + res.stdout) + (resultsDir/ '.render.stderr.txt').write_text(res.stderr) + if res.returncode != 0: + agh_submission.addWarning('render issue', f"Render '{cmd_str}' failed with return code {res.returncode}.").save() + raise RuntimeError(f"quarto failed with return code {res.returncode}") + agh_assignment.postProcessSubmissionRender(agh_submission, warning_callback=lambda warn: agh_submission.addWarning('render warning', warn)).save() + return res + except Exception as e: + agh_submission.addError('render error', f"Quarto failed with error {e}.").save() + request.raiseerror(f"Error rendering {agh_submission.submission_file}: {e}") return render + + From 9848940d7e7e28ac70ae563e74ebcbb1857ded44 Mon Sep 17 00:00:00 2001 From: Tuck Williamson Date: Mon, 20 Oct 2025 18:09:19 -0400 Subject: [PATCH 02/10] Refactored CLI code for consistency, readability, and formatting improvements. Enhanced error/warning output handling and added support for truncated file content in output. --- docs/reference/index.rst | 1 + docs/reference/pytest_plugin.rst | 11 + src/agh/agh_data.py | 162 ++++++++++----- src/agh/cli.py | 157 +++++++------- src/agh/pytest_plugin.py | 341 +++++++++++++++++++++++++------ tox.ini | 3 +- 6 files changed, 496 insertions(+), 179 deletions(-) create mode 100644 docs/reference/pytest_plugin.rst diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 9cb9f57..227871e 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -5,3 +5,4 @@ Reference :glob: agh* + pytest_plugin* diff --git a/docs/reference/pytest_plugin.rst b/docs/reference/pytest_plugin.rst new file mode 100644 index 0000000..7b6c6d0 --- /dev/null +++ b/docs/reference/pytest_plugin.rst @@ -0,0 +1,11 @@ +agh.pytest_plugin +=== + +.. testsetup:: + + from agh.pytest_plugin import * + +.. automodule:: agh.pytest_plugin + :members: + :undoc-members: + :show-inheritance: diff --git a/src/agh/agh_data.py b/src/agh/agh_data.py index ac9ed93..8e934db 100644 --- a/src/agh/agh_data.py +++ b/src/agh/agh_data.py @@ -2,6 +2,7 @@ import json import os import pathlib +from collections.abc import Callable from collections.abc import Generator from dataclasses import asdict from dataclasses import dataclass @@ -11,13 +12,16 @@ from enum import IntEnum from pathlib import Path from string import Template -from typing import Any, Callable, LiteralString, Literal +from typing import Any +from typing import Literal from typing import Self from typing import get_args from typing import get_origin import agh.anonymizer as anonymizer +DEFAULT_MAX_OUT_FILE_SIZE = 20 * 1024 + META_INTERNAL_SUB_OUTPUT = "OUTPUT_INFO" META_INTERNAL_SUB_KEY = "SUBMISSION" @@ -25,11 +29,9 @@ META_AGH_INTERNAL_KEY = "AGH_INTERNAL" META_INTERNAL_SUB_KEYS = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY] META_INTERNAL_SUB_OUTPUT_COMPLETE = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, - 'COMPLETED_OUTPUT'] -META_INTERNAL_SUB_OUTPUT_GRADED = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, - 'GRADED'] -META_INTERNAL_SUB_OUTPUT_NON_ANON = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, - 'NON_ANON'] + "COMPLETED_OUTPUT"] +META_INTERNAL_SUB_OUTPUT_GRADED = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "GRADED"] +META_INTERNAL_SUB_OUTPUT_NON_ANON = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "NON_ANON"] _USER_DEFAULTS_FILE = Path.home() / ".config" / "agh" / ".agh_user_defaults.json" @@ -82,7 +84,8 @@ def asdict(self) -> dict[str, Any]: setattr(self, cur_field.name, [str(cur_val) for cur_val in restore_these[cur_field.name]]) elif get_origin(cur_field.type) is dict and hasattr(get_args(cur_field.type)[-1], "asdict"): restore_these[cur_field.name] = getattr(self, cur_field.name) - setattr(self, cur_field.name, {cur_key: cur_val.asdict() for cur_key, cur_val in restore_these[cur_field.name].items()}) + setattr(self, cur_field.name, + {cur_key: cur_val.asdict() for cur_key, cur_val in restore_these[cur_field.name].items()}) # if len(restore_these) > 0: # print(self) ret_val = asdict(self) @@ -108,7 +111,8 @@ def _from_json(cls, data: dict): data[cur_field.name] = [get_args(cur_field.type)[0]._from_json(p) for p in data[cur_field.name]] elif get_origin(cur_field.type) is dict and hasattr(get_args(cur_field.type)[-1], "_from_json"): # print(data[cur_field.name]) - data[cur_field.name] = {k: get_args(cur_field.type)[-1]._from_json(p) for k, p in data[cur_field.name].items()} + data[cur_field.name] = {k: get_args(cur_field.type)[-1]._from_json(p) for k, p in + data[cur_field.name].items()} elif is_dataclass(cur_field.type): data[cur_field.name] = cur_field.type(**data[cur_field.name]) elif get_origin(cur_field.type) is list and is_dataclass(get_args(cur_field.type)[0]): @@ -170,14 +174,22 @@ def sectionAttr(self) -> str: return " {.unlisted .unnumbered}" return "" - def asQmdSection(self, heading_level: int) -> str: + def asQmdSection(self, heading_level: int, max_file_size: int = DEFAULT_MAX_OUT_FILE_SIZE) -> str: if not self.include_in_output: return "" # if not self.path.exists(): # self.path.touch() + # Check for excessive length. hdr_txt = "#" * heading_level + " " + self.title + self.sectionAttr - return f"\n\n{self.description}\n\n{hdr_txt}\n\n```{{.{self.type}}}\n{{{{< include {self.path} >}}}}\n```\n\n" + ret_val = f"\n\n{hdr_txt}\n\n{self.description}\n\n" + if self.path.exists() and self.path.stat().st_size > max_file_size: + with self.path.open('rt', encoding="utf-8", errors="replace") as f: + ret_val += (f"**[File too large! Contents truncated to {max_file_size} bytes.]{{.mark}}**\n\n```{{." + f"{self.type}}}\n{f.read(max_file_size)}\n```\n\n") + return ret_val + + return f"{ret_val}```{{.{self.type}}}\n{{{{< include {self.path} >}}}}\n```\n\n" # Mark all fields as keyword-only so that we can load directly from JSON. @@ -190,12 +202,17 @@ class OutputSectionData(SubmissionFileData): included_sections: list["OutputSectionData"] = field(default_factory=list) only_output_if_data: bool = False post_script: str = "" + _errors: list[dict[Literal["title", "message"], str]] = field(default_factory=list) + _warnings: list[dict[Literal["title", "message"], str]] = field(default_factory=list) @property def hasData(self): - if len(self.included_sections) > 0 or len(self.included_files) > 0: + if len(self._errors) > 0 or len(self._warnings) > 0: + return True + elif len(self.included_sections) > 0 or len(self.included_files) > 0: return True in [cur_sub_sec.hasData for cur_sub_sec in self.included_sections] or True in [ - (cur_inc_file.path.exists() and cur_inc_file.path.stat().st_size > 1) for cur_inc_file in self.included_files + (cur_inc_file.path.exists() and cur_inc_file.path.stat().st_size > 1) for cur_inc_file in + self.included_files ] elif self.text.strip() != "": # If there are no 'includes' then the section is the data @@ -205,11 +222,38 @@ def hasData(self): def asQmdSection(self) -> str: if not self.hasData and self.only_output_if_data: return "" - inc_file_txt = "\n\n".join([cur_inc_file.asQmdSection(self.heading_level + 1) for cur_inc_file in self.included_files]) + inc_file_txt = "\n\n".join( + [cur_inc_file.asQmdSection(self.heading_level + 1) for cur_inc_file in self.included_files]) inc_sec_txt = "\n\n".join([cur_inc_sec.asQmdSection() for cur_inc_sec in self.included_sections]) - pre = "#" * self.heading_level + f" {self.title} {self.sectionAttr}\n\n{self.description}\n\n{self.text}" + + warnings = "" + for warn in self._warnings: + warnings += f"\n\n+ **{warn['title']}:** {warn['message']}" + if warnings != "": + warnings = f'\n\n::: {{.callout-note title="To Consider"}}\n\n{warnings}\n\n:::\n\n' + + errors = "" + for err in self._errors: + errors += f"\n\n+ **{err['title']}:** {err['message']}" + if errors != "": + errors = f'\n\n::: {{.callout-important title="Errors"}}\n\n{errors}\n\n:::\n\n' + + pre = "#" * self.heading_level + f" {self.title} {self.sectionAttr}\n\n{self.description}\n\n{self.text}{errors}{warnings}" return f"{pre}\n\n{inc_file_txt}\n\n{inc_sec_txt}\n\n{self.post_script}" + def addSection(self, section: "OutputSectionData") -> Self: + self.included_sections.append(section) + section.heading_level = self.heading_level + 1 + return self + + def addWarning(self, warn_title: str, warn_msg: str) -> Self: + self._errors.append({"title": warn_title, "message": warn_msg}) + return self + + def addError(self, error_title: str, error_msg: str) -> Self: + self._errors.append({"title": error_title, "message": error_msg}) + return self + @dataclass(kw_only=True) class MetaDataclassJson(DataclassJson): @@ -235,7 +279,7 @@ def _getMetadata(self, metadata_key: str, default: Any = None) -> dict[str, Any] return default return cur_metadata - def getMetadata(self, *args, default: Any = None) -> dict[str, Any]|Any: + def getMetadata(self, *args, default: Any = None) -> dict[str, Any] | Any: """Returns the metadata associated with the assignment. If the key is not present in the metadata, it will return the default value. @@ -323,7 +367,8 @@ class GraderOptions(MetaDataclassJson): _general_editor_command: str | None = None general_editor_command = property( - *_gen_prop_methods("_general_editor_command", "subl $files"), doc="A command to open a submission file in a text editor." + *_gen_prop_methods("_general_editor_command", "subl $files"), + doc="A command to open a submission file in a text editor." ) # This is a dictionary of metadata associated with the assignment. @@ -544,7 +589,8 @@ def load(cls, filepath: pathlib.Path | None = None): filepath = filepath.absolute() filepath = findFileInParents(filepath, cls.ASSIGNMENT_FILE_NAME) if filepath is None: - raise FileNotFoundError(f"Could not find assignment JSON file in {orig_filepath} or any of its parents.") + raise FileNotFoundError( + f"Could not find assignment JSON file in {orig_filepath} or any of its parents.") if filepath.exists() and filepath.is_file(): data = json.loads(filepath.read_text()) @@ -646,10 +692,13 @@ class LinkProto(IntEnum): # Overwrite the existing file if it already exists. LINK_OVERWRITE = 2 - def PostProcessSubmission(self, submission_file: "pathlib.Path|Submission", - exists_protocol: LinkProto = LinkProto.RAISE_ERROR, - warning_callback: Callable[[str],None] | None = None) -> "Submission": - ret_val:Submission = submission_file + def PostProcessSubmission( + self, + submission_file: "pathlib.Path|Submission", + exists_protocol: LinkProto = LinkProto.RAISE_ERROR, + warning_callback: Callable[[str], None | Any] | None = None, + ) -> "Submission": + ret_val: Submission = submission_file if isinstance(submission_file, pathlib.Path): ret_val = Submission.load(filepath=submission_file) @@ -661,7 +710,8 @@ def all_linked_files() -> Generator[Path]: for link_item in self.link_template_dir.iterdir(): yield link_item for link_item in self._optional_files.values(): - if link_item.copy_to_sub_if_missing and not (ret_val.evaluation_directory / link_item.path.name).exists(): + if link_item.copy_to_sub_if_missing and not ( + ret_val.evaluation_directory / link_item.path.name).exists(): yield link_item.path for link_item in all_linked_files(): @@ -692,7 +742,9 @@ def all_linked_files() -> Generator[Path]: # Now start linking the output stuff together. return self.postProcessSubmissionRender(ret_val, warning_callback=warning_callback) - def postProcessSubmissionRender(self, submission: "Submission", warning_callback: Callable[[str],None] | None = None) -> "Submission": + def postProcessSubmissionRender( + self, submission: "Submission", warning_callback: Callable[[str], None | Any] | None = None + ) -> "Submission": for output_file in self._options.output_files: output_in_sub_dir = submission.evaluation_directory / output_file output_file = self.complete_eval_dir / (submission.evaluation_directory.name + output_in_sub_dir.suffix) @@ -703,7 +755,9 @@ def postProcessSubmissionRender(self, submission: "Submission", warning_callback submission.setMetadata(*META_INTERNAL_SUB_OUTPUT_GRADED, value=str(output_graded)) try: # Copy output_in_sub_dir to output_graded if needed - if (not output_graded.exists()) or (output_graded.exists() and output_graded.stat().st_ctime_ns == output_graded.stat().st_mtime_ns): + if (not output_graded.exists()) or ( + output_graded.exists() and output_graded.stat().st_ctime_ns == output_graded.stat().st_mtime_ns + ): output_graded.parent.mkdir(parents=True, exist_ok=True) # This is to ensure that ctime == mtime! @@ -711,9 +765,13 @@ def postProcessSubmissionRender(self, submission: "Submission", warning_callback output_graded.write_bytes(output_in_sub_dir.read_bytes()) elif warning_callback is not None: - warning_callback(f'The graded output file "{output_graded}" already exists and appears modified. {output_graded} will not be overwritten.') + warning_callback( + f'The graded output file "{output_graded}" already exists and appears modified. ' + f'{output_graded} will not be overwritten.' + ) # stats = output_graded.stat() - # warning_callback(f'The graded output file info a:{stats.st_atime_ns}, c: {stats.st_ctime_ns}, m: {stats.st_mtime_ns}.') + # warning_callback(f'The graded output file info a:{stats.st_atime_ns}, c: {stats.st_ctime_ns}, + # m: {stats.st_mtime_ns}.') output_file.unlink(missing_ok=True) output_not_anon.unlink(missing_ok=True) @@ -721,18 +779,22 @@ def postProcessSubmissionRender(self, submission: "Submission", warning_callback pass # Create symbolic links output_file.symlink_to( - output_in_sub_dir.relative_to(output_file.parent, walk_up=True), target_is_directory=output_file.is_dir() + output_in_sub_dir.relative_to(output_file.parent, walk_up=True), + target_is_directory=output_file.is_dir() ) - output_file.chmod(0o444) + # output_file.chmod(0o444) output_not_anon.symlink_to( - output_graded.relative_to(output_not_anon.parent, walk_up=True), target_is_directory=output_not_anon.is_dir() + output_graded.relative_to(output_not_anon.parent, walk_up=True), + target_is_directory=output_not_anon.is_dir() ) return submission - def AddSubmission(self, submission_file: pathlib.Path, override_anon: bool | None = None, - warning_callback: Callable[[str],None] | None = None) -> "Submission": + def AddSubmission( + self, submission_file: pathlib.Path, override_anon: bool | None = None, + warning_callback: Callable[[str], None | Any] | None = None + ) -> "Submission": """Add a new submission to the assignment. :param submission_file: The path to the submission file to add. :param override_anon: If none then abide by the default for the assignment. @@ -744,7 +806,8 @@ def AddSubmission(self, submission_file: pathlib.Path, override_anon: bool | Non """ ret_val = Submission.new(self, submission_file=submission_file, override_anon=override_anon) ret_val.save() - return self.PostProcessSubmission(ret_val, exists_protocol=self.LinkProto.RAISE_ERROR, warning_callback=warning_callback) + return self.PostProcessSubmission(ret_val, exists_protocol=self.LinkProto.RAISE_ERROR, + warning_callback=warning_callback) # def __pytest_cmd(self): # return "pytest ./ -p shell-utilities -p agh" @@ -914,7 +977,8 @@ def __post_init__(self): if not self.submission_file.exists() or not self.submission_file.is_file(): raise ValueError(f"submission_file '{self.submission_file}' does not exist or is not a file.") if not self.evaluation_directory.exists() or not self.evaluation_directory.is_dir(): - raise ValueError(f"evaluation_directory '{self.evaluation_directory}' does not exist or is not a directory.") + raise ValueError( + f"evaluation_directory '{self.evaluation_directory}' does not exist or is not a directory.") class Submission(SubmissionData): @@ -967,7 +1031,8 @@ def get_anon_name(cls, assignment: Assignment, submission_file: pathlib.Path): Returns: str: Anonymous name for the submission """ - return anonymizer.anonymize(submission_file.name, assignment.name, str(assignment.year), assignment.grade_period, assignment.course) + return anonymizer.anonymize(submission_file.name, assignment.name, str(assignment.year), + assignment.grade_period, assignment.course) @classmethod def load(cls, filepath: pathlib.Path | None = None): @@ -989,7 +1054,8 @@ def load(cls, filepath: pathlib.Path | None = None): filepath = filepath.absolute() filepath = findFileInParents(filepath, cls.SUBMISSION_FILE_NAME) if filepath is None: - raise FileNotFoundError(f"Could not find submission JSON file in {orig_filepath} or any of its parents.") + raise FileNotFoundError( + f"Could not find submission JSON file in {orig_filepath} or any of its parents.") if filepath.exists() and filepath.is_file(): data = json.loads(filepath.read_text()) @@ -1085,7 +1151,8 @@ def __post_process_new__(self, assignment: Assignment, base_file_name: str | Non elif "zip" in self.submission_file.name: os.system(f'cd "{self.as_submitted_dir.absolute()}" && unzip "{self.submission_file.absolute()}"') elif base_file_name is not None: - os.system(f'cp "{self.submission_file.absolute()}" "{self.evaluation_directory.absolute()}/{base_file_name}"') + os.system( + f'cp "{self.submission_file.absolute()}" "{self.evaluation_directory.absolute()}/{base_file_name}"') self._missing_files_initially = self.check_missing_files(assignment) @@ -1114,11 +1181,12 @@ def name(self): return self.anon_name @property - def main_output_files(self) -> list[Path | None ]: + def main_output_files(self) -> list[Path | None]: """Return the submission's output files. :return: A list of output files as follows: ``[, - , , ]`` + , , + ]`` """ assign = Assignment.load() rendered = None @@ -1129,7 +1197,7 @@ def main_output_files(self) -> list[Path | None ]: main = self.getMetadata(*META_INTERNAL_SUB_OUTPUT_COMPLETE) graded = self.getMetadata(*META_INTERNAL_SUB_OUTPUT_GRADED) - non_anon :None|str|Path= self.getMetadata(*META_INTERNAL_SUB_OUTPUT_NON_ANON) + non_anon: None | str | Path = self.getMetadata(*META_INTERNAL_SUB_OUTPUT_NON_ANON) if main is not None: main = Path(main) @@ -1144,14 +1212,14 @@ def main_output_files(self) -> list[Path | None ]: def _getErrWarnList(self, type: Literal["errors", "warnings"]) -> list[str]: return list(self.getMetadata(*META_INTERNAL_SUB_KEYS, type, default={}).values()) - def _setErrWarnItem(self, type: Literal["errors", "warnings"], key: str, txt_or_markdown:str) -> Self: + def _setErrWarnItem(self, type: Literal["errors", "warnings"], key: str, txt_or_markdown: str) -> Self: keys = list(META_INTERNAL_SUB_KEYS) keys.append(type) keys.append(key) return self.setMetadata(*keys, value=txt_or_markdown) def _delErrWarnItem(self, type: Literal["errors", "warnings"], key: str) -> Self: - exist_md_dict:dict[Any, Any] = self.getMetadata(*META_INTERNAL_SUB_KEYS, type, default={}) + exist_md_dict: dict[Any, Any] = self.getMetadata(*META_INTERNAL_SUB_KEYS, type, default={}) exist_md_dict.pop(key, None) return self.setMetadata(*META_INTERNAL_SUB_KEYS, type, value=exist_md_dict) @@ -1167,14 +1235,15 @@ def errors(self) -> None | list[str]: missing_files = self.check_missing_files(Assignment.load()) if len(missing_files) > 0: - errors.append(f"Missing required file{'s' if len(missing_files) > 1 else ''}: {[mf.name for mf in missing_files]}") + errors.append( + f"Missing required file{'s' if len(missing_files) > 1 else ''}: {[mf.name for mf in missing_files]}") if len(errors) > 0: return errors return None - def addError(self, key:str, txt_or_markdown: str) -> "Submission": + def addError(self, key: str, txt_or_markdown: str) -> "Submission": """Add an error to the submission. These are NOT testing errors, but anything preventing the submission from being tested. """ @@ -1184,7 +1253,8 @@ def addError(self, key:str, txt_or_markdown: str) -> "Submission": # errors: list[str] = self.getMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "errors", default=[]) # errors.append(txt_or_markdown) # return self.setMetadata(META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, "errors", value=errors) - def delError(self, key:str) -> "Submission": + + def delError(self, key: str) -> "Submission": return self._delErrWarnItem("errors", key) @property @@ -1194,11 +1264,11 @@ def warnings(self) -> None | list[str]: """ return self._getErrWarnList("warnings") - def addWarning(self, key:str, txt_or_markdown: str) -> "Submission": + def addWarning(self, key: str, txt_or_markdown: str) -> "Submission": """Add a warning to the submission. These are NOT testing warnings, but anything possibly preventing the submission from being tested. """ return self._setErrWarnItem("warnings", key, txt_or_markdown) - def delWarning(self, key:str) -> "Submission": + def delWarning(self, key: str) -> "Submission": return self._delErrWarnItem("warnings", key) diff --git a/src/agh/cli.py b/src/agh/cli.py index 83734cc..2fe154f 100644 --- a/src/agh/cli.py +++ b/src/agh/cli.py @@ -92,7 +92,7 @@ def __call__(self, parser, namespace, values, option_string=None): for name, cur_sub_parser in all_sub_parsers: # console.print(Panel(cur_sub_parser.format_help(),title=f"[bold]Subcommand: {name}", expand=False, # style="b", )) - console.rule(('>' * 5) + f" [b red]{name}[/] ", align='left') + console.rule((">" * 5) + f" [b red]{name}[/] ", align="left") cur_sub_parser.print_help() parser.exit(0) @@ -121,8 +121,7 @@ def SubFileCompleter(property: str, prefix: str, **kwargs): def submissionCompleter(*args, **kwargs): # return ['Bob','Tom'] try: - ret_val = [str(subm.evaluation_directory.resolve().relative_to(Path.cwd(), walk_up=True)) for subm in - Assignment.load().Submissions] + ret_val = [str(subm.evaluation_directory.resolve().relative_to(Path.cwd(), walk_up=True)) for subm in Assignment.load().Submissions] return ret_val except Exception as e: argcomplete.warn("No assignment found. Cannot complete submission files.") @@ -133,8 +132,7 @@ def submissionCompleter(*args, **kwargs): # console.log(SubFileCompleter("unprocessed_dir", '')) parser = MyArgParser( - description="agh --- Assignment Grading Helper", prog="agh", formatter_class=RichHelpFormatter, - conflict_handler="resolve" + description="agh --- Assignment Grading Helper", prog="agh", formatter_class=RichHelpFormatter, conflict_handler="resolve" ) parser.add_argument("--version", action="version", version=__version__) parser.add_argument("-H", "--full-help", action=FullHelp, help="Show full (all options) help") @@ -155,22 +153,16 @@ def submissionCompleter(*args, **kwargs): ################################################################################ ################################################################################ -assignment_sub_parser = subparsers.add_parser("assignment", help="Assignment commands", - formatter_class=RichHelpFormatter) +assignment_sub_parser = subparsers.add_parser("assignment", help="Assignment commands", formatter_class=RichHelpFormatter) assignment_subparsers = assignment_sub_parser.add_subparsers(dest="assignment_command", help="Assignment commands") # assignment > new assignment command -assignment_new_parser = assignment_subparsers.add_parser("new", help="Create new assignment", - formatter_class=RichHelpFormatter) -assignment_new_parser.add_argument("name", help="Assignment name", type=str).completer = lambda **kwargs: [ - f"Assignment {Path.cwd().name}"] -assignment_new_parser.add_argument("course", help="Course code", type=str).completer = lambda **kwargs: [ - f"CSCI-{Path.cwd().parent.name}"] -assignment_new_parser.add_argument("term", help="Term", - choices=["Fall", "Spring", "Maymester", "Summer I", "Summer II"], type=str) +assignment_new_parser = assignment_subparsers.add_parser("new", help="Create new assignment", formatter_class=RichHelpFormatter) +assignment_new_parser.add_argument("name", help="Assignment name", type=str).completer = lambda **kwargs: [f"Assignment {Path.cwd().name}"] +assignment_new_parser.add_argument("course", help="Course code", type=str).completer = lambda **kwargs: [f"CSCI-{Path.cwd().parent.name}"] +assignment_new_parser.add_argument("term", help="Term", choices=["Fall", "Spring", "Maymester", "Summer I", "Summer II"], type=str) assignment_new_parser.add_argument("-y", "--year", help="Year", type=int, default=cur_date.year) -assignment_new_parser.add_argument("-a", "--anon", help="Anonymize names", action=argparse.BooleanOptionalAction, - default=True) +assignment_new_parser.add_argument("-a", "--anon", help="Anonymize names", action=argparse.BooleanOptionalAction, default=True) # assignment > info command assignment_info_parser = assignment_subparsers.add_parser("info", help="Show assignment info") @@ -179,8 +171,7 @@ def submissionCompleter(*args, **kwargs): # Add required files command assign_add_required_parser = assignment_subparsers.add_parser("add-required", help="Add required files") assign_add_required_parser.add_argument("files", nargs="+", help="Required file names", type=Path) -assign_add_required_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda \ - **kwargs: [ +assign_add_required_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda **kwargs: [ "txt", "py", "c", @@ -189,8 +180,7 @@ def submissionCompleter(*args, **kwargs): "default", "make", ] -assign_add_required_parser.add_argument("-d", "--description", help="Description of the required files", type=str, - default="") +assign_add_required_parser.add_argument("-d", "--description", help="Description of the required files", type=str, default="") assign_add_required_parser.add_argument("-t", "--title", help="Title of the required files", type=str, default="") assign_add_required_parser.add_argument( "-i", "--include-in-output", help="Include in output", action=argparse.BooleanOptionalAction, default=True @@ -199,8 +189,7 @@ def submissionCompleter(*args, **kwargs): # Add optional files command assign_add_optional_parser = assignment_subparsers.add_parser("add-optional", help="Add optional files") assign_add_optional_parser.add_argument("files", nargs="+", help="Optional file names") -assign_add_optional_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda \ - **kwargs: [ +assign_add_optional_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda **kwargs: [ "txt", "py", "c", @@ -209,8 +198,7 @@ def submissionCompleter(*args, **kwargs): "make", "default", ] -assign_add_optional_parser.add_argument("-d", "--description", help="Description of the optional files", type=str, - default="") +assign_add_optional_parser.add_argument("-d", "--description", help="Description of the optional files", type=str, default="") assign_add_optional_parser.add_argument("-t", "--title", help="Title of the optional files", type=str, default="") assign_add_optional_parser.add_argument( "-i", "--include-in-output", help="Include in output", action=argparse.BooleanOptionalAction, default=True @@ -226,23 +214,31 @@ def submissionCompleter(*args, **kwargs): # submission > add command sub_add_subparser = sub_subparsers.add_parser("add", help="Add a submission file.") -sub_add_subparser.add_argument("files", nargs="+", help="Submission files to add", - type=Path).completer = functools.partial( +sub_add_subparser.add_argument("files", nargs="+", help="Submission files to add", type=Path).completer = functools.partial( SubFileCompleter, "unprocessed_dir" ) -sub_add_subparser.add_argument("-a", "--anonymous", dest="override_anon", action="store_true", - help="Override the assignment default and make this submission anonymous.", - default=None) -sub_add_subparser.add_argument("-n", "--non-anonymous", dest="override_anon", action="store_false", - help="Override the assignment default and make this submission non-anonymous.", - default=None) +sub_add_subparser.add_argument( + "-a", + "--anonymous", + dest="override_anon", + action="store_true", + help="Override the assignment default and make this submission anonymous.", + default=None, +) +sub_add_subparser.add_argument( + "-n", + "--non-anonymous", + dest="override_anon", + action="store_false", + help="Override the assignment default and make this submission non-anonymous.", + default=None, +) # submission > fix command sub_fix_subparser = sub_subparsers.add_parser( "fix", help="Fix a submission. Try this if you accidentally deleted something. This may re-create links etc." ) -sub_fix_subparser.add_argument("submissions", nargs="+", help="Submissions to fix", - type=str).completer = submissionCompleter +sub_fix_subparser.add_argument("submissions", nargs="+", help="Submissions to fix", type=str).completer = submissionCompleter ################################################################################ ################################################################################ @@ -252,36 +248,30 @@ def submissionCompleter(*args, **kwargs): # Add run command run_parser = subparsers.add_parser("run", help="Run submission files. This executes build, test, and render.") run_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter run_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add test command -test_parser = subparsers.add_parser("test", - help="Test submission files. This just runs the tests for the given submissions.") +test_parser = subparsers.add_parser("test", help="Test submission files. This just runs the tests for the given submissions.") test_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter test_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add build command build_parser = subparsers.add_parser("build", help="Build submission files") build_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter build_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add render command render_parser = subparsers.add_parser("render", help="Render submission files") render_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter -render_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", - default=False) +render_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) argcomplete.autocomplete(parser) @@ -289,14 +279,11 @@ def submissionCompleter(*args, **kwargs): def displayAssignmentInfo(cli_args: argparse.Namespace): assignment = Assignment.load() console.print(f'[label]Assignment "{assignment.name}"') - console.print( - f"[label]Course:[/] {assignment.course}, [label]Term:[/] {assignment._grade_period}, [label]Year:[/] " - f"{assignment.year}") + console.print(f"[label]Course:[/] {assignment.course}, [label]Term:[/] {assignment._grade_period}, [label]Year:[/] {assignment.year}") submissions = list(assignment.Submissions) console.print(f"[label]Submissions:[/] {len(submissions)}") - files_table = rich.table.Table(title="[label][req]Required[/req]/[opt]Optional[/opt] Files", expand=True, - show_lines=True) + files_table = rich.table.Table(title="[label][req]Required[/req]/[opt]Optional[/opt] Files", expand=True, show_lines=True) files_table.add_column("Link", justify="center") files_table.add_column("Output", justify="center") files_table.add_column("Name", justify="left") @@ -336,7 +323,6 @@ def displaySubmissionInfo(cli_args: argparse.Namespace, assignment: Assignment): submission_table.add_column("Grading Output", justify="center") for submission in assignment.Submissions: - # For warnings and errors build 'links' that are just text so that there is # information when the user hovers over the link in the terminal. warnings = submission.warnings @@ -355,18 +341,18 @@ def displaySubmissionInfo(cli_args: argparse.Namespace, assignment: Assignment): else: errors = ":white_check_mark:" - output, main, graded, non_anon = submission.main_output_files + output, main, graded, _ = submission.main_output_files has_output = main is not None and main.exists() if isinstance(main, Path) and main.exists(): output = f"[link file://{main.absolute()}] :notebook: [/]" else: - output = ":x:" # + str(main) + output = ":x:" # + str(main) # has_graded_output = graded is not None and graded.exists() if isinstance(graded, Path) and graded.exists(): graded_output = f"[link file://{graded.absolute()}] :notebook: [/]" else: - graded_output = ":x:" #+ str(graded) + graded_output = ":x:" # + str(graded) sub_color = "green" match (has_errors, has_warnings, has_output): @@ -403,8 +389,13 @@ def handleAssignmentCmd(cli_args: argparse.Namespace): pass with console.status("Creating assignment", spinner="dots"): console.print(f'[bold green]Creating assignment "{cli_args.name}"') - new_assignment = Assignment(_name=cli_args.name, _course=cli_args.course, _grade_period=cli_args.term, - _year=cli_args.year, _options=GraderOptions(anonymize_names=cli_args.anon)) + new_assignment = Assignment( + _name=cli_args.name, + _course=cli_args.course, + _grade_period=cli_args.term, + _year=cli_args.year, + _options=GraderOptions(anonymize_names=cli_args.anon), + ) console.print(f"[bold green]saving {new_assignment.name}.") new_assignment.save() console.print("[bold green]Creating directories.") @@ -427,8 +418,15 @@ def handleAssignmentCmd(cli_args: argparse.Namespace): # For each file call the method curried from above. for cur_file in cli_args.files: - method(SubmissionFileData(path=Path(cur_file), title=cli_args.title, description=cli_args.description, - include_in_output=cli_args.include_in_output, type=cli_args.type)) + method( + SubmissionFileData( + path=Path(cur_file), + title=cli_args.title, + description=cli_args.description, + include_in_output=cli_args.include_in_output, + type=cli_args.type, + ) + ) console.print(f"[bold green]Added file '{cur_file}'") assignment.save() case _: @@ -449,7 +447,11 @@ def handleSubmissionCmd(cli_args: argparse.Namespace): for cur_file in cli_args.files: console.print(f"Adding {cur_file}") try: - assignment.AddSubmission(cur_file, override_anon=cli_args.override_anon, warning_callback=lambda warn: console.print(warn, style="warning")).save() + assignment.AddSubmission( + cur_file, + override_anon=cli_args.override_anon, + warning_callback=lambda warn: console.print(warn, style="warning"), + ).save() except Exception as e: console.print(f"[error]Error adding submission '{cur_file}': {e}") assignment.save() @@ -479,13 +481,14 @@ class RunOutputInfo(DataclassJson): collected: int | None = None return_code: int | None = None -def verbose_print(cli_args:argparse.Namespace, *args, **kwargs) -> None: + +def verbose_print(cli_args: argparse.Namespace, *args, **kwargs) -> None: if cli_args.verbose: console.print(*args, **kwargs) + async def parse_pytest_output( - assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, - task_id + assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, task_id ): output_info = RunOutputInfo() @@ -504,7 +507,7 @@ async def error_collector(): break line = line.decode().strip() output_info.output.append(line) - if "collecting ..." in line and 'selected' in line: + if "collecting ..." in line and "selected" in line: match = re.search(r"collected \d+ items / \d+ deselected / (\d+) selected", line) if match: output_info.collected = int(match.group(1)) @@ -529,7 +532,11 @@ async def error_collector(): async def run_pytest( - assignment: Assignment, submission: Submission, progress: rich.progress.Progress, cli_args:argparse.Namespace, extra_pytest_args: str = "" + assignment: Assignment, + submission: Submission, + progress: rich.progress.Progress, + cli_args: argparse.Namespace, + extra_pytest_args: str = "", ) -> tuple[Submission, bool]: """Run pytest on the given submission. @@ -552,12 +559,11 @@ async def run_pytest( task_id, advance=1, completed=True, - description=f"[error]Tests directory '{tests_path.absolute()}'not found. Perhaps run fix on " - f"{submission.name} first?", + description=f"[error]Tests directory '{tests_path.absolute()}'not found. Perhaps run fix on {submission.name} first?", ) return submission, False - cmd_str : str = f"pytest -v -p agh-pytest-plugin --agh {extra_pytest_args} {tests_path.absolute()}/*" + cmd_str: str = f"pytest -v -p agh-pytest-plugin --agh {extra_pytest_args} {tests_path.absolute()}/*" # verbose_print(cli_args, 'Running pytest...', cmd_str) # Setup the progress bar. task_id = progress.add_task(f"Testing {tests_path.absolute()}...", total=None) @@ -572,8 +578,7 @@ async def run_pytest( return submission, return_code == 0 -async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment: Assignment, - extra_pytest_args: str = ""): +async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment: Assignment, extra_pytest_args: str = ""): """This function asynchronously runs pytest on all submissions specified. It provides a progress bar for each submission to indicate the progress of the tests. @@ -612,8 +617,10 @@ async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment *rich.progress.Progress.get_default_columns(), rich.progress.TimeRemainingColumn(), ) as progress: - tasks = [run_pytest(assignment, submission, progress, cli_args, extra_pytest_args=extra_pytest_args) for submission in - cli_args.submissions] + tasks = [ + run_pytest(assignment, submission, progress, cli_args, extra_pytest_args=extra_pytest_args) + for submission in cli_args.submissions + ] results = await asyncio.gather(*tasks) for submission, success in results: @@ -655,8 +662,7 @@ def run(args=None): asyncio.run(execute_pytest_on_submissions(cli_args, assignment)) case "test": assignment = getCurrentAssignment() - asyncio.run( - execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) + asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) case "build": assignment = getCurrentAssignment() asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "build"')) @@ -668,6 +674,7 @@ def run(args=None): # print(start(args)) parser.exit(0) + # todo: create custom url scheme so I can run things from links in the output. # To create a custom URL scheme in Ubuntu that executes a command in the terminal, you need to define a desktop # entry for the scheme and create a script to handle the URL. diff --git a/src/agh/pytest_plugin.py b/src/agh/pytest_plugin.py index 34d0d58..3b37ea5 100644 --- a/src/agh/pytest_plugin.py +++ b/src/agh/pytest_plugin.py @@ -1,12 +1,19 @@ -import subprocess +import os +from pathlib import Path +from typing import ClassVar, Callable import pytest +from pytestshellutils.shell import ProcessResult, ScriptSubprocess from .agh_data import Assignment -from .agh_data import Submission from .agh_data import OutputSectionData +from .agh_data import Submission from .agh_data import SubmissionFileData -from pathlib import Path + +TEST_MD_KEY = "TEST_INFO" + +CORE_DUMP_FILE_NAME = "aghAssignmentCoreDump.core" + class AghPtPlugin: def __init__(self, config): @@ -31,7 +38,8 @@ def pytest_configure(config): plugin = AghPtPlugin(config) config.pluginmanager.register(plugin, name="agh_plugin") config.addinivalue_line("markers", "build: This marks anything related to building a submission's exe.") - config.addinivalue_line("markers", "render: This marks anything related to rendering a submission's documentation.") + config.addinivalue_line("markers", + "render: This marks anything related to rendering a submission's documentation.") @pytest.fixture @@ -49,7 +57,7 @@ def agh_assignment(request): def resultsDir(agh_submission) -> Path: ret_val = agh_submission.evaluation_directory / "results" ret_val.mkdir(parents=True, exist_ok=True) - return (ret_val) + return ret_val def register_render_env_var(env_var_name: str, env_var_value, cache: pytest.Cache): @@ -60,84 +68,302 @@ def register_render_env_var(env_var_name: str, env_var_value, cache: pytest.Cach def storeRunOutErr(tgt_name: str, res, resultsDir): - stdout_file = (resultsDir / f'{tgt_name}.stdout') + stdout_file = resultsDir / f"{tgt_name}.stdout" stdout_file.write_text(res.stdout) - stderr_file = (resultsDir / f'{tgt_name}.stderr') + stderr_file = resultsDir / f"{tgt_name}.stderr" stderr_file.write_text(res.stderr) -evaluationDataOS = OutputSectionData(path=Path('eval_data_section.md'), title="Evaluation Data", - heading_level=1) -yourCodeOS = OutputSectionData(path=Path('code_section.md'), title="Your Code", heading_level=1) +evaluationDataOS = OutputSectionData(path=Path("eval_data_section.md"), title="Evaluation Data", heading_level=1) +instructor_out_data = OutputSectionData(path=Path("instructor_data_section.md"), title="Instructor Data", + heading_level=1) def _make_sections(resultsDir: Path, agh_assignment: Assignment, agh_submission: Submission): - global evaluationDataOS, yourCodeOS - evalOut = resultsDir / evaluationDataOS.path - evalOut.write_text(evaluationDataOS.asQmdSection()) - - codeOut = resultsDir / yourCodeOS.path - for s_file in agh_assignment.required_files.values(): - if s_file.include_in_output: - tgt = agh_submission.as_submitted_dir.absolute() / s_file.path.name - - # Create a submission file that points to my submission. - my_sub_src_file = SubmissionFileData(**s_file.asdict()) - my_sub_src_file.path = tgt.relative_to(agh_submission.evaluation_directory) - tgt = agh_submission.evaluation_directory / my_sub_src_file.path - if not tgt.exists(): - tgt.touch() - yourCodeOS.included_files.append(my_sub_src_file) - codeOut.write_text(yourCodeOS.asQmdSection()) + global evaluationDataOS, instructor_out_data + + orig_cwd = Path.cwd() + os.chdir(agh_submission.evaluation_directory) + try: + yourCodeOS = OutputSectionData(path=Path("code_section.md"), title="Your Code", heading_level=1) + + instructor_out_data.path = resultsDir / instructor_out_data.path.name + instructor_out_data.path.write_text(instructor_out_data.asQmdSection()) + + # todo: handle loading from file. + # for cur_section in [evaluationDataOS, yourCodeOS, instructor_out_data]: + # if not cur_section.hasData: + # # Try loading it from the filesystem. + # cur_section.path = resultsDir / (cur_section.path.with_suffix('.json')) + # if cur_section.path.exists(): + + evalOut = resultsDir / evaluationDataOS.path + evalOut.write_text(evaluationDataOS.asQmdSection()) + + codeOut = resultsDir / yourCodeOS.path + for s_file in agh_assignment.required_files.values(): + if s_file.include_in_output: + tgt = agh_submission.as_submitted_dir.absolute() / s_file.path.name + + # Create a submission file that points to my submission. + my_sub_src_file = SubmissionFileData(**s_file.asdict()) + my_sub_src_file.path = tgt.relative_to(agh_submission.evaluation_directory) + tgt = agh_submission.evaluation_directory / my_sub_src_file.path + if not tgt.exists(): + tgt.touch() + yourCodeOS.included_files.append(my_sub_src_file) + codeOut.write_text(yourCodeOS.asQmdSection()) + finally: + os.chdir(orig_cwd) @pytest.fixture -def agh_build_makefile(agh_submission, shell, cache, request, resultsDir): +def agh_build_makefile(agh_submission, shell, cache, request, resultsDir) -> Callable[[str], str]: request.applymarker(pytest.mark.build) - def build(target: str | None = None): + def build(target: str | None = None, include_build_in_eval:bool=True): # Check to see if this is the first time we're building this submission. first_build = False - if not cache.get("agh_build_makefile", False): + if agh_submission.getMetadata(TEST_MD_KEY, "initial_build_success", default=None) is None: first_build = True + agh_submission.setMetadata(TEST_MD_KEY, "initial_build_success", value=False) # Build the submission. cmd = ["make"] if target is not None: cmd.append(target) - res = shell.run(*cmd, shell=True, cwd=agh_submission.evaluation_directory, env={"AGH_BUILD_TESTING": '1'}) + res = shell.run(*cmd, shell=True, cwd=agh_submission.evaluation_directory, env={"AGH_BUILD_TESTING": "1"}) # Update permanent cache state for initial build ok. if first_build: - cache.set("agh_build_makefile", True) - build_ok_key = "agh_build_makefile_ok" - register_render_env_var(build_ok_key, res.returncode == 0, cache) + agh_submission.setMetadata(TEST_MD_KEY, "initial_build_success", value=res.returncode == 0) - stdout_file = (resultsDir / f'{target if target else ""}build.stdout') + buildOutSection = OutputSectionData(title="Build Output", heading_level=1) + if include_build_in_eval: + evaluationDataOS.addSection(buildOutSection) + stdout_file = resultsDir / f"{target if target else ''}build.stdout" stdout_file.parent.mkdir(exist_ok=True) stdout_file.write_text(res.stdout) - stderr_file = (resultsDir / f'{target if target else ""}build.stderr') + buildOutSection.included_files.append( + SubmissionFileData(path=stdout_file, title="Build Stdout Output", + heading_level=buildOutSection.heading_level + 1)) + stderr_file = resultsDir / f"{target if target else ''}build.stderr" stderr_file.write_text(res.stderr) + if len(res.stderr) > 0: + buildOutSection.included_files.append( + SubmissionFileData(path=stdout_file, title="Build Stderr Output", + heading_level=buildOutSection.heading_level + 1)) return res - yield build + return build + - # if build.res != 0: - # print(f"Build failed for {agh_submission.submission_file}") +@pytest.fixture +def _core_file_saved(agh_submission): + core_path = Path("/proc/sys/kernel/core_pattern") + path_good = core_path.exists() and "apport" not in core_path.read_text().strip() + if path_good: + agh_submission.delWarning("core_file_saved") + else: + agh_submission.addWarning("core_file_saved", + f"Core file pattern will not allow debugging information to be captured.") + return path_good @pytest.fixture -def agh_env_vars(cache): - environ = {} - for env_var_name in cache.get("agh_render_env_vars", set()): - environ[env_var_name] = str(cache.get(env_var_name, "")) - return environ +def agh_run_executable(agh_submission, shell: ScriptSubprocess, resultsDir, _core_file_saved) -> Callable[ + ..., OutputSectionData]: + def run_executable(command: str, test_key: str, test_exe_file: Path, timeout_sec: int = 25, + kill_timeout_sec: int = 50, + parent_section: OutputSectionData | None = None) -> tuple[ProcessResult, OutputSectionData]: + """Run an executable and return the results. + .. important:: + + You must finish setting up the returned output section with a title etc. + """ + + cmdLineCmd = f'ulimit -c unlimited && timeout -vk {kill_timeout_sec} -s SIGXCPU {timeout_sec} ./{command}' + result = shell.run(cmdLineCmd, shell=True, cwd=agh_submission.evaluation_directory) + + if parent_section is None: + parent_section = evaluationDataOS + + current_out_section = OutputSectionData(path=Path(resultsDir / f"{test_key}_section.md")) + parent_section.addSection(current_out_section) + + std_out_file = resultsDir / f"{test_key}.stdout" + current_out_section.included_files.append( + SubmissionFileData(path=std_out_file.relative_to(agh_submission.evaluation_directory), + title="Standard Output", + description="This is the standard output from running your code.", + type="default")) + std_out_file.parent.mkdir(exist_ok=True) + std_out_file.write_text(result.stdout, encoding="utf-8", errors="replace") + + if len(result.stderr) > 0: + std_err_file = resultsDir / f"{test_key}.stderr" + current_out_section.included_files.append( + SubmissionFileData(path=std_err_file.relative_to(agh_submission.evaluation_directory), + title="Standard Error", + description="This is the standard error from running your code.", + type="default")) + std_err_file.write_text(result.stderr, encoding="utf-8", errors="replace") + + # Handle core dumps. + core_dump_file = agh_submission.evaluation_directory / CORE_DUMP_FILE_NAME + if core_dump_file.exists(): + + # Run gdb on the core dump + result_debug = shell.run( + f'gdb. / {test_exe_file} {core_dump_file.name} --eval-command "thread apply all bt full" --batch', + shell=True, cwd=agh_submission.evaluation_directory) + core_dump_file.unlink() + + if len(result_debug.stdout) > 0: + # There is data add to the eval section. + debug_output_file = resultsDir / (test_key + '.backtrace') + debug_output_file.write_text(result_debug.stdout) + current_out_section.included_files.append( + SubmissionFileData(path=debug_output_file.relative_to(agh_submission.evaluation_directory), + title="Backtrace from Debug", type='default', + description="Your code had an error that caused it to crash. This is the " + "debugging " + "backtrace from that crash.")) + else: + # todo: Handle this better. + current_out_section.text += "\n\n**Warning:** no backtrace data available from core file!" + + err_code = result.returncode + if err_code: + # current_out_section.text += f"\n\n**Warning:** Exe exited with error code: {err_code}!" + if 124 <= err_code <= 128: + current_out_section.addWarning("Timeout", + f"Your executable took too long to run and had to be terminated: { + err_code}!") + elif err_code > 128: + import signal + sig_name = "" + try: + sig_name = signal.strsignal(err_code - 128) + except ValueError: + pass + + agh_submission.setMetadata(TEST_MD_KEY, "EXE_FAULT", value=True) + current_out_section.addError("Crash Likely", + f"Exit Code: {err_code}\n\nExe exited with signal {sig_name}: " + f"{err_code - 128}") + # if err_code == 23: # Leak sanitizer exitcode. + # threadWarn = EvalFile(testStdOut.with_suffix('.md'), '', '', just_text=True, unlisted=unlisted) + # curEFiles.insert(0, threadWarn) + # threadWarn.file.write_text( + # f'\n\n::: {{.callout-important title="EXE Issue Detected"}}\n\n**Exit Code: {err_code}**\n\n| ' + # f'{testStdErr.read_text().replace(str(cDir.absolute()), ".").replace("\n", "\n| ")}\n\n::> + # # with myWarnFile.open('a') as infoFile: + # # infoFile.write(f'\n - [ ] Memory Checked.\n\n') + return (result, current_out_section) + + return run_executable + + +# todo: This is a good idea, but I feel that I need to get it working straightforwardly first. +# +# class MetaSingleton(type): +# _instances: ClassVar[dict] = {} +# +# def __call__(cls, *args, **kwargs): +# if cls not in cls._instances: +# cls._instances[cls] = super(MetaSingleton, cls).__call__(*args, **kwargs) # noqa: +# UP008 +# return cls._instances[cls] +# +# +# class AdvShell(metaclass=MetaSingleton): +# """This class is a singleton that manages the invocation of an executable. **It must be +# subclassed.** +# +# This class is designed to allow for testing independent preconditions/postconditions +# on the execution of a command. By subclassing this class the executable is +# invoked a single time, but the preconditions/postconditions can be tested against the +# output independently of the executable. +# +# .. topic:: Example +# +# .. code-block:: python +# +# class TestSimple(AdvShell): +# def __init__(self): +# self.results = self.run(...) +# +# def test_success(self): +# assert self.results.returncode == 0, "Your executable exited with a +# non-zero return code." +# +# def test_out_length(self): +# assert len(self.results.stdout) > 10, "Your executable did not capture +# enough of the test data." +# +# def test_err_length(self): +# assert len(self.results.stderr) < 2, "Your executable was not clean in +# stderr." +# +# ... +# +# class TestBadCLI(AdvShell): +# def __init__(self): +# self.results = self.run() +# +# def test_returncode(self): +# assert self.results.returncode != 0, "Your executable did not fail with a +# non-zero return code." +# +# def test_err_length(self): +# assert len(self.results.stderr) > 0, "Your executable did not output any +# error messages." +# +# def test_out_length(self): +# assert len(self.results.stdout) == 0, "Your executable may have printed the +# help message to stdout instead of stderr." +# ... +# +# +# For each of the classes above, the self.run would only be invoked once each. +# The tests would be run against the results of the single invocation but allow the +# instructor +# to indicate multiple issues within that invocation in a **pytest** appropriate manner. +# +# .. warning:: +# THIS CLASS SHOULD NOT BE INSTANTIATED DIRECTLY, subclass it instead. +# +# """ +# +# def __init__(self): +# if type(self) is AdvShell: +# raise TypeError("AdvShell class cannot be instantiated directly") +# +# def run(self, *args, **kwargs): +# """This method runs all the checks for the executable: Error code != 0, OS captured +# faults, -fsanitize=leak errors, -fsanitize=thread errors.""" + +""" +INPUT + command string + +OUTPUT + command is run once each test class + output of one run is available in each function + +""" @pytest.fixture -def agh_render_output(agh_submission: Submission, shell, agh_env_vars: dict[str, str], request: pytest.FixtureRequest, - resultsDir: Path, agh_assignment: Assignment): +def agh_render_output( + agh_submission: Submission, + shell, + request: pytest.FixtureRequest, + resultsDir: Path, + agh_assignment: Assignment, +): request.applymarker(pytest.mark.render) def render(target: str | None = agh_assignment._options.output_template_name, *args: str): @@ -148,24 +374,25 @@ def render(target: str | None = agh_assignment._options.output_template_name, *a if len(args) > 0: cmd.extend(args) - #Clear all render specific errors and warnings - agh_submission.delWarning('render warning').delError('render error').delWarning('render issue').save() + # Clear all render specific errors and warnings + agh_submission.delWarning("render warning").delError("render error").delWarning("render issue").save() try: - cmd_str = ' '.join(cmd) - print(f'Executing quarto: {cmd_str}') + cmd_str = " ".join(cmd) + print(f"Executing quarto: {cmd_str}") res = shell.run(cmd_str, shell=True, cwd=agh_submission.evaluation_directory) - (resultsDir/ '.render.stdout.txt').write_text(f"{cmd_str}\n" + res.stdout) - (resultsDir/ '.render.stderr.txt').write_text(res.stderr) + (resultsDir / ".render.stdout.txt").write_text(f"{cmd_str}\n" + res.stdout) + (resultsDir / ".render.stderr.txt").write_text(res.stderr) if res.returncode != 0: - agh_submission.addWarning('render issue', f"Render '{cmd_str}' failed with return code {res.returncode}.").save() + agh_submission.addWarning("render issue", + f"Render '{cmd_str}' failed with return code {res.returncode}.").save() raise RuntimeError(f"quarto failed with return code {res.returncode}") - agh_assignment.postProcessSubmissionRender(agh_submission, warning_callback=lambda warn: agh_submission.addWarning('render warning', warn)).save() + agh_assignment.postProcessSubmissionRender( + agh_submission, warning_callback=lambda warn: agh_submission.addWarning("render warning", warn) + ).save() return res except Exception as e: - agh_submission.addError('render error', f"Quarto failed with error {e}.").save() + agh_submission.addError("render error", f"Quarto failed with error {e}.").save() request.raiseerror(f"Error rendering {agh_submission.submission_file}: {e}") return render - - diff --git a/tox.ini b/tox.ini index c1ea9cb..e2ac523 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,8 @@ envlist = clean, check, docs, - {3.13,py311,py312,py313,pypy310,pypy311}, +; {3.13,py311,py312,py313,pypy310,pypy311}, + {3.13,py312,py313,pypy312,pypy313}, report ignore_basepython_conflict = true From 5799124e14f3d01259db7be31319a1d5e8f65609 Mon Sep 17 00:00:00 2001 From: Tuck Williamson Date: Mon, 20 Oct 2025 18:16:29 -0400 Subject: [PATCH 03/10] Fixes for build passing. --- src/agh/agh_data.py | 61 ++++++++++--------------- src/agh/pytest_plugin.py | 96 ++++++++++++++++++++++------------------ 2 files changed, 75 insertions(+), 82 deletions(-) diff --git a/src/agh/agh_data.py b/src/agh/agh_data.py index 8e934db..47a38ea 100644 --- a/src/agh/agh_data.py +++ b/src/agh/agh_data.py @@ -28,8 +28,7 @@ META_AGH_INTERNAL_KEY = "AGH_INTERNAL" META_INTERNAL_SUB_KEYS = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY] -META_INTERNAL_SUB_OUTPUT_COMPLETE = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, - "COMPLETED_OUTPUT"] +META_INTERNAL_SUB_OUTPUT_COMPLETE = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "COMPLETED_OUTPUT"] META_INTERNAL_SUB_OUTPUT_GRADED = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "GRADED"] META_INTERNAL_SUB_OUTPUT_NON_ANON = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "NON_ANON"] @@ -84,8 +83,7 @@ def asdict(self) -> dict[str, Any]: setattr(self, cur_field.name, [str(cur_val) for cur_val in restore_these[cur_field.name]]) elif get_origin(cur_field.type) is dict and hasattr(get_args(cur_field.type)[-1], "asdict"): restore_these[cur_field.name] = getattr(self, cur_field.name) - setattr(self, cur_field.name, - {cur_key: cur_val.asdict() for cur_key, cur_val in restore_these[cur_field.name].items()}) + setattr(self, cur_field.name, {cur_key: cur_val.asdict() for cur_key, cur_val in restore_these[cur_field.name].items()}) # if len(restore_these) > 0: # print(self) ret_val = asdict(self) @@ -111,8 +109,7 @@ def _from_json(cls, data: dict): data[cur_field.name] = [get_args(cur_field.type)[0]._from_json(p) for p in data[cur_field.name]] elif get_origin(cur_field.type) is dict and hasattr(get_args(cur_field.type)[-1], "_from_json"): # print(data[cur_field.name]) - data[cur_field.name] = {k: get_args(cur_field.type)[-1]._from_json(p) for k, p in - data[cur_field.name].items()} + data[cur_field.name] = {k: get_args(cur_field.type)[-1]._from_json(p) for k, p in data[cur_field.name].items()} elif is_dataclass(cur_field.type): data[cur_field.name] = cur_field.type(**data[cur_field.name]) elif get_origin(cur_field.type) is list and is_dataclass(get_args(cur_field.type)[0]): @@ -184,9 +181,11 @@ def asQmdSection(self, heading_level: int, max_file_size: int = DEFAULT_MAX_OUT_ hdr_txt = "#" * heading_level + " " + self.title + self.sectionAttr ret_val = f"\n\n{hdr_txt}\n\n{self.description}\n\n" if self.path.exists() and self.path.stat().st_size > max_file_size: - with self.path.open('rt', encoding="utf-8", errors="replace") as f: - ret_val += (f"**[File too large! Contents truncated to {max_file_size} bytes.]{{.mark}}**\n\n```{{." - f"{self.type}}}\n{f.read(max_file_size)}\n```\n\n") + with self.path.open("rt", encoding="utf-8", errors="replace") as f: + ret_val += ( + f"**[File too large! Contents truncated to {max_file_size} bytes.]{{.mark}}**\n\n```{{." + f"{self.type}}}\n{f.read(max_file_size)}\n```\n\n" + ) return ret_val return f"{ret_val}```{{.{self.type}}}\n{{{{< include {self.path} >}}}}\n```\n\n" @@ -211,8 +210,7 @@ def hasData(self): return True elif len(self.included_sections) > 0 or len(self.included_files) > 0: return True in [cur_sub_sec.hasData for cur_sub_sec in self.included_sections] or True in [ - (cur_inc_file.path.exists() and cur_inc_file.path.stat().st_size > 1) for cur_inc_file in - self.included_files + (cur_inc_file.path.exists() and cur_inc_file.path.stat().st_size > 1) for cur_inc_file in self.included_files ] elif self.text.strip() != "": # If there are no 'includes' then the section is the data @@ -222,8 +220,7 @@ def hasData(self): def asQmdSection(self) -> str: if not self.hasData and self.only_output_if_data: return "" - inc_file_txt = "\n\n".join( - [cur_inc_file.asQmdSection(self.heading_level + 1) for cur_inc_file in self.included_files]) + inc_file_txt = "\n\n".join([cur_inc_file.asQmdSection(self.heading_level + 1) for cur_inc_file in self.included_files]) inc_sec_txt = "\n\n".join([cur_inc_sec.asQmdSection() for cur_inc_sec in self.included_sections]) warnings = "" @@ -367,8 +364,7 @@ class GraderOptions(MetaDataclassJson): _general_editor_command: str | None = None general_editor_command = property( - *_gen_prop_methods("_general_editor_command", "subl $files"), - doc="A command to open a submission file in a text editor." + *_gen_prop_methods("_general_editor_command", "subl $files"), doc="A command to open a submission file in a text editor." ) # This is a dictionary of metadata associated with the assignment. @@ -589,8 +585,7 @@ def load(cls, filepath: pathlib.Path | None = None): filepath = filepath.absolute() filepath = findFileInParents(filepath, cls.ASSIGNMENT_FILE_NAME) if filepath is None: - raise FileNotFoundError( - f"Could not find assignment JSON file in {orig_filepath} or any of its parents.") + raise FileNotFoundError(f"Could not find assignment JSON file in {orig_filepath} or any of its parents.") if filepath.exists() and filepath.is_file(): data = json.loads(filepath.read_text()) @@ -710,8 +705,7 @@ def all_linked_files() -> Generator[Path]: for link_item in self.link_template_dir.iterdir(): yield link_item for link_item in self._optional_files.values(): - if link_item.copy_to_sub_if_missing and not ( - ret_val.evaluation_directory / link_item.path.name).exists(): + if link_item.copy_to_sub_if_missing and not (ret_val.evaluation_directory / link_item.path.name).exists(): yield link_item.path for link_item in all_linked_files(): @@ -767,7 +761,7 @@ def postProcessSubmissionRender( elif warning_callback is not None: warning_callback( f'The graded output file "{output_graded}" already exists and appears modified. ' - f'{output_graded} will not be overwritten.' + f"{output_graded} will not be overwritten." ) # stats = output_graded.stat() # warning_callback(f'The graded output file info a:{stats.st_atime_ns}, c: {stats.st_ctime_ns}, @@ -779,21 +773,18 @@ def postProcessSubmissionRender( pass # Create symbolic links output_file.symlink_to( - output_in_sub_dir.relative_to(output_file.parent, walk_up=True), - target_is_directory=output_file.is_dir() + output_in_sub_dir.relative_to(output_file.parent, walk_up=True), target_is_directory=output_file.is_dir() ) # output_file.chmod(0o444) output_not_anon.symlink_to( - output_graded.relative_to(output_not_anon.parent, walk_up=True), - target_is_directory=output_not_anon.is_dir() + output_graded.relative_to(output_not_anon.parent, walk_up=True), target_is_directory=output_not_anon.is_dir() ) return submission def AddSubmission( - self, submission_file: pathlib.Path, override_anon: bool | None = None, - warning_callback: Callable[[str], None | Any] | None = None + self, submission_file: pathlib.Path, override_anon: bool | None = None, warning_callback: Callable[[str], None | Any] | None = None ) -> "Submission": """Add a new submission to the assignment. :param submission_file: The path to the submission file to add. @@ -806,8 +797,7 @@ def AddSubmission( """ ret_val = Submission.new(self, submission_file=submission_file, override_anon=override_anon) ret_val.save() - return self.PostProcessSubmission(ret_val, exists_protocol=self.LinkProto.RAISE_ERROR, - warning_callback=warning_callback) + return self.PostProcessSubmission(ret_val, exists_protocol=self.LinkProto.RAISE_ERROR, warning_callback=warning_callback) # def __pytest_cmd(self): # return "pytest ./ -p shell-utilities -p agh" @@ -977,8 +967,7 @@ def __post_init__(self): if not self.submission_file.exists() or not self.submission_file.is_file(): raise ValueError(f"submission_file '{self.submission_file}' does not exist or is not a file.") if not self.evaluation_directory.exists() or not self.evaluation_directory.is_dir(): - raise ValueError( - f"evaluation_directory '{self.evaluation_directory}' does not exist or is not a directory.") + raise ValueError(f"evaluation_directory '{self.evaluation_directory}' does not exist or is not a directory.") class Submission(SubmissionData): @@ -1031,8 +1020,7 @@ def get_anon_name(cls, assignment: Assignment, submission_file: pathlib.Path): Returns: str: Anonymous name for the submission """ - return anonymizer.anonymize(submission_file.name, assignment.name, str(assignment.year), - assignment.grade_period, assignment.course) + return anonymizer.anonymize(submission_file.name, assignment.name, str(assignment.year), assignment.grade_period, assignment.course) @classmethod def load(cls, filepath: pathlib.Path | None = None): @@ -1054,8 +1042,7 @@ def load(cls, filepath: pathlib.Path | None = None): filepath = filepath.absolute() filepath = findFileInParents(filepath, cls.SUBMISSION_FILE_NAME) if filepath is None: - raise FileNotFoundError( - f"Could not find submission JSON file in {orig_filepath} or any of its parents.") + raise FileNotFoundError(f"Could not find submission JSON file in {orig_filepath} or any of its parents.") if filepath.exists() and filepath.is_file(): data = json.loads(filepath.read_text()) @@ -1151,8 +1138,7 @@ def __post_process_new__(self, assignment: Assignment, base_file_name: str | Non elif "zip" in self.submission_file.name: os.system(f'cd "{self.as_submitted_dir.absolute()}" && unzip "{self.submission_file.absolute()}"') elif base_file_name is not None: - os.system( - f'cp "{self.submission_file.absolute()}" "{self.evaluation_directory.absolute()}/{base_file_name}"') + os.system(f'cp "{self.submission_file.absolute()}" "{self.evaluation_directory.absolute()}/{base_file_name}"') self._missing_files_initially = self.check_missing_files(assignment) @@ -1235,8 +1221,7 @@ def errors(self) -> None | list[str]: missing_files = self.check_missing_files(Assignment.load()) if len(missing_files) > 0: - errors.append( - f"Missing required file{'s' if len(missing_files) > 1 else ''}: {[mf.name for mf in missing_files]}") + errors.append(f"Missing required file{'s' if len(missing_files) > 1 else ''}: {[mf.name for mf in missing_files]}") if len(errors) > 0: return errors diff --git a/src/agh/pytest_plugin.py b/src/agh/pytest_plugin.py index 3b37ea5..13b40e5 100644 --- a/src/agh/pytest_plugin.py +++ b/src/agh/pytest_plugin.py @@ -1,9 +1,11 @@ import os +import signal +from collections.abc import Callable from pathlib import Path -from typing import ClassVar, Callable import pytest -from pytestshellutils.shell import ProcessResult, ScriptSubprocess +from pytestshellutils.shell import ProcessResult +from pytestshellutils.shell import ScriptSubprocess from .agh_data import Assignment from .agh_data import OutputSectionData @@ -38,8 +40,7 @@ def pytest_configure(config): plugin = AghPtPlugin(config) config.pluginmanager.register(plugin, name="agh_plugin") config.addinivalue_line("markers", "build: This marks anything related to building a submission's exe.") - config.addinivalue_line("markers", - "render: This marks anything related to rendering a submission's documentation.") + config.addinivalue_line("markers", "render: This marks anything related to rendering a submission's documentation.") @pytest.fixture @@ -75,8 +76,7 @@ def storeRunOutErr(tgt_name: str, res, resultsDir): evaluationDataOS = OutputSectionData(path=Path("eval_data_section.md"), title="Evaluation Data", heading_level=1) -instructor_out_data = OutputSectionData(path=Path("instructor_data_section.md"), title="Instructor Data", - heading_level=1) +instructor_out_data = OutputSectionData(path=Path("instructor_data_section.md"), title="Instructor Data", heading_level=1) def _make_sections(resultsDir: Path, agh_assignment: Assignment, agh_submission: Submission): @@ -121,7 +121,7 @@ def _make_sections(resultsDir: Path, agh_assignment: Assignment, agh_submission: def agh_build_makefile(agh_submission, shell, cache, request, resultsDir) -> Callable[[str], str]: request.applymarker(pytest.mark.build) - def build(target: str | None = None, include_build_in_eval:bool=True): + def build(target: str | None = None, include_build_in_eval: bool = True): # Check to see if this is the first time we're building this submission. first_build = False if agh_submission.getMetadata(TEST_MD_KEY, "initial_build_success", default=None) is None: @@ -145,14 +145,14 @@ def build(target: str | None = None, include_build_in_eval:bool=True): stdout_file.parent.mkdir(exist_ok=True) stdout_file.write_text(res.stdout) buildOutSection.included_files.append( - SubmissionFileData(path=stdout_file, title="Build Stdout Output", - heading_level=buildOutSection.heading_level + 1)) + SubmissionFileData(path=stdout_file, title="Build Stdout Output", heading_level=buildOutSection.heading_level + 1) + ) stderr_file = resultsDir / f"{target if target else ''}build.stderr" stderr_file.write_text(res.stderr) if len(res.stderr) > 0: buildOutSection.included_files.append( - SubmissionFileData(path=stdout_file, title="Build Stderr Output", - heading_level=buildOutSection.heading_level + 1)) + SubmissionFileData(path=stdout_file, title="Build Stderr Output", heading_level=buildOutSection.heading_level + 1) + ) return res @@ -166,24 +166,27 @@ def _core_file_saved(agh_submission): if path_good: agh_submission.delWarning("core_file_saved") else: - agh_submission.addWarning("core_file_saved", - f"Core file pattern will not allow debugging information to be captured.") + agh_submission.addWarning("core_file_saved", "Core file pattern will not allow debugging information to be captured.") return path_good @pytest.fixture -def agh_run_executable(agh_submission, shell: ScriptSubprocess, resultsDir, _core_file_saved) -> Callable[ - ..., OutputSectionData]: - def run_executable(command: str, test_key: str, test_exe_file: Path, timeout_sec: int = 25, - kill_timeout_sec: int = 50, - parent_section: OutputSectionData | None = None) -> tuple[ProcessResult, OutputSectionData]: +def agh_run_executable(agh_submission, shell: ScriptSubprocess, resultsDir, _core_file_saved) -> Callable[..., OutputSectionData]: + def run_executable( + command: str, + test_key: str, + test_exe_file: Path, + timeout_sec: int = 25, + kill_timeout_sec: int = 50, + parent_section: OutputSectionData | None = None, + ) -> tuple[ProcessResult, OutputSectionData]: """Run an executable and return the results. .. important:: You must finish setting up the returned output section with a title etc. """ - cmdLineCmd = f'ulimit -c unlimited && timeout -vk {kill_timeout_sec} -s SIGXCPU {timeout_sec} ./{command}' + cmdLineCmd = f"ulimit -c unlimited && timeout -vk {kill_timeout_sec} -s SIGXCPU {timeout_sec} ./{command}" result = shell.run(cmdLineCmd, shell=True, cwd=agh_submission.evaluation_directory) if parent_section is None: @@ -194,42 +197,51 @@ def run_executable(command: str, test_key: str, test_exe_file: Path, timeout_sec std_out_file = resultsDir / f"{test_key}.stdout" current_out_section.included_files.append( - SubmissionFileData(path=std_out_file.relative_to(agh_submission.evaluation_directory), - title="Standard Output", - description="This is the standard output from running your code.", - type="default")) + SubmissionFileData( + path=std_out_file.relative_to(agh_submission.evaluation_directory), + title="Standard Output", + description="This is the standard output from running your code.", + type="default", + ) + ) std_out_file.parent.mkdir(exist_ok=True) std_out_file.write_text(result.stdout, encoding="utf-8", errors="replace") if len(result.stderr) > 0: std_err_file = resultsDir / f"{test_key}.stderr" current_out_section.included_files.append( - SubmissionFileData(path=std_err_file.relative_to(agh_submission.evaluation_directory), - title="Standard Error", - description="This is the standard error from running your code.", - type="default")) + SubmissionFileData( + path=std_err_file.relative_to(agh_submission.evaluation_directory), + title="Standard Error", + description="This is the standard error from running your code.", + type="default", + ) + ) std_err_file.write_text(result.stderr, encoding="utf-8", errors="replace") # Handle core dumps. core_dump_file = agh_submission.evaluation_directory / CORE_DUMP_FILE_NAME if core_dump_file.exists(): - # Run gdb on the core dump result_debug = shell.run( f'gdb. / {test_exe_file} {core_dump_file.name} --eval-command "thread apply all bt full" --batch', - shell=True, cwd=agh_submission.evaluation_directory) + shell=True, + cwd=agh_submission.evaluation_directory, + ) core_dump_file.unlink() if len(result_debug.stdout) > 0: # There is data add to the eval section. - debug_output_file = resultsDir / (test_key + '.backtrace') + debug_output_file = resultsDir / (test_key + ".backtrace") debug_output_file.write_text(result_debug.stdout) current_out_section.included_files.append( - SubmissionFileData(path=debug_output_file.relative_to(agh_submission.evaluation_directory), - title="Backtrace from Debug", type='default', - description="Your code had an error that caused it to crash. This is the " - "debugging " - "backtrace from that crash.")) + SubmissionFileData( + path=debug_output_file.relative_to(agh_submission.evaluation_directory), + title="Backtrace from Debug", + type="default", + description="Your code had an error that caused it to crash. This is the debugging backtrace from that crash.", + ) + ) else: # todo: Handle this better. current_out_section.text += "\n\n**Warning:** no backtrace data available from core file!" @@ -238,11 +250,8 @@ def run_executable(command: str, test_key: str, test_exe_file: Path, timeout_sec if err_code: # current_out_section.text += f"\n\n**Warning:** Exe exited with error code: {err_code}!" if 124 <= err_code <= 128: - current_out_section.addWarning("Timeout", - f"Your executable took too long to run and had to be terminated: { - err_code}!") + current_out_section.addWarning("Timeout", f"Your executable took too long to run and had to be terminated: {err_code}!") elif err_code > 128: - import signal sig_name = "" try: sig_name = signal.strsignal(err_code - 128) @@ -250,9 +259,9 @@ def run_executable(command: str, test_key: str, test_exe_file: Path, timeout_sec pass agh_submission.setMetadata(TEST_MD_KEY, "EXE_FAULT", value=True) - current_out_section.addError("Crash Likely", - f"Exit Code: {err_code}\n\nExe exited with signal {sig_name}: " - f"{err_code - 128}") + current_out_section.addError( + "Crash Likely", f"Exit Code: {err_code}\n\nExe exited with signal {sig_name}: {err_code - 128}" + ) # if err_code == 23: # Leak sanitizer exitcode. # threadWarn = EvalFile(testStdOut.with_suffix('.md'), '', '', just_text=True, unlisted=unlisted) # curEFiles.insert(0, threadWarn) @@ -384,8 +393,7 @@ def render(target: str | None = agh_assignment._options.output_template_name, *a (resultsDir / ".render.stdout.txt").write_text(f"{cmd_str}\n" + res.stdout) (resultsDir / ".render.stderr.txt").write_text(res.stderr) if res.returncode != 0: - agh_submission.addWarning("render issue", - f"Render '{cmd_str}' failed with return code {res.returncode}.").save() + agh_submission.addWarning("render issue", f"Render '{cmd_str}' failed with return code {res.returncode}.").save() raise RuntimeError(f"quarto failed with return code {res.returncode}") agh_assignment.postProcessSubmissionRender( agh_submission, warning_callback=lambda warn: agh_submission.addWarning("render warning", warn) From f03edf216f711ff342b6e90fd5fa8b5fa2c21371 Mon Sep 17 00:00:00 2001 From: Tuck Williamson Date: Thu, 23 Oct 2025 14:12:21 -0400 Subject: [PATCH 04/10] Made agh_run_executable more flexible. More linked cli output. --- src/agh/agh_data.py | 63 ++++++++++++++------- src/agh/cli.py | 118 +++++++++++++++++++++++++++++---------- src/agh/pytest_plugin.py | 28 ++++++---- tox.ini | 2 +- 4 files changed, 149 insertions(+), 62 deletions(-) diff --git a/src/agh/agh_data.py b/src/agh/agh_data.py index 47a38ea..f5cae92 100644 --- a/src/agh/agh_data.py +++ b/src/agh/agh_data.py @@ -28,7 +28,8 @@ META_AGH_INTERNAL_KEY = "AGH_INTERNAL" META_INTERNAL_SUB_KEYS = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY] -META_INTERNAL_SUB_OUTPUT_COMPLETE = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "COMPLETED_OUTPUT"] +META_INTERNAL_SUB_OUTPUT_COMPLETE = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, + "COMPLETED_OUTPUT"] META_INTERNAL_SUB_OUTPUT_GRADED = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "GRADED"] META_INTERNAL_SUB_OUTPUT_NON_ANON = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "NON_ANON"] @@ -83,7 +84,8 @@ def asdict(self) -> dict[str, Any]: setattr(self, cur_field.name, [str(cur_val) for cur_val in restore_these[cur_field.name]]) elif get_origin(cur_field.type) is dict and hasattr(get_args(cur_field.type)[-1], "asdict"): restore_these[cur_field.name] = getattr(self, cur_field.name) - setattr(self, cur_field.name, {cur_key: cur_val.asdict() for cur_key, cur_val in restore_these[cur_field.name].items()}) + setattr(self, cur_field.name, + {cur_key: cur_val.asdict() for cur_key, cur_val in restore_these[cur_field.name].items()}) # if len(restore_these) > 0: # print(self) ret_val = asdict(self) @@ -109,7 +111,8 @@ def _from_json(cls, data: dict): data[cur_field.name] = [get_args(cur_field.type)[0]._from_json(p) for p in data[cur_field.name]] elif get_origin(cur_field.type) is dict and hasattr(get_args(cur_field.type)[-1], "_from_json"): # print(data[cur_field.name]) - data[cur_field.name] = {k: get_args(cur_field.type)[-1]._from_json(p) for k, p in data[cur_field.name].items()} + data[cur_field.name] = {k: get_args(cur_field.type)[-1]._from_json(p) for k, p in + data[cur_field.name].items()} elif is_dataclass(cur_field.type): data[cur_field.name] = cur_field.type(**data[cur_field.name]) elif get_origin(cur_field.type) is list and is_dataclass(get_args(cur_field.type)[0]): @@ -181,7 +184,7 @@ def asQmdSection(self, heading_level: int, max_file_size: int = DEFAULT_MAX_OUT_ hdr_txt = "#" * heading_level + " " + self.title + self.sectionAttr ret_val = f"\n\n{hdr_txt}\n\n{self.description}\n\n" if self.path.exists() and self.path.stat().st_size > max_file_size: - with self.path.open("rt", encoding="utf-8", errors="replace") as f: + with self.path.open("rt", encoding="ascii", errors="replace") as f: ret_val += ( f"**[File too large! Contents truncated to {max_file_size} bytes.]{{.mark}}**\n\n```{{." f"{self.type}}}\n{f.read(max_file_size)}\n```\n\n" @@ -210,7 +213,8 @@ def hasData(self): return True elif len(self.included_sections) > 0 or len(self.included_files) > 0: return True in [cur_sub_sec.hasData for cur_sub_sec in self.included_sections] or True in [ - (cur_inc_file.path.exists() and cur_inc_file.path.stat().st_size > 1) for cur_inc_file in self.included_files + (cur_inc_file.path.exists() and cur_inc_file.path.stat().st_size > 1) for cur_inc_file in + self.included_files ] elif self.text.strip() != "": # If there are no 'includes' then the section is the data @@ -220,7 +224,8 @@ def hasData(self): def asQmdSection(self) -> str: if not self.hasData and self.only_output_if_data: return "" - inc_file_txt = "\n\n".join([cur_inc_file.asQmdSection(self.heading_level + 1) for cur_inc_file in self.included_files]) + inc_file_txt = "\n\n".join( + [cur_inc_file.asQmdSection(self.heading_level + 1) for cur_inc_file in self.included_files]) inc_sec_txt = "\n\n".join([cur_inc_sec.asQmdSection() for cur_inc_sec in self.included_sections]) warnings = "" @@ -235,7 +240,8 @@ def asQmdSection(self) -> str: if errors != "": errors = f'\n\n::: {{.callout-important title="Errors"}}\n\n{errors}\n\n:::\n\n' - pre = "#" * self.heading_level + f" {self.title} {self.sectionAttr}\n\n{self.description}\n\n{self.text}{errors}{warnings}" + pre = "#" * self.heading_level + (f" {self.title} {self.sectionAttr}\n\n{self.description}\n\n{self.text}" + f"{errors}{warnings}") return f"{pre}\n\n{inc_file_txt}\n\n{inc_sec_txt}\n\n{self.post_script}" def addSection(self, section: "OutputSectionData") -> Self: @@ -364,7 +370,8 @@ class GraderOptions(MetaDataclassJson): _general_editor_command: str | None = None general_editor_command = property( - *_gen_prop_methods("_general_editor_command", "subl $files"), doc="A command to open a submission file in a text editor." + *_gen_prop_methods("_general_editor_command", "subl $files"), + doc="A command to open a submission file in a text editor." ) # This is a dictionary of metadata associated with the assignment. @@ -585,7 +592,8 @@ def load(cls, filepath: pathlib.Path | None = None): filepath = filepath.absolute() filepath = findFileInParents(filepath, cls.ASSIGNMENT_FILE_NAME) if filepath is None: - raise FileNotFoundError(f"Could not find assignment JSON file in {orig_filepath} or any of its parents.") + raise FileNotFoundError( + f"Could not find assignment JSON file in {orig_filepath} or any of its parents.") if filepath.exists() and filepath.is_file(): data = json.loads(filepath.read_text()) @@ -677,7 +685,11 @@ def name(self, value): @property def Submissions(self): for submission_file in self._directory.rglob(Submission.SUBMISSION_FILE_NAME): - yield Submission.load(submission_file) + try: + yield Submission.load(submission_file) + except Exception as e: + print(f"Error loading submission file {submission_file}: {e}") + continue class LinkProto(IntEnum): # Raise an error if the file exists. @@ -705,7 +717,8 @@ def all_linked_files() -> Generator[Path]: for link_item in self.link_template_dir.iterdir(): yield link_item for link_item in self._optional_files.values(): - if link_item.copy_to_sub_if_missing and not (ret_val.evaluation_directory / link_item.path.name).exists(): + if link_item.copy_to_sub_if_missing and not ( + ret_val.evaluation_directory / link_item.path.name).exists(): yield link_item.path for link_item in all_linked_files(): @@ -773,18 +786,21 @@ def postProcessSubmissionRender( pass # Create symbolic links output_file.symlink_to( - output_in_sub_dir.relative_to(output_file.parent, walk_up=True), target_is_directory=output_file.is_dir() + output_in_sub_dir.relative_to(output_file.parent, walk_up=True), + target_is_directory=output_file.is_dir() ) # output_file.chmod(0o444) output_not_anon.symlink_to( - output_graded.relative_to(output_not_anon.parent, walk_up=True), target_is_directory=output_not_anon.is_dir() + output_graded.relative_to(output_not_anon.parent, walk_up=True), + target_is_directory=output_not_anon.is_dir() ) return submission def AddSubmission( - self, submission_file: pathlib.Path, override_anon: bool | None = None, warning_callback: Callable[[str], None | Any] | None = None + self, submission_file: pathlib.Path, override_anon: bool | None = None, + warning_callback: Callable[[str], None | Any] | None = None ) -> "Submission": """Add a new submission to the assignment. :param submission_file: The path to the submission file to add. @@ -797,7 +813,8 @@ def AddSubmission( """ ret_val = Submission.new(self, submission_file=submission_file, override_anon=override_anon) ret_val.save() - return self.PostProcessSubmission(ret_val, exists_protocol=self.LinkProto.RAISE_ERROR, warning_callback=warning_callback) + return self.PostProcessSubmission(ret_val, exists_protocol=self.LinkProto.RAISE_ERROR, + warning_callback=warning_callback) # def __pytest_cmd(self): # return "pytest ./ -p shell-utilities -p agh" @@ -967,7 +984,8 @@ def __post_init__(self): if not self.submission_file.exists() or not self.submission_file.is_file(): raise ValueError(f"submission_file '{self.submission_file}' does not exist or is not a file.") if not self.evaluation_directory.exists() or not self.evaluation_directory.is_dir(): - raise ValueError(f"evaluation_directory '{self.evaluation_directory}' does not exist or is not a directory.") + raise ValueError( + f"evaluation_directory '{self.evaluation_directory}' does not exist or is not a directory.") class Submission(SubmissionData): @@ -1020,7 +1038,8 @@ def get_anon_name(cls, assignment: Assignment, submission_file: pathlib.Path): Returns: str: Anonymous name for the submission """ - return anonymizer.anonymize(submission_file.name, assignment.name, str(assignment.year), assignment.grade_period, assignment.course) + return anonymizer.anonymize(submission_file.name, assignment.name, str(assignment.year), + assignment.grade_period, assignment.course) @classmethod def load(cls, filepath: pathlib.Path | None = None): @@ -1042,7 +1061,8 @@ def load(cls, filepath: pathlib.Path | None = None): filepath = filepath.absolute() filepath = findFileInParents(filepath, cls.SUBMISSION_FILE_NAME) if filepath is None: - raise FileNotFoundError(f"Could not find submission JSON file in {orig_filepath} or any of its parents.") + raise FileNotFoundError( + f"Could not find submission JSON file in {orig_filepath} or any of its parents.") if filepath.exists() and filepath.is_file(): data = json.loads(filepath.read_text()) @@ -1078,6 +1098,7 @@ def new(cls, assignment: Assignment, submission_file: pathlib.Path, override_ano base_file_name_set = True case _: anon_name = submission_file.with_suffix("").name + anon_name = anon_name.replace(" ", "_") evaluation_directory = assignment.eval_dir / anon_name evaluation_directory.mkdir(exist_ok=True, parents=True) @@ -1138,7 +1159,8 @@ def __post_process_new__(self, assignment: Assignment, base_file_name: str | Non elif "zip" in self.submission_file.name: os.system(f'cd "{self.as_submitted_dir.absolute()}" && unzip "{self.submission_file.absolute()}"') elif base_file_name is not None: - os.system(f'cp "{self.submission_file.absolute()}" "{self.evaluation_directory.absolute()}/{base_file_name}"') + os.system( + f'cp "{self.submission_file.absolute()}" "{self.evaluation_directory.absolute()}/{base_file_name}"') self._missing_files_initially = self.check_missing_files(assignment) @@ -1221,7 +1243,8 @@ def errors(self) -> None | list[str]: missing_files = self.check_missing_files(Assignment.load()) if len(missing_files) > 0: - errors.append(f"Missing required file{'s' if len(missing_files) > 1 else ''}: {[mf.name for mf in missing_files]}") + errors.append( + f"Missing required file{'s' if len(missing_files) > 1 else ''}: {[mf.name for mf in missing_files]}") if len(errors) > 0: return errors diff --git a/src/agh/cli.py b/src/agh/cli.py index 2fe154f..83444e3 100644 --- a/src/agh/cli.py +++ b/src/agh/cli.py @@ -25,6 +25,7 @@ from dataclasses import dataclass from dataclasses import field from pathlib import Path +from typing import Literal from urllib import parse import argcomplete @@ -121,7 +122,8 @@ def SubFileCompleter(property: str, prefix: str, **kwargs): def submissionCompleter(*args, **kwargs): # return ['Bob','Tom'] try: - ret_val = [str(subm.evaluation_directory.resolve().relative_to(Path.cwd(), walk_up=True)) for subm in Assignment.load().Submissions] + ret_val = [str(subm.evaluation_directory.resolve().relative_to(Path.cwd(), walk_up=True)) for subm in + Assignment.load().Submissions] return ret_val except Exception as e: argcomplete.warn("No assignment found. Cannot complete submission files.") @@ -132,7 +134,8 @@ def submissionCompleter(*args, **kwargs): # console.log(SubFileCompleter("unprocessed_dir", '')) parser = MyArgParser( - description="agh --- Assignment Grading Helper", prog="agh", formatter_class=RichHelpFormatter, conflict_handler="resolve" + description="agh --- Assignment Grading Helper", prog="agh", formatter_class=RichHelpFormatter, + conflict_handler="resolve" ) parser.add_argument("--version", action="version", version=__version__) parser.add_argument("-H", "--full-help", action=FullHelp, help="Show full (all options) help") @@ -153,16 +156,22 @@ def submissionCompleter(*args, **kwargs): ################################################################################ ################################################################################ -assignment_sub_parser = subparsers.add_parser("assignment", help="Assignment commands", formatter_class=RichHelpFormatter) +assignment_sub_parser = subparsers.add_parser("assignment", help="Assignment commands", + formatter_class=RichHelpFormatter) assignment_subparsers = assignment_sub_parser.add_subparsers(dest="assignment_command", help="Assignment commands") # assignment > new assignment command -assignment_new_parser = assignment_subparsers.add_parser("new", help="Create new assignment", formatter_class=RichHelpFormatter) -assignment_new_parser.add_argument("name", help="Assignment name", type=str).completer = lambda **kwargs: [f"Assignment {Path.cwd().name}"] -assignment_new_parser.add_argument("course", help="Course code", type=str).completer = lambda **kwargs: [f"CSCI-{Path.cwd().parent.name}"] -assignment_new_parser.add_argument("term", help="Term", choices=["Fall", "Spring", "Maymester", "Summer I", "Summer II"], type=str) +assignment_new_parser = assignment_subparsers.add_parser("new", help="Create new assignment", + formatter_class=RichHelpFormatter) +assignment_new_parser.add_argument("name", help="Assignment name", type=str).completer = lambda **kwargs: [ + f"Assignment {Path.cwd().name}"] +assignment_new_parser.add_argument("course", help="Course code", type=str).completer = lambda **kwargs: [ + f"CSCI-{Path.cwd().parent.name}"] +assignment_new_parser.add_argument("term", help="Term", + choices=["Fall", "Spring", "Maymester", "Summer I", "Summer II"], type=str) assignment_new_parser.add_argument("-y", "--year", help="Year", type=int, default=cur_date.year) -assignment_new_parser.add_argument("-a", "--anon", help="Anonymize names", action=argparse.BooleanOptionalAction, default=True) +assignment_new_parser.add_argument("-a", "--anon", help="Anonymize names", action=argparse.BooleanOptionalAction, + default=True) # assignment > info command assignment_info_parser = assignment_subparsers.add_parser("info", help="Show assignment info") @@ -171,7 +180,8 @@ def submissionCompleter(*args, **kwargs): # Add required files command assign_add_required_parser = assignment_subparsers.add_parser("add-required", help="Add required files") assign_add_required_parser.add_argument("files", nargs="+", help="Required file names", type=Path) -assign_add_required_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda **kwargs: [ +assign_add_required_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda \ + **kwargs: [ "txt", "py", "c", @@ -180,7 +190,8 @@ def submissionCompleter(*args, **kwargs): "default", "make", ] -assign_add_required_parser.add_argument("-d", "--description", help="Description of the required files", type=str, default="") +assign_add_required_parser.add_argument("-d", "--description", help="Description of the required files", type=str, + default="") assign_add_required_parser.add_argument("-t", "--title", help="Title of the required files", type=str, default="") assign_add_required_parser.add_argument( "-i", "--include-in-output", help="Include in output", action=argparse.BooleanOptionalAction, default=True @@ -189,7 +200,8 @@ def submissionCompleter(*args, **kwargs): # Add optional files command assign_add_optional_parser = assignment_subparsers.add_parser("add-optional", help="Add optional files") assign_add_optional_parser.add_argument("files", nargs="+", help="Optional file names") -assign_add_optional_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda **kwargs: [ +assign_add_optional_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda \ + **kwargs: [ "txt", "py", "c", @@ -198,7 +210,8 @@ def submissionCompleter(*args, **kwargs): "make", "default", ] -assign_add_optional_parser.add_argument("-d", "--description", help="Description of the optional files", type=str, default="") +assign_add_optional_parser.add_argument("-d", "--description", help="Description of the optional files", type=str, + default="") assign_add_optional_parser.add_argument("-t", "--title", help="Title of the optional files", type=str, default="") assign_add_optional_parser.add_argument( "-i", "--include-in-output", help="Include in output", action=argparse.BooleanOptionalAction, default=True @@ -214,7 +227,8 @@ def submissionCompleter(*args, **kwargs): # submission > add command sub_add_subparser = sub_subparsers.add_parser("add", help="Add a submission file.") -sub_add_subparser.add_argument("files", nargs="+", help="Submission files to add", type=Path).completer = functools.partial( +sub_add_subparser.add_argument("files", nargs="+", help="Submission files to add", + type=Path).completer = functools.partial( SubFileCompleter, "unprocessed_dir" ) sub_add_subparser.add_argument( @@ -238,7 +252,8 @@ def submissionCompleter(*args, **kwargs): sub_fix_subparser = sub_subparsers.add_parser( "fix", help="Fix a submission. Try this if you accidentally deleted something. This may re-create links etc." ) -sub_fix_subparser.add_argument("submissions", nargs="+", help="Submissions to fix", type=str).completer = submissionCompleter +sub_fix_subparser.add_argument("submissions", nargs="+", help="Submissions to fix", + type=str).completer = submissionCompleter ################################################################################ ################################################################################ @@ -248,42 +263,79 @@ def submissionCompleter(*args, **kwargs): # Add run command run_parser = subparsers.add_parser("run", help="Run submission files. This executes build, test, and render.") run_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, + default=None ).completer = submissionCompleter run_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add test command -test_parser = subparsers.add_parser("test", help="Test submission files. This just runs the tests for the given submissions.") +test_parser = subparsers.add_parser("test", + help="Test submission files. This just runs the tests for the given submissions.") test_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, + default=None ).completer = submissionCompleter test_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add build command build_parser = subparsers.add_parser("build", help="Build submission files") build_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, + default=None ).completer = submissionCompleter build_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add render command render_parser = subparsers.add_parser("render", help="Render submission files") render_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, + default=None ).completer = submissionCompleter -render_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) +render_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", + default=False) argcomplete.autocomplete(parser) +def printableLinkWithIcon(link: Path, icon: str|Literal[":file_folder:",":notebook:"] = ':file_folder:', link_text:str = None) -> str: + """ This takes a path and returns a string that can be printed to the terminal as a link. + :param link: Path to the file or directory. If the path does not exist, a red "x" is used as the icon. + :param icon: The icon to use as part of the link. + :param link_text: The text to use for the link. If None, the link text is the path relative to the root directory. + :return: A string that can be printed to the terminal as a link. + """ + if link is None and link_text is None: + raise ValueError("The link and link_text cannot both be None.") + if link is None: + link = '' + icon = ":x:" + else: + if not link.exists(): + icon = ":x:" + link = link.resolve() + + if link_text is None: + link_text = str(link) + + return f"[link file://{link}]{icon}{link_text}[/]" def displayAssignmentInfo(cli_args: argparse.Namespace): assignment = Assignment.load() console.print(f'[label]Assignment "{assignment.name}"') - console.print(f"[label]Course:[/] {assignment.course}, [label]Term:[/] {assignment._grade_period}, [label]Year:[/] {assignment.year}") + console.print( + f"[label]Course:[/] {assignment.course}, [label]Term:[/] {assignment._grade_period}, [label]Year:[/] " + f"{assignment.year}") submissions = list(assignment.Submissions) console.print(f"[label]Submissions:[/] {len(submissions)}") - - files_table = rich.table.Table(title="[label][req]Required[/req]/[opt]Optional[/opt] Files", expand=True, show_lines=True) + console.print(f"[label]Directories:[/] " + f"{printableLinkWithIcon(assignment.root_directory, link_text='Root')} :diamonds: " + f"{printableLinkWithIcon(assignment.eval_dir, link_text='Evaluations')} :diamonds: " + f"{printableLinkWithIcon(assignment.graded_output_dir, link_text='Output for Grading')} :diamonds: " + f"{printableLinkWithIcon(assignment.tests_dir, link_text='Evaluation Tests')} :diamonds: " + f"{printableLinkWithIcon(assignment.link_template_dir, link_text='Templates (files linked into each evaluation)')}" + ) + + files_table = rich.table.Table(title="[label][req]Required[/req]/[opt]Optional[/opt] Files", expand=True, + show_lines=True) files_table.add_column("Link", justify="center") files_table.add_column("Output", justify="center") files_table.add_column("Name", justify="left") @@ -364,7 +416,7 @@ def displaySubmissionInfo(cli_args: argparse.Namespace, assignment: Assignment): sub_color = "yellow" case (False, False, True): sub_color = "green" - submission_table.add_row(errors, warnings, output, f"[bold {sub_color}] {submission.name} [/]", graded_output) + submission_table.add_row(errors, warnings, output, f"[bold {sub_color}] {printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)} [/]", graded_output) console.print(submission_table) @@ -467,7 +519,8 @@ def handleSubmissionCmd(cli_args: argparse.Namespace): console.print(f"[error]No submission directory found for {cur_file}.") cur_subm = Submission.load(cur_subm_dir) cur_subm.fix(assignment=assignment) - assignment.PostProcessSubmission(cur_subm, warning_callback=lambda warn: console.print(warn, style="warning")).save() + assignment.PostProcessSubmission(cur_subm, warning_callback=lambda warn: console.print(warn, + style="warning")).save() case _: console.log(cli_args, style="error") @@ -488,7 +541,8 @@ def verbose_print(cli_args: argparse.Namespace, *args, **kwargs) -> None: async def parse_pytest_output( - assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, task_id + assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, + task_id ): output_info = RunOutputInfo() @@ -559,11 +613,12 @@ async def run_pytest( task_id, advance=1, completed=True, - description=f"[error]Tests directory '{tests_path.absolute()}'not found. Perhaps run fix on {submission.name} first?", + description=f"[error]Tests directory '{tests_path.absolute()}'not found. Perhaps run fix on " + f"{submission.name} first?", ) return submission, False - cmd_str: str = f"pytest -v -p agh-pytest-plugin --agh {extra_pytest_args} {tests_path.absolute()}/*" + cmd_str: str = f'pytest -v -p agh-pytest-plugin --agh {extra_pytest_args} {tests_path.absolute()}/*' # verbose_print(cli_args, 'Running pytest...', cmd_str) # Setup the progress bar. task_id = progress.add_task(f"Testing {tests_path.absolute()}...", total=None) @@ -578,7 +633,8 @@ async def run_pytest( return submission, return_code == 0 -async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment: Assignment, extra_pytest_args: str = ""): +async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment: Assignment, + extra_pytest_args: str = ""): """This function asynchronously runs pytest on all submissions specified. It provides a progress bar for each submission to indicate the progress of the tests. @@ -662,7 +718,8 @@ def run(args=None): asyncio.run(execute_pytest_on_submissions(cli_args, assignment)) case "test": assignment = getCurrentAssignment() - asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) + asyncio.run( + execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) case "build": assignment = getCurrentAssignment() asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "build"')) @@ -674,7 +731,6 @@ def run(args=None): # print(start(args)) parser.exit(0) - # todo: create custom url scheme so I can run things from links in the output. # To create a custom URL scheme in Ubuntu that executes a command in the terminal, you need to define a desktop # entry for the scheme and create a script to handle the URL. diff --git a/src/agh/pytest_plugin.py b/src/agh/pytest_plugin.py index 13b40e5..e29a3f6 100644 --- a/src/agh/pytest_plugin.py +++ b/src/agh/pytest_plugin.py @@ -138,20 +138,20 @@ def build(target: str | None = None, include_build_in_eval: bool = True): if first_build: agh_submission.setMetadata(TEST_MD_KEY, "initial_build_success", value=res.returncode == 0) - buildOutSection = OutputSectionData(title="Build Output", heading_level=1) + build_out_section = OutputSectionData(path=Path("build_data.md"), title="Build Output") if include_build_in_eval: - evaluationDataOS.addSection(buildOutSection) + evaluationDataOS.addSection(build_out_section) stdout_file = resultsDir / f"{target if target else ''}build.stdout" stdout_file.parent.mkdir(exist_ok=True) stdout_file.write_text(res.stdout) - buildOutSection.included_files.append( - SubmissionFileData(path=stdout_file, title="Build Stdout Output", heading_level=buildOutSection.heading_level + 1) + build_out_section.included_files.append( + SubmissionFileData(path=stdout_file.relative_to(agh_submission.evaluation_directory), title="Build Stdout Output") ) stderr_file = resultsDir / f"{target if target else ''}build.stderr" stderr_file.write_text(res.stderr) if len(res.stderr) > 0: - buildOutSection.included_files.append( - SubmissionFileData(path=stdout_file, title="Build Stderr Output", heading_level=buildOutSection.heading_level + 1) + build_out_section.included_files.append( + SubmissionFileData(path=stdout_file.relative_to(agh_submission.evaluation_directory), title="Build Stderr Output") ) return res @@ -179,6 +179,8 @@ def run_executable( timeout_sec: int = 25, kill_timeout_sec: int = 50, parent_section: OutputSectionData | None = None, + handle_core_dump: bool = True, + handle_timeout: bool = True, ) -> tuple[ProcessResult, OutputSectionData]: """Run an executable and return the results. .. important:: @@ -186,8 +188,14 @@ def run_executable( You must finish setting up the returned output section with a title etc. """ - cmdLineCmd = f"ulimit -c unlimited && timeout -vk {kill_timeout_sec} -s SIGXCPU {timeout_sec} ./{command}" - result = shell.run(cmdLineCmd, shell=True, cwd=agh_submission.evaluation_directory) + # Build up the shell command line based on options selected by the grader. + shell_cmd_line = command + if handle_timeout: + shell_cmd_line = "timeout -vk {kill_timeout_sec} -s SIGXCPU {timeout_sec} " + shell_cmd_line + if handle_core_dump: + shell_cmd_line = f"ulimit -c unlimited && " + shell_cmd_line + + result = shell.run(shell_cmd_line, shell=True, cwd=agh_submission.evaluation_directory) if parent_section is None: parent_section = evaluationDataOS @@ -205,7 +213,7 @@ def run_executable( ) ) std_out_file.parent.mkdir(exist_ok=True) - std_out_file.write_text(result.stdout, encoding="utf-8", errors="replace") + std_out_file.write_text(result.stdout, encoding="ascii", errors="backslashreplace") if len(result.stderr) > 0: std_err_file = resultsDir / f"{test_key}.stderr" @@ -217,7 +225,7 @@ def run_executable( type="default", ) ) - std_err_file.write_text(result.stderr, encoding="utf-8", errors="replace") + std_err_file.write_text(result.stderr, encoding="ascii", errors="backslashreplace") # Handle core dumps. core_dump_file = agh_submission.evaluation_directory / CORE_DUMP_FILE_NAME diff --git a/tox.ini b/tox.ini index e2ac523..17048db 100644 --- a/tox.ini +++ b/tox.ini @@ -14,11 +14,11 @@ envlist = clean, check, docs, -; {3.13,py311,py312,py313,pypy310,pypy311}, {3.13,py312,py313,pypy312,pypy313}, report ignore_basepython_conflict = true +; {3.13,py311,py312,py313,pypy310,pypy311}, ;{py39,py310,py311,py312,py313,pypy39,pypy310,pypy311}, [testenv] From 3b0630bcc997bc0088f5e4a6973a9e75e26d07b4 Mon Sep 17 00:00:00 2001 From: Tuck Williamson Date: Thu, 23 Oct 2025 14:15:28 -0400 Subject: [PATCH 05/10] Fixes for build passing. --- src/agh/agh_data.py | 54 ++++++------------ src/agh/cli.py | 120 +++++++++++++++++---------------------- src/agh/pytest_plugin.py | 2 +- 3 files changed, 71 insertions(+), 105 deletions(-) diff --git a/src/agh/agh_data.py b/src/agh/agh_data.py index f5cae92..057fac9 100644 --- a/src/agh/agh_data.py +++ b/src/agh/agh_data.py @@ -28,8 +28,7 @@ META_AGH_INTERNAL_KEY = "AGH_INTERNAL" META_INTERNAL_SUB_KEYS = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY] -META_INTERNAL_SUB_OUTPUT_COMPLETE = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, - "COMPLETED_OUTPUT"] +META_INTERNAL_SUB_OUTPUT_COMPLETE = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "COMPLETED_OUTPUT"] META_INTERNAL_SUB_OUTPUT_GRADED = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "GRADED"] META_INTERNAL_SUB_OUTPUT_NON_ANON = [META_AGH_INTERNAL_KEY, META_INTERNAL_SUB_KEY, META_INTERNAL_SUB_OUTPUT, "NON_ANON"] @@ -84,8 +83,7 @@ def asdict(self) -> dict[str, Any]: setattr(self, cur_field.name, [str(cur_val) for cur_val in restore_these[cur_field.name]]) elif get_origin(cur_field.type) is dict and hasattr(get_args(cur_field.type)[-1], "asdict"): restore_these[cur_field.name] = getattr(self, cur_field.name) - setattr(self, cur_field.name, - {cur_key: cur_val.asdict() for cur_key, cur_val in restore_these[cur_field.name].items()}) + setattr(self, cur_field.name, {cur_key: cur_val.asdict() for cur_key, cur_val in restore_these[cur_field.name].items()}) # if len(restore_these) > 0: # print(self) ret_val = asdict(self) @@ -111,8 +109,7 @@ def _from_json(cls, data: dict): data[cur_field.name] = [get_args(cur_field.type)[0]._from_json(p) for p in data[cur_field.name]] elif get_origin(cur_field.type) is dict and hasattr(get_args(cur_field.type)[-1], "_from_json"): # print(data[cur_field.name]) - data[cur_field.name] = {k: get_args(cur_field.type)[-1]._from_json(p) for k, p in - data[cur_field.name].items()} + data[cur_field.name] = {k: get_args(cur_field.type)[-1]._from_json(p) for k, p in data[cur_field.name].items()} elif is_dataclass(cur_field.type): data[cur_field.name] = cur_field.type(**data[cur_field.name]) elif get_origin(cur_field.type) is list and is_dataclass(get_args(cur_field.type)[0]): @@ -213,8 +210,7 @@ def hasData(self): return True elif len(self.included_sections) > 0 or len(self.included_files) > 0: return True in [cur_sub_sec.hasData for cur_sub_sec in self.included_sections] or True in [ - (cur_inc_file.path.exists() and cur_inc_file.path.stat().st_size > 1) for cur_inc_file in - self.included_files + (cur_inc_file.path.exists() and cur_inc_file.path.stat().st_size > 1) for cur_inc_file in self.included_files ] elif self.text.strip() != "": # If there are no 'includes' then the section is the data @@ -224,8 +220,7 @@ def hasData(self): def asQmdSection(self) -> str: if not self.hasData and self.only_output_if_data: return "" - inc_file_txt = "\n\n".join( - [cur_inc_file.asQmdSection(self.heading_level + 1) for cur_inc_file in self.included_files]) + inc_file_txt = "\n\n".join([cur_inc_file.asQmdSection(self.heading_level + 1) for cur_inc_file in self.included_files]) inc_sec_txt = "\n\n".join([cur_inc_sec.asQmdSection() for cur_inc_sec in self.included_sections]) warnings = "" @@ -240,8 +235,7 @@ def asQmdSection(self) -> str: if errors != "": errors = f'\n\n::: {{.callout-important title="Errors"}}\n\n{errors}\n\n:::\n\n' - pre = "#" * self.heading_level + (f" {self.title} {self.sectionAttr}\n\n{self.description}\n\n{self.text}" - f"{errors}{warnings}") + pre = "#" * self.heading_level + (f" {self.title} {self.sectionAttr}\n\n{self.description}\n\n{self.text}{errors}{warnings}") return f"{pre}\n\n{inc_file_txt}\n\n{inc_sec_txt}\n\n{self.post_script}" def addSection(self, section: "OutputSectionData") -> Self: @@ -370,8 +364,7 @@ class GraderOptions(MetaDataclassJson): _general_editor_command: str | None = None general_editor_command = property( - *_gen_prop_methods("_general_editor_command", "subl $files"), - doc="A command to open a submission file in a text editor." + *_gen_prop_methods("_general_editor_command", "subl $files"), doc="A command to open a submission file in a text editor." ) # This is a dictionary of metadata associated with the assignment. @@ -592,8 +585,7 @@ def load(cls, filepath: pathlib.Path | None = None): filepath = filepath.absolute() filepath = findFileInParents(filepath, cls.ASSIGNMENT_FILE_NAME) if filepath is None: - raise FileNotFoundError( - f"Could not find assignment JSON file in {orig_filepath} or any of its parents.") + raise FileNotFoundError(f"Could not find assignment JSON file in {orig_filepath} or any of its parents.") if filepath.exists() and filepath.is_file(): data = json.loads(filepath.read_text()) @@ -717,8 +709,7 @@ def all_linked_files() -> Generator[Path]: for link_item in self.link_template_dir.iterdir(): yield link_item for link_item in self._optional_files.values(): - if link_item.copy_to_sub_if_missing and not ( - ret_val.evaluation_directory / link_item.path.name).exists(): + if link_item.copy_to_sub_if_missing and not (ret_val.evaluation_directory / link_item.path.name).exists(): yield link_item.path for link_item in all_linked_files(): @@ -786,21 +777,18 @@ def postProcessSubmissionRender( pass # Create symbolic links output_file.symlink_to( - output_in_sub_dir.relative_to(output_file.parent, walk_up=True), - target_is_directory=output_file.is_dir() + output_in_sub_dir.relative_to(output_file.parent, walk_up=True), target_is_directory=output_file.is_dir() ) # output_file.chmod(0o444) output_not_anon.symlink_to( - output_graded.relative_to(output_not_anon.parent, walk_up=True), - target_is_directory=output_not_anon.is_dir() + output_graded.relative_to(output_not_anon.parent, walk_up=True), target_is_directory=output_not_anon.is_dir() ) return submission def AddSubmission( - self, submission_file: pathlib.Path, override_anon: bool | None = None, - warning_callback: Callable[[str], None | Any] | None = None + self, submission_file: pathlib.Path, override_anon: bool | None = None, warning_callback: Callable[[str], None | Any] | None = None ) -> "Submission": """Add a new submission to the assignment. :param submission_file: The path to the submission file to add. @@ -813,8 +801,7 @@ def AddSubmission( """ ret_val = Submission.new(self, submission_file=submission_file, override_anon=override_anon) ret_val.save() - return self.PostProcessSubmission(ret_val, exists_protocol=self.LinkProto.RAISE_ERROR, - warning_callback=warning_callback) + return self.PostProcessSubmission(ret_val, exists_protocol=self.LinkProto.RAISE_ERROR, warning_callback=warning_callback) # def __pytest_cmd(self): # return "pytest ./ -p shell-utilities -p agh" @@ -984,8 +971,7 @@ def __post_init__(self): if not self.submission_file.exists() or not self.submission_file.is_file(): raise ValueError(f"submission_file '{self.submission_file}' does not exist or is not a file.") if not self.evaluation_directory.exists() or not self.evaluation_directory.is_dir(): - raise ValueError( - f"evaluation_directory '{self.evaluation_directory}' does not exist or is not a directory.") + raise ValueError(f"evaluation_directory '{self.evaluation_directory}' does not exist or is not a directory.") class Submission(SubmissionData): @@ -1038,8 +1024,7 @@ def get_anon_name(cls, assignment: Assignment, submission_file: pathlib.Path): Returns: str: Anonymous name for the submission """ - return anonymizer.anonymize(submission_file.name, assignment.name, str(assignment.year), - assignment.grade_period, assignment.course) + return anonymizer.anonymize(submission_file.name, assignment.name, str(assignment.year), assignment.grade_period, assignment.course) @classmethod def load(cls, filepath: pathlib.Path | None = None): @@ -1061,8 +1046,7 @@ def load(cls, filepath: pathlib.Path | None = None): filepath = filepath.absolute() filepath = findFileInParents(filepath, cls.SUBMISSION_FILE_NAME) if filepath is None: - raise FileNotFoundError( - f"Could not find submission JSON file in {orig_filepath} or any of its parents.") + raise FileNotFoundError(f"Could not find submission JSON file in {orig_filepath} or any of its parents.") if filepath.exists() and filepath.is_file(): data = json.loads(filepath.read_text()) @@ -1159,8 +1143,7 @@ def __post_process_new__(self, assignment: Assignment, base_file_name: str | Non elif "zip" in self.submission_file.name: os.system(f'cd "{self.as_submitted_dir.absolute()}" && unzip "{self.submission_file.absolute()}"') elif base_file_name is not None: - os.system( - f'cp "{self.submission_file.absolute()}" "{self.evaluation_directory.absolute()}/{base_file_name}"') + os.system(f'cp "{self.submission_file.absolute()}" "{self.evaluation_directory.absolute()}/{base_file_name}"') self._missing_files_initially = self.check_missing_files(assignment) @@ -1243,8 +1226,7 @@ def errors(self) -> None | list[str]: missing_files = self.check_missing_files(Assignment.load()) if len(missing_files) > 0: - errors.append( - f"Missing required file{'s' if len(missing_files) > 1 else ''}: {[mf.name for mf in missing_files]}") + errors.append(f"Missing required file{'s' if len(missing_files) > 1 else ''}: {[mf.name for mf in missing_files]}") if len(errors) > 0: return errors diff --git a/src/agh/cli.py b/src/agh/cli.py index 83444e3..9dbb7d5 100644 --- a/src/agh/cli.py +++ b/src/agh/cli.py @@ -122,8 +122,7 @@ def SubFileCompleter(property: str, prefix: str, **kwargs): def submissionCompleter(*args, **kwargs): # return ['Bob','Tom'] try: - ret_val = [str(subm.evaluation_directory.resolve().relative_to(Path.cwd(), walk_up=True)) for subm in - Assignment.load().Submissions] + ret_val = [str(subm.evaluation_directory.resolve().relative_to(Path.cwd(), walk_up=True)) for subm in Assignment.load().Submissions] return ret_val except Exception as e: argcomplete.warn("No assignment found. Cannot complete submission files.") @@ -134,8 +133,7 @@ def submissionCompleter(*args, **kwargs): # console.log(SubFileCompleter("unprocessed_dir", '')) parser = MyArgParser( - description="agh --- Assignment Grading Helper", prog="agh", formatter_class=RichHelpFormatter, - conflict_handler="resolve" + description="agh --- Assignment Grading Helper", prog="agh", formatter_class=RichHelpFormatter, conflict_handler="resolve" ) parser.add_argument("--version", action="version", version=__version__) parser.add_argument("-H", "--full-help", action=FullHelp, help="Show full (all options) help") @@ -156,22 +154,16 @@ def submissionCompleter(*args, **kwargs): ################################################################################ ################################################################################ -assignment_sub_parser = subparsers.add_parser("assignment", help="Assignment commands", - formatter_class=RichHelpFormatter) +assignment_sub_parser = subparsers.add_parser("assignment", help="Assignment commands", formatter_class=RichHelpFormatter) assignment_subparsers = assignment_sub_parser.add_subparsers(dest="assignment_command", help="Assignment commands") # assignment > new assignment command -assignment_new_parser = assignment_subparsers.add_parser("new", help="Create new assignment", - formatter_class=RichHelpFormatter) -assignment_new_parser.add_argument("name", help="Assignment name", type=str).completer = lambda **kwargs: [ - f"Assignment {Path.cwd().name}"] -assignment_new_parser.add_argument("course", help="Course code", type=str).completer = lambda **kwargs: [ - f"CSCI-{Path.cwd().parent.name}"] -assignment_new_parser.add_argument("term", help="Term", - choices=["Fall", "Spring", "Maymester", "Summer I", "Summer II"], type=str) +assignment_new_parser = assignment_subparsers.add_parser("new", help="Create new assignment", formatter_class=RichHelpFormatter) +assignment_new_parser.add_argument("name", help="Assignment name", type=str).completer = lambda **kwargs: [f"Assignment {Path.cwd().name}"] +assignment_new_parser.add_argument("course", help="Course code", type=str).completer = lambda **kwargs: [f"CSCI-{Path.cwd().parent.name}"] +assignment_new_parser.add_argument("term", help="Term", choices=["Fall", "Spring", "Maymester", "Summer I", "Summer II"], type=str) assignment_new_parser.add_argument("-y", "--year", help="Year", type=int, default=cur_date.year) -assignment_new_parser.add_argument("-a", "--anon", help="Anonymize names", action=argparse.BooleanOptionalAction, - default=True) +assignment_new_parser.add_argument("-a", "--anon", help="Anonymize names", action=argparse.BooleanOptionalAction, default=True) # assignment > info command assignment_info_parser = assignment_subparsers.add_parser("info", help="Show assignment info") @@ -180,8 +172,7 @@ def submissionCompleter(*args, **kwargs): # Add required files command assign_add_required_parser = assignment_subparsers.add_parser("add-required", help="Add required files") assign_add_required_parser.add_argument("files", nargs="+", help="Required file names", type=Path) -assign_add_required_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda \ - **kwargs: [ +assign_add_required_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda **kwargs: [ "txt", "py", "c", @@ -190,8 +181,7 @@ def submissionCompleter(*args, **kwargs): "default", "make", ] -assign_add_required_parser.add_argument("-d", "--description", help="Description of the required files", type=str, - default="") +assign_add_required_parser.add_argument("-d", "--description", help="Description of the required files", type=str, default="") assign_add_required_parser.add_argument("-t", "--title", help="Title of the required files", type=str, default="") assign_add_required_parser.add_argument( "-i", "--include-in-output", help="Include in output", action=argparse.BooleanOptionalAction, default=True @@ -200,8 +190,7 @@ def submissionCompleter(*args, **kwargs): # Add optional files command assign_add_optional_parser = assignment_subparsers.add_parser("add-optional", help="Add optional files") assign_add_optional_parser.add_argument("files", nargs="+", help="Optional file names") -assign_add_optional_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda \ - **kwargs: [ +assign_add_optional_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda **kwargs: [ "txt", "py", "c", @@ -210,8 +199,7 @@ def submissionCompleter(*args, **kwargs): "make", "default", ] -assign_add_optional_parser.add_argument("-d", "--description", help="Description of the optional files", type=str, - default="") +assign_add_optional_parser.add_argument("-d", "--description", help="Description of the optional files", type=str, default="") assign_add_optional_parser.add_argument("-t", "--title", help="Title of the optional files", type=str, default="") assign_add_optional_parser.add_argument( "-i", "--include-in-output", help="Include in output", action=argparse.BooleanOptionalAction, default=True @@ -227,8 +215,7 @@ def submissionCompleter(*args, **kwargs): # submission > add command sub_add_subparser = sub_subparsers.add_parser("add", help="Add a submission file.") -sub_add_subparser.add_argument("files", nargs="+", help="Submission files to add", - type=Path).completer = functools.partial( +sub_add_subparser.add_argument("files", nargs="+", help="Submission files to add", type=Path).completer = functools.partial( SubFileCompleter, "unprocessed_dir" ) sub_add_subparser.add_argument( @@ -252,8 +239,7 @@ def submissionCompleter(*args, **kwargs): sub_fix_subparser = sub_subparsers.add_parser( "fix", help="Fix a submission. Try this if you accidentally deleted something. This may re-create links etc." ) -sub_fix_subparser.add_argument("submissions", nargs="+", help="Submissions to fix", - type=str).completer = submissionCompleter +sub_fix_subparser.add_argument("submissions", nargs="+", help="Submissions to fix", type=str).completer = submissionCompleter ################################################################################ ################################################################################ @@ -263,41 +249,38 @@ def submissionCompleter(*args, **kwargs): # Add run command run_parser = subparsers.add_parser("run", help="Run submission files. This executes build, test, and render.") run_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter run_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add test command -test_parser = subparsers.add_parser("test", - help="Test submission files. This just runs the tests for the given submissions.") +test_parser = subparsers.add_parser("test", help="Test submission files. This just runs the tests for the given submissions.") test_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter test_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add build command build_parser = subparsers.add_parser("build", help="Build submission files") build_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter build_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add render command render_parser = subparsers.add_parser("render", help="Render submission files") render_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter -render_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", - default=False) +render_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) argcomplete.autocomplete(parser) -def printableLinkWithIcon(link: Path, icon: str|Literal[":file_folder:",":notebook:"] = ':file_folder:', link_text:str = None) -> str: - """ This takes a path and returns a string that can be printed to the terminal as a link. + +def printableLinkWithIcon( + link: Path, icon: str | Literal[":file_folder:", ":notebook:"] = ":file_folder:", link_text: str | None = None +) -> str: + """This takes a path and returns a string that can be printed to the terminal as a link. :param link: Path to the file or directory. If the path does not exist, a red "x" is used as the icon. :param icon: The icon to use as part of the link. :param link_text: The text to use for the link. If None, the link text is the path relative to the root directory. @@ -306,7 +289,7 @@ def printableLinkWithIcon(link: Path, icon: str|Literal[":file_folder:",":notebo if link is None and link_text is None: raise ValueError("The link and link_text cannot both be None.") if link is None: - link = '' + link = "" icon = ":x:" else: if not link.exists(): @@ -318,24 +301,23 @@ def printableLinkWithIcon(link: Path, icon: str|Literal[":file_folder:",":notebo return f"[link file://{link}]{icon}{link_text}[/]" + def displayAssignmentInfo(cli_args: argparse.Namespace): assignment = Assignment.load() console.print(f'[label]Assignment "{assignment.name}"') - console.print( - f"[label]Course:[/] {assignment.course}, [label]Term:[/] {assignment._grade_period}, [label]Year:[/] " - f"{assignment.year}") + console.print(f"[label]Course:[/] {assignment.course}, [label]Term:[/] {assignment._grade_period}, [label]Year:[/] {assignment.year}") submissions = list(assignment.Submissions) console.print(f"[label]Submissions:[/] {len(submissions)}") - console.print(f"[label]Directories:[/] " - f"{printableLinkWithIcon(assignment.root_directory, link_text='Root')} :diamonds: " - f"{printableLinkWithIcon(assignment.eval_dir, link_text='Evaluations')} :diamonds: " - f"{printableLinkWithIcon(assignment.graded_output_dir, link_text='Output for Grading')} :diamonds: " - f"{printableLinkWithIcon(assignment.tests_dir, link_text='Evaluation Tests')} :diamonds: " - f"{printableLinkWithIcon(assignment.link_template_dir, link_text='Templates (files linked into each evaluation)')}" - ) - - files_table = rich.table.Table(title="[label][req]Required[/req]/[opt]Optional[/opt] Files", expand=True, - show_lines=True) + console.print( + f"[label]Directories:[/] " + f"{printableLinkWithIcon(assignment.root_directory, link_text='Root')} :diamonds: " + f"{printableLinkWithIcon(assignment.eval_dir, link_text='Evaluations')} :diamonds: " + f"{printableLinkWithIcon(assignment.graded_output_dir, link_text='Output for Grading')} :diamonds: " + f"{printableLinkWithIcon(assignment.tests_dir, link_text='Evaluation Tests')} :diamonds: " + f"{printableLinkWithIcon(assignment.link_template_dir, link_text='Templates (files linked into each evaluation)')}" + ) + + files_table = rich.table.Table(title="[label][req]Required[/req]/[opt]Optional[/opt] Files", expand=True, show_lines=True) files_table.add_column("Link", justify="center") files_table.add_column("Output", justify="center") files_table.add_column("Name", justify="left") @@ -416,7 +398,13 @@ def displaySubmissionInfo(cli_args: argparse.Namespace, assignment: Assignment): sub_color = "yellow" case (False, False, True): sub_color = "green" - submission_table.add_row(errors, warnings, output, f"[bold {sub_color}] {printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)} [/]", graded_output) + submission_table.add_row( + errors, + warnings, + output, + f"[bold {sub_color}] {printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)} [/]", + graded_output, + ) console.print(submission_table) @@ -519,8 +507,7 @@ def handleSubmissionCmd(cli_args: argparse.Namespace): console.print(f"[error]No submission directory found for {cur_file}.") cur_subm = Submission.load(cur_subm_dir) cur_subm.fix(assignment=assignment) - assignment.PostProcessSubmission(cur_subm, warning_callback=lambda warn: console.print(warn, - style="warning")).save() + assignment.PostProcessSubmission(cur_subm, warning_callback=lambda warn: console.print(warn, style="warning")).save() case _: console.log(cli_args, style="error") @@ -541,8 +528,7 @@ def verbose_print(cli_args: argparse.Namespace, *args, **kwargs) -> None: async def parse_pytest_output( - assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, - task_id + assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, task_id ): output_info = RunOutputInfo() @@ -613,12 +599,11 @@ async def run_pytest( task_id, advance=1, completed=True, - description=f"[error]Tests directory '{tests_path.absolute()}'not found. Perhaps run fix on " - f"{submission.name} first?", + description=f"[error]Tests directory '{tests_path.absolute()}'not found. Perhaps run fix on {submission.name} first?", ) return submission, False - cmd_str: str = f'pytest -v -p agh-pytest-plugin --agh {extra_pytest_args} {tests_path.absolute()}/*' + cmd_str: str = f"pytest -v -p agh-pytest-plugin --agh {extra_pytest_args} {tests_path.absolute()}/*" # verbose_print(cli_args, 'Running pytest...', cmd_str) # Setup the progress bar. task_id = progress.add_task(f"Testing {tests_path.absolute()}...", total=None) @@ -633,8 +618,7 @@ async def run_pytest( return submission, return_code == 0 -async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment: Assignment, - extra_pytest_args: str = ""): +async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment: Assignment, extra_pytest_args: str = ""): """This function asynchronously runs pytest on all submissions specified. It provides a progress bar for each submission to indicate the progress of the tests. @@ -718,8 +702,7 @@ def run(args=None): asyncio.run(execute_pytest_on_submissions(cli_args, assignment)) case "test": assignment = getCurrentAssignment() - asyncio.run( - execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) + asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) case "build": assignment = getCurrentAssignment() asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "build"')) @@ -731,6 +714,7 @@ def run(args=None): # print(start(args)) parser.exit(0) + # todo: create custom url scheme so I can run things from links in the output. # To create a custom URL scheme in Ubuntu that executes a command in the terminal, you need to define a desktop # entry for the scheme and create a script to handle the URL. diff --git a/src/agh/pytest_plugin.py b/src/agh/pytest_plugin.py index e29a3f6..4a2bc95 100644 --- a/src/agh/pytest_plugin.py +++ b/src/agh/pytest_plugin.py @@ -193,7 +193,7 @@ def run_executable( if handle_timeout: shell_cmd_line = "timeout -vk {kill_timeout_sec} -s SIGXCPU {timeout_sec} " + shell_cmd_line if handle_core_dump: - shell_cmd_line = f"ulimit -c unlimited && " + shell_cmd_line + shell_cmd_line = "ulimit -c unlimited && " + shell_cmd_line result = shell.run(shell_cmd_line, shell=True, cwd=agh_submission.evaluation_directory) From 9c2c431db09fcb2825041765a4c6c7a528d527b6 Mon Sep 17 00:00:00 2001 From: Tuck Williamson Date: Tue, 4 Nov 2025 13:47:17 -0500 Subject: [PATCH 06/10] Fixes for build passing. --- pyproject.toml | 8 +- src/agh/agh_data.py | 33 ++++--- src/agh/cli.py | 194 +++++++++++++++++++++++++++++---------- src/agh/pytest_plugin.py | 7 +- tests/test_assignment.py | 6 +- 5 files changed, 173 insertions(+), 75 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8cc0d70..93ff1fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,13 +22,13 @@ classifiers = [ "Intended Audience :: Developers", "Operating System :: Unix", "Operating System :: POSIX", - "Operating System :: Microsoft :: Windows", +# "Operating System :: Microsoft :: Windows", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", # "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", +# "Programming Language :: Python :: 3.10", +# "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", @@ -42,7 +42,7 @@ classifiers = [ keywords = [ # eg: "keyword1", "keyword2", "keyword3", ] -requires-python = ">=3.9" +requires-python = ">=3.12" dependencies = [ "pytest>=8.4.2", "pytest-shell-utilities>=1.9.7", diff --git a/src/agh/agh_data.py b/src/agh/agh_data.py index 057fac9..419c526 100644 --- a/src/agh/agh_data.py +++ b/src/agh/agh_data.py @@ -181,7 +181,7 @@ def asQmdSection(self, heading_level: int, max_file_size: int = DEFAULT_MAX_OUT_ hdr_txt = "#" * heading_level + " " + self.title + self.sectionAttr ret_val = f"\n\n{hdr_txt}\n\n{self.description}\n\n" if self.path.exists() and self.path.stat().st_size > max_file_size: - with self.path.open("rt", encoding="ascii", errors="replace") as f: + with self.path.open("rt", encoding="ascii", errors="backslashreplace") as f: ret_val += ( f"**[File too large! Contents truncated to {max_file_size} bytes.]{{.mark}}**\n\n```{{." f"{self.type}}}\n{f.read(max_file_size)}\n```\n\n" @@ -687,7 +687,7 @@ class LinkProto(IntEnum): # Raise an error if the file exists. RAISE_ERROR = 0 # Ignore an error and continue with the rest of the links if the file exists (don't do anything). - IGNORE_ERROR = 1 + SKIP_FILE = 1 # Overwrite the existing file if it already exists. LINK_OVERWRITE = 2 @@ -710,7 +710,10 @@ def all_linked_files() -> Generator[Path]: yield link_item for link_item in self._optional_files.values(): if link_item.copy_to_sub_if_missing and not (ret_val.evaluation_directory / link_item.path.name).exists(): - yield link_item.path + ret_path = link_item.path + if not ret_path.exists(): + ret_path = self.link_template_dir / link_item.path.name + yield ret_path for link_item in all_linked_files(): link_tgt = ret_val.evaluation_directory / link_item.name @@ -720,20 +723,20 @@ def all_linked_files() -> Generator[Path]: link_item = link_item.readlink() # Depending on the protocol handle if there is already an existing link. - if link_tgt.exists(): + if link_tgt.exists(follow_symlinks=False): # Handle if the link exists but is pointing to the correct target already - continue. if link_tgt.is_symlink() and link_tgt.readlink() == link_item: - continue - - match exists_protocol: - case self.LinkProto.RAISE_ERROR: - raise FileExistsError(link_tgt) - case self.LinkProto.IGNORE_ERROR: - continue - case self.LinkProto.LINK_OVERWRITE: - link_tgt.unlink() - case _: - raise NotImplementedError("New existing link protocol added, but code not added") + link_tgt.unlink(missing_ok=True) + else: + match exists_protocol: + case self.LinkProto.RAISE_ERROR: + raise FileExistsError(link_tgt) + case self.LinkProto.SKIP_FILE: + continue + case self.LinkProto.LINK_OVERWRITE: + link_tgt.unlink(missing_ok=True) + case _: + raise NotImplementedError("New existing link protocol added, but code not added") link_tgt.symlink_to(link_item.absolute(), target_is_directory=link_item.is_dir()) diff --git a/src/agh/cli.py b/src/agh/cli.py index 9dbb7d5..f096d23 100644 --- a/src/agh/cli.py +++ b/src/agh/cli.py @@ -22,6 +22,7 @@ import functools import re import sys +from asyncio import CancelledError from dataclasses import dataclass from dataclasses import field from pathlib import Path @@ -122,7 +123,8 @@ def SubFileCompleter(property: str, prefix: str, **kwargs): def submissionCompleter(*args, **kwargs): # return ['Bob','Tom'] try: - ret_val = [str(subm.evaluation_directory.resolve().relative_to(Path.cwd(), walk_up=True)) for subm in Assignment.load().Submissions] + ret_val = [str(subm.evaluation_directory.resolve().relative_to(Path.cwd(), walk_up=True)) for subm in + Assignment.load().Submissions] return ret_val except Exception as e: argcomplete.warn("No assignment found. Cannot complete submission files.") @@ -133,10 +135,20 @@ def submissionCompleter(*args, **kwargs): # console.log(SubFileCompleter("unprocessed_dir", '')) parser = MyArgParser( - description="agh --- Assignment Grading Helper", prog="agh", formatter_class=RichHelpFormatter, conflict_handler="resolve" + description="agh --- Assignment Grading Helper", prog="agh", formatter_class=RichHelpFormatter, + conflict_handler="resolve" ) parser.add_argument("--version", action="version", version=__version__) parser.add_argument("-H", "--full-help", action=FullHelp, help="Show full (all options) help") +parser.add_argument("-D", "--debug-core-files", action="store_true", dest="debug_core_files", + help="This instructs the OS to store core files such that debugging information can be gleaned " + "from crashes. [b red u]THIS MUST BE CALLED WITH ROOT PERMISSIONS.[/b red u]Ex: `sudo agh -D`", + default=False) +parser.add_argument("--restore-default-core-location", action="store_true", dest="restore_default_core_location", + help="This restores the default core location after calling `sudo agh -D`. [b red u]THIS MUST BE " + "CALLED WITH ROOT PERMISSIONS.[/b red u]Ex: `sudo agh --restore-default-core-location`", + default=False) + subparsers = parser.add_subparsers(dest="command", help="Assignment/Submission/etc. commands") @@ -154,16 +166,22 @@ def submissionCompleter(*args, **kwargs): ################################################################################ ################################################################################ -assignment_sub_parser = subparsers.add_parser("assignment", help="Assignment commands", formatter_class=RichHelpFormatter) +assignment_sub_parser = subparsers.add_parser("assignment", help="Assignment commands", + formatter_class=RichHelpFormatter) assignment_subparsers = assignment_sub_parser.add_subparsers(dest="assignment_command", help="Assignment commands") # assignment > new assignment command -assignment_new_parser = assignment_subparsers.add_parser("new", help="Create new assignment", formatter_class=RichHelpFormatter) -assignment_new_parser.add_argument("name", help="Assignment name", type=str).completer = lambda **kwargs: [f"Assignment {Path.cwd().name}"] -assignment_new_parser.add_argument("course", help="Course code", type=str).completer = lambda **kwargs: [f"CSCI-{Path.cwd().parent.name}"] -assignment_new_parser.add_argument("term", help="Term", choices=["Fall", "Spring", "Maymester", "Summer I", "Summer II"], type=str) +assignment_new_parser = assignment_subparsers.add_parser("new", help="Create new assignment", + formatter_class=RichHelpFormatter) +assignment_new_parser.add_argument("name", help="Assignment name", type=str).completer = lambda **kwargs: [ + f"Assignment {Path.cwd().name}"] +assignment_new_parser.add_argument("course", help="Course code", type=str).completer = lambda **kwargs: [ + f"CSCI-{Path.cwd().parent.name}"] +assignment_new_parser.add_argument("term", help="Term", + choices=["Fall", "Spring", "Maymester", "Summer I", "Summer II"], type=str) assignment_new_parser.add_argument("-y", "--year", help="Year", type=int, default=cur_date.year) -assignment_new_parser.add_argument("-a", "--anon", help="Anonymize names", action=argparse.BooleanOptionalAction, default=True) +assignment_new_parser.add_argument("-a", "--anon", help="Anonymize names", action=argparse.BooleanOptionalAction, + default=True) # assignment > info command assignment_info_parser = assignment_subparsers.add_parser("info", help="Show assignment info") @@ -172,7 +190,8 @@ def submissionCompleter(*args, **kwargs): # Add required files command assign_add_required_parser = assignment_subparsers.add_parser("add-required", help="Add required files") assign_add_required_parser.add_argument("files", nargs="+", help="Required file names", type=Path) -assign_add_required_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda **kwargs: [ +assign_add_required_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda \ + **kwargs: [ "txt", "py", "c", @@ -181,7 +200,8 @@ def submissionCompleter(*args, **kwargs): "default", "make", ] -assign_add_required_parser.add_argument("-d", "--description", help="Description of the required files", type=str, default="") +assign_add_required_parser.add_argument("-d", "--description", help="Description of the required files", type=str, + default="") assign_add_required_parser.add_argument("-t", "--title", help="Title of the required files", type=str, default="") assign_add_required_parser.add_argument( "-i", "--include-in-output", help="Include in output", action=argparse.BooleanOptionalAction, default=True @@ -190,7 +210,8 @@ def submissionCompleter(*args, **kwargs): # Add optional files command assign_add_optional_parser = assignment_subparsers.add_parser("add-optional", help="Add optional files") assign_add_optional_parser.add_argument("files", nargs="+", help="Optional file names") -assign_add_optional_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda **kwargs: [ +assign_add_optional_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda \ + **kwargs: [ "txt", "py", "c", @@ -199,7 +220,8 @@ def submissionCompleter(*args, **kwargs): "make", "default", ] -assign_add_optional_parser.add_argument("-d", "--description", help="Description of the optional files", type=str, default="") +assign_add_optional_parser.add_argument("-d", "--description", help="Description of the optional files", type=str, + default="") assign_add_optional_parser.add_argument("-t", "--title", help="Title of the optional files", type=str, default="") assign_add_optional_parser.add_argument( "-i", "--include-in-output", help="Include in output", action=argparse.BooleanOptionalAction, default=True @@ -215,7 +237,8 @@ def submissionCompleter(*args, **kwargs): # submission > add command sub_add_subparser = sub_subparsers.add_parser("add", help="Add a submission file.") -sub_add_subparser.add_argument("files", nargs="+", help="Submission files to add", type=Path).completer = functools.partial( +sub_add_subparser.add_argument("files", nargs="+", help="Submission files to add", + type=Path).completer = functools.partial( SubFileCompleter, "unprocessed_dir" ) sub_add_subparser.add_argument( @@ -239,7 +262,8 @@ def submissionCompleter(*args, **kwargs): sub_fix_subparser = sub_subparsers.add_parser( "fix", help="Fix a submission. Try this if you accidentally deleted something. This may re-create links etc." ) -sub_fix_subparser.add_argument("submissions", nargs="+", help="Submissions to fix", type=str).completer = submissionCompleter +sub_fix_subparser.add_argument("submissions", nargs="+", help="Submissions to fix", + type=str).completer = submissionCompleter ################################################################################ ################################################################################ @@ -249,30 +273,36 @@ def submissionCompleter(*args, **kwargs): # Add run command run_parser = subparsers.add_parser("run", help="Run submission files. This executes build, test, and render.") run_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, + default=None ).completer = submissionCompleter run_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add test command -test_parser = subparsers.add_parser("test", help="Test submission files. This just runs the tests for the given submissions.") +test_parser = subparsers.add_parser("test", + help="Test submission files. This just runs the tests for the given submissions.") test_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, + default=None ).completer = submissionCompleter test_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add build command build_parser = subparsers.add_parser("build", help="Build submission files") build_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, + default=None ).completer = submissionCompleter build_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add render command render_parser = subparsers.add_parser("render", help="Render submission files") render_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, + default=None ).completer = submissionCompleter -render_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) +render_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", + default=False) argcomplete.autocomplete(parser) @@ -305,7 +335,9 @@ def printableLinkWithIcon( def displayAssignmentInfo(cli_args: argparse.Namespace): assignment = Assignment.load() console.print(f'[label]Assignment "{assignment.name}"') - console.print(f"[label]Course:[/] {assignment.course}, [label]Term:[/] {assignment._grade_period}, [label]Year:[/] {assignment.year}") + console.print( + f"[label]Course:[/] {assignment.course}, [label]Term:[/] {assignment._grade_period}, [label]Year:[/] " + f"{assignment.year}") submissions = list(assignment.Submissions) console.print(f"[label]Submissions:[/] {len(submissions)}") console.print( @@ -317,7 +349,8 @@ def displayAssignmentInfo(cli_args: argparse.Namespace): f"{printableLinkWithIcon(assignment.link_template_dir, link_text='Templates (files linked into each evaluation)')}" ) - files_table = rich.table.Table(title="[label][req]Required[/req]/[opt]Optional[/opt] Files", expand=True, show_lines=True) + files_table = rich.table.Table(title="[label][req]Required[/req]/[opt]Optional[/opt] Files", expand=True, + show_lines=True) files_table.add_column("Link", justify="center") files_table.add_column("Output", justify="center") files_table.add_column("Name", justify="left") @@ -402,7 +435,7 @@ def displaySubmissionInfo(cli_args: argparse.Namespace, assignment: Assignment): errors, warnings, output, - f"[bold {sub_color}] {printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)} [/]", + f"[bold {sub_color}]{printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)}[/]", graded_output, ) @@ -507,7 +540,10 @@ def handleSubmissionCmd(cli_args: argparse.Namespace): console.print(f"[error]No submission directory found for {cur_file}.") cur_subm = Submission.load(cur_subm_dir) cur_subm.fix(assignment=assignment) - assignment.PostProcessSubmission(cur_subm, warning_callback=lambda warn: console.print(warn, style="warning")).save() + assignment.PostProcessSubmission(cur_subm, + exists_protocol=assignment.LinkProto.SKIP_FILE, + warning_callback=lambda warn: console.print(warn, + style="warning")).save() case _: console.log(cli_args, style="error") @@ -528,7 +564,8 @@ def verbose_print(cli_args: argparse.Namespace, *args, **kwargs) -> None: async def parse_pytest_output( - assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, task_id + assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, + task_id ): output_info = RunOutputInfo() @@ -551,15 +588,17 @@ async def error_collector(): match = re.search(r"collected \d+ items / \d+ deselected / (\d+) selected", line) if match: output_info.collected = int(match.group(1)) - progress.update(task_id, total=output_info.collected) + progress.update(task_id, total=output_info.collected, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) elif "collecting ..." in line: match = re.search(r"collected (\d+) items", line) if match: output_info.collected = int(match.group(1)) - progress.update(task_id, total=output_info.collected) + progress.update(task_id, total=output_info.collected,name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) elif " PASSED " in line or " FAILED " in line or " SKIPPED " in line: - progress.update(task_id, advance=1) - progress.update(task_id, description=line) + progress.update(task_id, advance=1, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) + elif line == '': + continue + progress.update(task_id, description=line, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) await proc.wait() await err_coll_task @@ -594,31 +633,38 @@ async def run_pytest( # This resolves that path relative to the submission directory. tests_path = submission.evaluation_directory / assignment.tests_dir.name if not tests_path.exists(): - task_id = progress.add_task("Testing...", total=1) + task_id = progress.add_task("Testing...", total=1, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) progress.update( task_id, advance=1, completed=True, - description=f"[error]Tests directory '{tests_path.absolute()}'not found. Perhaps run fix on {submission.name} first?", + description=f"[error]Tests directory '{tests_path.absolute()}'not found. Perhaps run fix on " + f"{submission.name} first?", + name= printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name) ) return submission, False cmd_str: str = f"pytest -v -p agh-pytest-plugin --agh {extra_pytest_args} {tests_path.absolute()}/*" # verbose_print(cli_args, 'Running pytest...', cmd_str) # Setup the progress bar. - task_id = progress.add_task(f"Testing {tests_path.absolute()}...", total=None) + task_id = progress.add_task(f"Testing {tests_path.absolute()}...", total=None, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) - # Run pytest. - proc = await asyncio.create_subprocess_shell( - cmd_str, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - return_code = await parse_pytest_output(assignment, submission, proc, progress, task_id) + try: + # Run pytest. + proc = await asyncio.create_subprocess_shell( + cmd_str, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=assignment.eval_dir.absolute(), + ) + return_code = await parse_pytest_output(assignment, submission, proc, progress, task_id) + except CancelledError: + return_code = -1 return submission, return_code == 0 -async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment: Assignment, extra_pytest_args: str = ""): +async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment: Assignment, + extra_pytest_args: str = ""): """This function asynchronously runs pytest on all submissions specified. It provides a progress bar for each submission to indicate the progress of the tests. @@ -631,10 +677,12 @@ async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment # If there are no submissions specified, run on all submissions. if cli_args.submissions is None: cli_args.submissions = list(assignment.Submissions) + cli_args.submissions.sort(key=lambda x: x.name) else: # The CLI submissions provide the directory for the submission. We convert # to the submission objects and report any that don't exist. submission_list = [] + cli_args.submissions.sort() for submission in cli_args.submissions: try: submission_list.append(Submission.load(submission)) @@ -654,15 +702,18 @@ async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment with rich.progress.Progress( rich.progress.SpinnerColumn(spinner_name="dots"), + rich.progress.TextColumn("{task.fields[name]}", style="label", justify='center'), *rich.progress.Progress.get_default_columns(), - rich.progress.TimeRemainingColumn(), ) as progress: - tasks = [ - run_pytest(assignment, submission, progress, cli_args, extra_pytest_args=extra_pytest_args) - for submission in cli_args.submissions - ] - results = await asyncio.gather(*tasks) - + try: + tasks = [ + run_pytest(assignment, submission, progress, cli_args, extra_pytest_args=extra_pytest_args) + for submission in cli_args.submissions + ] + results = await asyncio.gather(*tasks) + except KeyboardInterrupt: + # sys.exit(0) + pass for submission, success in results: if success: console.print(f"[green]Tests passed for {submission.name}[/green]") @@ -688,6 +739,46 @@ def run(args=None): args = ["status"] cli_args = parser.parse_args(args=args) console.rule(f"[b i]agh[/] - Assignment Grading Helper - Version: [b i]{__version__}") + + #Core file Handling. + prior_pattern_path = getCurrentAssignment().root_directory / '.prior_core_pattern.txt' + core_pattern_path = Path('/proc/sys/kernel/core_pattern') + if (cli_args.debug_core_files or cli_args.restore_default_core_location) and cli_args.command is not None: + console.print("[error]The debug core files option can only be used without any other command.") + exit(1) + elif cli_args.debug_core_files: + # Handle setting up the system to store the core dump where testing scripts will pick it up. + target_core_loc = 'aghAssignmentCoreDump.core' + prior_loc = core_pattern_path.read_text() + if target_core_loc in prior_loc: + console.print("[error]Core dumps already enabled.") + exit(0) + else: + prior_pattern_path.write_text(prior_loc) + + try: + core_pattern_path.write_text(target_core_loc) + except Exception as e: + console.print(f"[error]Error setting core dump location: {e}") + exit(1) + console.print(f"[bold green]Core dump location set to '{target_core_loc}'") + console.print("[bold green]To restore the default core dump location, run 'agh debug-core-files --restore'") + exit(0) + elif cli_args.restore_default_core_location: + if not prior_pattern_path.exists(): + console.print("[error]No prior core dump location found.") + exit(1) + else: + prior_loc = prior_pattern_path.read_text() + try: + core_pattern_path.write_text(prior_loc) + except Exception as e: + console.print(f"[error]Error setting core dump location: {e}") + exit(1) + console.print(f"[bold green]Core dump location restored to '{prior_loc}'") + exit(0) + + # Command handling. match cli_args.command: case "status": assignment = getCurrentAssignment() @@ -699,10 +790,14 @@ def run(args=None): handleSubmissionCmd(cli_args) case "run": assignment = getCurrentAssignment() - asyncio.run(execute_pytest_on_submissions(cli_args, assignment)) + try: + asyncio.run(execute_pytest_on_submissions(cli_args, assignment)) + except KeyboardInterrupt: + pass case "test": assignment = getCurrentAssignment() - asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) + asyncio.run( + execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) case "build": assignment = getCurrentAssignment() asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "build"')) @@ -714,7 +809,6 @@ def run(args=None): # print(start(args)) parser.exit(0) - # todo: create custom url scheme so I can run things from links in the output. # To create a custom URL scheme in Ubuntu that executes a command in the terminal, you need to define a desktop # entry for the scheme and create a script to handle the URL. diff --git a/src/agh/pytest_plugin.py b/src/agh/pytest_plugin.py index 4a2bc95..e06de31 100644 --- a/src/agh/pytest_plugin.py +++ b/src/agh/pytest_plugin.py @@ -171,7 +171,7 @@ def _core_file_saved(agh_submission): @pytest.fixture -def agh_run_executable(agh_submission, shell: ScriptSubprocess, resultsDir, _core_file_saved) -> Callable[..., OutputSectionData]: +def agh_run_executable(agh_submission, shell: ScriptSubprocess, resultsDir, _core_file_saved) -> Callable[..., tuple[ProcessResult,OutputSectionData]]: def run_executable( command: str, test_key: str, @@ -181,6 +181,7 @@ def run_executable( parent_section: OutputSectionData | None = None, handle_core_dump: bool = True, handle_timeout: bool = True, + **kwargs, ) -> tuple[ProcessResult, OutputSectionData]: """Run an executable and return the results. .. important:: @@ -191,11 +192,11 @@ def run_executable( # Build up the shell command line based on options selected by the grader. shell_cmd_line = command if handle_timeout: - shell_cmd_line = "timeout -vk {kill_timeout_sec} -s SIGXCPU {timeout_sec} " + shell_cmd_line + shell_cmd_line = f"timeout -vk {kill_timeout_sec} -s SIGXCPU {timeout_sec} " + shell_cmd_line if handle_core_dump: shell_cmd_line = "ulimit -c unlimited && " + shell_cmd_line - result = shell.run(shell_cmd_line, shell=True, cwd=agh_submission.evaluation_directory) + result = shell.run(shell_cmd_line, shell=True, cwd=agh_submission.evaluation_directory, **kwargs) if parent_section is None: parent_section = evaluationDataOS diff --git a/tests/test_assignment.py b/tests/test_assignment.py index 9286e35..b119c14 100644 --- a/tests/test_assignment.py +++ b/tests/test_assignment.py @@ -345,7 +345,7 @@ def test_pps_links_test_files(temp_assignment, temp_submission_file): # def test_postprocesssubmission_ignores_existing_link(temp_assignment, temp_submission_file, tmp_path): -# """Test that PostProcessSubmission ignores existing link when protocol is IGNORE_ERROR.""" +# """Test that PostProcessSubmission ignores existing link when protocol is SKIP_FILE.""" # conflict_file = tmp_path / "tests" # conflict_file.mkdir() # @@ -353,9 +353,9 @@ def test_pps_links_test_files(temp_assignment, temp_submission_file): # (new_submission.evaluation_directory / "tests").symlink_to(conflict_file) # # try: -# temp_assignment.PostProcessSubmission(temp_submission_file, exists_protocol=LinkProto.IGNORE_ERROR) +# temp_assignment.PostProcessSubmission(temp_submission_file, exists_protocol=LinkProto.SKIP_FILE) # except FileExistsError: -# pytest.fail("PostProcessSubmission should not raise FileExistsError when IGNORE_ERROR is used.") +# pytest.fail("PostProcessSubmission should not raise FileExistsError when SKIP_FILE is used.") # def test_postprocesssubmission_overwrites_existing_link(temp_assignment, temp_submission_file, tmp_path): From 80cd008e7061f4ea470f4629216a3ac5cdf68fad Mon Sep 17 00:00:00 2001 From: Tuck Williamson Date: Tue, 4 Nov 2025 13:50:50 -0500 Subject: [PATCH 07/10] Fixes for build passing. --- pyproject.toml | 6 +- src/agh/cli.py | 150 +++++++++++++++++++-------------------- src/agh/pytest_plugin.py | 4 +- tests/test_assignment.py | 2 +- 4 files changed, 81 insertions(+), 81 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 93ff1fb..d274f51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,13 +22,13 @@ classifiers = [ "Intended Audience :: Developers", "Operating System :: Unix", "Operating System :: POSIX", -# "Operating System :: Microsoft :: Windows", + # "Operating System :: Microsoft :: Windows", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", # "Programming Language :: Python :: 3.9", -# "Programming Language :: Python :: 3.10", -# "Programming Language :: Python :: 3.11", + # "Programming Language :: Python :: 3.10", + # "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", diff --git a/src/agh/cli.py b/src/agh/cli.py index f096d23..9c9fea0 100644 --- a/src/agh/cli.py +++ b/src/agh/cli.py @@ -123,8 +123,7 @@ def SubFileCompleter(property: str, prefix: str, **kwargs): def submissionCompleter(*args, **kwargs): # return ['Bob','Tom'] try: - ret_val = [str(subm.evaluation_directory.resolve().relative_to(Path.cwd(), walk_up=True)) for subm in - Assignment.load().Submissions] + ret_val = [str(subm.evaluation_directory.resolve().relative_to(Path.cwd(), walk_up=True)) for subm in Assignment.load().Submissions] return ret_val except Exception as e: argcomplete.warn("No assignment found. Cannot complete submission files.") @@ -135,19 +134,27 @@ def submissionCompleter(*args, **kwargs): # console.log(SubFileCompleter("unprocessed_dir", '')) parser = MyArgParser( - description="agh --- Assignment Grading Helper", prog="agh", formatter_class=RichHelpFormatter, - conflict_handler="resolve" + description="agh --- Assignment Grading Helper", prog="agh", formatter_class=RichHelpFormatter, conflict_handler="resolve" ) parser.add_argument("--version", action="version", version=__version__) parser.add_argument("-H", "--full-help", action=FullHelp, help="Show full (all options) help") -parser.add_argument("-D", "--debug-core-files", action="store_true", dest="debug_core_files", - help="This instructs the OS to store core files such that debugging information can be gleaned " - "from crashes. [b red u]THIS MUST BE CALLED WITH ROOT PERMISSIONS.[/b red u]Ex: `sudo agh -D`", - default=False) -parser.add_argument("--restore-default-core-location", action="store_true", dest="restore_default_core_location", - help="This restores the default core location after calling `sudo agh -D`. [b red u]THIS MUST BE " - "CALLED WITH ROOT PERMISSIONS.[/b red u]Ex: `sudo agh --restore-default-core-location`", - default=False) +parser.add_argument( + "-D", + "--debug-core-files", + action="store_true", + dest="debug_core_files", + help="This instructs the OS to store core files such that debugging information can be gleaned " + "from crashes. [b red u]THIS MUST BE CALLED WITH ROOT PERMISSIONS.[/b red u]Ex: `sudo agh -D`", + default=False, +) +parser.add_argument( + "--restore-default-core-location", + action="store_true", + dest="restore_default_core_location", + help="This restores the default core location after calling `sudo agh -D`. [b red u]THIS MUST BE " + "CALLED WITH ROOT PERMISSIONS.[/b red u]Ex: `sudo agh --restore-default-core-location`", + default=False, +) subparsers = parser.add_subparsers(dest="command", help="Assignment/Submission/etc. commands") @@ -166,22 +173,16 @@ def submissionCompleter(*args, **kwargs): ################################################################################ ################################################################################ -assignment_sub_parser = subparsers.add_parser("assignment", help="Assignment commands", - formatter_class=RichHelpFormatter) +assignment_sub_parser = subparsers.add_parser("assignment", help="Assignment commands", formatter_class=RichHelpFormatter) assignment_subparsers = assignment_sub_parser.add_subparsers(dest="assignment_command", help="Assignment commands") # assignment > new assignment command -assignment_new_parser = assignment_subparsers.add_parser("new", help="Create new assignment", - formatter_class=RichHelpFormatter) -assignment_new_parser.add_argument("name", help="Assignment name", type=str).completer = lambda **kwargs: [ - f"Assignment {Path.cwd().name}"] -assignment_new_parser.add_argument("course", help="Course code", type=str).completer = lambda **kwargs: [ - f"CSCI-{Path.cwd().parent.name}"] -assignment_new_parser.add_argument("term", help="Term", - choices=["Fall", "Spring", "Maymester", "Summer I", "Summer II"], type=str) +assignment_new_parser = assignment_subparsers.add_parser("new", help="Create new assignment", formatter_class=RichHelpFormatter) +assignment_new_parser.add_argument("name", help="Assignment name", type=str).completer = lambda **kwargs: [f"Assignment {Path.cwd().name}"] +assignment_new_parser.add_argument("course", help="Course code", type=str).completer = lambda **kwargs: [f"CSCI-{Path.cwd().parent.name}"] +assignment_new_parser.add_argument("term", help="Term", choices=["Fall", "Spring", "Maymester", "Summer I", "Summer II"], type=str) assignment_new_parser.add_argument("-y", "--year", help="Year", type=int, default=cur_date.year) -assignment_new_parser.add_argument("-a", "--anon", help="Anonymize names", action=argparse.BooleanOptionalAction, - default=True) +assignment_new_parser.add_argument("-a", "--anon", help="Anonymize names", action=argparse.BooleanOptionalAction, default=True) # assignment > info command assignment_info_parser = assignment_subparsers.add_parser("info", help="Show assignment info") @@ -190,8 +191,7 @@ def submissionCompleter(*args, **kwargs): # Add required files command assign_add_required_parser = assignment_subparsers.add_parser("add-required", help="Add required files") assign_add_required_parser.add_argument("files", nargs="+", help="Required file names", type=Path) -assign_add_required_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda \ - **kwargs: [ +assign_add_required_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda **kwargs: [ "txt", "py", "c", @@ -200,8 +200,7 @@ def submissionCompleter(*args, **kwargs): "default", "make", ] -assign_add_required_parser.add_argument("-d", "--description", help="Description of the required files", type=str, - default="") +assign_add_required_parser.add_argument("-d", "--description", help="Description of the required files", type=str, default="") assign_add_required_parser.add_argument("-t", "--title", help="Title of the required files", type=str, default="") assign_add_required_parser.add_argument( "-i", "--include-in-output", help="Include in output", action=argparse.BooleanOptionalAction, default=True @@ -210,8 +209,7 @@ def submissionCompleter(*args, **kwargs): # Add optional files command assign_add_optional_parser = assignment_subparsers.add_parser("add-optional", help="Add optional files") assign_add_optional_parser.add_argument("files", nargs="+", help="Optional file names") -assign_add_optional_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda \ - **kwargs: [ +assign_add_optional_parser.add_argument("type", help="Type of the required file", type=str).completer = lambda **kwargs: [ "txt", "py", "c", @@ -220,8 +218,7 @@ def submissionCompleter(*args, **kwargs): "make", "default", ] -assign_add_optional_parser.add_argument("-d", "--description", help="Description of the optional files", type=str, - default="") +assign_add_optional_parser.add_argument("-d", "--description", help="Description of the optional files", type=str, default="") assign_add_optional_parser.add_argument("-t", "--title", help="Title of the optional files", type=str, default="") assign_add_optional_parser.add_argument( "-i", "--include-in-output", help="Include in output", action=argparse.BooleanOptionalAction, default=True @@ -237,8 +234,7 @@ def submissionCompleter(*args, **kwargs): # submission > add command sub_add_subparser = sub_subparsers.add_parser("add", help="Add a submission file.") -sub_add_subparser.add_argument("files", nargs="+", help="Submission files to add", - type=Path).completer = functools.partial( +sub_add_subparser.add_argument("files", nargs="+", help="Submission files to add", type=Path).completer = functools.partial( SubFileCompleter, "unprocessed_dir" ) sub_add_subparser.add_argument( @@ -262,8 +258,7 @@ def submissionCompleter(*args, **kwargs): sub_fix_subparser = sub_subparsers.add_parser( "fix", help="Fix a submission. Try this if you accidentally deleted something. This may re-create links etc." ) -sub_fix_subparser.add_argument("submissions", nargs="+", help="Submissions to fix", - type=str).completer = submissionCompleter +sub_fix_subparser.add_argument("submissions", nargs="+", help="Submissions to fix", type=str).completer = submissionCompleter ################################################################################ ################################################################################ @@ -273,36 +268,30 @@ def submissionCompleter(*args, **kwargs): # Add run command run_parser = subparsers.add_parser("run", help="Run submission files. This executes build, test, and render.") run_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter run_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add test command -test_parser = subparsers.add_parser("test", - help="Test submission files. This just runs the tests for the given submissions.") +test_parser = subparsers.add_parser("test", help="Test submission files. This just runs the tests for the given submissions.") test_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter test_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add build command build_parser = subparsers.add_parser("build", help="Build submission files") build_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter build_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) # Add render command render_parser = subparsers.add_parser("render", help="Render submission files") render_parser.add_argument( - "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, - default=None + "-s", "--submission", dest="submissions", nargs="+", help="Submissions to run (build, test, render).", type=Path, default=None ).completer = submissionCompleter -render_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", - default=False) +render_parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="Verbose output.", default=False) argcomplete.autocomplete(parser) @@ -335,9 +324,7 @@ def printableLinkWithIcon( def displayAssignmentInfo(cli_args: argparse.Namespace): assignment = Assignment.load() console.print(f'[label]Assignment "{assignment.name}"') - console.print( - f"[label]Course:[/] {assignment.course}, [label]Term:[/] {assignment._grade_period}, [label]Year:[/] " - f"{assignment.year}") + console.print(f"[label]Course:[/] {assignment.course}, [label]Term:[/] {assignment._grade_period}, [label]Year:[/] {assignment.year}") submissions = list(assignment.Submissions) console.print(f"[label]Submissions:[/] {len(submissions)}") console.print( @@ -349,8 +336,7 @@ def displayAssignmentInfo(cli_args: argparse.Namespace): f"{printableLinkWithIcon(assignment.link_template_dir, link_text='Templates (files linked into each evaluation)')}" ) - files_table = rich.table.Table(title="[label][req]Required[/req]/[opt]Optional[/opt] Files", expand=True, - show_lines=True) + files_table = rich.table.Table(title="[label][req]Required[/req]/[opt]Optional[/opt] Files", expand=True, show_lines=True) files_table.add_column("Link", justify="center") files_table.add_column("Output", justify="center") files_table.add_column("Name", justify="left") @@ -540,10 +526,11 @@ def handleSubmissionCmd(cli_args: argparse.Namespace): console.print(f"[error]No submission directory found for {cur_file}.") cur_subm = Submission.load(cur_subm_dir) cur_subm.fix(assignment=assignment) - assignment.PostProcessSubmission(cur_subm, - exists_protocol=assignment.LinkProto.SKIP_FILE, - warning_callback=lambda warn: console.print(warn, - style="warning")).save() + assignment.PostProcessSubmission( + cur_subm, + exists_protocol=assignment.LinkProto.SKIP_FILE, + warning_callback=lambda warn: console.print(warn, style="warning"), + ).save() case _: console.log(cli_args, style="error") @@ -564,8 +551,7 @@ def verbose_print(cli_args: argparse.Namespace, *args, **kwargs) -> None: async def parse_pytest_output( - assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, - task_id + assignment: Assignment, submission: Submission, proc: asyncio.subprocess.Process, progress: rich.progress.Progress, task_id ): output_info = RunOutputInfo() @@ -588,15 +574,23 @@ async def error_collector(): match = re.search(r"collected \d+ items / \d+ deselected / (\d+) selected", line) if match: output_info.collected = int(match.group(1)) - progress.update(task_id, total=output_info.collected, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) + progress.update( + task_id, + total=output_info.collected, + name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name), + ) elif "collecting ..." in line: match = re.search(r"collected (\d+) items", line) if match: output_info.collected = int(match.group(1)) - progress.update(task_id, total=output_info.collected,name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) + progress.update( + task_id, + total=output_info.collected, + name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name), + ) elif " PASSED " in line or " FAILED " in line or " SKIPPED " in line: progress.update(task_id, advance=1, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) - elif line == '': + elif line == "": continue progress.update(task_id, description=line, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) await proc.wait() @@ -633,21 +627,26 @@ async def run_pytest( # This resolves that path relative to the submission directory. tests_path = submission.evaluation_directory / assignment.tests_dir.name if not tests_path.exists(): - task_id = progress.add_task("Testing...", total=1, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) + task_id = progress.add_task( + "Testing...", total=1, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name) + ) progress.update( task_id, advance=1, completed=True, - description=f"[error]Tests directory '{tests_path.absolute()}'not found. Perhaps run fix on " - f"{submission.name} first?", - name= printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name) + description=f"[error]Tests directory '{tests_path.absolute()}'not found. Perhaps run fix on {submission.name} first?", + name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name), ) return submission, False cmd_str: str = f"pytest -v -p agh-pytest-plugin --agh {extra_pytest_args} {tests_path.absolute()}/*" # verbose_print(cli_args, 'Running pytest...', cmd_str) # Setup the progress bar. - task_id = progress.add_task(f"Testing {tests_path.absolute()}...", total=None, name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name)) + task_id = progress.add_task( + f"Testing {tests_path.absolute()}...", + total=None, + name=printableLinkWithIcon(submission.evaluation_directory, link_text=submission.name), + ) try: # Run pytest. @@ -663,8 +662,7 @@ async def run_pytest( return submission, return_code == 0 -async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment: Assignment, - extra_pytest_args: str = ""): +async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment: Assignment, extra_pytest_args: str = ""): """This function asynchronously runs pytest on all submissions specified. It provides a progress bar for each submission to indicate the progress of the tests. @@ -702,7 +700,7 @@ async def execute_pytest_on_submissions(cli_args: argparse.Namespace, assignment with rich.progress.Progress( rich.progress.SpinnerColumn(spinner_name="dots"), - rich.progress.TextColumn("{task.fields[name]}", style="label", justify='center'), + rich.progress.TextColumn("{task.fields[name]}", style="label", justify="center"), *rich.progress.Progress.get_default_columns(), ) as progress: try: @@ -740,15 +738,15 @@ def run(args=None): cli_args = parser.parse_args(args=args) console.rule(f"[b i]agh[/] - Assignment Grading Helper - Version: [b i]{__version__}") - #Core file Handling. - prior_pattern_path = getCurrentAssignment().root_directory / '.prior_core_pattern.txt' - core_pattern_path = Path('/proc/sys/kernel/core_pattern') + # Core file Handling. + prior_pattern_path = getCurrentAssignment().root_directory / ".prior_core_pattern.txt" + core_pattern_path = Path("/proc/sys/kernel/core_pattern") if (cli_args.debug_core_files or cli_args.restore_default_core_location) and cli_args.command is not None: console.print("[error]The debug core files option can only be used without any other command.") exit(1) elif cli_args.debug_core_files: # Handle setting up the system to store the core dump where testing scripts will pick it up. - target_core_loc = 'aghAssignmentCoreDump.core' + target_core_loc = "aghAssignmentCoreDump.core" prior_loc = core_pattern_path.read_text() if target_core_loc in prior_loc: console.print("[error]Core dumps already enabled.") @@ -796,8 +794,7 @@ def run(args=None): pass case "test": assignment = getCurrentAssignment() - asyncio.run( - execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) + asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "not build and not render"')) case "build": assignment = getCurrentAssignment() asyncio.run(execute_pytest_on_submissions(cli_args, assignment, extra_pytest_args='-m "build"')) @@ -809,6 +806,7 @@ def run(args=None): # print(start(args)) parser.exit(0) + # todo: create custom url scheme so I can run things from links in the output. # To create a custom URL scheme in Ubuntu that executes a command in the terminal, you need to define a desktop # entry for the scheme and create a script to handle the URL. diff --git a/src/agh/pytest_plugin.py b/src/agh/pytest_plugin.py index e06de31..30f8f44 100644 --- a/src/agh/pytest_plugin.py +++ b/src/agh/pytest_plugin.py @@ -171,7 +171,9 @@ def _core_file_saved(agh_submission): @pytest.fixture -def agh_run_executable(agh_submission, shell: ScriptSubprocess, resultsDir, _core_file_saved) -> Callable[..., tuple[ProcessResult,OutputSectionData]]: +def agh_run_executable( + agh_submission, shell: ScriptSubprocess, resultsDir, _core_file_saved +) -> Callable[..., tuple[ProcessResult, OutputSectionData]]: def run_executable( command: str, test_key: str, diff --git a/tests/test_assignment.py b/tests/test_assignment.py index b119c14..690e1ba 100644 --- a/tests/test_assignment.py +++ b/tests/test_assignment.py @@ -23,7 +23,7 @@ def test_defaults(self): # Defaults self.assertEqual(a._name, "assignment") self.assertEqual(a._year, 2025) - self.assertEqual(a._grade_period, "Fall") + # self.assertEqual(a._grade_period, "Fall") This now switches based on the time of year. self.assertEqual(a._course, "CSCI-340") # self.assertEqual(a._submission_files, []) self.assertEqual(a._required_files, {}) From f6662ca1a0c1e7c970cc70c25de0624ed5e54495 Mon Sep 17 00:00:00 2001 From: Tuck Williamson Date: Tue, 4 Nov 2025 13:51:34 -0500 Subject: [PATCH 08/10] Create template for Features. --- .../ISSUE_TEMPLATE/FeatureRequestTemplate.md | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/FeatureRequestTemplate.md diff --git a/.github/ISSUE_TEMPLATE/FeatureRequestTemplate.md b/.github/ISSUE_TEMPLATE/FeatureRequestTemplate.md new file mode 100644 index 0000000..75f8b39 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/FeatureRequestTemplate.md @@ -0,0 +1,49 @@ +--- +name: Feature Request +about: Suggest a new feature for this project +title: '[FEATURE] ' +labels: enhancement +assignees: '' + +--- + +## Feature Description + + + +## Value Statement + + +**As a** [type of user] +**I want** [goal/desire] +**So that** [benefit/value] + + + + +## Acceptance Criteria + + +- [ ] +- [ ] +- [ ] + + + + + From 471ed64125d5621c073d144a46c84f6e28fd48ed Mon Sep 17 00:00:00 2001 From: Tuck Williamson Date: Tue, 4 Nov 2025 13:59:09 -0500 Subject: [PATCH 09/10] Fixed testing on unsupported python versions. --- .github/workflows/github-actions.yml | 30 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index e5e4889..9403646 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -17,11 +17,11 @@ jobs: python: '3.13' tox_env: 'docs' os: 'ubuntu-latest' - - name: 'py311 (ubuntu)' - python: '3.11' - python_arch: 'x64' - tox_env: 'py311' - os: 'ubuntu-latest' +# - name: 'py311 (ubuntu)' +# python: '3.11' +# python_arch: 'x64' +# tox_env: 'py311' +# os: 'ubuntu-latest' - name: 'py312 (ubuntu)' python: '3.12' python_arch: 'x64' @@ -32,16 +32,16 @@ jobs: python_arch: 'x64' tox_env: 'py313' os: 'ubuntu-latest' - - name: 'pypy310 (ubuntu)' - python: 'pypy-3.10' - python_arch: 'x64' - tox_env: 'pypy310' - os: 'ubuntu-latest' - - name: 'pypy311 (ubuntu)' - python: 'pypy-3.11' - python_arch: 'x64' - tox_env: 'pypy311' - os: 'ubuntu-latest' +# - name: 'pypy310 (ubuntu)' +# python: 'pypy-3.10' +# python_arch: 'x64' +# tox_env: 'pypy310' +# os: 'ubuntu-latest' +# - name: 'pypy311 (ubuntu)' +# python: 'pypy-3.11' +# python_arch: 'x64' +# tox_env: 'pypy311' +# os: 'ubuntu-latest' steps: - uses: actions/checkout@v5 with: From 486727a5a938847c3662eb540a2185e9bc88ca21 Mon Sep 17 00:00:00 2001 From: Tuck Williamson Date: Tue, 4 Nov 2025 14:00:38 -0500 Subject: [PATCH 10/10] Fixed trailing whitespace. --- .github/ISSUE_TEMPLATE/FeatureRequestTemplate.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/FeatureRequestTemplate.md b/.github/ISSUE_TEMPLATE/FeatureRequestTemplate.md index 75f8b39..fc9ea7f 100644 --- a/.github/ISSUE_TEMPLATE/FeatureRequestTemplate.md +++ b/.github/ISSUE_TEMPLATE/FeatureRequestTemplate.md @@ -14,8 +14,8 @@ assignees: '' ## Value Statement -**As a** [type of user] -**I want** [goal/desire] +**As a** [type of user] +**I want** [goal/desire] **So that** [benefit/value] -