diff --git a/.gitignore b/.gitignore index 358eca0..e97d461 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ out/ # Compiled class file *.class +# Generated libraries +lib/ + # Log file *.log @@ -45,3 +48,6 @@ out/ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* + +# backup files +*~ diff --git a/CHANGELOG.md b/CHANGELOG.md index a01c6c0..4128072 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,3 +89,9 @@ _4.15.2019_ **Repository** - Switched CI to GitLab + +#### 2.1.1 +_12.12.2022_ +- Fixed Gradescope example +- Updated library versions +- Improved error handling and reporting diff --git a/README.md b/README.md index cb3a88d..5738f7a 100644 --- a/README.md +++ b/README.md @@ -1,300 +1,12 @@ - # JGrade -_A library for grading Java assignments_ - - -[![pipeline status](https://gitlab.com/tkutcher/jgrade/badges/dev/pipeline.svg)](https://gitlab.com/tkutcher/jgrade/pinelines/dev/latest) - - docs - - - version - - - -
- - - -[API Documentation](https://tkutcher.gitlab.io/jgrade/api) - - ---- - - -NOTE - I've moved the CI to GitLab and am using GitLab to host the API docs (https://tkutcher.gitlab.io/jgrade), but this -will remain the primary repository. GitLab will just mirror the master and dev branches. - - -:bangbang: Help Wanted :bangbang: - -Once upon a time, it was my priority to be grading intermediate-level Java 8 assignments - but after graduating that -priority has gone down a bit :grimacing: . I would be happy to help familiarize anyone who is interested in contributing -and keeping this maintained. Submit an issue in the project if you are interested or have any ideas! - - -- [Overview](#overview) -- [Quick Start](#quick-start) -- [Features and Usage](#features-and-usage) -- [Development](#development) - - [Ideas / Wishlist](#wishlist) - ---- - -## Overview -JGrade is a helper tool with various classes designed to assist in course instructors "autograding" an assignment, -inspired by the [Gradescope Autograder](https://gradescope-autograders.readthedocs.io/en/latest/). There are classes -that the client can integrate with directly, or use the jar's main method (and provide a class with annotations) that -wraps a lot of common functionality (see [examples](https://github.com/tkutcher/jgrade/tree/development/examples)). -It was designed to produce the output needed for Gradescope while being extensible enough to produce different -outputs and configure the specific JSON output Gradescope is looking for. - - -## Quick Start - -To make use of this, you first need to grab the jar file from the [Releases](https://github.com/tkutcher/jgrade/releases) page. -This includes many classes you can make use of, as well as a main method for running and producing grading output. - -With this, you could have the following setup: - -A class that runs some unit tests we want to treat their success as a grade (these would import student code): - -```java -import com.github.tkutcher.jgrade.gradedtest.GradedTest; -import org.junit.Test; - -import static com.github.tkutcher.jgrade.gradedtest.GradedTestResult.HIDDEN; -import static com.github.tkutcher.jgrade.gradedtest.GradedTestResult.VISIBLE; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class ExampleGradedTests { - @Test - @GradedTest(name="True is true", points=2.0, visibility=VISIBLE) - public void trueIsTrue() { - assertTrue(true); - } - - @Test - @GradedTest(name="False is false", number="2", points=3.0, visibility=HIDDEN) - public void falseIsFalse() { - assertFalse(false); - } - - @Test - @GradedTest(name="Captures output") - public void capturesOutput() { - System.out.println("hello"); - } - - @Test - @GradedTest(name="This test should fail") - public void badTest() { - fail(); - } -} -``` - -and a main method with some other grading-related non-unit-testing logic `MyGrader.java`: - -```java -import com.github.tkutcher.jgrade.BeforeGrading; -import com.github.tkutcher.jgrade.AfterGrading; -import com.github.tkutcher.jgrade.Grade; -import com.github.tkutcher.jgrade.Grader; -import com.github.tkutcher.jgrade.gradedtest.GradedTestResult; - -import static com.github.tkutcher.jgrade.gradedtest.GradedTestResult.HIDDEN; - - -public class BasicGraderExample { - - /* All @Grade/@BeforeGrading/@AfterGrading methods must take exactly one parameter - * of type Grader. This parameter is the same grader throughout. - * - * @BeforeGrading methods are run before others. - */ - @BeforeGrading - public void initGrader(Grader grader) { - grader.startTimer(); - } - - /* You can run unit tests that are annotated with @GradedTest to add - * GradedTestResults to the Grader in this way. - */ - @Grade - public void runGradedUnitTests(Grader grader) { - grader.runJUnitGradedTests(ExampleGradedTests.class); - } - /* You can also manually add GradedTestResults you create to the grader. */ - @Grade - public void singleTestResult(Grader grader) { - grader.addGradedTestResult( - new GradedTestResult("manual test", "1", 1.0, HIDDEN) - ); - } - - /* Grader.startTimer() and Grader.stopTimer() can be used to time the grader */ - @Grade - public void loopForTime(Grader grader) { - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTime < 1000); - } - - /* @AfterGrading methods are run after all other methods. */ - @AfterGrading - public void endGrader(Grader grader) { - grader.stopTimer(); - } -} -``` - -Then, you could run - -```shell script -java -jar ../lib/jgrade-1.1-all.jar -c MyGrader -o results.json -``` - -and get GradeScope-formatted json. See the [examples](/examples) for more complete examples and how to set up a script -to work with GradeScope, and expand the usage below to see the arguments you can provide this main program. - -
Usage -

- -``` --c,--classname arg the class containing annotated methods to grade --f,--format output-format specify output, one of 'json' (default) or 'txt' --h,--help
- --no-output don't produce any output (if user overriding) --o destination save output to another file (if not specified, - prints to standard out) - --pretty-print pretty-print output (when format is json) --v,--version - -``` - -

-
- - -## Features and Usage - -The way I used this library is to have a base class for the course (for example, a `_226Grader`) that contains -annotated methods for functionality/grading parts that are consistent across all assignments. For example, the -`@BeforeGrading` method starts a timer and the `@AfterGrading` method stops it. There is a `@Grade` method that -does the "grading" of style with checkstyle. Subclasses, for example `Assignment1Grader` (or `Assignment0Grader` - I suppose :wink:), extend this and add `@Grade` methods to add assignment-specific grading. -See the gradescope folder in the examples for a rough example setup. - -### Features - -See the [API Docs](https://tkutcher.gitlab.io/jgrade/api) for more complete documentation. - -#### `CheckstyleGrader` - -With the `CheckstyleGrader` you can specify grading deductions for checkstyle errors. This method below, for example, -would check the students files and deduct a point for each checkstyle error type (missing javadoc, require this, etc.). - -```java - @Grade - public void runCheckstyle(Grader grader) { - CheckstyleGrader checker = new CheckstyleGrader(5.0, 1.0, MY_CHECKSTYLE_JAR, STUDENTFILES); - checker.setConfig(MY_CHECKSTYLE_CONFIG); - GradedTestResult result = checker.runForGradedTestResult(); - result.setScore(Math.max(0, 5 - checker.getErrorTypeCount())); - grader.addGradedTestResult(result); - } -``` - -#### `DeductiveGraderStrategy` - -You can use this strategy to make failed tests deduct points from a total. So say in the current assignment there are two -parts, A and B, each worth 25 points. If someone fails 30 tests for part B each worth one point, you don't want that to cut -in to the assignment A portion: - -```java -public class GradeAssignment7 extends Grade226Assignment { - - private static final int AVL_POINTS = 30; - private static final int TREAP_POINTS = 20; - - @Grade - public void gradeAvlTree(Grader grader) { - grader.setGraderStrategy(new DeductiveGraderStrategy(AVL_POINTS, "AvlTreeMap")); - grader.runJUnitGradedTests(GradeAvlTreeMap.class); - } - - @Grade - public void gradeBinaryHeapPQ(Grader grader) { - grader.setGraderStrategy(new DeductiveGraderStrategy(TREAP_POINTS, "TreapMap")); - grader.runJUnitGradedTests(GradeTreapMap.class); - } -} -``` - - -#### `DeductiveGraderStrategy` - -You can use this strategy to make failed tests deduct points from a total. So say in the current assignment there are two -parts, A and B, each worth 25 points. If someone fails 30 tests for part B each worth one point, you don't want that to cut -in to the assignment A portion: - -```java -public class GradeAssignment7 extends Grade226Assignment { - - private static final int AVL_POINTS = 30; - private static final int TREAP_POINTS = 20; - - @Grade - public void gradeAvlTree(Grader grader) { - grader.setGraderStrategy(new DeductiveGraderStrategy(AVL_POINTS, "AvlTreeMap")); - grader.runJUnitGradedTests(GradeAvlTreeMap.class); - } - - @Grade - public void gradeBinaryHeapPQ(Grader grader) { - grader.setGraderStrategy(new DeductiveGraderStrategy(TREAP_POINTS, "TreapMap")); - grader.runJUnitGradedTests(GradeTreapMap.class); - } -} -``` - -#### `CLITester` - -A class to help wrap testing command line programs. You subclass `CLITester`, then implement -the `getInvocation()` method for how the command line program is invoked, then you can use -`runCommand(String)` to get the output in an object that you can test for expected output. - - ---- - -## Development - -- `mvn install` to compile -- `mvn test` to run unit tests -- `mvn checkstyle:checkstyle` to run checkstyle -- `mvn javadoc:jar` to generate API docs. - -Check out [contributing](/CONTRIBUTING.md) for more. - - -### Requirements -JGrade is written in [Java 8](https://www.oracle.com/technetwork/java/javase/overview/java8-2100321.html). -Since the library has classes designed to run alongside JUnit, [JUnit 4](https://junit.org/junit4/) is a dependency -for the entire project (as opposed to just for running the projects own unit tests). -The [org.json](https://mvnrepository.com/artifact/org.json/json) package is used in producing correctly formatted -JSON output, and the [Apache Commons CLI](https://commons.apache.org/proper/commons-cli/) library is used for -reading the command line in the main program. - -For simplicity, the main jar (appended with "-all") includes all of these dependencies. +_A library for grading Java assignments_ -### Wishlist -- Feedback for required files - - In our autograder, we built in something that took a list of required files and created a visible test case worth 0 points of what files were missing - this helped students debug. - - Could try and move some of this there. -- Actual Observer pattern - - Allow for people to specify custom handlers whenever things like new graded test results are added - - Old "observer" terminology not really an observer +This is a fork of [tkutcher/JGrade](https://github.com/tkutcher/jgrade), +created by Tim Kutcher. Because he has moved on to other projects, I +forked it so I could run it with the current (2022) version of Gradescope. +I have made minor changes to the JGrade library and significant changes +to the [Gradescope example](examples/gradescope/README.md). See +[my changes](https://github.com/tkutcher/jgrade/compare/tkutcher:jgrade:dev...espertus:jgrade:dev). +**This has been superseded by [Jacquard](https://github.com/espertus/jacquard).** diff --git a/examples/gradescope/.gitignore b/examples/gradescope/.gitignore new file mode 100644 index 0000000..7c581e0 --- /dev/null +++ b/examples/gradescope/.gitignore @@ -0,0 +1 @@ +zips/*.zip diff --git a/examples/gradescope/README.md b/examples/gradescope/README.md index 1ab9568..2fbdac3 100644 --- a/examples/gradescope/README.md +++ b/examples/gradescope/README.md @@ -1,17 +1,42 @@ # JGrade Gradescope Example -This is a full example that works on gradescope, and models much of the setup from the original [java example](https://github.com/gradescope/autograder_samples/tree/master/java) Gradescope links to. +This demonstrates a Gradescope autograder that uses checkstyle and JUnit. -It compiles all files in to a created `classes/` directory (not tracked). The `lib/` folder contains all jars and library files - for this example just `jgrade-1.0.0-all.jar` (which includes JUnit, etc.), and `checkstyle-8.12.jar`. The `res/` directory is for resources (like the checkstyle configuration file). `src/` is the main source code, and `test_submissions/` are submissions to test with on Gradescope. +[![Watch the video](https://img.youtube.com/vi/o1FHbHZwyUY/maxresdefault.jpg)](https://youtu.be/o1FHbHZwyUY) -The source has 2 main packages, `staff` and `student`. The staff package contains the unit tests, a solution (to debug with) and the code to do the grading. +These are the files and directories: +* `make_autograder.sh`: zips up files for uploading to Gradescope +* `makefile`: provides an alternative to directly sourcing the shell scripts +* `setup.sh` [required by Gradescope]: sets up the environment by installing a recent JDK +* `run_autograder` [required by Gradescope]: attempts to copy student code into `/autograder/source` directory, compile the grading code with the student code, and run the grader +* `compile.sh`: compiles the combined code, copying `compilation_error.json` into the `results` directory if compilation fails +* `compilation_error.json`: the JSON for indicating the [student's] code did not compile +* `run.sh`: runs JGrade, passing in the assignment-specific grading class (`GradeHello`) +* `README.md`: this file +* `classes`: the destination for compiled files +* `lib/`: the location of needed libraries, which are not checked into git + * `checkstyle-10.5.0-all.jar` [which you need to download if you want] + * `jgrade-2.1.1-all.jar` [which you need to build yourself for now] + * `README.md`: documentation +* `res/`: the location of resources + * `sun_checks.xml`: a configuration file needed by checkstyle +* `src/main/java/` + * `staff/hello/`: code provided by the instructor + * `GradeHello.java`: the controller (package is `staff.hello`) + * `Greeting.java`: code imported by the student (package is `student.hello`) + * `Hello.java`: model solution to the assignment (package is `student.hello`) + * `HelloTest`: test cases using JUnit (package is `staff.hello`) + * `student/hello/`: the student's code + * `Hello.java`: skeletal code that students need to complete +* `test_submissions/`: zip files of student submissions, to be manually provided to Gradescope for testing the autograder + * `correct.zip`: a fully functional project with checkstyle errors + * `errors.zip`: a project that fails some tests + * `nocompile.zip`: a project that has compile errors +* `zips/`: where `build_autograder.sh` places the zipped autograder it builds -To build the autograder, run either `$ sh make_autograder.sh` or `$ make autograder` which will place it in the `zips/` folder. - -While debugging, a makefile is provided for compiling and running. `make output` will start fresh and run the autograder, pretty-printing the output to the console. +To test (and debug) the autograder before uploading it, execute: +``` +./run_autograder --local +``` -- `setup.sh`: Installs correct JDK -- `run_autograder`: Main script for the autograder. Copies in submission, compiles, and runs. -- `compile.sh`: Compiles all of the source into a classes directory -- `run.sh`: Runs JGrade, passing in the `GradeHello` file, writing output - - If run with `--local` then prints output to console, else to the results/results.json file. \ No newline at end of file +To build the autograder, run either `$ sh make_autograder.sh` or `$ make autograder` which will place it in the `zips/` folder. diff --git a/examples/gradescope/compilation_error.json b/examples/gradescope/compilation_error.json new file mode 100644 index 0000000..e826b16 --- /dev/null +++ b/examples/gradescope/compilation_error.json @@ -0,0 +1,12 @@ +{ + "tests": [ + { + "output": "", + "score": 0, + "number": "", + "visibility": "visible", + "max_score": 15, + "name": "compiler error" + } + ] +} diff --git a/examples/gradescope/compile.sh b/examples/gradescope/compile.sh index c3b2432..16f9b79 100755 --- a/examples/gradescope/compile.sh +++ b/examples/gradescope/compile.sh @@ -8,6 +8,17 @@ mkdir -p classes # Compile all java files in src directory java_files=$(find src -name "*.java") echo "compiling java files..." -javac -cp lib/*:. -d classes ${java_files} -echo "---" -echo "DONE" \ No newline at end of file +javac -cp lib/jgrade-2.2.0-all.jar:. -d classes ${java_files} +if [ $? -eq 0 ]; then + echo "---" + echo "DONE" + exit 0 # success +else + echo "Compilation failed" + if [ "$1" = "--local" ]; then + cat compilation_error.json + else + cp compilation_error.json /autograder/results/results.json + fi + exit 1 # failure +fi diff --git a/examples/gradescope/lib/README.md b/examples/gradescope/lib/README.md index f0480a5..f89bcd9 100644 --- a/examples/gradescope/lib/README.md +++ b/examples/gradescope/lib/README.md @@ -1,4 +1,4 @@ -Here you would include all jars that the execution depends on. For this example: +The location for needed jar files. This example requires: -- checkstyle-8.12-all.jar -- jgrade-1.1-all.jar +* checkstyle-10.5.0-all.jar +* jgrade-2.1.1-all.jar diff --git a/examples/gradescope/make_autograder.sh b/examples/gradescope/make_autograder.sh index 76dc9f4..54efdef 100644 --- a/examples/gradescope/make_autograder.sh +++ b/examples/gradescope/make_autograder.sh @@ -1,9 +1,12 @@ #!/usr/bin/env bash +echo "Removing carriage returns from scripts..." +dos2unix *.sh + echo "Building Autograder..." -zip -r hello_autograder.zip lib/ res/ src/ compile.sh run.sh setup.sh run_autograder +zip -r hello_autograder.zip lib/ res/ src compile.sh run.sh setup.sh run_autograder compilation_error.json mv hello_autograder.zip zips/ echo "---" -echo "DONE. Locate it in the zips directory." \ No newline at end of file +echo "DONE. Locate it in the zips directory." diff --git a/examples/gradescope/res/sun_checks.xml b/examples/gradescope/res/sun_checks.xml index a73e6c8..eed537b 100644 --- a/examples/gradescope/res/sun_checks.xml +++ b/examples/gradescope/res/sun_checks.xml @@ -1,172 +1,198 @@ + "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN" + "https://checkstyle.org/dtds/configuration_1_3.dtd"> - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - \ No newline at end of file + diff --git a/examples/gradescope/run.sh b/examples/gradescope/run.sh index b11b6fc..2030b80 100755 --- a/examples/gradescope/run.sh +++ b/examples/gradescope/run.sh @@ -6,9 +6,9 @@ cd classes # invoke the main program of JGrade passing the class GradeHello as the main parameter # sending output to the results.json file (or stdout if running locally). if [ "$1" = "--local" ]; then - java -jar ../lib/jgrade-1.1-all.jar -c staff.hello.GradeHello --pretty-print + java -jar ../lib/jgrade-2.2.0-all.jar -c staff.hello.GradeHello --pretty-print else - java -jar ../lib/jgrade-1.1-all.jar -c staff.hello.GradeHello -o /autograder/results/results.json + java -jar ../lib/jgrade-2.2.0-all.jar -c staff.hello.GradeHello -o /autograder/results/results.json fi # return to original cwd diff --git a/examples/gradescope/run_autograder b/examples/gradescope/run_autograder index 0601f9f..0825965 100755 --- a/examples/gradescope/run_autograder +++ b/examples/gradescope/run_autograder @@ -1,11 +1,17 @@ #!/usr/bin/env bash -cd /autograder/source - -cp /autograder/submission/Hello.java /autograder/source/src/main/java/student/hello/ +if [ "$1" != "--local" ]; then + cd /autograder/source + cp /autograder/submission/Hello.java /autograder/source/src/main/java/student/hello/ +fi # compilation -sh compile.sh +sh compile.sh $1 +if [ $? -eq 0 ]; then + # execution + sh run.sh $1 +fi +exit $? + + -# execution -sh run.sh \ No newline at end of file diff --git a/examples/gradescope/setup.sh b/examples/gradescope/setup.sh index 74b0907..b599db4 100644 --- a/examples/gradescope/setup.sh +++ b/examples/gradescope/setup.sh @@ -1,4 +1,3 @@ #!/usr/bin/env bash -# Java 8 -apt-get -y install openjdk-8-jdk +apt-get -y install openjdk-18-jdk diff --git a/examples/gradescope/src/main/java/staff/hello/GradeHello.java b/examples/gradescope/src/main/java/staff/hello/GradeHello.java index f5e412c..20ba293 100644 --- a/examples/gradescope/src/main/java/staff/hello/GradeHello.java +++ b/examples/gradescope/src/main/java/staff/hello/GradeHello.java @@ -4,6 +4,7 @@ import com.github.tkutcher.jgrade.Grade; import com.github.tkutcher.jgrade.Grader; import com.github.tkutcher.jgrade.gradedtest.GradedTestResult; +import com.github.tkutcher.jgrade.gradedtest.Visibility; public class GradeHello { @@ -12,7 +13,7 @@ public void debugMode(Grader grader) { if (HelloTest.DEBUG) { GradedTestResult r = new GradedTestResult( "Debug Mode Warning", "", - 0.0, GradedTestResult.HIDDEN + 0.0, Visibility.HIDDEN ); r.addOutput("WARNING: Autograder in DEBUG mode, not " + "checking student submission. If seeing this on " + @@ -31,7 +32,7 @@ public void runUnitTests(Grader grader) { public void runCheckstyle(Grader grader) { // FIXME - Better than knowing running from the classes directory... CheckstyleGrader checker = new CheckstyleGrader(10.0, 1.0, - "../lib/checkstyle-8.12-all.jar", "../src/main/java/student/hello/"); + "../lib/checkstyle-10.5.0-all.jar", "../src/main/java/student/hello/"); checker.setConfig("../res/sun_checks.xml"); GradedTestResult result = checker.runForGradedTestResult(); grader.addGradedTestResult(result); diff --git a/examples/gradescope/src/main/java/student/hello/Greeting.java b/examples/gradescope/src/main/java/staff/hello/Greeting.java similarity index 100% rename from examples/gradescope/src/main/java/student/hello/Greeting.java rename to examples/gradescope/src/main/java/staff/hello/Greeting.java diff --git a/examples/gradescope/src/main/java/staff/hello/HelloTest.java b/examples/gradescope/src/main/java/staff/hello/HelloTest.java index 8dfaa86..cb69aee 100644 --- a/examples/gradescope/src/main/java/staff/hello/HelloTest.java +++ b/examples/gradescope/src/main/java/staff/hello/HelloTest.java @@ -1,5 +1,8 @@ package staff.hello; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + import com.github.tkutcher.jgrade.gradedtest.GradedTest; import org.junit.Before; import org.junit.Rule; @@ -23,27 +26,32 @@ public class HelloTest { private static final String GREETING = "Hello"; private Greeting unit; - // Makes it so can verify tests work for instructor solution. + // Running with the DEBUG flag enabled tests the staff solution. @Before public void initUnit() { this.unit = DEBUG ? new Hello(GREETING) : new student.hello.Hello(GREETING); } @Test - @GradedTest(name="greet() works") + @GradedTest(name = "greet() works") public void defaultGreeting() { assertEquals(GREETING, unit.greet()); } @Test - @GradedTest(name="greet(String who) works", points=2.0) + @GradedTest(name = "greet(String who) works", points = 2.0) public void greetSomebody() { assertEquals(GREETING + ", World!", unit.greet("World")); } @Test - @GradedTest(name="prints greeting", points=0.0) + @GradedTest(name = "prints greeting", points = 2.0) public void printGreeting() { + PrintStream realStdout = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); unit.printGreeting(); + assertEquals(GREETING, baos.toString().trim()); + System.setOut(realStdout); } } diff --git a/examples/gradescope/test_submissions/correct-hello-only.zip b/examples/gradescope/test_submissions/correct-hello-only.zip new file mode 100644 index 0000000..25c2f6d Binary files /dev/null and b/examples/gradescope/test_submissions/correct-hello-only.zip differ diff --git a/examples/gradescope/test_submissions/hello/correct.zip b/examples/gradescope/test_submissions/correct.zip similarity index 100% rename from examples/gradescope/test_submissions/hello/correct.zip rename to examples/gradescope/test_submissions/correct.zip diff --git a/examples/gradescope/test_submissions/hello/errors.zip b/examples/gradescope/test_submissions/errors.zip similarity index 100% rename from examples/gradescope/test_submissions/hello/errors.zip rename to examples/gradescope/test_submissions/errors.zip diff --git a/examples/gradescope/test_submissions/hello/nocompile.zip b/examples/gradescope/test_submissions/nocompile.zip similarity index 100% rename from examples/gradescope/test_submissions/hello/nocompile.zip rename to examples/gradescope/test_submissions/nocompile.zip diff --git a/pom.xml b/pom.xml index 5c9e3f0..d17a5a5 100644 --- a/pom.xml +++ b/pom.xml @@ -29,12 +29,12 @@ junit junit - 4.13.1 + 4.13.2 org.hamcrest hamcrest-library - 1.3 + 2.2 org.json @@ -45,12 +45,18 @@ commons-cli commons-cli - 1.4 + 1.5.0 + + + + org.apache.commons + commons-exec + 1.3 - jgrade-1.1 + jgrade-2.2.0 org.apache.maven.plugins @@ -85,7 +91,7 @@ single - jgrade-1.1-all + jgrade-2.2.0-all jar-with-dependencies @@ -116,4 +122,4 @@ - \ No newline at end of file + diff --git a/src/main/java/com/github/tkutcher/jgrade/CheckstyleGrader.java b/src/main/java/com/github/tkutcher/jgrade/CheckstyleGrader.java index 3c1760e..187cd0e 100644 --- a/src/main/java/com/github/tkutcher/jgrade/CheckstyleGrader.java +++ b/src/main/java/com/github/tkutcher/jgrade/CheckstyleGrader.java @@ -1,6 +1,10 @@ package com.github.tkutcher.jgrade; import com.github.tkutcher.jgrade.gradedtest.GradedTestResult; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.exec.PumpStreamHandler; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NamedNodeMap; @@ -12,26 +16,25 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.StringReader; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import java.util.Map; import java.util.TreeMap; -import static com.github.tkutcher.jgrade.gradedtest.GradedTestResult.VISIBLE; +import static com.github.tkutcher.jgrade.gradedtest.Visibility.VISIBLE; /** * Class to assist in getting a {@link GradedTestResult} for checkstyle. It * sets a max score, and a points per deduction, then deducts up to the max - * score number of points. Assumes is checking an entire directory but + * score number of points. Assumes it is checking an entire directory but * excludes any files with "test" in the name by default (since we don't * really require JUnit files be checkstyle compliant). To configure it for * specific files the client has to override {@link #isFileToCheck(Path)}. + * * @version 1.0.0 */ public class CheckstyleGrader { @@ -56,9 +59,10 @@ public class CheckstyleGrader { /** * Instantiate a new CheckstyleGrader. - * @param points The total number of points for the checkstyle test. - * @param deduct The number of points to deduct per error. - * @param pathToJar The path to the checkstyle jar executable. + * + * @param points The total number of points for the checkstyle test. + * @param deduct The number of points to deduct per error. + * @param pathToJar The path to the checkstyle jar executable. * @param dirToCheck The directory of files to check. */ public CheckstyleGrader(double points, double deduct, @@ -73,6 +77,7 @@ public CheckstyleGrader(double points, double deduct, /** * Set a configuration file to use for the checkstyle run. + * * @param config - The file to use as the -c argument to checkstyle. */ public void setConfig(String config) { @@ -85,26 +90,40 @@ public void setConfig(String config) { * specified from {@link #setConfig(String)} then it will add the config * to the command. Will include all files that {@link #isFileToCheck(Path)} * returns true for, which by default is any java file not containing - * "test" in it's name. Will deduct to 0 points for each error. + * "test" in its name. Will deduct to 0 points for each error. + * * @return The generated result. */ public GradedTestResult runForGradedTestResult() { - List command = new ArrayList<>(Arrays.asList("java", "-jar", - this.pathToJar, "-f", CHECKSTYLE_FORMAT)); - if (this.config != null) { - command.add("-c"); - command.add(this.config); + if (this.config == null) { + throw new RuntimeException("config was null"); } - try { + CommandLine cmd = new CommandLine("java"); + String[] args = {"-jar", this.pathToJar, "-f", CHECKSTYLE_FORMAT, "-c", this.config}; + cmd.addArguments(args); Files.walk(Paths.get(dirToCheck)) .filter(CheckstyleGrader::isFileToCheck) - .forEach(path -> command.add(path.toString())); - String xmlOutput = CLITester.executeProcess( - new ProcessBuilder(command)) - .getOutput(CLIResult.STREAM.STDOUT); + .forEach(path -> cmd.addArgument(path.toString())); + // Capture both output streams in case something goes wrong. + // https://stackoverflow.com/a/34571800/631051 + DefaultExecutor executor = new DefaultExecutor(); + ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + ByteArrayOutputStream stderr = new ByteArrayOutputStream(); + PumpStreamHandler handler = new PumpStreamHandler(stdout, stderr); + executor.setStreamHandler(handler); + try { + executor.execute(cmd); + } catch (ExecuteException e) { + // The exit code is the number of errors found. + // That's normal, not exceptional, so ignore. + } + String xmlOutput = stdout.toString().trim(); + if (xmlOutput.isEmpty()) { + throw new InternalError(stderr.toString()); + } return xmlToGradedTestResult(xmlOutput); - } catch (InternalError | IOException e) { + } catch (IOException | InternalError e) { e.printStackTrace(); e.printStackTrace(System.err); return internalErrorResult(e.toString()); @@ -113,6 +132,7 @@ public GradedTestResult runForGradedTestResult() { /** * Get the map of error types to their count. + * * @return The map of error types to their count. */ public Map getErrorTypes() { @@ -121,6 +141,7 @@ public Map getErrorTypes() { /** * Get the number of different error types encountered. + * * @return The number of different error types encountered. */ public int getErrorTypeCount() { @@ -129,12 +150,14 @@ public int getErrorTypeCount() { // FIXME - Alternative to make this take some interface that calls the // static boolean function. + /** * Boolean function for whether or not a file should be included in * checkstyle's run. By default it only includes files that are * java files and excludes any containing "test" in the directory. * If a client wanted to include more or exclude others they would * have to subclass and override this. + * * @param path The file to consider. * @return True if it should be checked. */ @@ -217,7 +240,7 @@ private String getOutputForErrorNode(NamedNodeMap attributes) { return String.format("\t%-20s - %s [%s]\n", getAttributeValue("line: ", lineAttribute) - + getAttributeValue(", column", columnAttribute), + + getAttributeValue(", column", columnAttribute), getAttributeValue(messageAttribute), errorTypeAttribute); } diff --git a/src/main/java/com/github/tkutcher/jgrade/DeductiveGraderStrategy.java b/src/main/java/com/github/tkutcher/jgrade/DeductiveGraderStrategy.java index 7656ee6..0164597 100644 --- a/src/main/java/com/github/tkutcher/jgrade/DeductiveGraderStrategy.java +++ b/src/main/java/com/github/tkutcher/jgrade/DeductiveGraderStrategy.java @@ -1,6 +1,7 @@ package com.github.tkutcher.jgrade; import com.github.tkutcher.jgrade.gradedtest.GradedTestResult; +import com.github.tkutcher.jgrade.gradedtest.Visibility; import java.util.List; @@ -55,7 +56,7 @@ public void grade(List l) { } // Since scores get set to 0 ... GradedTestResult baseScore = new GradedTestResult(sectionName, "", - startingScore, GradedTestResult.HIDDEN); + startingScore, Visibility.HIDDEN); baseScore.setScore(startingScore); l.add(baseScore); } diff --git a/src/main/java/com/github/tkutcher/jgrade/gradedtest/GradedTest.java b/src/main/java/com/github/tkutcher/jgrade/gradedtest/GradedTest.java index 3cf6a69..9f6bcff 100644 --- a/src/main/java/com/github/tkutcher/jgrade/gradedtest/GradedTest.java +++ b/src/main/java/com/github/tkutcher/jgrade/gradedtest/GradedTest.java @@ -9,7 +9,7 @@ * The GradedTest annotation is primarily based off of capturing the data for * a test object in the Gradescope JSON output. They are used to create * corresponding {@link GradedTestResult} objects. - * + *

* Uses a String for the name of the test (default "Unnamed test") , a String * for the question number (default ""), a double for the number of points the * test is worth(defaults to 1.0), and a String for the visibility of the test @@ -20,25 +20,29 @@ public @interface GradedTest { /** * The name of the test. + * * @return The name of the test. */ String name() default GradedTestResult.DEFAULT_NAME; /** * The number corresponding to the test. + * * @return The number corresponding to the test. */ String number() default GradedTestResult.DEFAULT_NUMBER; /** * The number of points the test is worth. + * * @return The number of points the test is worth. */ double points() default GradedTestResult.DEFAULT_POINTS; /** * The visibility level of the test. + * * @return The visibility level of the test. */ - String visibility() default GradedTestResult.DEFAULT_VISIBILITY; + Visibility visibility() default Visibility.VISIBLE; } diff --git a/src/main/java/com/github/tkutcher/jgrade/gradedtest/GradedTestResult.java b/src/main/java/com/github/tkutcher/jgrade/gradedtest/GradedTestResult.java index f513cb6..ee1bc50 100644 --- a/src/main/java/com/github/tkutcher/jgrade/gradedtest/GradedTestResult.java +++ b/src/main/java/com/github/tkutcher/jgrade/gradedtest/GradedTestResult.java @@ -4,37 +4,22 @@ * A class for the data that models a graded test. Primarily based on the * data needed for Gradescope's Autograder JSON. When creating * {@link GradedTestResult}s and working with visibility, use the public - * constants {@link GradedTestResult#VISIBLE}, {@link GradedTestResult#HIDDEN}, - * etc. + * constants in {@link Visibility}. */ public class GradedTestResult { - // - - /** Visible to the student always. */ - public static final String VISIBLE = "visible"; - - /** Never visible to the student. */ - public static final String HIDDEN = "hidden"; - - /** Visible to the student only after the due date. */ - public static final String AFTER_DUE_DATE = "after_due_date"; - - /** Visible to the student only after grades have been released. */ - public static final String AFTER_PUBLISHED = "after_published"; - - // - // GradedTest annotation defaults static final String DEFAULT_NAME = "Unnamed Test"; static final String DEFAULT_NUMBER = ""; static final double DEFAULT_POINTS = 1.0; - static final String DEFAULT_VISIBILITY = VISIBLE; + // This duplicates the default value in the GradedTestResult annotation, + // but there is no clean way to get its value for a static constant. + static final Visibility DEFAULT_VISIBILITY = Visibility.VISIBLE; private String name; private String number; private double points; - private String visibility; + private Visibility visibility; private double score; private StringBuilder output; @@ -48,18 +33,11 @@ public class GradedTestResult { * @param visibility The visibility setting of the test. * @throws IllegalArgumentException If the visibility is not valid. */ - public GradedTestResult(String name, String number, double points, String visibility) + public GradedTestResult(String name, String number, double points, Visibility visibility) throws IllegalArgumentException { this.name = name; this.number = number; this.points = points; - - if (!(visibility.equals(HIDDEN) || visibility.equals(VISIBLE) - || visibility.equals(AFTER_DUE_DATE) || visibility.equals(AFTER_PUBLISHED))) { - throw new IllegalArgumentException("visibility should be one of 'hidden', 'visible', " - + "'after_due_date', or 'after_published'"); - } - this.visibility = visibility; this.score = 0; this.output = new StringBuilder(); @@ -132,7 +110,7 @@ public double getPoints() { * Get the visibility setting of the test. * @return The visibility setting of the test. */ - public String getVisibility() { + public Visibility getVisibility() { return visibility; } diff --git a/src/main/java/com/github/tkutcher/jgrade/gradedtest/Visibility.java b/src/main/java/com/github/tkutcher/jgrade/gradedtest/Visibility.java new file mode 100644 index 0000000..cd75393 --- /dev/null +++ b/src/main/java/com/github/tkutcher/jgrade/gradedtest/Visibility.java @@ -0,0 +1,18 @@ +package com.github.tkutcher.jgrade.gradedtest; + +public enum Visibility { + VISIBLE("visible"), + HIDDEN("hidden"), + AFTER_DUE_DATE("after_due_date"), + AFTER_PUBLISHED("after_published"); + + private String text; + + Visibility(String text) { + this.text = text; + } + + public String getText() { + return text; + } +} diff --git a/src/main/java/com/github/tkutcher/jgrade/gradescope/GradescopeJsonFormatter.java b/src/main/java/com/github/tkutcher/jgrade/gradescope/GradescopeJsonFormatter.java index a2b0cd6..ff43d91 100644 --- a/src/main/java/com/github/tkutcher/jgrade/gradescope/GradescopeJsonFormatter.java +++ b/src/main/java/com/github/tkutcher/jgrade/gradescope/GradescopeJsonFormatter.java @@ -3,18 +3,13 @@ import com.github.tkutcher.jgrade.Grader; import com.github.tkutcher.jgrade.OutputFormatter; import com.github.tkutcher.jgrade.gradedtest.GradedTestResult; +import com.github.tkutcher.jgrade.gradedtest.Visibility; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import java.util.List; -import static com.github.tkutcher.jgrade.gradedtest.GradedTestResult.AFTER_DUE_DATE; -import static com.github.tkutcher.jgrade.gradedtest.GradedTestResult.AFTER_PUBLISHED; -import static com.github.tkutcher.jgrade.gradedtest.GradedTestResult.HIDDEN; -import static com.github.tkutcher.jgrade.gradedtest.GradedTestResult.VISIBLE; - - /** * A concrete formatter for a {@link Grader} where the output it produces * is the JSON a Gradescope Autograder can work with. @@ -33,8 +28,8 @@ public class GradescopeJsonFormatter implements OutputFormatter { private JSONObject json; private int prettyPrint; - private String visibility; - private String stdoutVisibility; + private Visibility visibility; + private Visibility stdoutVisibility; /** * Creates an instance of the formatter. By default the pretty-print @@ -57,25 +52,21 @@ private boolean hasStdoutVisibility() { /** * Sets the visibility for all of the test cases. + * * @param visibility The top-level visibility to use for all test cases. * @throws GradescopeJsonException If visibility not valid. */ - public void setVisibility(String visibility) throws GradescopeJsonException { - if (!isValidVisibility(visibility)) { - throw new GradescopeJsonException(visibility + " is not a valid visibility"); - } + public void setVisibility(Visibility visibility) throws GradescopeJsonException { this.visibility = visibility; } /** * Sets the visibility for standard out during the run. + * * @param visibility The visibility to set for standard out. * @throws GradescopeJsonException If visibility is not valid. */ - public void setStdoutVisibility(String visibility) throws GradescopeJsonException { - if (!isValidVisibility(visibility)) { - throw new GradescopeJsonException(visibility + " is not a valid visibility"); - } + public void setStdoutVisibility(Visibility visibility) throws GradescopeJsonException { this.stdoutVisibility = visibility; } @@ -84,6 +75,7 @@ public void setStdoutVisibility(String visibility) throws GradescopeJsonExceptio * spaces to add for each indent level. A negative integer corresponds to * disabling pretty-print. If non-negative, simply calls * {@link JSONObject#toString(int)} + * * @param prettyPrint The integer for how much to indent */ public void setPrettyPrint(int prettyPrint) { @@ -112,7 +104,7 @@ private JSONObject assemble(GradedTestResult r) { .put(MAX_SCORE, r.getPoints()) .put(NUMBER, r.getNumber()) .put(OUTPUT, r.getOutput()) - .put(VISIBILITY, r.getVisibility()); + .put(VISIBILITY, r.getVisibility().getText()); } catch (JSONException e) { throw new InternalError(e); } @@ -142,10 +134,10 @@ private void assemble(Grader grader, JSONObject json) throws GradescopeJsonExcep json.put(OUTPUT, grader.getOutput()); } if (this.hasVisibility()) { - json.put(VISIBILITY, this.visibility); + json.put(VISIBILITY, this.visibility.getText()); } if (this.hasStdoutVisibility()) { - json.put(STDOUT_VISIBILITY, this.stdoutVisibility); + json.put(STDOUT_VISIBILITY, this.stdoutVisibility.getText()); } if (grader.hasGradedTestResults()) { json.put(TESTS, this.assemble(grader.getGradedTestResults())); @@ -159,27 +151,5 @@ private void validateGrader(Grader grader) { if (!(grader.hasScore() || grader.hasGradedTestResults())) { throw new GradescopeJsonException("Gradescope Json must have either tests or score set"); } - - /* The following checks ~should~ all pass because they would have been checked when set. */ - assert isValidVisibility(this.visibility); - assert isValidVisibility(this.stdoutVisibility); - assert allValidVisibility(grader.getGradedTestResults()); - } - - private static boolean allValidVisibility(List results) { - for (GradedTestResult r : results) { - if (!isValidVisibility(r.getVisibility())) { - return false; - } - } - return true; - } - - private static boolean isValidVisibility(String visibility) { - return visibility == null // Just wasn't set, which is OK - || visibility.equals(VISIBLE) - || visibility.equals(HIDDEN) - || visibility.equals(AFTER_DUE_DATE) - || visibility.equals(AFTER_PUBLISHED); } } diff --git a/src/test/java/com/github/tkutcher/jgrade/DeductiveGraderStrategyTest.java b/src/test/java/com/github/tkutcher/jgrade/DeductiveGraderStrategyTest.java index a642942..fc02cdf 100644 --- a/src/test/java/com/github/tkutcher/jgrade/DeductiveGraderStrategyTest.java +++ b/src/test/java/com/github/tkutcher/jgrade/DeductiveGraderStrategyTest.java @@ -1,13 +1,13 @@ package com.github.tkutcher.jgrade; import com.github.tkutcher.jgrade.gradedtest.GradedTestResult; +import com.github.tkutcher.jgrade.gradedtest.Visibility; import org.junit.Before; import org.junit.Test; import java.util.ArrayList; import java.util.List; -import static com.github.tkutcher.jgrade.gradedtest.GradedTestResult.HIDDEN; import static org.junit.Assert.assertEquals; public class DeductiveGraderStrategyTest { @@ -23,13 +23,13 @@ public void initUnit() { } private static GradedTestResult failedGradedTestResult(double points) { - GradedTestResult r = new GradedTestResult("", "", points, HIDDEN); + GradedTestResult r = new GradedTestResult("", "", points, Visibility.HIDDEN); r.setPassed(false); return r; } private static GradedTestResult successfulGradedTestResult(double points) { - GradedTestResult r = new GradedTestResult("", "", points, HIDDEN); + GradedTestResult r = new GradedTestResult("", "", points, Visibility.HIDDEN); r.setPassed(true); r.setScore(points); return r; diff --git a/src/test/java/com/github/tkutcher/jgrade/GraderTest.java b/src/test/java/com/github/tkutcher/jgrade/GraderTest.java index 911ab45..4014161 100644 --- a/src/test/java/com/github/tkutcher/jgrade/GraderTest.java +++ b/src/test/java/com/github/tkutcher/jgrade/GraderTest.java @@ -1,6 +1,7 @@ package com.github.tkutcher.jgrade; import com.github.tkutcher.jgrade.gradedtest.GradedTestResult; +import com.github.tkutcher.jgrade.gradedtest.Visibility; import org.junit.Before; import org.junit.Test; @@ -54,7 +55,7 @@ public void canAddOutput() { @Test public void canAddGradedTestResults() { - GradedTestResult t = new GradedTestResult("", "", 0.0, "visible"); + GradedTestResult t = new GradedTestResult("", "", 0.0, Visibility.VISIBLE); unit.addGradedTestResult(t); assertTrue(unit.hasGradedTestResults()); List results = unit.getGradedTestResults(); diff --git a/src/test/java/com/github/tkutcher/jgrade/JGradeCommandLineTest.java b/src/test/java/com/github/tkutcher/jgrade/JGradeCommandLineTest.java index a42b6ea..f026949 100644 --- a/src/test/java/com/github/tkutcher/jgrade/JGradeCommandLineTest.java +++ b/src/test/java/com/github/tkutcher/jgrade/JGradeCommandLineTest.java @@ -1,6 +1,7 @@ package com.github.tkutcher.jgrade; import com.github.tkutcher.jgrade.gradedtest.GradedTestResult; +import com.github.tkutcher.jgrade.gradedtest.Visibility; import org.json.JSONException; import org.json.JSONObject; import org.junit.After; @@ -87,7 +88,7 @@ public void graderMethod(Grader g) { "Test GradedTestResult", "1", 25.0, - GradedTestResult.VISIBLE + Visibility.VISIBLE )); } } diff --git a/src/test/java/com/github/tkutcher/jgrade/gradedtest/GradedTestListenerTest.java b/src/test/java/com/github/tkutcher/jgrade/gradedtest/GradedTestListenerTest.java index 5af5a96..5992373 100644 --- a/src/test/java/com/github/tkutcher/jgrade/gradedtest/GradedTestListenerTest.java +++ b/src/test/java/com/github/tkutcher/jgrade/gradedtest/GradedTestListenerTest.java @@ -70,7 +70,7 @@ public void addsCustomGradedTestResult() { assertEquals(EXAMPLE_NAME, result.getName()); assertEquals(EXAMPLE_NUMBER, result.getNumber()); assertEquals(EXAMPLE_POINTS, result.getPoints(), 0.0); - Assert.assertEquals(GradedTestResult.HIDDEN, result.getVisibility()); + Assert.assertEquals(Visibility.HIDDEN, result.getVisibility()); } @Test @@ -148,7 +148,7 @@ public static class SingleCustomGradedTest { name=EXAMPLE_NAME, number=EXAMPLE_NUMBER, points=EXAMPLE_POINTS, - visibility= GradedTestResult.HIDDEN) + visibility=Visibility.HIDDEN) public void gradedTest() { assertTrue(true); } } diff --git a/src/test/java/com/github/tkutcher/jgrade/gradedtest/GradedTestResultTest.java b/src/test/java/com/github/tkutcher/jgrade/gradedtest/GradedTestResultTest.java index 7298347..f97ffe2 100644 --- a/src/test/java/com/github/tkutcher/jgrade/gradedtest/GradedTestResultTest.java +++ b/src/test/java/com/github/tkutcher/jgrade/gradedtest/GradedTestResultTest.java @@ -63,9 +63,4 @@ public void canAddScore() { public void cannotAddScoreGreaterThanPoints() { unit.setScore(15.0); } - - @Test(expected=IllegalArgumentException.class) - public void visibilityMustBeValid() { - new GradedTestResult(GradedTestResult.DEFAULT_NAME, GradedTestResult.DEFAULT_NUMBER, GradedTestResult.DEFAULT_POINTS, "INVALID"); - } } diff --git a/src/test/java/com/github/tkutcher/jgrade/gradescope/GradescopeJsonFormatterTest.java b/src/test/java/com/github/tkutcher/jgrade/gradescope/GradescopeJsonFormatterTest.java index c1696b7..acb092c 100644 --- a/src/test/java/com/github/tkutcher/jgrade/gradescope/GradescopeJsonFormatterTest.java +++ b/src/test/java/com/github/tkutcher/jgrade/gradescope/GradescopeJsonFormatterTest.java @@ -2,6 +2,7 @@ import com.github.tkutcher.jgrade.Grader; import com.github.tkutcher.jgrade.gradedtest.GradedTestResult; +import com.github.tkutcher.jgrade.gradedtest.Visibility; import org.json.JSONException; import org.json.JSONObject; import org.junit.Before; @@ -43,17 +44,7 @@ public void validIfScoreSet() throws JSONException { @Test public void validIfTests() throws JSONException { - grader.addGradedTestResult(new GradedTestResult("", "", 20.0, "visible")); + grader.addGradedTestResult(new GradedTestResult("", "", 20.0, Visibility.VISIBLE)); assertValidJson(unit.format(grader)); } - - @Test(expected=GradescopeJsonException.class) - public void catchesInvalidVisibility() { - unit.setVisibility("invisible"); - } - - @Test(expected=GradescopeJsonException.class) - public void catchesInvalidStdoutVisibility() { - unit.setStdoutVisibility("invisible"); - } }