Modified test utils to allow inline errors

Instead of using separate files or stating at the top of a class file the error that it should report, now it is read and all correctly defined errors are added to a list. It is then checked against the first error that occurred and reports back. Still only works with a single error reported, however.

Author: cheestree

liquidjava-verifier/src/test/java/liquidjava/api/tests/TestExamples.java

@@ -7,6 +7,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.List;
 import java.util.Optional;
 import java.util.stream.Stream;
 import liquidjava.api.CommandLineLauncher;
@@ -43,41 +44,38 @@ public void testPath(final Path path) {
                     + diagnostics.getErrorOutput());
             fail();
         }
-        // verification should fail, check if it failed as expected (we assume each error test has exactly one error)
+        // verification should fail, check if it failed as expected (multiple errors can be found)
         else if (shouldFail(pathName)) {
             if (!diagnostics.foundError()) {
                 System.out.println("Error in: " + pathName + " --- should fail but no errors were found. \n"
                         + diagnostics.getErrorOutput());
                 fail();
             } else {
                 // check if expected error was found
-                Optional<String> expected = isDirectory ? getExpectedErrorFromDirectory(path)
-                        : getExpectedErrorFromFile(path);
-                if (diagnostics.getErrors().size() > 1) {
-                    System.out.println("Multiple errors found in: " + pathName + " --- expected exactly one error. \n"
-                            + diagnostics.getErrorOutput());
-                    System.out.println(diagnostics.getErrorOutput());
+                List<String> expected = isDirectory ? getExpectedErrorsFromDirectory(path)
+                        : getExpectedErrorsFromFile(path);
+                if (diagnostics.getErrors().size() > expected.size()) {
+                    System.out.println("Multiple errors found in: " + pathName + " --- expected exactly "
+                            + expected.size() + " errors. \n" + diagnostics.getErrorOutput());
                     fail();
                 }
                 LJError error = diagnostics.getErrors().iterator().next();
                 if (error.getPosition() == null) {
                     System.out.println("Error in: " + pathName + " --- error has no position information. \n"
                             + diagnostics.getErrorOutput());
-                    System.out.println(diagnostics.getErrorOutput());
                     fail();
                 }
-                if (expected.isPresent()) {
-                    String expectedError = expected.get();
+                if (!expected.isEmpty()) {
                     String foundError = error.getTitle();
-                    if (!foundError.equalsIgnoreCase(expectedError)) {
-                        System.out.println("Error in: " + pathName + " --- expected error: " + expectedError
+                    if (!expected.contains(foundError)) {
+                        System.out.println("Error in: " + pathName + " --- expected errors: " + expected
                                 + ", but found: " + foundError + ". \n" + diagnostics.getErrorOutput());
                         fail();
                     }
                 } else {
-                    System.out.println("No expected error message found for: " + pathName);
-                    System.out.println("Please provide an expected error in " + (isDirectory
-                            ? "a .expected file in the directory" : "the first line of the test file as a comment"));
+                    System.out.println("No expected error messages found for: " + pathName);
+                    System.out.println(
+                            "Please provide the expected errors in the test file as comments starting with // at the end of the supposed error line.");
                     fail();
                 }
             }

liquidjava-verifier/src/test/java/liquidjava/utils/TestUtils.java

@@ -1,10 +1,11 @@
 package liquidjava.utils;
 
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Optional;
-import java.util.stream.Stream;
+import java.util.ArrayList;
+import java.util.List;
 
 public class TestUtils {
 
@@ -27,41 +28,55 @@ public static boolean shouldFail(String path) {
     }
 
     /**
-     * Reads the expected error message from the first line of the given test file
+     * Reads the expected error messages from the given file by looking for a comment containing the expected error
+     * message.
      * 
      * @param filePath
      * 
-     * @return optional containing the expected error message if present, otherwise empty
+     * @return list of expected error messages found in the file, or empty list if there was an error reading the file
+     *         or if there are no expected error messages in the file
      */
-    public static Optional<String> getExpectedErrorFromFile(Path filePath) {
-        try (Stream<String> lines = Files.lines(filePath)) {
-            Optional<String> first = lines.findFirst();
-            if (first.isPresent() && first.get().startsWith("//")) {
-                return Optional.of(first.get().substring(2).trim());
-            } else {
-                return Optional.empty();
+    public static List<String> getExpectedErrorsFromFile(Path filePath) {
+        List<String> expectedErrors = new ArrayList<>();
+        try (BufferedReader reader = Files.newBufferedReader(filePath)) {
+            String line;
+            while ((line = reader.readLine()) != null) {
+                int idx = line.indexOf("//");
+                if (idx != -1 && line.substring(idx).contains("Error")) {
+                    // only expects the error type, NOT the actual refinement error message, depends on deterministic
+                    // variable names
+                    String comment = line.substring(idx + 2).trim();
+                    int dotIdx = comment.indexOf(":");
+                    if (dotIdx != -1) {
+                        comment = comment.substring(0, dotIdx).trim();
+                    }
+                    expectedErrors.add(comment);
+                }
             }
-        } catch (Exception e) {
-            return Optional.empty();
+        } catch (IOException e) {
+            return List.of();
         }
+        return expectedErrors;
     }
 
     /**
-     * Reads the expected error message from a .expected file in the given directory
+     * Reads the expected error messages from all files in the given directory and combines them into a single list
      * 
      * @param dirPath
      * 
-     * @return optional containing the expected error message if present, otherwise empty
+     * @return list of expected error messages from all files in the directory, or empty list if there was an error
+     *         reading the directory or if there are no files in the directory
      */
-    public static Optional<String> getExpectedErrorFromDirectory(Path dirPath) {
-        Path expectedFilePath = dirPath.resolve(".expected");
-        if (Files.exists(expectedFilePath)) {
-            try (Stream<String> lines = Files.lines(expectedFilePath)) {
-                return lines.findFirst().map(String::trim);
-            } catch (IOException e) {
-                return Optional.empty();
+    public static List<String> getExpectedErrorsFromDirectory(Path dirPath) {
+        List<String> expectedErrors = new ArrayList<>();
+        try {
+            List<Path> files = Files.list(dirPath).filter(Files::isRegularFile).toList();
+            for (Path file : files) {
+                getExpectedErrorsFromFile(file).forEach(expectedErrors::add);
             }
+        } catch (IOException e) {
+            return List.of();
         }
-        return Optional.empty();
+        return expectedErrors;
     }
 }