Improved test error tracking

Errors are now gathered with their line number and compared based on the message and position. Added a Pair record to

Author: cheestree

liquidjava-verifier/src/test/java/liquidjava/api/tests/TestExamples.java

@@ -1,6 +1,7 @@
 package liquidjava.api.tests;
 
 import static liquidjava.utils.TestUtils.*;
+
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
@@ -10,14 +11,19 @@
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Stream;
+
 import liquidjava.api.CommandLineLauncher;
 import liquidjava.diagnostics.Diagnostics;
 
 import liquidjava.diagnostics.errors.LJError;
+import liquidjava.utils.TestUtils.Pair;
+
 import org.junit.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
 
+import liquidjava.diagnostics.ErrorPosition;
+
 public class TestExamples {
 
     Diagnostics diagnostics = Diagnostics.getInstance();
@@ -52,25 +58,26 @@ else if (shouldFail(pathName)) {
                 fail();
             } else {
                 // check if expected error was found
-                List<String> expected = isDirectory ? getExpectedErrorsFromDirectory(path)
+                List<Pair> expected = isDirectory ? getExpectedErrorsFromDirectory(path)
                         : getExpectedErrorsFromFile(path);
                 if (diagnostics.getErrors().size() > expected.size()) {
                     System.out.println("Multiple errors found in: " + pathName + " --- expected exactly "
                             + expected.size() + " errors. \n" + diagnostics.getErrorOutput());
                     fail();
                 }
-                LJError error = diagnostics.getErrors().iterator().next();
-                if (error.getPosition() == null) {
-                    System.out.println("Error in: " + pathName + " --- error has no position information. \n"
-                            + diagnostics.getErrorOutput());
-                    fail();
-                }
                 if (!expected.isEmpty()) {
-                    String foundError = error.getTitle();
-                    if (!expected.contains(foundError)) {
-                        System.out.println("Error in: " + pathName + " --- expected errors: " + expected
-                                + ", but found: " + foundError + ". \n" + diagnostics.getErrorOutput());
-                        fail();
+                    for (LJError e : diagnostics.getErrors()) {
+                        String foundError = e.getTitle();
+                        int errorPosition = e.getPosition().lineStart();
+                        boolean match = expected.stream().anyMatch(
+                                pair -> pair.errorMessage().equals(foundError) && pair.lineNumber() == errorPosition);
+
+                        if (!match) {
+                            System.out.println("Error in: " + pathName + " --- expected errors: " + expected
+                                    + ", but found: " + foundError + " at " + errorPosition + ". \n"
+                                    + diagnostics.getErrorOutput());
+                            fail();
+                        }
                     }
                 } else {
                     System.out.println("No expected error messages found for: " + pathName);

liquidjava-verifier/src/test/java/liquidjava/utils/TestUtils.java

@@ -8,6 +8,15 @@
 import java.util.List;
 
 public class TestUtils {
+    /*
+     * A simple record to store an error message and its line number, used for expected errors read from test files
+     */
+    public record Pair(String errorMessage, int lineNumber) {
+        @Override
+        public String toString() {
+            return errorMessage + " at " + lineNumber;
+        };
+    }
 
     /**
      * Determines if the given path indicates that the test should pass
@@ -36,11 +45,13 @@ public static boolean shouldFail(String path) {
      * @return list of expected error messages found in the file, or empty list if there was an error reading the file
      *         or if there are no expected error messages in the file
      */
-    public static List<String> getExpectedErrorsFromFile(Path filePath) {
-        List<String> expectedErrors = new ArrayList<>();
+    public static List<Pair> getExpectedErrorsFromFile(Path filePath) {
+        List<Pair> expectedErrors = new ArrayList<>();
         try (BufferedReader reader = Files.newBufferedReader(filePath)) {
             String line;
+            int lineNumber = 0;
             while ((line = reader.readLine()) != null) {
+                lineNumber++;
                 int idx = line.indexOf("//");
                 if (idx != -1 && line.substring(idx).contains("Error")) {
                     // only expects the error type, NOT the actual refinement error message, depends on deterministic
@@ -50,7 +61,7 @@ public static List<String> getExpectedErrorsFromFile(Path filePath) {
                     if (dotIdx != -1) {
                         comment = comment.substring(0, dotIdx).trim();
                     }
-                    expectedErrors.add(comment);
+                    expectedErrors.add(new Pair(comment, lineNumber));
                 }
             }
         } catch (IOException e) {
@@ -67,8 +78,8 @@ public static List<String> getExpectedErrorsFromFile(Path filePath) {
      * @return list of expected error messages from all files in the directory, or empty list if there was an error
      *         reading the directory or if there are no files in the directory
      */
-    public static List<String> getExpectedErrorsFromDirectory(Path dirPath) {
-        List<String> expectedErrors = new ArrayList<>();
+    public static List<Pair> getExpectedErrorsFromDirectory(Path dirPath) {
+        List<Pair> expectedErrors = new ArrayList<>();
         try {
             List<Path> files = Files.list(dirPath).filter(Files::isRegularFile).toList();
             for (Path file : files) {