fix(core): handle Markdown backreference lookbehind

org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExp.java

@@ -94,7 +94,7 @@ private Regex parsePattern(final String pattern, final boolean ignoreCase) throw
 	}
 
 	/**
-	 * Rewrites the given pattern to work around limitations of the Joni library, which does not support variable-length lookbehinds.
+	 * Rewrites the given pattern to work around lookbehind limitations of the Joni library.
 	 *
 	 * Strategy:
 	 * <ul>
@@ -107,6 +107,7 @@ private Regex parsePattern(final String pattern, final boolean ignoreCase) throw
 	 * </ul>
 	 *
 	 * @see <a href="https://github.com/eclipse-tm4e/tm4e/issues/677">github.com/eclipse-tm4e/tm4e/issues/677</a>
+	 * @see <a href="https://github.com/eclipse-tm4e/tm4e/issues/1027">github.com/eclipse-tm4e/tm4e/issues/1027</a>
 	 */
 	private String rewritePatternIfRequired(final String pattern) {
 		if (pattern.isEmpty())
@@ -144,7 +145,12 @@ private String rewritePatternIfRequired(final String pattern) {
 		if (pattern.startsWith(negLB))
 			return "(?<!\\.)\\s*" + pattern.substring(negLB.length());
 
-		return pattern;
+		// --- Backreferences in lookbehinds ----------------------------------------
+		// Used in markdown.tmLanguage.json: (\1)(?!(?<=_\1)\w) ==> (?!(?<=_)\1\w)(\1)
+		// Joni rejects backreferences inside lookbehinds. Test Markdown's closing delimiter before consuming it so the
+		// backreference stays outside the lookbehind and the original capture groups and boundary semantics are preserved.
+		// Keep this rewrite narrow because arbitrary backreference lookbehinds cannot be moved without changing their meaning.
+		return pattern.replace("(\\1)(?!(?<=_\\1)\\w)", "(?!(?<=_)\\1\\w)(\\1)");
 	}
 
 	/**

org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/internal/oniguruma/OnigRegExpTest.java

@@ -98,4 +98,22 @@ void testNegativeLookBehinds() {
 		assertOnigRegExpSearch("(?<=\\s*\\.)\\w+", ".foo", 0, true, ".foo");
 		assertOnigRegExpSearch("(?<=\\s*\\.)\\w+", "  .foo", 0, true, "  .foo");
 	}
+
+	@Test
+	void testMarkdownBackReferenceInLookBehind() {
+		// Keep the exact Markdown strikethrough pattern so this test exercises the same Joni compatibility path as the grammar.
+		final var pattern =
+				"(?<!\\\\)(~{2,})(?!(?<=\\w~~)_)((?:[^~]|(?!(?<![~\\\\])\\1(?!~))~)*+)(\\1)(?!(?<=_\\1)\\w)";
+
+		// Capture groups are part of the grammar contract and must stay unchanged when the assertion is rewritten.
+		assertOnigRegExpSearch(pattern, "~~text~~", 0, true, "~~text~~", "~~", "text", "~~");
+
+		// Underscores next to either delimiter must not create intraword strikethrough.
+		assertOnigRegExpSearch(pattern, "abc~~_~~x", 0, false);
+		assertOnigRegExpSearch(pattern, "~~foo_~~bar", 0, false);
+		assertOnigRegExpSearch(pattern, "~~foo_~~!", 0, true, "~~foo_~~", "~~", "foo_", "~~");
+
+		// Escaping the opening delimiter must continue to suppress the match after the rewrite.
+		assertOnigRegExpSearch(pattern, "\\~~text~~", 0, false);
+	}
 }

org.eclipse.tm4e.core/src/test/java/org/eclipse/tm4e/core/internal/parser/TMParserTest.java

@@ -26,6 +26,7 @@
 import java.util.Collection;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Pattern;
 
 import org.eclipse.jdt.annotation.Nullable;
 import org.eclipse.tm4e.core.Data;
@@ -42,6 +43,8 @@
 
 @TestMethodOrder(MethodOrderer.MethodName.class)
 class TMParserTest {
+	// Mirrors RegExpSource's package-private runtime check for begin-capture references.
+	private static final Pattern HAS_BACK_REFERENCES = Pattern.compile("\\\\(\\d+)");
 
 	private void validateCaptures(final RawGrammar grammar) {
 		assertThat(grammar.getPatterns()).isNotNull();
@@ -208,26 +211,30 @@ void testParseYAML() throws Exception {
 	private void assertParseablePattern(final @Nullable String pattern) {
 		if (pattern == null)
 			return;
-		try {
-			assertThat(new OnigRegExp(pattern)).isNotNull();
-		} catch (final RuntimeException ex) {
-			final var msg = ex.getMessage();
-			if (msg != null && msg.contains("invalid backref number/name")) {
-				// ignore
-			} else
-				throw ex;
-		}
+		assertThat(new OnigRegExp(pattern)).isNotNull();
+	}
+
+	private void assertParseableEndOrWhilePattern(final @Nullable String pattern) {
+		if (pattern == null)
+			return;
+		// Runtime replacements are regex-escaped begin captures. A fixed literal preserves
+		// the surrounding syntax and gives look-behinds a concrete width for this check.
+		assertParseablePattern(HAS_BACK_REFERENCES.matcher(pattern).replaceAll("x"));
+	}
+
+	private void assertParseableRule(final IRawRule rule) {
+		assertParseablePattern(rule.getBegin());
+		assertParseableEndOrWhilePattern(rule.getEnd());
+		assertParseablePattern(rule.getMatch());
+		assertParseableEndOrWhilePattern(rule.getWhile());
 	}
 
 	private void assertParseablePatterns(final @Nullable Collection<IRawRule> patterns) {
 		if (patterns == null || patterns.isEmpty())
 			return;
 
 		for (final var rule : patterns) {
-			assertParseablePattern(rule.getBegin());
-			assertParseablePattern(rule.getEnd());
-			assertParseablePattern(rule.getMatch());
-			assertParseablePattern(rule.getWhile());
+			assertParseableRule(rule);
 			assertParseablePatterns(rule.getPatterns());
 		}
 	}
@@ -254,6 +261,10 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr
 						final var patterns = castNonNull(rawGrammar.getPatterns());
 						assertThat(patterns).isNotEmpty();
 						assertParseablePatterns(patterns);
+						// TextMate include rules reference named repository entries by string, so walking
+						// getPatterns() never reaches their regex fields. Keep this check to the named rule;
+						// recursively expanding repository subtrees exposes separate existing incompatibilities.
+						rawGrammar.getRepository().putEntries((name, rule) -> assertParseableRule(rule));
 
 						final var reg = new Registry();
 						final var grammar = reg.addGrammar(IGrammarSource.fromFile(file));