From 75de03bcd45f9b58d1ff9d0ed9cea4961e121521 Mon Sep 17 00:00:00 2001 From: Thibaud Colas Date: Tue, 7 Jul 2026 07:12:17 +0100 Subject: [PATCH] Implement first-draft Markdown importer to ContentState --- docs/ROADMAP.md | 4 + docs/contributing/architecture.md | 16 +- docs/markdown.md | 110 ++- draftjs_exporter/__init__.py | 9 + draftjs_exporter/markdown/importer.py | 717 ++++++++++++++++ pyproject.toml | 4 +- tests/markdown/test_importer.py | 1112 +++++++++++++++++++++++++ tests/strategies.py | 101 +++ tests/test_properties.py | 66 +- 9 files changed, 2131 insertions(+), 8 deletions(-) create mode 100644 draftjs_exporter/markdown/importer.py create mode 100644 tests/markdown/test_importer.py diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index b0f6d2d..ecd1132 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -30,6 +30,10 @@ Provide a cookbook showing how to prompt LLMs to draft block maps/entity decorat > Possible changes that require R&D, and high-risk ideas that could bring large benefits but with likely trade-offs. +### Markdown importer + +`markdown_to_content_state(markdown, options)` parses Markdown back into Draft.js ContentState, enabling round-trip workflows (ContentState → Markdown → ContentState). The parser is dependency-free and mirrors the exporter's configurable markers. It supports the Markdown subset the exporter produces, plus common CommonMark edge cases (escapes, HR variants, link titles, code fence info strings, multi-line blockquotes, `)` ordered-list delimiters, `~~~` fences). Inline HTML tags (``, ``, etc.) emitted by the exporter for non-Markdown styles are parsed back into `inlineStyleRanges`. + ### Rust-backed DOM engine Prototype a Rust extension or PyO3 module for DOM construction to outperform the current string/lxml/html5lib engines on large documents. diff --git a/docs/contributing/architecture.md b/docs/contributing/architecture.md index bdfb0be..6203dbf 100644 --- a/docs/contributing/architecture.md +++ b/docs/contributing/architecture.md @@ -25,7 +25,8 @@ draftjs_exporter/ html5lib.py # BeautifulSoup / html5lib engine. lxml.py # lxml engine. markdown.py # Non-escaping string engine for Markdown output. - markdown/ # Markdown-specific components, config builder, and helpers. + markdown/ # Markdown-specific components, config builder, importer, and helpers. + importer.py # Markdown importer – parses Markdown into ContentState. utils/ module_loading.py # import_string() for dotted-path class resolution. ``` @@ -46,6 +47,19 @@ The core flow lives in `HTML.render()` and proceeds as follows: 5. **Wrapper resolution** – `wrapper_state.element_for()` resolves each block to a DOM element, managing nesting of wrapper elements (e.g. `
    `/`
      ` for list items) based on block depth. 6. **Final rendering** – All block elements are appended to the document fragment, then `DOM.render()` serialises the virtual DOM tree to the output string (HTML or Markdown). +## Importer pipeline + +The reverse direction — Markdown → Draft.js ContentState — lives in `markdown/importer.py` and is exposed as `markdown_to_content_state(markdown, options=None)`. It is a dependency-free parser that does not reuse the export pipeline's `Command`/`EntityState`/`StyleState` abstractions (those consume ranges; the importer must _produce_ ranges from a different source representation). + +The flow: + +1. **Block splitting** – `_split_blocks()` splits Markdown into raw block strings, handling code fences, paragraph joining, and continuation lines. +2. **Block parsing** – `_parse_block()` dispatches each raw block to the right handler (heading, blockquote, list, code fence, HR, image, or unstyled fallback) via regex matching. +3. **Inline parsing** – `_InlineParser` walks block text character-by-character, tracking opening/closing of style markers, inline HTML tags, and link syntax. It produces plain text, `inlineStyleRanges`, `entityRanges`, and entity definitions. +4. **Style merging** – `_merge_style_ranges()` merges adjacent or overlapping ranges of the same style. This compensates for the exporter's inline style nesting behavior: when styles partially overlap (e.g. bold 0-5, italic 3-8), the exporter must close and reopen the outer marker to produce valid Markdown (`**Bold ****_Italic_**`), producing two BOLD ranges in the imported output that should be a single continuous range. This is an intentional round-trip trade-off, coupling the importer to the exporter's marker-emission quirks. + +The importer accepts `MarkdownImporterOptions` with the three inline style markers (`bold`, `italic`, `strikethrough`) and a custom `html_style_tags` mapping. Block-level syntax (list markers, horizontal rule variants, code fence variants) is recognized polymorphically rather than configured – the importer accepts any valid variant the exporter might produce. See [Markdown support](../markdown.md#importer) for the user-facing options reference. + ## Engine system The exporter uses a **Strategy pattern** for output generation. All engines implement the `DOMEngine` interface (five static methods: `create_tag`, `parse_html`, `append_child`, `render`, `render_debug`). The `DOM` class delegates to the active engine, selected at runtime via dotted-path strings stored as class constants (`DOM.STRING`, `DOM.HTML5LIB`, `DOM.LXML`, `DOM.MARKDOWN`, `DOM.STRING_COMPAT`). Engine selection is thread-safe through a `ContextVar`. diff --git a/docs/markdown.md b/docs/markdown.md index dfba2df..b50470f 100644 --- a/docs/markdown.md +++ b/docs/markdown.md @@ -150,12 +150,112 @@ config = { ## Unsupported -The Markdown exporter has inherent limitations compared to the HTML exporter. +The Markdown exporter has inherent limitations compared to the HTML exporter. Where the importer can repair or absorb these limitations for round-trip workflows, that's noted inline; see [Importer](#importer) for the full picture. -- **No underline, subscript, or other HTML-only styles**: The exporter’s default configuration falls through to inline HTML like `text`. +- **No underline, subscript, or other HTML-only styles**: The exporter's default configuration falls through to inline HTML like `text`. The importer parses these back into inline style ranges. - **No reference-style links**: Links are always rendered inline as `[text](url)`. - **No table support**: Draft.js has no built-in table block type, and the exporter does not attempt to generate Markdown tables from custom block types. - **No Setext-style headings**: Headings always use ATX style (`# Heading`). -- **Entity data fidelity**: The Markdown link syntax `[text](url)` only preserves the URL. -- **No HTML escaping in text**: If your Draft.js content contains literal HTML characters like like `<` or `>`, the Markdown output will include them unescaped, which may be interpreted as HTML by Markdown renderers. -- **Inline style nesting edge cases**: When bold and italic styles partially overlap, the exporter may produce markers that some strict Markdown parsers reject (e.g. `**Bold **_Italic_**`). Most renderers handle this correctly, but it is not guaranteed by the CommonMark spec. +- **Entity data fidelity**: The Markdown link syntax `[text](url)` preserves the URL and optional title, but other entity data (e.g. `rel`, `data-*` attributes) is lost. +- **No HTML escaping in text**: If your Draft.js content contains literal HTML characters like `<` or `>`, the Markdown output will include them unescaped, which may be interpreted as HTML by Markdown renderers. +- **Inline style nesting edge cases**: When bold and italic styles partially overlap, the exporter may produce markers that some strict Markdown parsers reject (e.g. `**Bold **_Italic_**`). Most renderers handle this correctly, but it is not guaranteed by the CommonMark spec. The importer repairs these by merging adjacent ranges of the same style. + +## Importer + +> ⚠️ The Markdown importer is **experimental**. There is no guarantee of API stability at this time. + +The importer parses Markdown back into Draft.js ContentState, enabling round-trip workflows. It is dependency-free and understands the same subset of Markdown that the exporter produces, including the limitations documented above. + +### Quick start + +```python +from draftjs_exporter import markdown_to_content_state + +content_state = markdown_to_content_state(markdown_text) +``` + +### Configuring input markers + +Pass `MarkdownImporterOptions` matching the options used with `build_markdown_config` to round-trip customized output: + +```python +from draftjs_exporter import build_markdown_config, HTML, markdown_to_content_state + +options = {"bold": "__", "italic": "*"} +config = build_markdown_config(options) +exporter = HTML(config) + +markdown = exporter.render(content_state) +# Pass the same options to the importer. +content_state = markdown_to_content_state(markdown, options) +``` + +### Importer options + +| Option | Choices | Default | +| ----------------- | ------------------------------ | -------------------- | +| `bold` | `**`, `__` | `**` | +| `italic` | `*`, `_` | `_` | +| `strikethrough` | `~`, `~~` | `~` | +| `html_style_tags` | `dict[str, str]` (tag → style) | Inverse of STYLE_MAP | + +Only inline style markers are configurable. Block-level syntax (list markers, horizontal rule variants, code fence variants, ordered list delimiters) is **recognized polymorphically** – the importer accepts any of `-`/`*`/`+` for unordered lists, `.`/`)` for ordered lists, `---`/`***`/`___` for horizontal rules, and ` ``` `/`~~~` for code fences. Any block-level keys you pass through from `MarkdownOptions` (`unordered_list_marker`, `ordered_list_delimiter`, `horizontal_rule`, `code_fence`) are simply ignored. + +The `html_style_tags` option extends the default HTML tag → style mapping (e.g. `` → `UNDERLINE`). Custom tags are merged with the defaults, so built-in tags still work alongside user-defined ones. This lets you round-trip non-default inline styles configured on the exporter side: + +```python +from draftjs_exporter import HTML, build_markdown_config, markdown_to_content_state +from draftjs_exporter.defaults import STYLE_MAP +from draftjs_exporter.dom import DOM + +# Register a custom "KBD" inline style on the exporter side. +# (defaults.py ships "KEYBOARD" -> ; here we re-map it to a custom name.) +STYLE_MAP["KBD"] = lambda props: DOM.create_element("kbd", {}, props["children"]) + +options = {"html_style_tags": {"kbd": "KBD"}} +config = build_markdown_config(options) +exporter = HTML(config) + +markdown = exporter.render(content_state) +# The importer parses back as "KBD" rather than the default "KEYBOARD". +content_state = markdown_to_content_state(markdown, options) +``` + +### Fallback behavior + +If the importer encounters a block-level construct it doesn't recognize (anything that doesn't match the heading, list, blockquote, code fence, horizontal rule, or image patterns), the block falls through to `unstyled` and is logged at debug level. This is the expected path for hand-written Markdown (plain paragraphs don't match any of the block-level patterns), so the importer doesn't emit warnings on normal input. Inline syntax that isn't recognized (unknown HTML tags, mismatched markers, unterminated links) is emitted as literal text rather than dropping content. + +### Round-trip guarantees + +`ContentState → Markdown → ContentState` preserves the following when the default exporter config is used: + +- Block types (`unstyled`, headings 1-6, blockquote, list items with depth, code-block, atomic). +- Block text, including soft line breaks within block-level elements. +- `inlineStyleRanges` (offsets, lengths, and style names) for the four Markdown markers (`BOLD`, `ITALIC`, `CODE`, `STRIKETHROUGH`) and the ten inline HTML tag styles (`UNDERLINE`, `SUPERSCRIPT`, `SUBSCRIPT`, `MARK`, `QUOTATION`, `SMALL`, `SAMPLE`, `INSERT`, `DELETE`, `KEYBOARD`). +- `entityRanges` (offsets, lengths, and key mappings) for `LINK`, `IMAGE`, and `HORIZONTAL_RULE` entities. +- Entity `data.url` and, for links, an optional `data.title`. + +The following are **not** preserved, because Markdown's link syntax cannot represent them: + +- Entity data beyond `url` and `title` (e.g. `rel`, `target`, `data-*` attributes on links). +- Structural entities added by composite decorators during export (e.g. linkified URLs). The importer parses them back as `LINK` entities, but they didn't exist in the original ContentState. +- Atomic block internal structure (e.g. embed payloads). +- Block keys (the importer generates random 5-character keys; Draft.js reassigns keys client-side when loading content into an editor anyway). + +### Supported Markdown features + +- ATX headings (`#` through `######`) +- Blockquotes (single-line and multi-line) +- Unordered lists (`-`, `*`, `+` with 2-space indent per depth) +- Ordered lists (`.` and `)` delimiters) +- Fenced code blocks (` ``` ` and `~~~`, with optional info strings) +- Horizontal rules (`---`, `***`, `___`) +- Images (`![alt](src)`) +- Links (`[text](url)`, with optional `"title"`) +- Inline styles: bold, italic, code, strikethrough +- Inline HTML tags: ``, ``, ``, ``, ``, ``, ``, ``, ``, `` +- Backslash escapes for all CommonMark punctuation +- Nested links and styles within link text +- Soft line breaks within block-level elements + +The importer does not validate URL schemes. Custom protocols such as `wagtail://core.Page.89` round-trip unchanged in the entity `data.url`. Whether to accept or reject dangerous schemes (e.g. `javascript:`) is the integrating application's responsibility, matching the exporter's behavior – see [SECURITY.md](SECURITY.md). diff --git a/draftjs_exporter/__init__.py b/draftjs_exporter/__init__.py index 34e0485..77f2061 100644 --- a/draftjs_exporter/__init__.py +++ b/draftjs_exporter/__init__.py @@ -23,6 +23,12 @@ from draftjs_exporter.markdown import CONFIG as MARKDOWN_CONFIG from draftjs_exporter.markdown import MarkdownOptions as MarkdownOptions from draftjs_exporter.markdown import build_markdown_config as build_markdown_config +from draftjs_exporter.markdown.importer import ( + MarkdownImporterOptions as MarkdownImporterOptions, +) +from draftjs_exporter.markdown.importer import ( + markdown_to_content_state as markdown_to_content_state, +) from draftjs_exporter.types import Block as Block from draftjs_exporter.types import Component as Component from draftjs_exporter.types import CompositeDecorators as CompositeDecorators @@ -60,6 +66,9 @@ "MARKDOWN_CONFIG", "MarkdownOptions", "build_markdown_config", + # Importer + "MarkdownImporterOptions", + "markdown_to_content_state", # Constants "BLOCK_TYPES", "ENTITY_TYPES", diff --git a/draftjs_exporter/markdown/importer.py b/draftjs_exporter/markdown/importer.py new file mode 100644 index 0000000..587b3fc --- /dev/null +++ b/draftjs_exporter/markdown/importer.py @@ -0,0 +1,717 @@ +"""Markdown importer: parse Markdown into Draft.js ContentState.""" + +import logging +import random +import re +import string +from typing import Any, Literal, TypedDict + +from draftjs_exporter.constants import BLOCK_TYPES, ENTITY_TYPES, INLINE_STYLES +from draftjs_exporter.types import ( + Block, + ContentState, + Entity, + EntityRange, + InlineStyleRange, + Mutability, +) + +logger = logging.getLogger(__name__) + +HEADING_RE = re.compile(r"^(#{1,6}) (.+)$") +BLOCKQUOTE_RE = re.compile(r"^> (.*)$") +UL_RE = re.compile(r"^(\s*)[*\-+] (.*)$") +OL_RE = re.compile(r"^(\s*)(\d+)[.)] (.*)$") +HR_RE = re.compile(r"^(---|\*\*\*|___)$") +IMAGE_RE = re.compile(r"^!\[([^\]]*)\]\(([^)]+)\)$") + +HEADING_TYPES = { + 1: BLOCK_TYPES.HEADER_ONE, + 2: BLOCK_TYPES.HEADER_TWO, + 3: BLOCK_TYPES.HEADER_THREE, + 4: BLOCK_TYPES.HEADER_FOUR, + 5: BLOCK_TYPES.HEADER_FIVE, + 6: BLOCK_TYPES.HEADER_SIX, +} + +# Inline HTML tags the Markdown exporter emits for styles that have no +# Markdown marker equivalent. This is the inverse of defaults.STYLE_MAP, +# excluding BOLD/CODE/ITALIC/STRIKETHROUGH which are overridden by +# Markdown marker syntax. +DEFAULT_HTML_STYLE_TAGS: dict[str, str] = { + "u": INLINE_STYLES.UNDERLINE, + "sup": INLINE_STYLES.SUPERSCRIPT, + "sub": INLINE_STYLES.SUBSCRIPT, + "mark": INLINE_STYLES.MARK, + "q": INLINE_STYLES.QUOTATION, + "small": INLINE_STYLES.SMALL, + "samp": INLINE_STYLES.SAMPLE, + "ins": INLINE_STYLES.INSERT, + "del": INLINE_STYLES.DELETE, + "kbd": INLINE_STYLES.KEYBOARD, +} + +_HTML_OPEN_RE = re.compile(r"<(\w+)>") +_HTML_CLOSE_RE = re.compile(r"") + +_KEY_CHARS = string.ascii_lowercase + string.digits + +# Characters that can be escaped with a backslash (per CommonMark spec). +_ESCAPE_CHARS = frozenset('\\`*_{}[]()#+-.!|~>"') + + +class MarkdownImporterOptions(TypedDict, total=False): + """Options for customizing Markdown importer parsing behavior. + + Only inline style markers are mirrored from the exporter's + ``MarkdownOptions``: the importer's block-level regexes accept all + variants polymorphically (any of ``-``/``*``/``+`` for unordered + lists, ``.``/``)`` for ordered lists, ``---``/``***``/``___`` for + rules, and `` ``` ``/``~~~`` for fences), so there is no need to + pass through ``unordered_list_marker`` / ``ordered_list_delimiter`` + / ``horizontal_rule`` / ``code_fence``. The same options dict can + still be passed to ``markdown_to_content_state`` if it was produced + by user code for ``build_markdown_config`` — the extra keys are + simply ignored. + + Attributes: + bold: The bold marker used by the exporter. Defaults to ``"**"``. + italic: The italic marker used by the exporter. Defaults to ``"_"``. + strikethrough: The strikethrough marker. Defaults to ``"~"``. + html_style_tags: Custom mapping of HTML tag names to Draft.js + inline style names. Merged with the defaults, so unknown + tags fall back to plain text. + """ + + bold: Literal["**", "__"] + italic: Literal["*", "_"] + strikethrough: Literal["~", "~~"] + html_style_tags: dict[str, str] + + +def _build_style_markers( + options: MarkdownImporterOptions | None, +) -> list[tuple[str, str]]: + """Build the style marker list from options, ordered longest-first. + + Parameters: + options: Importer options, or ``None`` for defaults. + + Returns: + A list of ``(marker, style)`` tuples sorted by marker length + descending, so longer markers are tried first during parsing. + """ + opts = options or {} + markers = [ + (opts.get("bold", "**"), INLINE_STYLES.BOLD), + (opts.get("italic", "_"), INLINE_STYLES.ITALIC), + ("`", INLINE_STYLES.CODE), + (opts.get("strikethrough", "~"), INLINE_STYLES.STRIKETHROUGH), + ] + markers.sort(key=lambda m: len(m[0]), reverse=True) + return markers + + +def _build_html_style_tags( + options: MarkdownImporterOptions | None, +) -> dict[str, str]: + """Build the HTML tag mapping from options. + + Parameters: + options: Importer options, or ``None`` for defaults. + + Returns: + A dict mapping HTML tag names to Draft.js inline style names. + """ + opts = options or {} + tags = dict(DEFAULT_HTML_STYLE_TAGS) + if "html_style_tags" in opts: + tags.update(opts["html_style_tags"]) + return tags + + +def _gen_key() -> str: + # Draft.js assigns keys client-side when content is loaded into an + # editor; the importer's keys are decorative and only need to be + # unique within a single ContentState. Random choices are sufficient + # and let us avoid a module-level counter that would leak state + # across calls. + return "".join(random.choices(_KEY_CHARS, k=5)) # noqa: S311 + + +class _InlineParser: + """Parses inline markdown (styles and entities) from a text string. + + Walks through the text character by character, tracking opening/closing + of style markers and link syntax, and builds up the plain text, + style ranges, entity ranges, and entity definitions. + """ + + __slots__ = ( + "plain", + "styles", + "entity_ranges", + "entities", + "entity_counter", + "_open_styles", + "_style_markers", + "_html_tags", + ) + + def __init__( + self, + entity_counter: int = 0, + style_markers: list[tuple[str, str]] | None = None, + html_tags: dict[str, str] | None = None, + ) -> None: + self.plain = "" + self.styles: list[InlineStyleRange] = [] + self.entity_ranges: list[EntityRange] = [] + self.entities: dict[str, Entity] = {} + self.entity_counter = entity_counter + self._open_styles: dict[str, int] = {} + self._style_markers = style_markers or _DEFAULT_STYLE_MARKERS + self._html_tags = html_tags or DEFAULT_HTML_STYLE_TAGS + + def parse(self, text: str) -> None: + i = 0 + while i < len(text): + consumed = self._try_escape(text, i) + if consumed: + i += consumed + continue + + consumed = self._try_link(text, i) + if consumed: + i += consumed + continue + + consumed = self._try_html_tag(text, i) + if consumed: + i += consumed + continue + + consumed = self._try_style_marker(text, i) + if consumed: + i += consumed + continue + + self.plain += text[i] + i += 1 + + def _try_escape(self, text: str, i: int) -> int: + """Try to parse a backslash escape at position i. + + A backslash followed by an escapable punctuation character emits + the character literally into the plain text. + + Parameters: + text: The text being parsed. + i: Current position in the text. + + Returns: + Number of characters consumed, or 0 if no escape matched. + """ + if text[i] != "\\" or i + 1 >= len(text): + return 0 + if text[i + 1] not in _ESCAPE_CHARS: + return 0 + self.plain += text[i + 1] + return 2 + + def _try_link(self, text: str, i: int) -> int: + """Try to parse a [text](url) link at position i. Returns chars consumed, or 0.""" + if text[i] != "[": + return 0 + + depth = 1 + j = i + 1 + while j < len(text) and depth > 0: + if text[j] == "[": + depth += 1 + elif text[j] == "]": + depth -= 1 + j += 1 + + if j >= len(text) or text[j] != "(": + return 0 + + k = j + 1 + paren_depth = 1 + while k < len(text) and paren_depth > 0: + if text[k] == "(": + paren_depth += 1 + elif text[k] == ")": + paren_depth -= 1 + k += 1 + + link_text = text[i + 1 : j - 1] + raw_url = text[j + 1 : k - 1] + + # Parse optional title: url "title" or url 'title'. + url = raw_url + title: str | None = None + for quote_char in ('"', "'"): + if raw_url.endswith(quote_char) and " " + quote_char in raw_url: + space_pos = raw_url.rfind(" " + quote_char) + url = raw_url[:space_pos] + title = raw_url[space_pos + 2 : -1] + break + + start = len(self.plain) + + # Parse the link text first so any nested entities claim their + # keys before we reserve one for the outer link. This avoids the + # pre-increment dance of guessing the outer key before parsing. + inner = _InlineParser(self.entity_counter, self._style_markers, self._html_tags) + inner.parse(link_text) + entity_key = inner.entity_counter + self.entity_counter = entity_key + 1 + + link_data: dict[str, Any] = {"url": url} + if title is not None: + link_data["title"] = title + self.entities[str(entity_key)] = { + "type": ENTITY_TYPES.LINK, + "mutability": "MUTABLE", + "data": link_data, + } + self.entity_ranges.append( + {"offset": start, "length": len(inner.plain), "key": entity_key} + ) + + for s in inner.styles: + self.styles.append( + { + "offset": s["offset"] + start, + "length": s["length"], + "style": s["style"], + } + ) + + for ek, ev in inner.entities.items(): + self.entities[ek] = ev + self.entity_ranges.extend(inner.entity_ranges) + + self.plain += inner.plain + return k - i + + def _try_html_tag(self, text: str, i: int) -> int: + """Try to parse an inline HTML style tag at position i. + + Matches ```` opening and ```` closing tags against the + configured HTML tag mapping. Uses the same open/close tracking + as style markers (``_open_styles`` dict). + + Parameters: + text: The text being parsed. + i: Current position in the text. + + Returns: + Number of characters consumed, or 0 if no HTML tag matched. + """ + m = _HTML_CLOSE_RE.match(text, i) + if m and m.group(1) in self._html_tags: + style = self._html_tags[m.group(1)] + if style in self._open_styles: + start = self._open_styles.pop(style) + length = len(self.plain) - start + if length > 0: + self.styles.append( + {"offset": start, "length": length, "style": style} + ) + return m.end() - i + return 0 + + m = _HTML_OPEN_RE.match(text, i) + if m and m.group(1) in self._html_tags: + style = self._html_tags[m.group(1)] + self._open_styles[style] = len(self.plain) + return m.end() - i + + return 0 + + def _try_style_marker(self, text: str, i: int) -> int: + """Try to match a style marker at position i. Returns chars consumed, or 0.""" + for marker, style in self._style_markers: + if text[i : i + len(marker)] != marker: + continue + + # Single-char markers like _ must be at a word boundary to avoid + # matching inside words like some_var_name or URLs with underscores. + if marker == "_": + prev_is_word = i > 0 and (text[i - 1].isalnum() or text[i - 1] == "_") + next_is_word = i + 1 < len(text) and ( + text[i + 1].isalnum() or text[i + 1] == "_" + ) + if prev_is_word and next_is_word: + return 0 + + if style in self._open_styles: + start = self._open_styles.pop(style) + length = len(self.plain) - start + if length > 0: + self.styles.append( + {"offset": start, "length": length, "style": style} + ) + return len(marker) + else: + self._open_styles[style] = len(self.plain) + return len(marker) + + return 0 + + +# Default style markers, ordered longest-first for correct matching. +_DEFAULT_STYLE_MARKERS = _build_style_markers(None) + +_BLOCK_PATTERNS = [HEADING_RE, BLOCKQUOTE_RE, UL_RE, OL_RE, HR_RE, IMAGE_RE] + + +def _is_block_level(line: str) -> bool: + """Check if a line is a standalone block-level element (not a plain paragraph).""" + return any(p.match(line) for p in _BLOCK_PATTERNS) + + +def _split_blocks(markdown: str) -> list[str]: + r"""Split markdown into raw block strings. + + Handles code fences, joins consecutive plain lines into a single + paragraph block (standard markdown soft line breaks), and attaches + continuation lines to the preceding block-level element (preserving + ``\n`` in block text for soft line breaks within list items, etc.). + """ + blocks: list[str] = [] + in_code = False + code_lines: list[str] = [] + paragraph_lines: list[str] = [] + # Whether the last emitted block was block-level (heading, list, etc.) + # and no blank line has appeared since — continuation lines should be + # appended to it rather than starting a new paragraph. + continuation = False + + def flush_paragraph() -> None: + if paragraph_lines: + blocks.append(" ".join(paragraph_lines)) + paragraph_lines.clear() + + for line in markdown.split("\n"): + if not in_code and (line.startswith("```") or line.startswith("~~~")): + flush_paragraph() + continuation = False + in_code = True + code_lines = [line] + continue + + if in_code and (line == "```" or line == "~~~"): + code_lines.append(line) + blocks.append("\n".join(code_lines)) + code_lines = [] + in_code = False + continue + + if in_code: + code_lines.append(line) + continue + + if line == "": + flush_paragraph() + continuation = False + continue + + if _is_block_level(line): + # Consecutive blockquote lines (each starting with "> ") are + # treated as a continuation of the same block rather than + # separate blocks. + if ( + continuation + and blocks + and blocks[-1].startswith("> ") + and line.startswith("> ") + ): + blocks[-1] += "\n" + line + else: + flush_paragraph() + blocks.append(line) + continuation = True + elif continuation: + # Continuation of a block-level element (e.g. list item with + # a soft line break in the text). Join with \n to preserve the + # original line break. + blocks[-1] += "\n" + line + else: + paragraph_lines.append(line) + + flush_paragraph() + return blocks + + +def _inline_block( + text: str, + block_type: str, + depth: int, + entity_counter: int, + style_markers: list[tuple[str, str]] | None = None, + html_tags: dict[str, str] | None = None, +) -> tuple[Block, dict[str, Entity], int]: + """Parse inline content and build a block with the given type and depth. + + Parameters: + text: The raw text to parse. + block_type: The Draft.js block type. + depth: The block nesting depth. + entity_counter: The current entity key counter. + style_markers: Configurable style markers, or ``None`` for defaults. + html_tags: Configurable HTML tag mapping, or ``None`` for defaults. + + Returns: + A tuple of (block, entities, updated entity counter). + """ + parser = _InlineParser(entity_counter, style_markers, html_tags) + parser.parse(text) + block: Block = { + "key": _gen_key(), + "text": parser.plain, + "type": block_type, + "depth": depth, + "inlineStyleRanges": _merge_style_ranges(parser.styles), + "entityRanges": parser.entity_ranges, + } + return block, parser.entities, parser.entity_counter + + +def _atomic_block( + entity_type: str, + mutability: Mutability, + data: dict[str, Any], + entity_counter: int, +) -> tuple[Block, dict[str, Entity], int]: + """Build an atomic block with a single entity (HR, IMAGE, etc.). + + Parameters: + entity_type: The Draft.js entity type. + mutability: The entity mutability. + data: The entity data dict. + entity_counter: The current entity key counter. + + Returns: + A tuple of (block, entities, updated entity counter). + """ + block: Block = { + "key": _gen_key(), + "text": " ", + "type": BLOCK_TYPES.ATOMIC, + "depth": 0, + "inlineStyleRanges": [], + "entityRanges": [{"offset": 0, "length": 1, "key": entity_counter}], + } + entities: dict[str, Entity] = { + str(entity_counter): { + "type": entity_type, + "mutability": mutability, + "data": data, + } + } + return block, entities, entity_counter + 1 + + +def _parse_block( + raw: str, + entity_counter: int, + style_markers: list[tuple[str, str]] | None = None, + html_tags: dict[str, str] | None = None, +) -> tuple[Block, dict[str, Entity], int]: + """Parse a raw block string into a Block, entities, and updated entity counter. + + Parameters: + raw: The raw block string to parse. + entity_counter: The current entity key counter. + style_markers: Configurable style markers, or ``None`` for defaults. + html_tags: Configurable HTML tag mapping, or ``None`` for defaults. + + Returns: + A tuple of (block, entities, updated entity counter). + """ + if HR_RE.match(raw): + return _atomic_block( + ENTITY_TYPES.HORIZONTAL_RULE, "IMMUTABLE", {}, entity_counter + ) + + m = IMAGE_RE.match(raw) + if m: + alt = m.group(1) + src = m.group(2) + data: dict[str, Any] = {"src": src} + if alt: + data["alt"] = alt + return _atomic_block(ENTITY_TYPES.IMAGE, "IMMUTABLE", data, entity_counter) + + is_backtick_fence = raw.startswith("```") and raw.endswith("```") + is_tilde_fence = raw.startswith("~~~") and raw.endswith("~~~") + if is_backtick_fence or is_tilde_fence: + lines = raw.split("\n") + # Strip opening fence (+ optional info string) and closing fence. + content = "\n".join(lines[1:-1]) if len(lines) >= 3 else "" + block: Block = { + "key": _gen_key(), + "text": content, + "type": BLOCK_TYPES.CODE, + "depth": 0, + "inlineStyleRanges": [], + "entityRanges": [], + } + return block, {}, entity_counter + + # For block-level elements, the regex matches the first line only. + # Any continuation lines after \n are appended to preserve soft line breaks. + newline_pos = raw.find("\n") + first_line = raw[:newline_pos] if newline_pos != -1 else raw + rest = raw[newline_pos:] if newline_pos != -1 else "" + + m = HEADING_RE.match(first_line) + if m: + level = len(m.group(1)) + return _inline_block( + m.group(2) + rest, + HEADING_TYPES[level], + 0, + entity_counter, + style_markers, + html_tags, + ) + + m = BLOCKQUOTE_RE.match(first_line) + if m: + # Strip "> " prefix from continuation lines (multi-line blockquotes). + if rest: + rest = rest.replace("\n> ", "\n") + return _inline_block( + m.group(1) + rest, + BLOCK_TYPES.BLOCKQUOTE, + 0, + entity_counter, + style_markers, + html_tags, + ) + + m = UL_RE.match(first_line) + if m: + depth = len(m.group(1)) // 2 + return _inline_block( + m.group(2) + rest, + BLOCK_TYPES.UNORDERED_LIST_ITEM, + depth, + entity_counter, + style_markers, + html_tags, + ) + + m = OL_RE.match(first_line) + if m: + depth = len(m.group(1)) // 2 + return _inline_block( + m.group(3) + rest, + BLOCK_TYPES.ORDERED_LIST_ITEM, + depth, + entity_counter, + style_markers, + html_tags, + ) + + # Any non-empty text that doesn't match a block-level pattern becomes a + # plain paragraph. This is the expected path for hand-written Markdown, + # so we log at debug level rather than warning (a paragraph isn't an + # anomaly). Malformed block-level syntax (e.g. "#NoSpace") also lands + # here and is treated as a paragraph. + logger.debug('Treating markdown as unstyled paragraph: "%s"', first_line) + return _inline_block( + raw, + BLOCK_TYPES.UNSTYLED, + 0, + entity_counter, + style_markers, + html_tags, + ) + + +def _merge_style_ranges( + styles: list[InlineStyleRange], +) -> list[InlineStyleRange]: + """Merge adjacent or overlapping ranges of the same style. + + This compensates for the exporter's inline style nesting behavior. + When styles partially overlap (e.g. bold 0-5, italic 3-8), the + exporter must close and reopen the outer marker to produce valid + Markdown (e.g. ``**Bold ****_Italic_**``). This produces two BOLD + ranges that should be a single continuous range. + + This is an intentional round-trip trade-off: the importer repairs + the exporter's marker-emission quirks so that + ``ContentState -> Markdown -> ContentState`` preserves the original + style ranges. + """ + if not styles: + return styles + + by_style: dict[str, list[InlineStyleRange]] = {} + for s in styles: + by_style.setdefault(s["style"], []).append(s) + + merged: list[InlineStyleRange] = [] + for style, ranges in by_style.items(): + sorted_ranges = sorted(ranges, key=lambda r: r["offset"]) + cur_offset = sorted_ranges[0]["offset"] + cur_length = sorted_ranges[0]["length"] + for r in sorted_ranges[1:]: + cur_end = cur_offset + cur_length + if r["offset"] <= cur_end: + new_end = max(cur_end, r["offset"] + r["length"]) + cur_length = new_end - cur_offset + else: + merged.append( + {"offset": cur_offset, "length": cur_length, "style": style} + ) + cur_offset = r["offset"] + cur_length = r["length"] + merged.append({"offset": cur_offset, "length": cur_length, "style": style}) + + return merged + + +def markdown_to_content_state( + markdown: str, + options: MarkdownImporterOptions | None = None, +) -> ContentState: + """Convert a Markdown string to a Draft.js ContentState. + + Supports the same subset of Markdown that the exporter produces: + block-level formatting, inline styles, and entities. + + Parameters: + markdown: The Markdown string to parse. + options: Optional importer options mirroring the exporter's + ``MarkdownOptions``. Pass the same options used with + ``build_markdown_config`` to round-trip customized output. + + Returns: + A Draft.js ContentState dict with blocks and entityMap. + """ + markdown = markdown.rstrip() + + raw_blocks = _split_blocks(markdown) + + style_markers = _build_style_markers(options) + html_tags = _build_html_style_tags(options) + + blocks: list[Block] = [] + entity_map: dict[str, Entity] = {} + entity_counter = 0 + + for raw in raw_blocks: + block, block_entities, entity_counter = _parse_block( + raw, entity_counter, style_markers, html_tags + ) + blocks.append(block) + entity_map.update(block_entities) + + return {"blocks": blocks, "entityMap": entity_map} diff --git a/pyproject.toml b/pyproject.toml index b5f8de4..d821701 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -168,7 +168,9 @@ no_implicit_reexport = true [[tool.mypy.overrides]] module = "tests.*" -# For test cases, we don’t need to annotate functions. +# For test cases, we don't need to annotate functions. disallow_untyped_defs = false +# Allow calling untyped functions from test methods (metaclass-generated tests, helpers). +disallow_untyped_calls = false # disallow_incomplete_defs = false # warn_no_return = false diff --git a/tests/markdown/test_importer.py b/tests/markdown/test_importer.py new file mode 100644 index 0000000..1bd6a0b --- /dev/null +++ b/tests/markdown/test_importer.py @@ -0,0 +1,1112 @@ +import json +import os +import unittest + +from draftjs_exporter.html import HTML +from draftjs_exporter.markdown import CONFIG as MARKDOWN_CONFIG +from draftjs_exporter.markdown import build_markdown_config +from draftjs_exporter.markdown.importer import markdown_to_content_state + +fixtures_path = os.path.join(os.path.dirname(__file__), "..", "test_exports.json") +with open(fixtures_path) as f: + fixtures = json.loads(f.read()) + +FIXTURES_BY_LABEL = {f["label"]: f for f in fixtures} + +# Cases where the markdown output is a lossless representation of the content_state, +# meaning the importer can fully reconstruct block types, text, inline styles, +# entity offsets/lengths, and depth. +LOSSLESS_CASES = [ + "Plain text", + "Single inline style", + "Nested inline styles", + "Nested inline styles (inverted)", + "Partially nested inline styles", + "Adjacent inline styles", + "Adjacent entities", + "Entity with inline style", + "Ordered list", + "All plain HTML elements we need", + "Style map defaults", + "HTML entities escaping", + "Same content multiple times", +] + + +def _blocks_match(expected, actual): + """Compare blocks ignoring keys (which are randomly generated).""" + if len(expected) != len(actual): + return False, f"block count: expected {len(expected)}, got {len(actual)}" + + for i, (eb, ab) in enumerate(zip(expected, actual)): + if eb["type"] != ab["type"]: + return False, f"block {i} type: expected {eb['type']!r}, got {ab['type']!r}" + if eb["text"] != ab["text"]: + return False, f"block {i} text: expected {eb['text']!r}, got {ab['text']!r}" + if eb.get("depth", 0) != ab.get("depth", 0): + return ( + False, + f"block {i} depth: expected {eb.get('depth', 0)}, got {ab.get('depth', 0)}", + ) + + exp_styles = sorted( + eb.get("inlineStyleRanges", []), + key=lambda s: (s["offset"], s["style"]), + ) + got_styles = sorted( + ab.get("inlineStyleRanges", []), + key=lambda s: (s["offset"], s["style"]), + ) + if exp_styles != got_styles: + return False, f"block {i} styles: expected {exp_styles}, got {got_styles}" + + exp_er = eb.get("entityRanges", []) + got_er = ab.get("entityRanges", []) + if len(exp_er) != len(got_er): + return ( + False, + f"block {i} entity count: expected {len(exp_er)}, got {len(got_er)}", + ) + for j, (ee, ge) in enumerate(zip(exp_er, got_er)): + if ee["offset"] != ge["offset"] or ee["length"] != ge["length"]: + return ( + False, + f"block {i} entity {j}: expected off={ee['offset']} len={ee['length']}, got off={ge['offset']} len={ge['length']}", + ) + + return True, "" + + +class ImporterTestMeta(type): + """Generates importer test cases from test_exports.json fixtures.""" + + def __new__(mcs, name, bases, tests): + for fixture in fixtures: + label = fixture["label"] + if label not in LOSSLESS_CASES: + continue + if "markdown" not in fixture["output"]: + continue + + test_label = label.lower().replace(" ", "_") + test_name = f"test_import_{test_label}" + + md = fixture["output"]["markdown"] + expected = fixture["content_state"] + + def gen_test(markdown, content_state): + def test(self): + result = markdown_to_content_state(markdown) + ok, msg = _blocks_match(content_state["blocks"], result["blocks"]) + self.assertTrue(ok, msg) + + return test + + tests[test_name] = gen_test(md, expected) + + return type.__new__(mcs, name, bases, tests) + + +class TestImporter(unittest.TestCase, metaclass=ImporterTestMeta): + """Tests that markdown input produces the expected content_state blocks.""" + + +class TestImporterLossyEntityData(unittest.TestCase): + """Tests for cases where entity data is partially lost in markdown. + + The markdown link syntax [text](url) only preserves the url, so extra + entity data (title, rel, data-* attributes) is lost. These tests verify + that the importer still reconstructs the correct block structure and + entity offsets/lengths. + """ + + def test_import_entity_preserves_structure(self): + fix = FIXTURES_BY_LABEL["Entity"] + md = fix["output"]["markdown"] + result = markdown_to_content_state(md) + self.assertEqual(len(result["blocks"]), 1) + block = result["blocks"][0] + self.assertEqual(block["text"], "a") + self.assertEqual(block["type"], "unstyled") + self.assertEqual(len(block["entityRanges"]), 1) + self.assertEqual(block["entityRanges"][0]["offset"], 0) + self.assertEqual(block["entityRanges"][0]["length"], 1) + self.assertEqual( + sorted(block["inlineStyleRanges"], key=lambda s: s["offset"]), + [{"offset": 0, "length": 1, "style": "ITALIC"}], + ) + + def test_import_entity_with_data_star_preserves_structure(self): + fix = FIXTURES_BY_LABEL["Entity with data-*"] + md = fix["output"]["markdown"] + result = markdown_to_content_state(md) + block = result["blocks"][0] + self.assertEqual(block["text"], "a") + entity_key = str(block["entityRanges"][0]["key"]) + self.assertEqual(result["entityMap"][entity_key]["data"], {"url": "/"}) + + +class TestImporterDecoratorCases(unittest.TestCase): + """Tests for cases where composite decorators add structure during export. + + For example, the linkify decorator auto-links URLs, creating link syntax + in the markdown output that wasn't an entity in the original content_state. + The importer correctly parses these as entities. + """ + + def test_import_multiple_decorators(self): + fix = FIXTURES_BY_LABEL["Multiple decorators"] + md = fix["output"]["markdown"] + result = markdown_to_content_state(md) + block = result["blocks"][0] + # The original had no entities, but the linkify decorator created a link + # in the markdown output. The importer correctly parses it as an entity. + self.assertEqual( + block["text"], + "search http://www.google.com#world for the #world", + ) + self.assertEqual(len(block["entityRanges"]), 1) + er = block["entityRanges"][0] + self.assertEqual(er["offset"], 7) + self.assertEqual(er["length"], 27) + entity_key = str(er["key"]) + self.assertEqual( + result["entityMap"][entity_key]["data"]["url"], + "http://www.google.com#world", + ) + + +class TestImporterBlockTypes(unittest.TestCase): + """Direct tests for block types not covered by test_exports.json fixtures.""" + + def test_horizontal_rule(self): + result = markdown_to_content_state("---\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "atomic") + self.assertEqual(block["text"], " ") + entity = result["entityMap"][str(block["entityRanges"][0]["key"])] + self.assertEqual(entity["type"], "HORIZONTAL_RULE") + self.assertEqual(entity["mutability"], "IMMUTABLE") + + def test_image_with_alt(self): + result = markdown_to_content_state( + "![alt text](http://example.com/img.png)\n\n" + ) + block = result["blocks"][0] + self.assertEqual(block["type"], "atomic") + self.assertEqual(block["text"], " ") + entity = result["entityMap"][str(block["entityRanges"][0]["key"])] + self.assertEqual(entity["type"], "IMAGE") + self.assertEqual(entity["data"]["src"], "http://example.com/img.png") + self.assertEqual(entity["data"]["alt"], "alt text") + + def test_image_without_alt(self): + result = markdown_to_content_state("![](http://example.com/img.png)\n\n") + entity = result["entityMap"][str(result["blocks"][0]["entityRanges"][0]["key"])] + self.assertNotIn("alt", entity["data"]) + self.assertEqual(entity["data"]["src"], "http://example.com/img.png") + + def test_code_block(self): + result = markdown_to_content_state("```\ndef foo():\n pass\n```\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "code-block") + self.assertEqual(block["text"], "def foo():\n pass") + self.assertEqual(block["inlineStyleRanges"], []) + self.assertEqual(block["entityRanges"], []) + + def test_empty_code_block(self): + result = markdown_to_content_state("```\n```\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "code-block") + self.assertEqual(block["text"], "") + + def test_code_block_with_surrounding_blank_lines(self): + result = markdown_to_content_state("```\n\ncontent\n\n```\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "code-block") + self.assertEqual(block["text"], "\ncontent\n") + + def test_code_block_preserves_markdown_syntax(self): + result = markdown_to_content_state( + "```\n**not bold** [not a link](url)\n```\n\n" + ) + block = result["blocks"][0] + self.assertEqual(block["type"], "code-block") + self.assertEqual(block["text"], "**not bold** [not a link](url)") + self.assertEqual(block["inlineStyleRanges"], []) + self.assertEqual(block["entityRanges"], []) + + def test_code_block_text_ending_with_newline(self): + md = "```\ndef foo():\n pass\n\n```\n\n" + result = markdown_to_content_state(md) + block = result["blocks"][0] + self.assertEqual(block["type"], "code-block") + self.assertEqual(block["text"], "def foo():\n pass\n") + + def test_nested_unordered_list_depth(self): + md = "- depth 0\n - depth 1\n - depth 2\n\n" + result = markdown_to_content_state(md) + depths = [b["depth"] for b in result["blocks"]] + self.assertEqual(depths, [0, 1, 2]) + + def test_nested_ordered_list_depth(self): + md = "1. depth 0\n 1. depth 1\n 1. depth 2\n\n" + result = markdown_to_content_state(md) + depths = [b["depth"] for b in result["blocks"]] + self.assertEqual(depths, [0, 1, 2]) + + def test_list_item_with_soft_line_break(self): + md = "- Convert line breaks to `
      `\nelements.\n\n" + result = markdown_to_content_state(md) + self.assertEqual(len(result["blocks"]), 1) + block = result["blocks"][0] + self.assertEqual(block["type"], "unordered-list-item") + self.assertEqual(block["text"], "Convert line breaks to
      \nelements.") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 23, "length": 4, "style": "CODE"}], + ) + + def test_ordered_list_item_with_soft_line_break(self): + md = "1. First line\nsecond line\n\n" + result = markdown_to_content_state(md) + self.assertEqual(len(result["blocks"]), 1) + block = result["blocks"][0] + self.assertEqual(block["type"], "ordered-list-item") + self.assertEqual(block["text"], "First line\nsecond line") + + def test_blockquote_with_soft_line_break(self): + md = "> First line\nsecond line\n\n" + result = markdown_to_content_state(md) + self.assertEqual(len(result["blocks"]), 1) + block = result["blocks"][0] + self.assertEqual(block["type"], "blockquote") + self.assertEqual(block["text"], "First line\nsecond line") + + def test_blockquote(self): + result = markdown_to_content_state("> quoted text\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "blockquote") + self.assertEqual(block["text"], "quoted text") + + def test_heading_levels(self): + for level in range(1, 7): + prefix = "#" * level + result = markdown_to_content_state(f"{prefix} Heading {level}\n\n") + block = result["blocks"][0] + self.assertEqual( + block["type"], + f"header-{['one', 'two', 'three', 'four', 'five', 'six'][level - 1]}", + ) + self.assertEqual(block["text"], f"Heading {level}") + + +class TestImporterInlineStyles(unittest.TestCase): + """Tests for each inline style marker individually.""" + + def test_inline_code(self): + result = markdown_to_content_state("hello `code` world\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello code world") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 6, "length": 4, "style": "CODE"}], + ) + + def test_inline_strikethrough(self): + result = markdown_to_content_state("hello ~strike~ world\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello strike world") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 6, "length": 6, "style": "STRIKETHROUGH"}], + ) + + def test_multiple_different_styles(self): + result = markdown_to_content_state("**bold** and _italic_ text\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "bold and italic text") + styles = sorted(block["inlineStyleRanges"], key=lambda s: s["offset"]) + self.assertEqual(styles[0], {"offset": 0, "length": 4, "style": "BOLD"}) + self.assertEqual(styles[1], {"offset": 9, "length": 6, "style": "ITALIC"}) + + def test_inline_styles_in_heading(self): + result = markdown_to_content_state("## **bold** heading\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "header-two") + self.assertEqual(block["text"], "bold heading") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 0, "length": 4, "style": "BOLD"}], + ) + + def test_underscore_inside_word_is_literal(self): + result = markdown_to_content_state("some_variable_name\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "some_variable_name") + self.assertEqual(block["inlineStyleRanges"], []) + + def test_underscore_in_url_is_literal(self): + result = markdown_to_content_state( + "[link](http://example.com/path_to/page)\n\n" + ) + block = result["blocks"][0] + self.assertEqual(block["text"], "link") + entity = result["entityMap"][str(block["entityRanges"][0]["key"])] + self.assertEqual(entity["data"]["url"], "http://example.com/path_to/page") + + def test_underscore_at_word_boundary_is_italic(self): + result = markdown_to_content_state("hello _world_ test\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello world test") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 6, "length": 5, "style": "ITALIC"}], + ) + + def test_inline_styles_in_blockquote(self): + result = markdown_to_content_state("> _italic_ quote\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "blockquote") + self.assertEqual(block["text"], "italic quote") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 0, "length": 6, "style": "ITALIC"}], + ) + + def test_inline_styles_in_list_item(self): + result = markdown_to_content_state("- **bold** item\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "unordered-list-item") + self.assertEqual(block["text"], "bold item") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 0, "length": 4, "style": "BOLD"}], + ) + + +class TestImporterInlineHtmlTags(unittest.TestCase): + """Tests for raw inline HTML tag parsing (non-Markdown styles).""" + + def test_underline_tag(self): + result = markdown_to_content_state("hello under world\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello under world") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 6, "length": 5, "style": "UNDERLINE"}], + ) + + def test_superscript_tag(self): + result = markdown_to_content_state("x2 + y\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "x2 + y") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 1, "length": 1, "style": "SUPERSCRIPT"}], + ) + + def test_subscript_tag(self): + result = markdown_to_content_state("H2O\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "H2O") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 1, "length": 1, "style": "SUBSCRIPT"}], + ) + + def test_mark_tag(self): + result = markdown_to_content_state("hello world\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello world") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 6, "length": 5, "style": "MARK"}], + ) + + def test_quotation_tag(self): + result = markdown_to_content_state("He said hi\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "He said hi") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 8, "length": 2, "style": "QUOTATION"}], + ) + + def test_small_tag(self): + result = markdown_to_content_state("hello world\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello world") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 6, "length": 5, "style": "SMALL"}], + ) + + def test_sample_tag(self): + result = markdown_to_content_state("hello world\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello world") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 6, "length": 5, "style": "SAMPLE"}], + ) + + def test_insert_tag(self): + result = markdown_to_content_state("hello world\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello world") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 6, "length": 5, "style": "INSERT"}], + ) + + def test_delete_tag(self): + result = markdown_to_content_state("hello world\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello world") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 6, "length": 5, "style": "DELETE"}], + ) + + def test_keyboard_tag(self): + result = markdown_to_content_state("Press Enter\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "Press Enter") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 6, "length": 5, "style": "KEYBOARD"}], + ) + + def test_nested_html_tags(self): + result = markdown_to_content_state("under both\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "under both") + styles = sorted(block["inlineStyleRanges"], key=lambda s: s["offset"]) + self.assertEqual(styles[0], {"offset": 0, "length": 10, "style": "UNDERLINE"}) + self.assertEqual(styles[1], {"offset": 6, "length": 4, "style": "MARK"}) + + def test_html_tag_with_markdown_styles(self): + result = markdown_to_content_state("**bold** under text\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "bold under text") + styles = sorted(block["inlineStyleRanges"], key=lambda s: s["offset"]) + self.assertEqual(styles[0], {"offset": 0, "length": 4, "style": "BOLD"}) + self.assertEqual(styles[1], {"offset": 5, "length": 5, "style": "UNDERLINE"}) + + def test_unknown_html_tag_is_literal(self): + result = markdown_to_content_state("hello tag\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello tag") + self.assertEqual(block["inlineStyleRanges"], []) + + def test_unmatched_closing_tag_is_literal(self): + result = markdown_to_content_state("hello       world\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello       world") + self.assertEqual(block["inlineStyleRanges"], []) + + def test_html_tags_in_heading(self): + result = markdown_to_content_state("## Underlined heading\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "header-two") + self.assertEqual(block["text"], "Underlined heading") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 0, "length": 18, "style": "UNDERLINE"}], + ) + + def test_html_tags_in_blockquote(self): + result = markdown_to_content_state("> Underlined quote\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "blockquote") + self.assertEqual(block["text"], "Underlined quote") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 0, "length": 16, "style": "UNDERLINE"}], + ) + + def test_html_tags_in_list_item(self): + result = markdown_to_content_state("- underlined item\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "unordered-list-item") + self.assertEqual(block["text"], "underlined item") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 0, "length": 15, "style": "UNDERLINE"}], + ) + + def test_empty_html_tag_pair(self): + """Adjacent open+close HTML tags (e.g. ) produce no style range.""" + result = markdown_to_content_state("beforeafter\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "beforeafter") + self.assertEqual(block["inlineStyleRanges"], []) + + +class TestImporterEscapes(unittest.TestCase): + """Tests for backslash escape handling.""" + + def test_escaped_star_is_literal(self): + result = markdown_to_content_state("hello \\*world\\* text\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello *world* text") + self.assertEqual(block["inlineStyleRanges"], []) + + def test_escaped_underscore_is_literal(self): + result = markdown_to_content_state("hello \\_world\\_ text\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello _world_ text") + self.assertEqual(block["inlineStyleRanges"], []) + + def test_escaped_backtick_is_literal(self): + result = markdown_to_content_state("hello \\`code\\` text\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello `code` text") + self.assertEqual(block["inlineStyleRanges"], []) + + def test_escaped_tilde_is_literal(self): + result = markdown_to_content_state("hello \\~strike\\~ text\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello ~strike~ text") + self.assertEqual(block["inlineStyleRanges"], []) + + def test_escaped_hash_is_literal(self): + result = markdown_to_content_state("\\# Not a heading\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "unstyled") + self.assertEqual(block["text"], "# Not a heading") + + def test_escaped_brackets_are_literal(self): + result = markdown_to_content_state("\\[not a link\\](not a url)\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "[not a link](not a url)") + self.assertEqual(block["entityRanges"], []) + + def test_escaped_backslash_is_literal(self): + result = markdown_to_content_state("hello \\\\ world\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello \\ world") + + def test_unescaped_backslash_passthrough(self): + result = markdown_to_content_state("hello \\n world\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello \\n world") + + def test_escape_at_end_of_string(self): + result = markdown_to_content_state("text\\\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "text\\") + + +class TestImporterHorizontalRuleVariants(unittest.TestCase): + """Tests for horizontal rule variants (---, ***, ___).""" + + def test_hr_dashes(self): + result = markdown_to_content_state("---\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "atomic") + entity = result["entityMap"][str(block["entityRanges"][0]["key"])] + self.assertEqual(entity["type"], "HORIZONTAL_RULE") + + def test_hr_asterisks(self): + result = markdown_to_content_state("***\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "atomic") + entity = result["entityMap"][str(block["entityRanges"][0]["key"])] + self.assertEqual(entity["type"], "HORIZONTAL_RULE") + + def test_hr_underscores(self): + result = markdown_to_content_state("___\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "atomic") + entity = result["entityMap"][str(block["entityRanges"][0]["key"])] + self.assertEqual(entity["type"], "HORIZONTAL_RULE") + + +class TestImporterLinkTitles(unittest.TestCase): + """Tests for link title attribute parsing.""" + + def test_link_with_double_quoted_title(self): + result = markdown_to_content_state('[link](http://example.com "My Title")\n\n') + block = result["blocks"][0] + entity = result["entityMap"][str(block["entityRanges"][0]["key"])] + self.assertEqual(entity["data"]["url"], "http://example.com") + self.assertEqual(entity["data"]["title"], "My Title") + + def test_link_with_single_quoted_title(self): + result = markdown_to_content_state("[link](http://example.com 'My Title')\n\n") + block = result["blocks"][0] + entity = result["entityMap"][str(block["entityRanges"][0]["key"])] + self.assertEqual(entity["data"]["url"], "http://example.com") + self.assertEqual(entity["data"]["title"], "My Title") + + def test_link_without_title(self): + result = markdown_to_content_state("[link](http://example.com)\n\n") + block = result["blocks"][0] + entity = result["entityMap"][str(block["entityRanges"][0]["key"])] + self.assertEqual(entity["data"]["url"], "http://example.com") + self.assertNotIn("title", entity["data"]) + + def test_link_with_title_and_special_chars_in_url(self): + result = markdown_to_content_state( + '[link](http://example.com/path?a=1&b=2 "Title")\n\n' + ) + block = result["blocks"][0] + entity = result["entityMap"][str(block["entityRanges"][0]["key"])] + self.assertEqual(entity["data"]["url"], "http://example.com/path?a=1&b=2") + self.assertEqual(entity["data"]["title"], "Title") + + +class TestImporterCodeFenceVariants(unittest.TestCase): + """Tests for code fence handling (info strings, ~~~ fences).""" + + def test_code_block_with_info_string(self): + result = markdown_to_content_state("```python\ndef foo():\n pass\n```\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "code-block") + self.assertEqual(block["text"], "def foo():\n pass") + + def test_tilde_code_fence(self): + result = markdown_to_content_state("~~~\ncode\n~~~\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "code-block") + self.assertEqual(block["text"], "code") + + def test_tilde_code_fence_with_info_string(self): + result = markdown_to_content_state("~~~python\ncode\n~~~\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "code-block") + self.assertEqual(block["text"], "code") + + def test_empty_code_block_with_info_string(self): + result = markdown_to_content_state("```python\n```\n\n") + block = result["blocks"][0] + self.assertEqual(block["type"], "code-block") + self.assertEqual(block["text"], "") + + +class TestImporterMultiLineBlockquotes(unittest.TestCase): + """Tests for multi-line blockquote handling.""" + + def test_two_line_blockquote(self): + result = markdown_to_content_state("> Line one\n> Line two\n\n") + blocks = result["blocks"] + self.assertEqual(len(blocks), 1) + block = blocks[0] + self.assertEqual(block["type"], "blockquote") + self.assertEqual(block["text"], "Line one\nLine two") + + def test_three_line_blockquote(self): + result = markdown_to_content_state("> First\n> Second\n> Third\n\n") + blocks = result["blocks"] + self.assertEqual(len(blocks), 1) + self.assertEqual(blocks[0]["text"], "First\nSecond\nThird") + + def test_blockquote_with_inline_style_across_lines(self): + result = markdown_to_content_state("> **bold** line one\n> line two\n\n") + blocks = result["blocks"] + self.assertEqual(len(blocks), 1) + block = blocks[0] + self.assertEqual(block["type"], "blockquote") + self.assertEqual(block["text"], "bold line one\nline two") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 0, "length": 4, "style": "BOLD"}], + ) + + +class TestImporterInlineEdgeCases(unittest.TestCase): + """Tests for inline parsing edge cases.""" + + def test_bracket_not_followed_by_paren(self): + """A [bracket] without (url) should be treated as plain text.""" + result = markdown_to_content_state("[not a link] here\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "[not a link] here") + self.assertEqual(block["entityRanges"], []) + + def test_nested_brackets_in_link(self): + result = markdown_to_content_state("[text [inner]](http://example.com)\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "text [inner]") + self.assertEqual(len(block["entityRanges"]), 1) + + def test_nested_parens_in_url(self): + result = markdown_to_content_state("[link](http://example.com/path_(test))\n\n") + block = result["blocks"][0] + entity = result["entityMap"][str(block["entityRanges"][0]["key"])] + self.assertEqual(entity["data"]["url"], "http://example.com/path_(test)") + + def test_empty_style_markers_ignored(self): + """Adjacent open+close markers (e.g. ****) with no content produce no style range.""" + result = markdown_to_content_state("before****after\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "beforeafter") + self.assertEqual(block["inlineStyleRanges"], []) + + def test_non_adjacent_same_style_ranges(self): + result = markdown_to_content_state("**one** middle **two**\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "one middle two") + styles = sorted(block["inlineStyleRanges"], key=lambda s: s["offset"]) + self.assertEqual(len(styles), 2) + self.assertEqual(styles[0], {"offset": 0, "length": 3, "style": "BOLD"}) + self.assertEqual(styles[1], {"offset": 11, "length": 3, "style": "BOLD"}) + + def test_link_with_inner_styles(self): + result = markdown_to_content_state("[**bold link**](http://example.com)\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "bold link") + self.assertEqual(len(block["entityRanges"]), 1) + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 0, "length": 9, "style": "BOLD"}], + ) + + def test_nested_link_in_link_text(self): + """A link whose text contains another link produces two entities.""" + result = markdown_to_content_state( + "[before [inner](http://inner.com) after](http://outer.com)\n\n" + ) + block = result["blocks"][0] + self.assertEqual(block["text"], "before inner after") + self.assertEqual(len(block["entityRanges"]), 2) + # Outer link covers everything. + outer = block["entityRanges"][0] + self.assertEqual(outer["offset"], 0) + self.assertEqual(outer["length"], 18) + self.assertEqual( + result["entityMap"][str(outer["key"])]["data"]["url"], + "http://outer.com", + ) + # Inner link covers just "inner". + inner = block["entityRanges"][1] + self.assertEqual(inner["offset"], 7) + self.assertEqual(inner["length"], 5) + self.assertEqual( + result["entityMap"][str(inner["key"])]["data"]["url"], + "http://inner.com", + ) + + +class TestImporterMultiBlock(unittest.TestCase): + """Tests for multi-block documents with mixed block types.""" + + def test_mixed_blocks(self): + md = "# Title\n\nParagraph\n\n> Quote\n\n- Item\n\n---\n\n" + result = markdown_to_content_state(md) + types = [b["type"] for b in result["blocks"]] + self.assertEqual( + types, + [ + "header-one", + "unstyled", + "blockquote", + "unordered-list-item", + "atomic", + ], + ) + + def test_code_block_between_paragraphs(self): + md = "Before\n\n```\ncode\n```\n\nAfter\n\n" + result = markdown_to_content_state(md) + self.assertEqual( + [(b["type"], b["text"]) for b in result["blocks"]], + [("unstyled", "Before"), ("code-block", "code"), ("unstyled", "After")], + ) + + def test_entity_counter_across_blocks(self): + md = "[a](http://a.com)\n\n[b](http://b.com)\n\n" + result = markdown_to_content_state(md) + keys = [b["entityRanges"][0]["key"] for b in result["blocks"]] + self.assertEqual(keys[0], 0) + self.assertEqual(keys[1], 1) + self.assertEqual(result["entityMap"]["0"]["data"]["url"], "http://a.com") + self.assertEqual(result["entityMap"]["1"]["data"]["url"], "http://b.com") + + def test_empty_input(self): + result = markdown_to_content_state("") + self.assertEqual(result["blocks"], []) + self.assertEqual(result["entityMap"], {}) + + def test_whitespace_only_input(self): + result = markdown_to_content_state("\n\n\n") + self.assertEqual(result["blocks"], []) + self.assertEqual(result["entityMap"], {}) + + def test_input_without_trailing_newlines(self): + result = markdown_to_content_state("hello") + self.assertEqual(len(result["blocks"]), 1) + self.assertEqual(result["blocks"][0]["text"], "hello") + self.assertEqual(result["blocks"][0]["type"], "unstyled") + + def test_consecutive_lines_joined_as_paragraph(self): + md = "This is a long\nparagraph across\nmultiple lines\n\n" + result = markdown_to_content_state(md) + self.assertEqual(len(result["blocks"]), 1) + self.assertEqual( + result["blocks"][0]["text"], + "This is a long paragraph across multiple lines", + ) + self.assertEqual(result["blocks"][0]["type"], "unstyled") + + def test_consecutive_lines_separated_by_block_level(self): + md = "First para\nstill first\n\n# Heading\n\nSecond para\nstill second\n\n" + result = markdown_to_content_state(md) + self.assertEqual( + [(b["type"], b["text"]) for b in result["blocks"]], + [ + ("unstyled", "First para still first"), + ("header-one", "Heading"), + ("unstyled", "Second para still second"), + ], + ) + + def test_paragraph_before_code_block(self): + md = "First line\nsecond line\n\n```\ncode\n```\n\n" + result = markdown_to_content_state(md) + self.assertEqual( + [(b["type"], b["text"]) for b in result["blocks"]], + [("unstyled", "First line second line"), ("code-block", "code")], + ) + + +class TestImporterDirectRoundTrip(unittest.TestCase): + """Round-trip tests for block types not covered by test_exports.json fixtures.""" + + exporter = HTML(MARKDOWN_CONFIG) + + def _roundtrip(self, md: str) -> None: + content_state = markdown_to_content_state(md) + re_exported = self.exporter.render(content_state) + self.assertEqual(re_exported, md) + + def test_roundtrip_horizontal_rule(self): + self._roundtrip("---\n\n") + + def test_roundtrip_image_with_alt(self): + self._roundtrip("![alt text](http://example.com/img.png)\n\n") + + def test_roundtrip_image_without_alt(self): + self._roundtrip("![](http://example.com/img.png)\n\n") + + def test_roundtrip_code_block(self): + self._roundtrip("```\ndef foo():\n pass\n```\n\n") + + def test_roundtrip_empty_code_block(self): + self._roundtrip("```\n\n```\n\n") + + def test_roundtrip_blockquote(self): + self._roundtrip("> quoted text\n\n") + + def test_roundtrip_heading(self): + for level in range(1, 7): + prefix = "#" * level + self._roundtrip(f"{prefix} Heading\n\n") + + def test_roundtrip_nested_unordered_list(self): + self._roundtrip("- depth 0\n - depth 1\n - depth 2\n\n") + + def test_roundtrip_nested_ordered_list(self): + self._roundtrip("1. depth 0\n 1. depth 1\n 1. depth 2\n\n") + + def test_roundtrip_inline_code(self): + self._roundtrip("hello `code` world\n\n") + + def test_roundtrip_inline_strikethrough(self): + self._roundtrip("hello ~strike~ world\n\n") + + def test_roundtrip_link(self): + self._roundtrip("[link text](http://example.com)\n\n") + + def test_roundtrip_mixed_document(self): + self._roundtrip( + "# Title\n\n" + "A paragraph with **bold** and _italic_ text.\n\n" + "> A blockquote\n\n" + "- Item one\n- Item two\n\n" + "1. First\n2. Second\n\n" + "---\n\n" + ) + + +class RoundTripTestMeta(type): + """Generates exporter -> importer -> exporter round-trip test cases.""" + + def __new__(mcs, name, bases, tests): + exporter = HTML(MARKDOWN_CONFIG) + + for fixture in fixtures: + label = fixture["label"] + if label not in LOSSLESS_CASES: + continue + if "markdown" not in fixture["output"]: + continue + + test_label = label.lower().replace(" ", "_") + test_name = f"test_roundtrip_{test_label}" + + md = fixture["output"]["markdown"] + + def gen_test(markdown, html_exporter): + def test(self): + # Import the markdown. + content_state = markdown_to_content_state(markdown) + # Re-export to markdown. + re_exported = html_exporter.render(content_state) + self.assertEqual(re_exported, markdown) + + return test + + tests[test_name] = gen_test(md, exporter) + + return type.__new__(mcs, name, bases, tests) + + +class TestRoundTrip(unittest.TestCase, metaclass=RoundTripTestMeta): + """Tests that export -> import -> export produces identical markdown.""" + + +class TestImporterConfigurableMarkers(unittest.TestCase): + """Tests for configurable style markers via MarkdownImporterOptions.""" + + def test_custom_bold_marker(self): + result = markdown_to_content_state("__bold__ text\n\n", {"bold": "__"}) + block = result["blocks"][0] + self.assertEqual(block["text"], "bold text") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 0, "length": 4, "style": "BOLD"}], + ) + + def test_custom_italic_marker(self): + result = markdown_to_content_state("*italic* text\n\n", {"italic": "*"}) + block = result["blocks"][0] + self.assertEqual(block["text"], "italic text") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 0, "length": 6, "style": "ITALIC"}], + ) + + def test_custom_strikethrough_marker(self): + result = markdown_to_content_state( + "~~strike~~ text\n\n", {"strikethrough": "~~"} + ) + block = result["blocks"][0] + self.assertEqual(block["text"], "strike text") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 0, "length": 6, "style": "STRIKETHROUGH"}], + ) + + def test_custom_bold_and_italic_markers(self): + result = markdown_to_content_state( + "__bold__ and *italic*\n\n", + {"bold": "__", "italic": "*"}, + ) + block = result["blocks"][0] + self.assertEqual(block["text"], "bold and italic") + styles = sorted(block["inlineStyleRanges"], key=lambda s: s["offset"]) + self.assertEqual(styles[0], {"offset": 0, "length": 4, "style": "BOLD"}) + self.assertEqual(styles[1], {"offset": 9, "length": 6, "style": "ITALIC"}) + + def test_default_markers_still_work_without_options(self): + result = markdown_to_content_state("**bold** _italic_\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "bold italic") + styles = sorted(block["inlineStyleRanges"], key=lambda s: s["offset"]) + self.assertEqual(styles[0], {"offset": 0, "length": 4, "style": "BOLD"}) + self.assertEqual(styles[1], {"offset": 5, "length": 6, "style": "ITALIC"}) + + def test_round_trip_with_custom_bold(self): + exporter = HTML(build_markdown_config({"bold": "__"})) + md = "__bold__ text\n\n" + cs = markdown_to_content_state(md, {"bold": "__"}) + re_exported = exporter.render(cs) + self.assertEqual(re_exported, md) + + def test_round_trip_with_custom_italic(self): + exporter = HTML(build_markdown_config({"italic": "*"})) + md = "*italic* text\n\n" + cs = markdown_to_content_state(md, {"italic": "*"}) + re_exported = exporter.render(cs) + self.assertEqual(re_exported, md) + + def test_round_trip_with_custom_strikethrough(self): + exporter = HTML(build_markdown_config({"strikethrough": "~~"})) + md = "~~strike~~ text\n\n" + cs = markdown_to_content_state(md, {"strikethrough": "~~"}) + re_exported = exporter.render(cs) + self.assertEqual(re_exported, md) + + +class TestImporterCustomHtmlTags(unittest.TestCase): + """Tests for custom HTML tag mappings via MarkdownImporterOptions.""" + + def test_custom_html_tag_mapping(self): + result = markdown_to_content_state( + "hello tag\n\n", + {"html_style_tags": {"custom": "CUSTOM"}}, + ) + block = result["blocks"][0] + self.assertEqual(block["text"], "hello tag") + self.assertEqual( + block["inlineStyleRanges"], + [{"offset": 6, "length": 3, "style": "CUSTOM"}], + ) + + def test_custom_html_tag_extends_defaults(self): + result = markdown_to_content_state( + "under and tag\n\n", + {"html_style_tags": {"custom": "CUSTOM"}}, + ) + block = result["blocks"][0] + self.assertEqual(block["text"], "under and tag") + styles = sorted(block["inlineStyleRanges"], key=lambda s: s["offset"]) + self.assertEqual(styles[0], {"offset": 0, "length": 5, "style": "UNDERLINE"}) + self.assertEqual(styles[1], {"offset": 10, "length": 3, "style": "CUSTOM"}) + + def test_unknown_tag_without_mapping_is_literal(self): + result = markdown_to_content_state("hello tag\n\n") + block = result["blocks"][0] + self.assertEqual(block["text"], "hello tag") + self.assertEqual(block["inlineStyleRanges"], []) + + +class TestImporterOrderedListItemDelimiters(unittest.TestCase): + """Tests for ordered list with ) delimiter.""" + + def test_ordered_list_with_paren_delimiter(self): + result = markdown_to_content_state("1) First\n2) Second\n\n") + blocks = result["blocks"] + self.assertEqual(len(blocks), 2) + self.assertEqual(blocks[0]["type"], "ordered-list-item") + self.assertEqual(blocks[0]["text"], "First") + self.assertEqual(blocks[1]["type"], "ordered-list-item") + self.assertEqual(blocks[1]["text"], "Second") + + +class TestImporterPublicAPI(unittest.TestCase): + """Tests that the importer is accessible from the top-level package.""" + + def test_import_from_top_level(self): + from draftjs_exporter import markdown_to_content_state as top_level_fn + + result = top_level_fn("hello\n\n") + self.assertEqual(len(result["blocks"]), 1) + self.assertEqual(result["blocks"][0]["text"], "hello") + + def test_import_options_from_top_level(self): + from draftjs_exporter import MarkdownImporterOptions as Options + + opts: Options = {"bold": "__"} + result = markdown_to_content_state("__bold__\n\n", opts) + self.assertEqual( + result["blocks"][0]["inlineStyleRanges"], + [{"offset": 0, "length": 4, "style": "BOLD"}], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/strategies.py b/tests/strategies.py index f3d3001..3a1424d 100644 --- a/tests/strategies.py +++ b/tests/strategies.py @@ -151,6 +151,107 @@ def content_states(draw: st.DrawFn, max_blocks: int = 6) -> dict[str, Any]: return {"entityMap": entity_map, "blocks": block_list} +# Markdown-syntax fragments used to exercise every parser branch in the +# importer (styles, links, images, inline HTML tags, block-level syntax, +# escapes). Combined randomly they produce adversarial inputs that +# stress edge cases the hand-written unit tests don't cover: unterminated +# links, mismatched markers, malformed HTML tags, deeply nested syntax. +MARKDOWN_FRAGMENTS = [ + # Style markers (open/close asymmetries, nesting, intraword). + "**", + "__", + "*", + "_", + "~", + "~~", + "`", + "*a", + "_b", + "**c", + "~d", + "`e", + # Link / image syntax: well-formed, broken, and exotic schemes. + "[", + "]", + "(", + ")", + "![", + "](http://", + "![alt](", + "](url)", + "http://example.com", + "wagtail://core.Page.89", + ' "title"', + " 'title'", + '"', + "'", + # Inline HTML tags: known style mappings and unknown tags. + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "<", + ">", + "", + # Block-level prefixes. + "#", + "##", + "###", + "####", + "#####", + "######", + "- ", + "* ", + "+ ", + "1. ", + "2. ", + "1) ", + "2) ", + "> ", + ">", + "---", + "***", + "___", + "```", + "```python", + "~~~", + "~~~python", + # Escapes (CommonMark set, sampled). + "\\", + "\\*", + "\\_", + "\\#", + "\\[", + "\\]", + "\\(", + "\\)", + "\\`", + "\\~", + # Soft line breaks and plain text fillers. + " ", + "\n", + "\n\n", + "text", + "hello", + "world", + "abc", +] + +markdown_text = st.lists( + st.sampled_from(MARKDOWN_FRAGMENTS), min_size=0, max_size=30 +).map("".join) +"""Random Markdown built from syntax fragments for importer crash-safety.""" + + # Common HTML/JS injection fragments, used to check that block text and # entity `data` can never break out of the tag/attribute they're rendered # into (see docs/SECURITY.md#tampering). Each fragment contains at least one diff --git a/tests/test_properties.py b/tests/test_properties.py index d5ad964..800878b 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -27,7 +27,8 @@ from draftjs_exporter.dom import DOM from draftjs_exporter.html import HTML, ExporterConfig from draftjs_exporter.markdown import CONFIG as MARKDOWN_CONFIG -from tests.strategies import content_states, dangerous_content_states +from draftjs_exporter.markdown.importer import markdown_to_content_state +from tests.strategies import content_states, dangerous_content_states, markdown_text from tests.test_entities import link CONFIG: ExporterConfig = { @@ -166,3 +167,66 @@ def test_dangerous_fragments_never_become_markup(self, content_state): self.assertEqual(links[0].get_text(), text) else: self.assertEqual(parsed.get_text(), text) + + +class TestImporterCrashSafety(unittest.TestCase): + """`markdown_to_content_state` must not raise on adversarial input. + + The importer is the reverse direction of the export pipeline: it walks + untrusted Markdown text character-by-character to produce ranges. + Unlike ContentState (which Draft.js editors produce in a constrained + shape), Markdown is often hand-written or generated by other tools, + so the importer must degrade gracefully on adversarial inputs - + unterminated links, mismatched style markers, malformed HTML tags, + deeply nested syntax - rather than crash or produce malformed output. + + This mirrors `TestRenderCrashSafety` for the export direction. + """ + + @given(markdown=markdown_text) + @settings(deadline=None) + # Pinned examples of known-tricky inputs so they always run even if + # Hypothesis's random search would not stumble on them again. + @example(markdown="") + @example(markdown="\n") + @example(markdown="[unclosed") + @example(markdown="*unclosed") + @example(markdown="**unclosed") + @example(markdown="") + @example(markdown="```\nunclosed code fence") + @example(markdown="[text](unclosed") + @example(markdown="unclosed tag") + @example(markdown="\\") # Trailing backslash at end of input. + @example(markdown="![alt](unclosed") + @example(markdown="> unclosed blockquote") + def test_importer_never_raises(self, markdown): + result = markdown_to_content_state(markdown) + + # Result is always well-formed ContentState, regardless of input. + self.assertIsInstance(result, dict) + self.assertIsInstance(result.get("blocks"), list) + self.assertIsInstance(result.get("entityMap"), dict) + + for block in result["blocks"]: + self.assertIsInstance(block.get("key"), str) + self.assertIsInstance(block.get("text"), str) + self.assertIsInstance(block.get("type"), str) + self.assertIsInstance(block.get("depth"), int) + self.assertIsInstance(block.get("inlineStyleRanges"), list) + self.assertIsInstance(block.get("entityRanges"), list) + + # Style ranges have valid offset/length/style fields. + for style_range in block["inlineStyleRanges"]: + self.assertIsInstance(style_range.get("offset"), int) + self.assertIsInstance(style_range.get("length"), int) + self.assertIsInstance(style_range.get("style"), str) + + # Entity ranges have valid offset/length/key fields. + for entity_range in block["entityRanges"]: + self.assertIsInstance(entity_range.get("offset"), int) + self.assertIsInstance(entity_range.get("length"), int) + self.assertIsInstance(entity_range.get("key"), int) + + # Every entity range's key resolves in the entity map. + for entity_range in block["entityRanges"]: + self.assertIn(str(entity_range["key"]), result["entityMap"])