From 540ec7a0fc724905bf1be628125924f4269cbf64 Mon Sep 17 00:00:00 2001 From: dybucc <149513579+dybucc@users.noreply.github.com> Date: Sun, 7 Dec 2025 09:06:56 +0100 Subject: [PATCH 1/3] perf: Rework extractor and fn signature parser The extractor needed some fine tuning to only pick up top level docstrings instead of (possibly wrong) locally scoped docstrings outside the reach of library users. The function signature parser should now be a tad bit more efficient. It performs no operations on the whole array that gets passed beyond those strictly required by however as many elements (lines) are part of the function signature. Prior to this, there were a bunch of `enumerate` and `join` calls that wouldn't exactly be efficient for, say, arrays containing thousands of lines for some of the source code. --- docs/genhtml.py | 92 +++++++++---------- docs/typlodocus/extractor.typ | 6 +- docs/typlodocus/parser.typ | 164 ++++++++++++++++++---------------- 3 files changed, 138 insertions(+), 124 deletions(-) diff --git a/docs/genhtml.py b/docs/genhtml.py index c0e2d9f6..cc59627c 100755 --- a/docs/genhtml.py +++ b/docs/genhtml.py @@ -37,7 +37,7 @@ def generate_output_from_typst(code, output_path, format="svg", cetz_path=DEFAUL if result.returncode != 0: print(f"Error generating {format.upper()}: {result.stderr}") return False - + return True except Exception as e: print(f"Exception generating {format.upper()}: {e}") @@ -58,7 +58,7 @@ def convert_typst_to_html_content(text): """ tmp.write(typst_content) tmp.flush() - + with tempfile.NamedTemporaryFile(suffix='.html', delete=True) as html_tmp: result = subprocess.run( ["typst", "compile", tmp.name, html_tmp.name, @@ -69,11 +69,11 @@ def convert_typst_to_html_content(text): if result.returncode != 0: print(f"Error generating HTML: {result.stderr}") return None - + # Read the generated HTML and extract body content with open(html_tmp.name, 'r') as f: html_content = f.read() - + # Extract content between tags import re body_match = re.search(r']*>(.*?)', html_content, re.DOTALL) @@ -81,7 +81,7 @@ def convert_typst_to_html_content(text): return body_match.group(1).strip() else: return html_content - + except Exception as e: print(f"Exception converting Typst to HTML: {e}") return None @@ -91,14 +91,14 @@ def html_to_mdx(html_content): """Convert HTML content to MDX-compatible format.""" if not html_content: return "" - + # Clean up the HTML for MDX compatibility: # - Remove style attributes # - Convert self closing to closing html_content = re.sub(r'\s+style="[^"]*"', '', html_content) html_content = re.sub(r'
', '
', html_content) html_content = re.sub(r'
', '
', html_content) - + return html_content.strip() @@ -125,10 +125,10 @@ def generate_mdx_file(func_data, output_path, cetz_path=DEFAULT_CETZ_VERSION): comment = func_data.get("comment", {}) signature = func_data.get("signature", {}) func_name = signature.get("name", "unknown") if signature else "unknown" - + # Start MDX content mdx_lines = [] - + # Build parameters object for Function component parameters = {} arguments = comment.get("arguments", []) @@ -142,24 +142,24 @@ def generate_mdx_file(func_data, output_path, cetz_path=DEFAULT_CETZ_VERSION): default_value = default_value[2:] param_info["default"] = default_value parameters[name] = param_info - + # Function component import json parameters_json = json.dumps(parameters) mdx_lines.append(f''.replace('null', 'undefined')) - + # Convert main text through Typst -> HTML -> MDX main_text = comment.get("text", "") if main_text: # Remove example blocks from main text for separate processing text_without_examples = re.sub(r'```(?:typc?\s+)?(?:example|example-vertical)\s*\n.*?```', '', main_text, flags=re.DOTALL) - + html_content = convert_typst_to_html_content(text_without_examples) if html_content: mdx_content = html_to_mdx(html_content) mdx_lines.append(mdx_content) mdx_lines.append('') - + # Add example blocks examples = extract_example_blocks(main_text) if examples: @@ -168,7 +168,7 @@ def generate_mdx_file(func_data, output_path, cetz_path=DEFAULT_CETZ_VERSION): mdx_lines.append(example_code) mdx_lines.append('```') mdx_lines.append('') - + # Add parameter documentation if arguments: for arg in arguments: @@ -176,7 +176,7 @@ def generate_mdx_file(func_data, output_path, cetz_path=DEFAULT_CETZ_VERSION): types = format_types(arg.get("types", [])) default_value = arg.get("default-value", "") text = arg.get("text", "") - + if name: param_attrs = [f'name="{name}"'] if types: @@ -185,19 +185,19 @@ def generate_mdx_file(func_data, output_path, cetz_path=DEFAULT_CETZ_VERSION): if default_value.startswith("= "): default_value = default_value[2:] param_attrs.append(f'default_value="{escape_html_attr(default_value)}"') - + mdx_lines.append(f'') - + if text: # Convert parameter description through Typst -> HTML -> MDX html_content = convert_typst_to_html_content(text) if html_content: mdx_content = html_to_mdx(html_content) mdx_lines.append(mdx_content) - + mdx_lines.append('') mdx_lines.append('') - + try: with open(output_path, 'w') as f: f.write('\n'.join(mdx_lines)) @@ -211,33 +211,33 @@ def extract_example_blocks(text): """Extract all example code blocks from text with proper indentation.""" if not text: return [] - + # Find all ```example or ```example-vertical blocks pattern = r'```(?:typc?\s+)?(?:example|example-vertical)\s*\n(.*?)```' matches = re.findall(pattern, text, re.DOTALL) - + cleaned_blocks = [] for match in matches: lines = match.split('\n') if not lines: continue - + # Remove leading/trailing empty lines while lines and not lines[0].strip(): lines.pop(0) while lines and not lines[-1].strip(): lines.pop() - + if not lines: continue - + # Find the minimum indentation (excluding empty lines) min_indent = float('inf') for line in lines: if line.strip(): # Skip empty lines indent = len(line) - len(line.lstrip()) min_indent = min(min_indent, indent) - + # Remove the common indentation from all lines if min_indent != float('inf') and min_indent > 0: dedented_lines = [] @@ -249,7 +249,7 @@ def extract_example_blocks(text): cleaned_blocks.append('\n'.join(dedented_lines)) else: cleaned_blocks.append('\n'.join(lines)) - + return cleaned_blocks @@ -259,7 +259,7 @@ def main(): description='Generate MDX documentation from CeTZ docs with optional SVG examples' ) parser.add_argument( - 'json_file', + 'json_file', nargs='?', help='Path to docs.json file (reads from stdin if not provided)' ) @@ -281,16 +281,16 @@ def main(): const='svg_output', help='Generate SVG files from examples. Optional: specify output directory (default: svg_output)' ) - + args = parser.parse_args() - + # Load JSON data if args.json_file: json_file = Path(args.json_file) if not json_file.exists(): print(f"Error: {json_file} not found.") sys.exit(1) - + with open(json_file, 'r') as f: data = json.load(f) else: @@ -300,32 +300,32 @@ def main(): except: print("Error: Could not read from stdin.") sys.exit(1) - + mdx_dir = Path(args.output) mdx_dir.mkdir(exist_ok=True) svg_dir = None if args.svg: svg_dir = Path(args.svg) svg_dir.mkdir(exist_ok=True) - + if isinstance(data, list) and len(data) > 0: data = data[0] - + # Process each file in the JSON for file_path, functions in data.items(): print(f"\nProcessing {file_path}") - + # Create directory structure matching source # e.g., "src/draw/shapes.typ" -> "draw/shapes/" path_parts = file_path.replace("src/", "").replace(".typ", "").split("/") - + current_mdx_dir = mdx_dir for part in path_parts: current_mdx_dir = current_mdx_dir / part current_mdx_dir.mkdir(exist_ok=True) - + file_base = file_path.replace("src/", "").replace("/", "_").replace(".typ", "") - + # Track functions for combined file generation function_names = [] for func_data in functions: @@ -333,39 +333,39 @@ def main(): func_name = signature.get("name", "unknown") if signature else "unknown" comment = func_data.get("comment", {}) text = comment.get("text", "") - + if not func_name.startswith("_"): function_names.append(func_name) - + # Generate MDX files mdx_filename = f"{func_name}.mdx" mdx_path = current_mdx_dir / mdx_filename - + if generate_mdx_file(func_data, mdx_path, args.cetz): print(f"[ OK] Generated MDX: {'/'.join(path_parts)}/{mdx_filename}") else: print(f"[ERROR] Failed to generate MDX: {'/'.join(path_parts)}/{mdx_filename}") - + # Generate SVGs if svg_dir: examples = extract_example_blocks(text) - + if examples: for i, example_code in enumerate(examples): svg_filename = f"{file_base}_{func_name}_{i}.svg" svg_path = svg_dir / svg_filename - + if generate_output_from_typst(example_code, svg_path, "svg", args.cetz): print(f"[ OK] Generated SVG: {svg_filename} ({i+1}/{len(examples)})") else: print(f"[ERROR] Failed to generate SVG: {svg_filename}") - + # Generate combined MDX file if function_names: #combined_filename = f"{path_parts[-1]}-combined.mdx" combined_filename = f"-combined.mdx" combined_path = current_mdx_dir / combined_filename - + combined_imports = [] combined_lines = [] for func_name in function_names: @@ -373,7 +373,7 @@ def main(): combined_imports.append(f'import {import_name} from "./{func_name}.mdx"') combined_lines.append(f'## {func_name}') combined_lines.append(f'<{import_name}/>\n') - + try: with open(combined_path, 'w') as f: f.write('\n'.join(combined_imports) + '\n\n' + '\n'.join(combined_lines) + '\n') diff --git a/docs/typlodocus/extractor.typ b/docs/typlodocus/extractor.typ index 69adc7f9..77885525 100644 --- a/docs/typlodocus/extractor.typ +++ b/docs/typlodocus/extractor.typ @@ -14,13 +14,13 @@ let i = 0 while i < lines.len() { - let line = lines.at(i).trim() + let line = lines.at(i).trim(at: end) if line.starts-with("///") { in-comment = true - current-comment += line.slice(3) + "\n" + current-comment += line.slice(3).trim() + "\n" } else if in-comment { - if line.starts-with(regex("\#?let\s+")) { + if line.starts-with(regex(`#let\s+`.text)) { let function = parser.parse-function-signature(lines.slice(i)) comments.push(( comment: parser.parse-docstring(current-comment), diff --git a/docs/typlodocus/parser.typ b/docs/typlodocus/parser.typ index 6092e2f8..033b9fd9 100644 --- a/docs/typlodocus/parser.typ +++ b/docs/typlodocus/parser.typ @@ -15,10 +15,13 @@ } else if char == "\"" and not escape-next { in-str = not in-str } else if not in-str { - if char == "(" { depth += 1 } - else if char == ")" { depth -= 1 } - else if char == "[" { in-content = true } - else if char == "," and depth == 0 { + if char == "(" { + depth += 1 + } else if char == ")" { + depth -= 1 + } else if char == "[" { + in-content = true + } else if char == "," and depth == 0 { arguments.push(( name: current-arg.trim(), default-value: current-default.trim(), @@ -54,84 +57,95 @@ } #let parse-function-signature(lines) = { - let let-re = regex("^#?let\s+") - let identifier-re = regex("^([_a-zA-Z]+[_\w-]*)") - - let combined-line = lines.join(" ") - combined-line = combined-line.trim() - - // Cut off #let or let - let let-m = combined-line.match(let-re) - if let-m == none { - return none + let ident = lines + .first() + .replace( + regex(`#let\s+([_[:alnum:]-]+).*`.text), + match => match.captures.first(), + count: 1, + ) + + lines.first() = lines + .first() + .trim( + regex(`#let\s+[_[:alnum:]-]+\(?`.text), + at: start, + repeat: false, + ) + + if lines.first().starts-with(regex(`\s*=`.text)) { + return ( + name: ident, + arguments: (), + ) } - combined-line = combined-line.slice(let-m.end) - - // Parse function name - let identifier-m = combined-line.match(identifier-re) - if identifier-m == none { - return none - } - - let name = identifier-m.captures.at(0) - combined-line = combined-line.slice(identifier-m.end) - - // Check if we have an argument list - let paren-match = combined-line.match(regex("^\s*\(")) - if paren-match == none { - return none - } - - // Parse the entire argument list - let args-start = paren-match.end - let paren-depth = 1 - let in-str = false - let in-content = false - let escape-next = false - let args-end = none - - // Find the argument list bounds - for ((i, char)) in combined-line.slice(paren-match.end).codepoints().enumerate() { - if escape-next { - escape-next = false - } else if in-content { - if char == "]" { - in-content = false + + let param-end-line = -1 + let param-end-char-pos + let escape-hit = false + let delim-stack = ("(",) + let delim-lut = ( + start: ( + "\"", + "(", + "[", + ), + end: ( + "\"", + ")", + "]", + ), + ) + + for line in lines { + param-end-char-pos = -1 + + for char in line.codepoints() { + if char == "\\" { + escape-hit = true + continue } - } else if char == "\\" { - escape-next = true - } else if char == "\"" { - in-str = not in-str - } else if not in-str { - if char == "(" { - paren-depth += 1 - } else if char == ")" { - paren-depth -= 1 - if paren-depth == 0 { - args-end = args-start + i - break + if escape-hit { + escape-hit = false + continue + } + + let stack-top = delim-stack.last() + if stack-top == delim-lut.start.first() { + if char == delim-lut.end.first() { delim-stack.pop() } + } else if stack-top == delim-lut.start.last() { + if char == delim-lut.end.last() { delim-stack.pop() } + } else { + if char in delim-lut.start { + delim-stack.push(char) + } else if char in delim-lut.end { + delim-stack.pop() } - } else if char == "[" { - in-content = true } + + param-end-char-pos += 1 + if delim-stack.len() == 0 { break } } - } - - if args-end == none { - return none - } - - // Parse arguments - let arguments = () - let args-str = combined-line.slice(args-start, args-end) - if args-str.trim() != "" { - arguments = parse-argument-list(args-str) + param-end-line += 1 + if delim-stack.len() == 0 { break } } - + + let param-span = lines + .slice(0, param-end-line + 1) + .enumerate() + .fold("", (accum, (line-num, line)) => { + if line-num == param-end-line { + accum + " " + line.slice(0, param-end-char-pos) + } else { + accum + " " + line + } + }) + .trim() + return ( - name: name, - arguments: arguments + name: ident, + arguments: parse-argument-list(param-span), ) } @@ -192,7 +206,7 @@ in-result = true result.push(( type: result-m.captures.at(0).trim(), - text: result-m.captures.at(1).trim() + text: result-m.captures.at(1).trim(), )) } else { text += line + "\n" From 93e3c59e90a7527b942a0fad6d2f653823dbe700 Mon Sep 17 00:00:00 2001 From: dybucc <149513579+dybucc@users.noreply.github.com> Date: Sun, 21 Dec 2025 16:52:15 +0100 Subject: [PATCH 2/3] perf: Finish fn signature and parameter parser Following the plan in the PR this branch is part of, the function signature parser rework is done. The conclusion is that the signature gets parsed now all in a single pass instead of having to separately consider the function and parameter span, and then parse the parameter span. The gains in efficency on the prior commit are still kept, so no operations are performed on the whole array; only those elements of the array with all the source lines corresponding to the function are parsed. There is, though, a small performance loss in the fact that a custom runtime regex is built to more accurately have the whitespace indentation of some parameters' default values be represented in the final output. Because the project doesn't use an autoformatter, if some lines contain an indentation that is beyond a single multiple of 2 (according to the .editorconfig file) then the parser will correctly recognize that the contents of the named argument, if the named argument is not a string or Typst content type, should be deindented by however as much whitespace was detected at the start of the parameter name. Now the parameter parser also correctly implements function default arguments. --- docs/typlodocus/parser.typ | 175 +++++++++++++++++++------------------ 1 file changed, 88 insertions(+), 87 deletions(-) diff --git a/docs/typlodocus/parser.typ b/docs/typlodocus/parser.typ index 033b9fd9..cb9586e7 100644 --- a/docs/typlodocus/parser.typ +++ b/docs/typlodocus/parser.typ @@ -1,61 +1,3 @@ -#let parse-argument-list(string) = { - let arguments = () - let current-arg = "" - let current-default = "" - let named-arg = false - - let depth = 0 - let in-str = false - let in-content = false - let escape-next = false - - for char in string.codepoints() { - if in-content and not escape-next { - if char == "]" { in-content = false } - } else if char == "\"" and not escape-next { - in-str = not in-str - } else if not in-str { - if char == "(" { - depth += 1 - } else if char == ")" { - depth -= 1 - } else if char == "[" { - in-content = true - } else if char == "," and depth == 0 { - arguments.push(( - name: current-arg.trim(), - default-value: current-default.trim(), - has-default: named-arg, - )) - - current-arg = "" - current-default = "" - named-arg = false - continue - } else if char == ":" and depth == 0 { - named-arg = true - continue - } - } - - if named-arg { - current-default += char - } else { - current-arg += char - } - } - - if current-arg.trim() != "" { - arguments.push(( - name: current-arg.trim(), - default-value: current-default.trim(), - has-default: named-arg, - )) - } - - return arguments -} - #let parse-function-signature(lines) = { let ident = lines .first() @@ -80,8 +22,6 @@ ) } - let param-end-line = -1 - let param-end-char-pos let escape-hit = false let delim-stack = ("(",) let delim-lut = ( @@ -89,16 +29,46 @@ "\"", "(", "[", + "{", ), end: ( "\"", ")", "]", + "}", ), ) + let arguments = () + let current-arg = ( + name: "", + default-value: "", + has-default: false, + ) + let marker-lut = ( + ":", + ",", + ) + let in-named-arg = false + let indent-ws + let indent-re = regex(`^(\s+).*`.text) for line in lines { - param-end-char-pos = -1 + if delim-stack.len() == 1 { + let tmp = line.match(indent-re) + if tmp != none { indent-ws = tmp.captures.first().len() } + + line = line.trim(at: start) + } + if in-named-arg { + let stack-top = delim-stack.last() + + if stack-top == delim-lut.start.at(1) or stack-top == delim-lut.start.at(3) { + let tmp = line.match(regex(`^(\s{`.text + str(indent-ws) + `}).*`.text)) + if tmp != none { line = line.slice(2) } + } + + current-arg.default-value += "\n" + } for char in line.codepoints() { if char == "\\" { @@ -111,41 +81,72 @@ } let stack-top = delim-stack.last() - if stack-top == delim-lut.start.first() { - if char == delim-lut.end.first() { delim-stack.pop() } - } else if stack-top == delim-lut.start.last() { - if char == delim-lut.end.last() { delim-stack.pop() } - } else { - if char in delim-lut.start { - delim-stack.push(char) - } else if char in delim-lut.end { - delim-stack.pop() + + if char in delim-lut.start or char in delim-lut.end { + if stack-top == delim-lut.start.at(0) { + if char == delim-lut.end.at(0) { delim-stack.pop() } + if in-named-arg { current-arg.default-value += char } + } else if stack-top == delim-lut.start.at(2) { + if char == delim-lut.end.at(2) { delim-stack.pop() } + if in-named-arg { current-arg.default-value += char } + } else if stack-top == delim-lut.start.at(3) { + if char == delim-lut.end.at(3) { delim-stack.pop() } + if in-named-arg { current-arg.default-value += char } + } else { + if char in delim-lut.start { + delim-stack.push(char) + if in-named-arg { current-arg.default-value += char } + } else if char in delim-lut.end { + if in-named-arg and delim-stack.len() > 1 { current-arg.default-value += char } + delim-stack.pop() + } } - } - param-end-char-pos += 1 - if delim-stack.len() == 0 { break } + if delim-stack.len() == 0 { + if current-arg.name.len() != 0 { + current-arg.name = current-arg.name.trim() + if current-arg.has-default { + current-arg.default-value = current-arg.default-value.trim() + } + + arguments.push(current-arg) + } + break + } + } else if delim-stack.len() == 1 { + if char not in marker-lut and not in-named-arg { + current-arg.name += char + } else if char == marker-lut.first() { + in-named-arg = true + current-arg.has-default = true + } else if char == marker-lut.last() { + in-named-arg = false + + current-arg.name = current-arg.name.trim() + if current-arg.has-default { + current-arg.default-value = current-arg.default-value.trim() + } + arguments.push(current-arg) + + current-arg = ( + name: "", + default-value: "", + has-default: false, + ) + } else if in-named-arg { + current-arg.default-value += char + } + } else if in-named-arg { + current-arg.default-value += char + } } - param-end-line += 1 if delim-stack.len() == 0 { break } } - let param-span = lines - .slice(0, param-end-line + 1) - .enumerate() - .fold("", (accum, (line-num, line)) => { - if line-num == param-end-line { - accum + " " + line.slice(0, param-end-char-pos) - } else { - accum + " " + line - } - }) - .trim() - return ( name: ident, - arguments: parse-argument-list(param-span), + arguments: arguments, ) } From 8a2e9fbc1b059f5c6f532b6501178c1e5f5f62aa Mon Sep 17 00:00:00 2001 From: dybucc <149513579+dybucc@users.noreply.github.com> Date: Sat, 3 Jan 2026 19:32:44 +0100 Subject: [PATCH 3/3] refactor: keep working on wip docstring parser The parser is mostly done. This commit will be ammended/fixed up once the docstring parser is completely done, so more details can be found in the accompanying PR. --- docs/typlodocus/extractor.typ | 7 +- docs/typlodocus/parser.typ | 324 ++++++++++++++++++++++++++++++++++ 2 files changed, 327 insertions(+), 4 deletions(-) diff --git a/docs/typlodocus/extractor.typ b/docs/typlodocus/extractor.typ index 77885525..44cd70be 100644 --- a/docs/typlodocus/extractor.typ +++ b/docs/typlodocus/extractor.typ @@ -18,13 +18,12 @@ if line.starts-with("///") { in-comment = true - current-comment += line.slice(3).trim() + "\n" + current-comment += line.slice(3) + "\n" } else if in-comment { if line.starts-with(regex(`#let\s+`.text)) { - let function = parser.parse-function-signature(lines.slice(i)) comments.push(( - comment: parser.parse-docstring(current-comment), - signature: function, + comment: parser.parse-docstring-alt(current-comment), + signature: parser.parse-function-signature(lines.slice(i)), )) } diff --git a/docs/typlodocus/parser.typ b/docs/typlodocus/parser.typ index cb9586e7..55f10c02 100644 --- a/docs/typlodocus/parser.typ +++ b/docs/typlodocus/parser.typ @@ -1,3 +1,18 @@ +// TODO: see if we can perform documentation diagnostics: +// (1) the diagnostics would include: +// - default argument type validation (would have to look into how do +// this with custom types but it's feasible,) +// - argument name validation (feasible considering we preprocess the +// function signature prior to the docstring, though it would +// require further rework,) +// (2) the diagnostics should panic such that querying fails and the error +// is reported during the doc building process (to better diagnose +// issues whenever this happens in CI, it would be best if the errors +// were also written to some log file or alternatively replaced the +// query operation, and the Python script reported the actual error +// and wrote to file if a specific json schema in the expected query +// results is detected.) + #let parse-function-signature(lines) = { let ident = lines .first() @@ -150,6 +165,315 @@ ) } +// TODO: change the `symbol` key in the corresponding dictionaries with the +// `token` key. +// TODO: see if repetitive code blocks at different stages of the parsing +// process can be refactored into a set of functions that handle such +// functionality in an isolated manner. +// NOTE: a current limitation of this parser is that Typst code that would get +// parsed as some output 'A', will get parsed as an output 'B' instead +// because whitespace removal is performed eagerly at the start of all +// lines. As an example, take Typst's lists; They use whitespace to denote +// offsetting from top-level items, but this won't be the case after this +// gets parsed. +#let parse-docstring-alt(string) = { + let indent-ws = 0 + let example-fields = ( + symbol: ( + open: "```example", + close: "```", + ), + inside: false, + ) + let parameter-fields = ( + symbol: ( + param: "-", + param-list-open: "(", + param-list-close: ")", + ), + inside: false, + inside-param-list: false, + re: ( + line: regex( + // TODO: rework the part of the regex parsing the optional type list to + // make it accept only comma-optional separated sequences. + `^-(\s+)((?:\.{2})?[[:alnum:]_-]+)((?:\s+)(?:\()([[:alnum:][:blank:]-_\.,]+)(?:\)))?((?:\s+=\s+)([^:]+))?(:\s+(.+))?`.text, + ), + parameter-list: regex(`([[:alnum:]-_\.]+)(?:,){0,1}`.text), + default-param: regex(`\)(\s+=\s+([^:]+))?:(?:\s*)?(.*)?`.text), + ), + indent-ws: 0, + ) + let result-fields = ( + symbol: "->", + inside: false, + re: regex(`^->(\s+)\(?([[:alnum:]-_]+)\)?((?:\s+)(.+))?`.text), + indent-ws: 0, + ) + // NOTE: this matches the empty string on non-matching haystacks, so it's + // never `none`. + let comment-ws-re = regex(`^([[:blank:]]*).*`.text) + let arguments = () + let result = () + let text = () + // NOTE: this is meant to store the contents of (possibly) parameter + // documentation in case it turns out to be Typst native list syntax + // instead of a multiline parameter type list. + let tmp-buffer = none + + for line in string.split("\n") { + if not (example-fields.inside or parameter-fields.inside or result-fields.inside) { + indent-ws = line.match(comment-ws-re).captures.first().len() + line = line.slice(indent-ws) + + if line.len() == 0 and text.last().len() == 0 { + continue + } else if line.starts-with(example-fields.symbol.open) { + example-fields.inside = true + text.push(line.trim(at: end)) + } else if line.starts-with(result-fields.symbol) { + let re-result = line.match(result-fields.re) + if re-result != none { + result-fields.indent-ws = re-result.captures.first().len() + result-fields.inside = true + + result.push(( + type: re-result.captures.at(1), + text: if re-result.captures.at(3) != none { + (re-result.captures.at(3).trim(at: end),) + } else { + () + }, + )) + + continue + } + } else if line.starts-with(parameter-fields.symbol.param) { + let param = line.match(parameter-fields.re.line) + if param != none { + parameter-fields.indent-ws = param.captures.first().len() + parameter-fields.inside = true + + if ( + param.captures.at(2) == none + and param.captures.at(3) == none + and param.captures.at(4) == none + ) { tmp-buffer = (line,) } + + arguments.push(( + name: param.captures.at(1), + types: if param.captures.at(3) != none { + param + .captures + .at(3) + .matches(parameter-fields.re.parameter-list) + .map(it => it.captures.first()) + } else { + () + }, + default-value: if param.captures.at(5) != none { param.captures.at(5) } else { none }, + text: if param.captures.at(7) != none { + (param.captures.at(7).trim(at: end),) + } else { + () + }, + )) + + continue + } + } else { + text.push(line.trim(at: end)) + } + } else if example-fields.inside { + let tmp-ws = line.match(comment-ws-re).captures.first().len() + line = line.slice(tmp-ws) + + if line.starts-with(example-fields.symbol.close) { example-fields.inside = false } + text.push(line) + } else if parameter-fields.inside { + let tmp-ws = line.match(comment-ws-re).captures.first().len() + line = line.slice(tmp-ws) + + if tmp-ws < indent-ws + parameter-fields.indent-ws + parameter-fields.symbol.param.len() { + parameter-fields.inside = false + indent-ws = tmp-ws + + if tmp-buffer != none { + arguments.pop() + for buf in tmp-buffer { text.push(buf) } + + tmp-buffer = none + } + + if line.starts-with(result-fields.symbol) { + let re-result = line.match(result-fields.re) + if re-result != none { + result-fields.indent-ws = re-result.captures.first().len() + result-fields.inside = true + + result.push(( + type: re-result.captures.at(1), + text: if re-result.captures.at(3) != none { + (re-result.captures.at(3).trim(at: end),) + } else { + () + }, + )) + } + } else if line.starts-with(parameter-fields.symbol.param) { + let param = line.match(parameter-fields.re.line) + if param != none { + parameter-fields.indent-ws = param.captures.first().len() + parameter-fields.inside = true + + if ( + param.captures.at(2) == none + or param.captures.at(3) == none + or param.captures.at(4) == none + ) { tmp-buffer = (line,) } + + arguments.push(( + name: param.captures.at(1), + types: if param.captures.at(3) != none { + param + .captures + .at(3) + .matches(parameter-fields.re.parameter-list) + .map(it => it.captures.first()) + } else { + () + }, + default-value: if param.captures.at(5) != none { param.captures.at(5) } else { none }, + text: if param.captures.at(7) != none { + (param.captures.at(7).trim(at: end),) + } else { + () + }, + )) + } + } else { + if text.last().len() == 0 and line.len() == 0 { continue } + text.push(line.trim(at: end)) + } + } else { + if parameter-fields.inside-param-list { + if line.starts-with(parameter-fields.symbol.param-list-close) { + parameter-fields.inside-param-list = false + + let result = line.match(parameter-fields.re.default-param) + if result != none { arguments.last().text.push(result.captures.at(1)) } + } else { + let result = line.match(parameter-fields.re.parameter-list) + if result != none { arguments.last().types.push(result.captures.first()) } + } + } else if arguments.last().types.len() == 0 { + if tmp-buffer.len() == 1 and line.first() == parameter-fields.symbol.param-list-open { + parameter-fields.inside-param-list = true + } else { + tmp-buffer.push(line) + } + } else { + arguments.last().text.push(line.trim(at: end)) + } + } + } else if result-fields.inside { + let tmp-ws = line.match(comment-ws-re).captures.first().len() + line = line.slice(tmp-ws) + + if tmp-ws < indent-ws + result-fields.indent-ws + result-fields.symbol.len() { + result-fields.inside = false + indent-ws = tmp-ws + + if line.starts-with(result-fields.symbol) { + let re-result = line.match(result-fields.re) + if re-result != none { + result-fields.indent-ws = re-result.captures.first().len() + result-fields.inside = true + + result.push(( + type: re-result.captures.at(1), + text: if re-result.captures.at(3) != none { + (re-result.captures.at(3).trim(at: end),) + } else { + () + }, + )) + } + } else if line.starts-with(parameter-fields.symbol.param) { + let param = line.match(parameter-fields.re.line) + if param != none { + parameter-fields.indent-ws = param.captures.first().len() + parameter-fields.inside = true + + if ( + param.captures.at(2) == none + or param.captures.at(3) == none + or param.captures.at(4) == none + ) { tmp-buffer = (line,) } + + arguments.push(( + name: param.captures.at(1), + types: if param.captures.at(3) != none { + param + .captures + .at(3) + .matches(parameter-fields.re.parameter-list) + .map(it => it.captures.first()) + } else { + () + }, + default-value: if param.captures.at(5) != none { param.captures.at(5) } else { none }, + text: if param.captures.at(7) != none { + (param.captures.at(7).trim(at: end),) + } else { + () + }, + )) + } + } else if line.starts-with(example-fields.symbol.open) { + example-fields.inside = true + text.push(line.trim(at: end)) + } else { + if text.last().len() == 0 and line.len() == 0 { continue } + text.push(line.trim(at: end)) + } + } else { + result.last().text.push(line.trim(at: end)) + } + } + } + + if tmp-buffer != none { + for buf in tmp-buffer { text.push(buf) } + arguments.pop() + } + + return ( + raw: string, + text: text.join("\n", default: "").trim(), + arguments: arguments.map(it => ( + ..it, + text: it + .text + .join( + "\n", + default: "", + ) + .trim(), + )), + result: result.map(it => ( + ..it, + text: it + .text + .join( + "\n", + default: "", + ) + .trim(), + )), + ) +} + #let parse-docstring(string) = { let argument-re = regex("-\s+(\.*[_a-zA-Z]+[-\w]*)\s+(\\(.*?\\))?(\s*=\s*.*?)?:(.*)") let result-re = regex("->\s+\(?([_a-zA-Z]+[-\w]*)\)?\s*(.*)")