diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 02fab06..73665d8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: "26.0.2" - gleam-version: "1.11.1" + gleam-version: "1.13.0" rebar3-version: "3" - run: gleam deps download - run: gleam build --warnings-as-errors diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e297a9..21c8532 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## Unreleased + +### Deprecated +- `file` was deprecated for clarity reasons. Use `document` instead. + +### Added + +- Adds support for replacing snippets in code comments, + using `comments_in` instead of `document`. + ## v2.0.0 - 2025-08-02 This is a complete rewrite. Instead of checking or running gleam code via checkmark, diff --git a/README.md b/README.md index a1cf208..e8119bb 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ import simplifile pub fn example_test() { assert checkmark.new(simplifile.read, simplifile.write) - |> checkmark.file("README.md") + |> checkmark.document("README.md") |> checkmark.should_contain_contents_of("./example.sh", tagged: "sh") |> checkmark.should_contain_contents_of( "./test/example_test.gleam", diff --git a/src/checkmark.gleam b/src/checkmark.gleam index e641012..14397e7 100644 --- a/src/checkmark.gleam +++ b/src/checkmark.gleam @@ -22,7 +22,12 @@ pub opaque type Checker(e) { /// A markdown file to check, which can be linked to snippets in multiple other files. /// The error type depends on the IO library used. pub opaque type File(e) { - File(name: String, checker: Checker(e), expectations: List(#(String, String))) + File( + name: String, + checker: Checker(e), + check_in_comments: Bool, + expectations: List(#(String, String)), + ) } /// Any error that can happen during checking or updating a markdown file. @@ -44,8 +49,19 @@ pub fn new( } /// Configures a markdown file to be checked or updated. +@deprecated("Use document instead") pub fn file(checker: Checker(e), filename: String) -> File(e) { - File(filename, checker, []) + document(checker, filename) +} + +/// Configures a markdown document to be checked or updated. +pub fn document(checker: Checker(e), filename: String) -> File(e) { + File(filename, checker, False, []) +} + +/// Configures comments in a gleam source file to be checked or updated. +pub fn comments_in(checker: Checker(e), filename: String) -> File(e) { + File(filename, checker, True, []) } /// Specify that the markdown file should contain the contents of another file as a code block. @@ -119,10 +135,11 @@ fn render_replacements( contents |> list.map(fn(section) { case section { - parser.FencedCode(_, content, start_fence, end_fence) -> { + parser.FencedCode(content:, start_fence:, end_fence:, prefix:, ..) -> { case dict.get(replacements, string.trim(start_fence.info)) { - Ok(replacement) -> render_code(start_fence, replacement, end_fence) - Error(_) -> render_code(start_fence, content, end_fence) + Ok(replacement) -> + render_code(start_fence, prefix, replacement, end_fence) + Error(_) -> render_code(start_fence, prefix, content, end_fence) } } parser.Other(_, content) -> content @@ -133,26 +150,27 @@ fn render_replacements( fn render_code( start_fence: Fence, + prefix: String, content: String, end_fence: Option(Fence), ) -> String { // This is not very optimized, but we probably don't need to care... - let content = case start_fence.indent { - 0 -> content - amount -> + let content = case start_fence.indent, prefix { + 0, "" -> content + amount, _ -> indent( splitter.new(["\n", "\r\n"]), - string.repeat(" ", amount), + prefix <> string.repeat(" ", amount), content, [], ) } - let without_end = render_fence(start_fence) <> content + let without_end = render_fence(prefix, start_fence) <> content case end_fence { option.None -> without_end - option.Some(end_fence) -> without_end <> render_fence(end_fence) + option.Some(end_fence) -> without_end <> render_fence(prefix, end_fence) } } @@ -171,8 +189,8 @@ fn indent( } } -fn render_fence(fence: Fence) -> String { - string.repeat(" ", fence.indent) <> fence.fence <> fence.info +fn render_fence(prefix: String, fence: Fence) -> String { + prefix <> string.repeat(" ", fence.indent) <> fence.fence <> fence.info } fn parse_file( @@ -180,7 +198,7 @@ fn parse_file( ) -> Result(List(parser.Section), List(CheckError(e))) { read_file(file.checker, file.name) |> result.map_error(list.wrap) - |> result.map(parser.parse) + |> result.map(parser.parse(_, file.check_in_comments)) } fn read_file( @@ -223,10 +241,13 @@ fn find_match( )) } - [parser.FencedCode(line, content, start_fence, end_fence), ..rest] -> { + [ + parser.FencedCode(start_line:, content:, start_fence:, end_fence:, ..), + ..rest + ] -> { case string.trim(start_fence.info) == tag { True -> { - let result = #(line, Snippet(content, start_fence, end_fence)) + let result = #(start_line, Snippet(content, start_fence, end_fence)) find_match(rest, tag, [result, ..found]) } False -> find_match(rest, tag, found) diff --git a/src/checkmark/internal/parser.gleam b/src/checkmark/internal/parser.gleam index 14b0911..513038b 100644 --- a/src/checkmark/internal/parser.gleam +++ b/src/checkmark/internal/parser.gleam @@ -8,6 +8,7 @@ pub type Section { Other(start_line: Int, content: String) FencedCode( start_line: Int, + prefix: String, content: String, start_fence: Fence, end_fence: Option(Fence), @@ -20,104 +21,171 @@ pub type Fence { type SectionBuilder { OtherBuilder(start_line: Int, parts: List(String)) - FencedCodeBuilder(start_line: Int, parts: List(String), start_fence: Fence) + FencedCodeBuilder( + start_line: Int, + parts: List(String), + prefix: String, + start_fence: Fence, + ) +} + +type LineContent { + LineContent(prefix: String, content: String, ending: String) +} + +fn initial_content(line: LineContent) -> List(String) { + [line.ending, line.content, line.prefix] } -pub fn parse(content: String) -> List(Section) { +pub fn parse(content: String, search_in_comments: Bool) -> List(Section) { use <- bool.guard(when: content == "", return: []) let line_ends = splitter.new(["\n", "\r\n"]) let #(line, ending, rest) = splitter.split(line_ends, content) - parse_lines(line_ends, 1, [], None, line, ending, rest) + parse_lines(line_ends, search_in_comments, 1, [], None, line, ending, rest) +} + +fn add_parts(builder: SectionBuilder, line: LineContent) { + case builder { + FencedCodeBuilder(..) -> + FencedCodeBuilder(..builder, parts: [ + line.ending, + line.content, + ..builder.parts + ]) + OtherBuilder(..) -> + OtherBuilder(..builder, parts: [ + line.ending, + line.content, + line.prefix, + ..builder.parts + ]) + } +} + +fn to_section(builder: SectionBuilder, end_fence: Option(Fence)) { + case builder { + FencedCodeBuilder(start_line:, parts:, start_fence:, prefix:) -> + FencedCode( + start_line, + parts_to_string(parts, start_fence.indent), + prefix:, + start_fence:, + end_fence:, + ) + + OtherBuilder(start_line:, parts:) -> + Other(start_line, parts_to_string(parts, 0)) + } } fn parse_lines( splitter: Splitter, - line: Int, + search_in_comments: Bool, + line_number: Int, sections: List(Section), - current_section: Option(SectionBuilder), - content: String, + current_builder: Option(SectionBuilder), + line: String, ending: String, rest: String, -) { - let #(sections, current_section) = case parse_fence(content, ending) { - None -> { - let current_section = case current_section { - None -> OtherBuilder(line, [ending, content]) - Some(builder) -> { - let parts = [ending, content, ..builder.parts] +) -> List(Section) { + let #(prefix, line, is_comment) = case search_in_comments { + True -> parse_comment(line) + False -> #("", line, False) + } + let line = LineContent(prefix, line, ending) + + // Check special cases for comments + let #(sections, current_section) = case search_in_comments && !is_comment { + // Line should NOT be included in code blocks, as it is not a comment + True -> + case current_builder { + None -> #( + sections, + Some(OtherBuilder(line_number, initial_content(line))), + ) + Some(builder) -> case builder { - FencedCodeBuilder(line, _, start_fence:) -> - FencedCodeBuilder(line, parts, start_fence) - OtherBuilder(line, _) -> OtherBuilder(line, parts) + // End code block if comment ends + FencedCodeBuilder(..) -> #( + [to_section(builder, None), ..sections], + Some(OtherBuilder(line_number, initial_content(line))), + ) + + // Otherwise add to current + OtherBuilder(..) -> #(sections, Some(add_parts(builder, line))) } - } } - #(sections, Some(current_section)) - } - Some(fence) -> - case current_section { - None -> #(sections, Some(FencedCodeBuilder(line, [], fence))) - Some(OtherBuilder(prev_line, parts)) -> #( - [Other(prev_line, parts_to_string(parts, 0)), ..sections], - Some(FencedCodeBuilder(line, [], fence)), - ) - Some(FencedCodeBuilder(start_line:, parts:, start_fence:)) -> - case should_close(start_fence, fence) { - True -> #( - [ - FencedCode( - start_line, - parts_to_string(parts, start_fence.indent), - start_fence, - Some(fence), - ), - ..sections - ], - None, - ) - False -> #( + // Otherwise follow normal parsing + False -> + case parse_fence(line) { + // No fence, add to current, or start new builder: + None -> { + let current_section = case current_builder { + None -> OtherBuilder(line_number, initial_content(line)) + Some(builder) -> add_parts(builder, line) + } + #(sections, Some(current_section)) + } + + // Found fence: + Some(current_fence) -> { + case current_builder { + // No current builder, start new fenced code: + None -> #( sections, Some(FencedCodeBuilder( - start_line, - [ending, content, ..parts], - start_fence, + line_number, + [], + line.prefix, + current_fence, )), ) + + // Other content: finalize it, start new fenced code + Some(OtherBuilder(..) as other_builder) -> #( + [to_section(other_builder, None), ..sections], + Some(FencedCodeBuilder( + line_number, + [], + line.prefix, + current_fence, + )), + ) + + // Fenced code, check if it should be closed or not: + Some(FencedCodeBuilder(start_fence:, ..) as fence_builder) -> + case should_close(start_fence, current_fence) { + True -> #( + [to_section(fence_builder, Some(current_fence)), ..sections], + None, + ) + + False -> #(sections, Some(add_parts(fence_builder, line))) + } } + } } } case rest { + // Nothing left, finalize the current builder, if any: "" -> { let sections = case current_section { None -> sections - Some(builder) -> - case builder { - FencedCodeBuilder(start_line:, start_fence:, parts:) -> [ - FencedCode( - start_line, - parts_to_string(parts, start_fence.indent), - start_fence, - None, - ), - ..sections - ] - OtherBuilder(start_line:, parts:) -> [ - Other(start_line, parts_to_string(parts, 0)), - ..sections - ] - } + Some(builder) -> [to_section(builder, None), ..sections] } list.reverse(sections) } + // More to parse, recurse on next line: rest -> { let #(content, ending, rest) = splitter.split(splitter, rest) parse_lines( splitter, - line + 1, + search_in_comments, + line_number + 1, sections, current_section, content, @@ -147,20 +215,29 @@ fn remove_indent_up_to(string: String, indent: Int) -> String { } } -fn parse_fence(line: String, ending: String) -> Option(Fence) { +/// Checks for a doc or module comment, and splits it into a prefix if found. +fn parse_comment(line: String) -> #(String, String, Bool) { + case line { + "/// " <> rest -> #("/// ", rest, True) + "//// " <> rest -> #("//// ", rest, True) + _ -> #("", line, False) + } +} + +fn parse_fence(line: LineContent) -> Option(Fence) { // A code fence is a sequence of at least three consecutive backtick // characters (`) or tildes (~). (Tildes and backticks cannot be mixed.) // A fenced code block begins with a code fence, // preceded by up to three spaces of indentation. - case line { - "```" <> rest -> Some(build_fence("`", 0, 3, rest, ending)) - " ```" <> rest -> Some(build_fence("`", 1, 3, rest, ending)) - " ```" <> rest -> Some(build_fence("`", 2, 3, rest, ending)) - " ```" <> rest -> Some(build_fence("`", 3, 3, rest, ending)) - "~~~" <> rest -> Some(build_fence("~", 0, 3, rest, ending)) - " ~~~" <> rest -> Some(build_fence("~", 1, 3, rest, ending)) - " ~~~" <> rest -> Some(build_fence("~", 2, 3, rest, ending)) - " ~~~" <> rest -> Some(build_fence("~", 3, 3, rest, ending)) + case line.content { + "```" <> rest -> Some(build_fence("`", 0, 3, rest, line)) + " ```" <> rest -> Some(build_fence("`", 1, 3, rest, line)) + " ```" <> rest -> Some(build_fence("`", 2, 3, rest, line)) + " ```" <> rest -> Some(build_fence("`", 3, 3, rest, line)) + "~~~" <> rest -> Some(build_fence("~", 0, 3, rest, line)) + " ~~~" <> rest -> Some(build_fence("~", 1, 3, rest, line)) + " ~~~" <> rest -> Some(build_fence("~", 2, 3, rest, line)) + " ~~~" <> rest -> Some(build_fence("~", 3, 3, rest, line)) _ -> None } } @@ -170,11 +247,14 @@ fn build_fence( indent: Int, delimiters: Int, rest: String, - ending: String, + original: LineContent, ) -> Fence { case string.pop_grapheme(rest) { Ok(#(first, rest)) if first == character -> - build_fence(character, indent, delimiters + 1, rest, ending) - _ -> Fence(string.repeat(character, delimiters), rest <> ending, indent) + build_fence(character, indent, delimiters + 1, rest, original) + _ -> { + let fence = string.repeat(character, delimiters) + Fence(fence, rest <> original.ending, indent) + } } } diff --git a/test/checkmark/internal/parser_test.gleam b/test/checkmark/internal/parser_test.gleam index 3b5282d..f2acf08 100644 --- a/test/checkmark/internal/parser_test.gleam +++ b/test/checkmark/internal/parser_test.gleam @@ -1,22 +1,55 @@ import checkmark/internal/parser.{Fence, FencedCode, Other} import gleam/option.{None, Some} +import gleam/string + +fn lines(lines: List(String)) -> String { + lines |> string.join("\n") +} + +fn comment_agnostic(body: fn(Bool) -> Nil) -> Nil { + body(False) + body(True) +} pub fn empty_string_test() { - assert parser.parse("") == [] + use in_comments <- comment_agnostic() + assert parser.parse("", in_comments) == [] } pub fn no_snippets_test() { - let text = "one\ntwo\r\nthree" - assert parser.parse(text) == [parser.Other(1, text)] + use in_comments <- comment_agnostic() + + let text = + lines([ + "one", + "two\r", + "three", + ]) + assert parser.parse(text, in_comments) == [parser.Other(1, text)] } pub fn basic_snippet_test() { - assert parser.parse("start\n```gleam\ncode\r\nmore_code\n``` \nrest") + assert parser.parse( + lines([ + "start", + "```gleam", + "code\r", + "more_code", + "``` ", + "rest", + ]), + False, + ) == [ Other(1, "start\n"), FencedCode( 2, - "code\r\nmore_code\n", + "", + lines([ + "code\r", + "more_code", + "", + ]), Fence("```", "gleam\n", 0), Some(Fence("```", " \n", 0)), ), @@ -24,14 +57,131 @@ pub fn basic_snippet_test() { ] } +pub fn doc_comment_test() { + assert parser.parse( + lines([ + "pub const answer = 42", + "", + "/// start", + "/// ```gleam", + "/// code\r", + "/// more_code", + "/// ``` ", + "pub const answer_str = \"*\"", + ]), + True, + ) + == [ + Other( + 1, + lines([ + "pub const answer = 42", + "", + "/// start", + "", + ]), + ), + FencedCode( + 4, + "/// ", + lines([ + "code\r", + "more_code", + "", + ]), + Fence("```", "gleam\n", 0), + Some(Fence("```", " \n", 0)), + ), + Other(8, "pub const answer_str = \"*\""), + ] +} + +pub fn module_comment_test() { + assert parser.parse( + lines([ + "//// start", + "//// ```gleam", + "//// code", + "//// ``` ", + "//// rest", + ]), + True, + ) + == [ + Other( + 1, + lines([ + "//// start", + "", + ]), + ), + FencedCode( + 2, + "//// ", + lines([ + "code", + "", + ]), + Fence("```", "gleam\n", 0), + Some(Fence("```", " \n", 0)), + ), + Other(5, "//// rest"), + ] +} + +pub fn unfinished_comment_block_test() { + assert parser.parse( + lines([ + "//// start", + "//// ```gleam", + "//// code", + "rest", + ]), + True, + ) + == [ + Other( + 1, + lines([ + "//// start", + "", + ]), + ), + FencedCode( + 2, + "//// ", + lines([ + "code", + "", + ]), + Fence("```", "gleam\n", 0), + None, + ), + Other(4, "rest"), + ] +} + pub fn indented_snippet_test() { assert parser.parse( - " ```gleam\n code\n more_code\n not indented enough\n ```", + lines([ + " ```gleam", + " code", + " more_code", + " not indented enough", + " ```", + ]), + False, ) == [ FencedCode( 1, - "code\n more_code\nnot indented enough\n", + "", + lines([ + "code", + " more_code", + "not indented enough", + "", + ]), Fence("```", "gleam\n", 3), Some(Fence("```", "", 2)), ), @@ -39,11 +189,26 @@ pub fn indented_snippet_test() { } pub fn non_matching_fences_test() { - assert parser.parse("````\n```\n~~~\ncode\n````") + assert parser.parse( + lines([ + "````", + "```", + "~~~", + "code", + "````", + ]), + False, + ) == [ FencedCode( 1, - "```\n~~~\ncode\n", + "", + lines([ + "```", + "~~~", + "code", + "", + ]), Fence("````", "\n", 0), Some(Fence("````", "", 0)), ), @@ -51,25 +216,67 @@ pub fn non_matching_fences_test() { } pub fn missing_end_fence_test() { - assert parser.parse("```\ncode") - == [FencedCode(1, "code", Fence("```", "\n", 0), None)] + assert parser.parse( + lines([ + "```", + "code", + ]), + False, + ) + == [FencedCode(1, "", "code", Fence("```", "\n", 0), None)] } pub fn empty_fence_test() { - assert parser.parse("```info") - == [FencedCode(1, "", Fence("```", "info", 0), None)] + assert parser.parse("```info", False) + == [FencedCode(1, "", "", Fence("```", "info", 0), None)] } pub fn emtpy_line_preservation_test() { - assert parser.parse("\ntext\n\n```\n\ncode\n\n```\n\n\n") + assert parser.parse( + lines([ + "", + "text", + "", + "```", + "", + "code", + "", + "```", + "", + "", + "", + ]), + False, + ) == [ - Other(1, "\ntext\n\n"), + Other( + 1, + lines([ + "", + "text", + "", + "", + ]), + ), FencedCode( 4, - "\ncode\n\n", + "", + lines([ + "", + "code", + "", + "", + ]), Fence("```", "\n", 0), Some(Fence("```", "\n", 0)), ), - Other(9, "\n\n"), + Other( + 9, + lines([ + "", + "", + "", + ]), + ), ] } diff --git a/test/checkmark_test.gleam b/test/checkmark_test.gleam index 929b0f3..086bba5 100644 --- a/test/checkmark_test.gleam +++ b/test/checkmark_test.gleam @@ -1,4 +1,4 @@ -import checkmark.{CouldNotReadFile, MultipleTagsFound, TagNotFound} +import checkmark.{type Checker, CouldNotReadFile, MultipleTagsFound, TagNotFound} import gleam/erlang/process import gleeunit import simplifile @@ -9,7 +9,7 @@ pub fn main() { pub fn check_existing_file_test() { assert checkmark.new(simplifile.read, simplifile.write) - |> checkmark.file("./test/test.md") + |> checkmark.document("./test/test.md") |> checkmark.should_contain_contents_of( "./test/test_content.txt", tagged: "multiple", @@ -28,33 +28,57 @@ pub fn check_existing_file_test() { pub fn check_missing_markdown_file_test() { assert checkmark.new(simplifile.read, simplifile.write) - |> checkmark.file("this-file-does-not-exist") + |> checkmark.document("this-file-does-not-exist") |> checkmark.check() == Error([CouldNotReadFile(simplifile.Enoent)]) } pub fn check_missing_source_file_test() { assert checkmark.new(simplifile.read, simplifile.write) - |> checkmark.file("./test/test.md") + |> checkmark.document("./test/test.md") |> checkmark.should_contain_contents_of("this-file-does-not-exist", "") |> checkmark.check() == Error([CouldNotReadFile(simplifile.Enoent)]) } -pub fn update_test() { +pub fn update_markdown_test() { + use checker <- update_test("./test/expected_updated.md") + + checker + |> checkmark.document("./test/update.md") + |> checkmark.should_contain_contents_of("./test/test_content.txt", "update") + |> checkmark.update() +} + +pub fn update_code_test() { + use checker <- update_test("./test/expected_updated.gleam.txt") + + checker + |> checkmark.comments_in("./test/test.gleam.txt") + |> checkmark.should_contain_contents_of( + "./test/test_content.txt", + "gleam module", + ) + |> checkmark.should_contain_contents_of( + "./test/test_content.txt", + "gleam value", + ) + |> checkmark.update() +} + +fn update_test( + expected: String, + using: fn(Checker(simplifile.FileError)) -> Result(Nil, a), +) -> Nil { let self = process.new_subject() let write = fn(_, content) { process.send(self, content) Ok(Nil) } - assert checkmark.new(simplifile.read, write) - |> checkmark.file("./test/update.md") - |> checkmark.should_contain_contents_of("./test/test_content.txt", "update") - |> checkmark.update() - == Ok(Nil) + assert using(checkmark.new(simplifile.read, write)) == Ok(Nil) let assert Ok(written) = process.receive(self, 0) - let assert Ok(expected) = simplifile.read("./test/expected_updated.md") + let assert Ok(expected) = simplifile.read(expected) assert written == expected } diff --git a/test/example_test.gleam b/test/example_test.gleam index 52a8171..b4bad13 100644 --- a/test/example_test.gleam +++ b/test/example_test.gleam @@ -4,7 +4,7 @@ import simplifile pub fn example_test() { assert checkmark.new(simplifile.read, simplifile.write) - |> checkmark.file("README.md") + |> checkmark.document("README.md") |> checkmark.should_contain_contents_of("./example.sh", tagged: "sh") |> checkmark.should_contain_contents_of( "./test/example_test.gleam", diff --git a/test/expected_updated.gleam.txt b/test/expected_updated.gleam.txt new file mode 100644 index 0000000..23d1dbf --- /dev/null +++ b/test/expected_updated.gleam.txt @@ -0,0 +1,12 @@ +//// Module +//// ```gleam module +//// test +//// content +//// ``` + +/// Value +/// ```gleam value +/// test +/// content +/// ``` +pub const meaning = 42 diff --git a/test/test.gleam.txt b/test/test.gleam.txt new file mode 100644 index 0000000..e179626 --- /dev/null +++ b/test/test.gleam.txt @@ -0,0 +1,10 @@ +//// Module +//// ```gleam module +//// should be replaced +//// ``` + +/// Value +/// ```gleam value +/// should be replaced +/// ``` +pub const meaning = 42