diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index bb0a7da..7449fc3 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -21,6 +21,7 @@ codegen CSPRNG ctx cuz +dedenting Deframer downcasted downcasting diff --git a/src/fpy/compiler.py b/src/fpy/compiler.py index 1643577..c781e50 100644 --- a/src/fpy/compiler.py +++ b/src/fpy/compiler.py @@ -93,6 +93,29 @@ maybe_placeholders=True, ) + +def _parse_fpy(text: str, **kwargs): + """Parse fpy source into a Lark tree, tolerating a missing final newline. + + The indenter (PythonIndenter) derives INDENT/DEDENT tokens from the + whitespace each _NEWLINE token carries. When the source does not end in a + newline, the final line's leading whitespace is never followed by a newline, + so at end-of-input it is misread as a brand-new indentation level: a trailing + tab or an over-indented comment on the last line then emits a spurious INDENT + (or a bogus dedent) and an otherwise-valid sequence fails to compile + (https://github.com/fprime-community/fpy/issues/61). + + Appending a newline when one is absent makes the last line's indentation + resolve to column 0 -- closing any open blocks cleanly -- exactly as + CPython's tokenizer supplies an implicit NEWLINE before end-of-input. The + newline is added only at the very end, so it shifts no positions and leaves + error line/column reporting (and text.splitlines()) unchanged. + """ + if not text.endswith("\n"): + text = text + "\n" + return _fpy_parser.parse(text, **kwargs) + + # Load builtin time.fpy functions at module level _builtin_time_path = Path(__file__).parent / "builtin" / "time.fpy" _builtin_time_text = _builtin_time_path.read_text(encoding="utf-8") @@ -112,7 +135,7 @@ def _get_builtin_library_ast(): fpy.error.input_text = _builtin_time_text fpy.error.input_lines = _builtin_time_text.splitlines() - tree = _fpy_parser.parse(_builtin_time_text) + tree = _parse_fpy(_builtin_time_text) _builtin_library_ast = FpyTransformer().transform(tree) # Restore error state @@ -129,7 +152,7 @@ def text_to_ast(text: str): fpy.error.input_text = text fpy.error.input_lines = text.splitlines() try: - tree = _fpy_parser.parse(text, on_error=handle_lark_error) + tree = _parse_fpy(text, on_error=handle_lark_error) except LarkError as e: handle_lark_error(e) return None diff --git a/test/fpy/test_parsing.py b/test/fpy/test_parsing.py index a37b15a..ac4a54b 100644 --- a/test/fpy/test_parsing.py +++ b/test/fpy/test_parsing.py @@ -663,3 +663,174 @@ def test_multiline_assert(self, fprime_test_api): ) """ assert_run_success(fprime_test_api, seq) + + +class TestTrailingWhitespaceAtEndOfFile: + """A file that ends *without* a final newline must still compile, even when + the last line is a lone tab, trailing spaces, or an (over/under-indented) + comment. + + Regression tests for https://github.com/fprime-community/fpy/issues/61 + ("Tabbed New Line does not compile at the end of a if/else statement"). The + indenter derives INDENT/DEDENT from the whitespace each _NEWLINE token + carries; with no trailing newline the last line's indentation used to be + misread as a fresh indentation level at end-of-file, emitting a spurious + INDENT (trailing tab / over-indented comment), a bogus dedent + (under-indented comment), or a stray blank logical line inside a block. + Python ignores such trailing blank/comment lines, and now so does fpy. + + The sequences are built with explicit ``\\n``/``\\t`` escapes (not + triple-quoted literals) so the significant trailing whitespace survives + editors and formatters. + """ + + def test_trailing_tab_after_statement(self, fprime_test_api): + # A lone tab on the final line, no trailing newline. + seq = "x: U32 = 1\nassert x == 1\n\t" + assert_run_success(fprime_test_api, seq) + + def test_trailing_spaces_after_statement(self, fprime_test_api): + seq = "x: U32 = 1\nassert x == 1\n " + assert_run_success(fprime_test_api, seq) + + def test_trailing_over_indented_comment(self, fprime_test_api): + seq = "x: U32 = 1\nassert x == 1\n # trailing comment" + assert_run_success(fprime_test_api, seq) + + def test_trailing_tab_then_comment(self, fprime_test_api): + # The exact shape called out in the issue: a tab, then a comment, at EOF. + seq = "x: U32 = 1\nassert x == 1\n\t# there is a tab here" + assert_run_success(fprime_test_api, seq) + + def test_trailing_comment_at_body_indent(self, fprime_test_api): + # Comment aligned with the block body (used to leave a stray blank + # logical line inside the block, breaking it). + seq = "if True:\n x: U32 = 1\n # aligned comment" + assert_run_success(fprime_test_api, seq) + + def test_trailing_under_indented_comment(self, fprime_test_api): + # Comment indented between column 0 and the body (used to raise a + # DedentError for dedenting to an unknown column). + seq = "if True:\n x: U32 = 1\n # under-indented comment" + assert_run_success(fprime_test_api, seq) + + def test_trailing_tab_after_if_elif(self, fprime_test_api): + seq = "x: U32 = 0\nif True:\n x = 1\nelif False:\n x = 2\n\t" + assert_run_success(fprime_test_api, seq) + + def test_trailing_tab_after_if_else(self, fprime_test_api): + seq = "x: U32 = 0\nif True:\n x = 1\nelse:\n x = 2\n\t" + assert_run_success(fprime_test_api, seq) + + def test_trailing_tab_after_for_body(self, fprime_test_api): + seq = "s: I64 = 0\nfor i in 0 .. 3:\n s = s + i\n\t" + assert_run_success(fprime_test_api, seq) + + def test_trailing_tab_after_while_body(self, fprime_test_api): + seq = "i: U64 = 0\nwhile i < 3:\n i = i + 1\n\t" + assert_run_success(fprime_test_api, seq) + + def test_trailing_tab_after_def_body(self, fprime_test_api): + seq = "def f() -> U32:\n return 1\n\t" + assert_run_success(fprime_test_api, seq) + + def test_trailing_tab_after_check_body(self, fprime_test_api): + seq = "x: bool = False\ncheck True timeout never:\n x = True\n\t" + assert_run_success(fprime_test_api, seq) + + def test_trailing_tabs_after_nested_blocks(self, fprime_test_api): + # Several dedent levels to close at once, with an over-indented last line. + seq = "x: U32 = 0\nif True:\n if True:\n x = 1\n\t\t\t" + assert_run_success(fprime_test_api, seq) + + def test_trailing_crlf_then_tab(self, fprime_test_api): + # CRLF line endings, trailing tab, no final newline. + seq = "x: U32 = 1\r\nassert x == 1\r\n\t" + assert_run_success(fprime_test_api, seq) + + def test_only_whitespace_file(self, fprime_test_api): + # A file that is nothing but a tab must compile to an empty sequence. + seq = "\t" + assert_run_success(fprime_test_api, seq) + + def test_only_indented_comment_file(self, fprime_test_api): + seq = " # just a comment, indented, no newline" + assert_run_success(fprime_test_api, seq) + + def test_issue_61_reported_example(self, fprime_test_api): + # Faithful adaptation of the snippet in the issue: an if/elif whose final + # branch is followed by a tab + comment at end of file, with a multiline + # log() call (no backslash) inside a branch body. + seq = ( + "temp: I64 = -1\n" + "if temp < 0:\n" + ' log("temperature sensor invalid reading",\n' + " Fw.LogSeverity.WARNING_HI)\n" + "elif temp > 100:\n" + ' log("temp high", Fw.LogSeverity.WARNING_HI)\n' + "\t# there is a tab here" + ) + assert_run_success(fprime_test_api, seq) + + +class TestMultilineConstructorsWithoutBackslash: + """Multiline function calls / type constructors must not require a backslash + at the end of each line: inside ``()``, ``[]`` or ``{}`` the expression + continues implicitly, exactly like Python. + + Regression tests for https://github.com/fprime-community/fpy/issues/68 + ("Require fewer backslashes in multiline exprs"). Existing backslash + continuation must keep working too. + """ + + def test_constructor_args_on_separate_lines(self, fprime_test_api): + # The issue's shape: constructor arguments on separate lines, aligned + # under the open paren, with no backslashes. + seq = ( + "v: Fw.TimeIntervalValue = Fw.TimeIntervalValue(10,\n" + " 500)\n" + "assert v.seconds == 10\n" + "assert v.useconds == 500\n" + ) + assert_run_success(fprime_test_api, seq) + + def test_constructor_multiline_is_last_statement_no_newline(self, fprime_test_api): + # Multiline constructor as the final statement, file ends with no + # trailing newline (exercises issue #61 and #68 together). + seq = "v: Fw.TimeIntervalValue = Fw.TimeIntervalValue(\n 10,\n 500\n)" + assert_run_success(fprime_test_api, seq) + + def test_constructor_multiline_then_trailing_tab(self, fprime_test_api): + seq = "v: Fw.TimeIntervalValue = Fw.TimeIntervalValue(\n 10,\n 500\n)\n\t" + assert_run_success(fprime_test_api, seq) + + def test_constructor_multiline_with_backslash_still_works(self, fprime_test_api): + # Redundant backslashes inside the parens must not break. + seq = ( + "v: Fw.TimeIntervalValue = Fw.TimeIntervalValue(10, \\\n" + " 500)\n" + "assert v.useconds == 500\n" + ) + assert_run_success(fprime_test_api, seq) + + def test_constructor_multiline_backslash_on_every_line(self, fprime_test_api): + # The full "backslash at the end of each line" style users reached for; + # it must remain valid alongside the now-unnecessary implicit form. + seq = ( + "v: Fw.TimeIntervalValue = Fw.TimeIntervalValue( \\\n" + " 10, \\\n" + " 500 \\\n" + ")\n" + "assert v.seconds == 10\n" + ) + assert_run_success(fprime_test_api, seq) + + def test_nested_multiline_constructor(self, fprime_test_api): + seq = ( + "v: Fw.TimeIntervalValue = Fw.TimeIntervalValue(\n" + " [10, 20, 30][0],\n" + " 500,\n" + ")\n" + "assert v.seconds == 10\n" + ) + assert_run_success(fprime_test_api, seq)