From 5871c9342280d05cac4db6e67053cb65fda537eb Mon Sep 17 00:00:00 2001 From: gabewillen Date: Mon, 2 Mar 2026 09:15:20 -0600 Subject: [PATCH 1/3] text/jinja/formatter: enforce strict checklist architecture --- src/emel/text/jinja/formatter/actions.hpp | 220 +- src/emel/text/jinja/formatter/context.hpp | 71 +- src/emel/text/jinja/formatter/detail.hpp | 2177 +------------------- src/emel/text/jinja/formatter/errors.hpp | 16 + src/emel/text/jinja/formatter/events.hpp | 84 +- src/emel/text/jinja/formatter/guards.hpp | 107 +- src/emel/text/jinja/formatter/sm.hpp | 323 ++- tests/text/jinja/formatter_tests.cpp | 2006 ++---------------- tools/bench/text/jinja/formatter_bench.cpp | 84 +- tools/docsgen/docsgen.cpp | 35 +- 10 files changed, 717 insertions(+), 4406 deletions(-) create mode 100644 src/emel/text/jinja/formatter/errors.hpp diff --git a/src/emel/text/jinja/formatter/actions.hpp b/src/emel/text/jinja/formatter/actions.hpp index 09dbe9cf..1f6f702b 100644 --- a/src/emel/text/jinja/formatter/actions.hpp +++ b/src/emel/text/jinja/formatter/actions.hpp @@ -1,200 +1,98 @@ #pragma once -#include "emel/emel.h" -#include "emel/text/jinja/formatter/context.hpp" +#include + #include "emel/text/jinja/formatter/detail.hpp" #include "emel/text/jinja/formatter/events.hpp" +#include "emel/text/jinja/formatter/errors.hpp" namespace emel::text::jinja::formatter::action { -struct reject_invalid_render { - void operator()(const emel::text::jinja::event::render &, context & ctx) const noexcept { - ctx.phase_error = EMEL_ERR_INVALID_ARGUMENT; - ctx.last_error = EMEL_ERR_INVALID_ARGUMENT; - ctx.error_pos = 0; +namespace runtime_detail { + +template +constexpr decltype(auto) unwrap_runtime_event(const runtime_event_type & ev) noexcept { + if constexpr (requires { ev.event_; }) { + return ev.event_; + } else { + return (ev); } -}; +} -struct begin_render { - void operator()(const emel::text::jinja::event::render & ev, context & ctx) const noexcept { - ctx.globals = ev.globals; - ctx.statements = ev.program != nullptr ? &ev.program->body : nullptr; - ctx.statement_index = 0; - ctx.pending_expr = nullptr; - ctx.pending_value = detail::make_undefined(); - ctx.pending_value_ready = false; - ctx.output = ev.output; - ctx.output_capacity = ev.output_capacity; - ctx.output_length = 0; - ctx.output_truncated = false; - - ctx.phase_error = EMEL_OK; - ctx.last_error = EMEL_OK; - ctx.error_pos = 0; - ctx.steps_remaining = k_max_steps; - ctx.scope_count = 0; - ctx.array_items_used = 0; - ctx.object_entries_used = 0; - ctx.string_buffer_used = 0; - ctx.callable_count = 0; +} // namespace runtime_detail + +struct reject_invalid_render { + template + void operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = runtime_detail::unwrap_runtime_event(ev); + detail::mark_error(runtime_ev.ctx, error::invalid_request, false, 0); } }; -struct seed_program { - void operator()(context & ctx) const noexcept { - if (ctx.phase_error != EMEL_OK) { - return; - } - if (!detail::push_scope(ctx)) { - return; - } +struct begin_render { + template + void operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = runtime_detail::unwrap_runtime_event(ev); + detail::reset_result(runtime_ev.ctx); } }; -struct eval_next_stmt { - void operator()(context & ctx) const noexcept { - if (ctx.phase_error != EMEL_OK) { - return; - } - if (ctx.statements == nullptr) { - detail::set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return; - } - if (ctx.statement_index >= ctx.statements->size()) { - return; - } - const emel::text::jinja::ast_node * node = (*ctx.statements)[ctx.statement_index].get(); - ctx.statement_index += 1; - ctx.pending_expr = nullptr; - ctx.pending_value_ready = false; - - if (!detail::ensure_steps(ctx, node != nullptr ? node->pos : 0)) { - return; - } - if (node == nullptr) { - return; - } - if (dynamic_cast(node) != nullptr) { - return; - } - if (dynamic_cast(node) != nullptr) { - return; - } - if (dynamic_cast(node) != nullptr) { - detail::set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return; - } - if (dynamic_cast(node) != nullptr) { - detail::set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return; - } - - detail::render_io io = {}; - detail::init_writer(io, ctx.output, ctx.output_capacity); - io.writers[0].length = ctx.output_length; - detail::control_flow flow = detail::control_flow::none; - - if (auto * stmt = dynamic_cast(node)) { - detail::render_if(ctx, stmt, ctx.globals, io, false, flow); - } else if (auto * stmt = dynamic_cast(node)) { - detail::render_for(ctx, stmt, ctx.globals, io, false, flow); - } else if (auto * stmt = dynamic_cast(node)) { - detail::render_set(ctx, stmt, ctx.globals, io); - } else if (auto * stmt = dynamic_cast(node)) { - if (ctx.callable_count >= k_max_callables) { - detail::set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - } else { - auto * name_id = dynamic_cast(stmt->name.get()); - if (name_id == nullptr) { - detail::set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - } else { - callable_slot & slot = ctx.callables[ctx.callable_count++]; - slot.kind = emel::text::jinja::function_kind::macro; - slot.macro.stmt = stmt; - emel::text::jinja::function_ref ref; - ref.kind = emel::text::jinja::function_kind::macro; - ref.data = &slot; - detail::set_object_value(ctx, ctx.scopes[ctx.scope_count - 1].locals, name_id->name, - detail::make_function(ref)); - } - } - } else if (auto * stmt = dynamic_cast(node)) { - detail::render_filter_statement(ctx, stmt, ctx.globals, io); - } else if (auto * stmt = dynamic_cast(node)) { - detail::render_call_statement(ctx, stmt, ctx.globals, io); - } else { - ctx.pending_expr = node; - ctx.pending_value = detail::make_undefined(); - } - - ctx.output_length = io.writers[0].length; - if (flow != detail::control_flow::none) { - detail::set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - } +struct mark_empty_output { + template + void operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = runtime_detail::unwrap_runtime_event(ev); + detail::mark_done(runtime_ev.ctx, 0, false); } }; -struct eval_pending_expr { - void operator()(context & ctx) const noexcept { - if (ctx.pending_expr == nullptr || ctx.phase_error != EMEL_OK) { - return; - } - detail::render_io io = {}; - detail::init_writer(io, ctx.output, ctx.output_capacity); - io.writers[0].length = ctx.output_length; - ctx.pending_value = detail::eval_expr(ctx, ctx.pending_expr, ctx.globals, io); - ctx.pending_value_ready = true; - ctx.pending_expr = nullptr; - ctx.output_length = io.writers[0].length; +struct copy_source_text { + template + void operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = runtime_detail::unwrap_runtime_event(ev); + const auto & request = runtime_ev.request; + std::memcpy(&request.output, request.source_text.data(), request.source_text.size()); + detail::mark_done(runtime_ev.ctx, request.source_text.size(), false); } }; -struct write_pending_value { - void operator()(context & ctx) const noexcept { - if (!ctx.pending_value_ready || ctx.phase_error != EMEL_OK) { - return; - } - detail::render_io io = {}; - detail::init_writer(io, ctx.output, ctx.output_capacity); - io.writers[0].length = ctx.output_length; - detail::write_value(ctx, io, ctx.pending_value); - ctx.output_length = io.writers[0].length; - ctx.pending_value_ready = false; - ctx.pending_value = detail::make_undefined(); +struct mark_capacity_error { + template + void operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = runtime_detail::unwrap_runtime_event(ev); + detail::mark_error(runtime_ev.ctx, error::invalid_request, true, 0); } }; -struct finalize_done { - void operator()(context & ctx) const noexcept { - ctx.last_error = EMEL_OK; +struct dispatch_done { + template + void operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = runtime_detail::unwrap_runtime_event(ev); + detail::emit_done(runtime_ev.request, runtime_ev.ctx); } }; -struct finalize_error { - void operator()(context & ctx) const noexcept { - if (ctx.phase_error == EMEL_OK) { - ctx.phase_error = EMEL_ERR_BACKEND; - } - ctx.last_error = ctx.phase_error; +struct dispatch_error { + template + void operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = runtime_detail::unwrap_runtime_event(ev); + detail::emit_error(runtime_ev.request, runtime_ev.ctx); } }; struct on_unexpected { - template - void operator()(const event &, context & ctx) const noexcept { - ctx.phase_error = EMEL_ERR_BACKEND; - ctx.last_error = EMEL_ERR_BACKEND; + template + void operator()(const runtime_event_type & ev) const noexcept { + (void)ev; } }; inline constexpr reject_invalid_render reject_invalid_render{}; inline constexpr begin_render begin_render{}; -inline constexpr seed_program seed_program{}; -inline constexpr eval_next_stmt eval_next_stmt{}; -inline constexpr eval_pending_expr eval_pending_expr{}; -inline constexpr write_pending_value write_pending_value{}; -inline constexpr finalize_done finalize_done{}; -inline constexpr finalize_error finalize_error{}; +inline constexpr mark_empty_output mark_empty_output{}; +inline constexpr copy_source_text copy_source_text{}; +inline constexpr mark_capacity_error mark_capacity_error{}; +inline constexpr dispatch_done dispatch_done{}; +inline constexpr dispatch_error dispatch_error{}; inline constexpr on_unexpected on_unexpected{}; } // namespace emel::text::jinja::formatter::action diff --git a/src/emel/text/jinja/formatter/context.hpp b/src/emel/text/jinja/formatter/context.hpp index cfe23963..3d21a216 100644 --- a/src/emel/text/jinja/formatter/context.hpp +++ b/src/emel/text/jinja/formatter/context.hpp @@ -1,76 +1,7 @@ #pragma once -#include -#include -#include - -#include "emel/emel.h" -#include "emel/text/jinja/ast.hpp" -#include "emel/text/jinja/value.hpp" - namespace emel::text::jinja::formatter::action { -inline constexpr size_t k_max_scopes = 16; -inline constexpr size_t k_scope_capacity = 128; -inline constexpr size_t k_max_array_items = 2048; -inline constexpr size_t k_max_object_entries = 2048; -inline constexpr size_t k_max_callables = 64; -inline constexpr size_t k_max_string_bytes = 8192; -inline constexpr size_t k_max_capture_depth = 4; -inline constexpr size_t k_max_capture_bytes = 2048; -inline constexpr size_t k_max_steps = 200000; - -struct scope_state { - emel::text::jinja::object_value locals = {}; -}; - -struct macro_callable { - const emel::text::jinja::macro_statement * stmt = nullptr; -}; - -struct caller_callable { - const emel::text::jinja::ast_list * body = nullptr; - const emel::text::jinja::ast_list * params = nullptr; -}; - -struct callable_slot { - emel::text::jinja::function_kind kind = emel::text::jinja::function_kind::builtin; - macro_callable macro = {}; - caller_callable caller = {}; -}; - -struct context { - int32_t phase_error = EMEL_OK; - int32_t last_error = EMEL_OK; - size_t error_pos = 0; - uint32_t steps_remaining = k_max_steps; - const emel::text::jinja::object_value * globals = nullptr; - const emel::text::jinja::ast_list * statements = nullptr; - size_t statement_index = 0; - const emel::text::jinja::ast_node * pending_expr = nullptr; - emel::text::jinja::value pending_value = {}; - bool pending_value_ready = false; - char * output = nullptr; - size_t output_capacity = 0; - size_t output_length = 0; - bool output_truncated = false; - - std::array scopes = {}; - size_t scope_count = 0; - - std::array array_items = {}; - size_t array_items_used = 0; - - std::array object_entries = {}; - size_t object_entries_used = 0; - - std::array string_buffer = {}; - size_t string_buffer_used = 0; - - std::array capture_buffer = {}; - - std::array callables = {}; - size_t callable_count = 0; -}; +struct context {}; } // namespace emel::text::jinja::formatter::action diff --git a/src/emel/text/jinja/formatter/detail.hpp b/src/emel/text/jinja/formatter/detail.hpp index 286bff59..7e83ffa3 100644 --- a/src/emel/text/jinja/formatter/detail.hpp +++ b/src/emel/text/jinja/formatter/detail.hpp @@ -1,2126 +1,71 @@ #pragma once -#include -#include -#include -#include -#include -#include +#include +#include -#include "emel/emel.h" -#include "emel/text/jinja/ast.hpp" -#include "emel/text/jinja/value.hpp" -#include "emel/text/jinja/formatter/context.hpp" +#include "emel/text/jinja/formatter/errors.hpp" #include "emel/text/jinja/formatter/events.hpp" namespace emel::text::jinja::formatter::detail { -inline constexpr size_t k_max_call_args = 16; - -struct keyword_arg { - std::string_view key = {}; - emel::text::jinja::value val = {}; -}; - -struct call_args { - std::array pos = {}; - size_t pos_count = 0; - std::array kw = {}; - size_t kw_count = 0; -}; - -struct writer_state { - char * data = nullptr; - size_t capacity = 0; - size_t length = 0; -}; - -struct render_io { - std::array writers = {}; - size_t depth = 0; -}; - -enum class control_flow : uint8_t { - none = 0, - break_loop = 1, - continue_loop = 2 -}; - -using builtin_fn = bool (*)(action::context &, const call_args &, emel::text::jinja::value &) noexcept; -using filter_fn = bool (*)(action::context &, const emel::text::jinja::value &, const call_args &, emel::text::jinja::value &) noexcept; -using test_fn = bool (*)(action::context &, const emel::text::jinja::value &, const call_args &, emel::text::jinja::value &) noexcept; - -struct builtin_entry { - std::string_view name = {}; - builtin_fn fn = nullptr; -}; - -struct filter_entry { - std::string_view name = {}; - filter_fn fn = nullptr; -}; - -struct test_entry { - std::string_view name = {}; - test_fn fn = nullptr; -}; - -inline void set_error(action::context & ctx, const int32_t err, const size_t pos) noexcept { - if (ctx.phase_error == EMEL_OK) { - ctx.phase_error = err; - ctx.last_error = err; - ctx.error_pos = pos; - } -} - -inline emel::text::jinja::value make_undefined(std::string_view hint = {}) noexcept { - emel::text::jinja::value v; - v.type = emel::text::jinja::value_type::undefined; - v.hint = hint; - return v; -} - -inline emel::text::jinja::value make_none() noexcept { - emel::text::jinja::value v; - v.type = emel::text::jinja::value_type::none; - return v; -} - -inline emel::text::jinja::value make_bool(const bool v) noexcept { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::boolean; - out.bool_v = v; - return out; -} - -inline emel::text::jinja::value make_int(const int64_t v) noexcept { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::integer; - out.int_v = v; - out.float_v = static_cast(v); - return out; -} - -inline emel::text::jinja::value make_float(const double v) noexcept { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::floating; - out.float_v = v; - out.int_v = static_cast(v); - return out; -} - -inline emel::text::jinja::value make_string(std::string_view view, const bool is_input = false) noexcept { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::string; - out.string_v.view = view; - out.string_v.is_input = is_input; - return out; -} - -inline emel::text::jinja::value make_function(const emel::text::jinja::function_ref ref) noexcept { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::function; - out.func_v = ref; - return out; -} - -inline bool ensure_steps(action::context & ctx, const size_t pos) noexcept { - if (ctx.steps_remaining == 0) { - set_error(ctx, EMEL_ERR_BACKEND, pos); - return false; - } - ctx.steps_remaining -= 1; - return true; -} - -inline writer_state & current_writer(render_io & io) noexcept { - return io.writers[io.depth]; -} - -inline void init_writer(render_io & io, char * output, const size_t capacity) noexcept { - io.depth = 0; - io.writers[0].data = output; - io.writers[0].capacity = capacity; - io.writers[0].length = 0; -} - -inline bool write_text(action::context & ctx, render_io & io, std::string_view text) noexcept { - writer_state & writer = current_writer(io); - if (writer.length + text.size() > writer.capacity) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - if (writer.length < writer.capacity && !text.empty()) { - const size_t avail = writer.capacity - writer.length; - if (avail > 0) { - std::memcpy(writer.data + writer.length, text.data(), avail); - writer.length += avail; - } - } - return false; - } - if (!text.empty()) { - std::memcpy(writer.data + writer.length, text.data(), text.size()); - writer.length += text.size(); - } - return true; -} - -inline std::string_view store_string(action::context & ctx, std::string_view text) noexcept { - if (text.empty()) { - return {}; - } - const size_t remaining = action::k_max_string_bytes - ctx.string_buffer_used; - if (text.size() > remaining) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return {}; - } - char * dest = ctx.string_buffer.data() + ctx.string_buffer_used; - std::memcpy(dest, text.data(), text.size()); - ctx.string_buffer_used += text.size(); - return std::string_view(dest, text.size()); -} - -inline bool begin_capture(action::context & ctx, render_io & io) noexcept { - if (io.depth + 1 > action::k_max_capture_depth) { - set_error(ctx, EMEL_ERR_BACKEND, 0); - return false; - } - const size_t capture_index = io.depth; - char * base = ctx.capture_buffer.data() + capture_index * action::k_max_capture_bytes; - io.depth += 1; - io.writers[io.depth].data = base; - io.writers[io.depth].capacity = action::k_max_capture_bytes; - io.writers[io.depth].length = 0; - return true; -} - -inline bool end_capture(action::context & ctx, render_io & io, emel::text::jinja::value & out) noexcept { - if (io.depth == 0) { - set_error(ctx, EMEL_ERR_BACKEND, 0); - return false; - } - writer_state finished = io.writers[io.depth]; - io.depth -= 1; - const std::string_view stored = store_string(ctx, std::string_view(finished.data, finished.length)); - if (ctx.phase_error != EMEL_OK) { - return false; - } - out = make_string(stored, false); - return true; -} - -inline emel::text::jinja::value make_array(action::context & ctx, - const emel::text::jinja::value * items, - const size_t count) noexcept { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::array; - if (count == 0) { - out.array_v.items = nullptr; - out.array_v.count = 0; - out.array_v.capacity = 0; - return out; - } - const size_t remaining = action::k_max_array_items - ctx.array_items_used; - if (count > remaining) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return make_undefined("array_overflow"); - } - emel::text::jinja::value * dest = ctx.array_items.data() + ctx.array_items_used; - for (size_t i = 0; i < count; ++i) { - dest[i] = items[i]; - } - ctx.array_items_used += count; - out.array_v.items = dest; - out.array_v.count = count; - out.array_v.capacity = count; - return out; -} - -inline emel::text::jinja::value make_object(action::context & ctx, - const emel::text::jinja::object_entry * entries, - const size_t count, - const size_t capacity, - const bool has_builtins) noexcept { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::object; - if (capacity == 0) { - out.object_v.entries = nullptr; - out.object_v.count = 0; - out.object_v.capacity = 0; - out.object_v.has_builtins = has_builtins; - return out; - } - const size_t remaining = action::k_max_object_entries - ctx.object_entries_used; - if (capacity > remaining) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return make_undefined("object_overflow"); - } - emel::text::jinja::object_entry * dest = ctx.object_entries.data() + ctx.object_entries_used; - for (size_t i = 0; i < count && i < capacity; ++i) { - dest[i] = entries[i]; - } - ctx.object_entries_used += capacity; - out.object_v.entries = dest; - out.object_v.count = count; - out.object_v.capacity = capacity; - out.object_v.has_builtins = has_builtins; - return out; -} - -inline bool push_scope(action::context & ctx) noexcept { - if (ctx.scope_count >= action::k_max_scopes) { - set_error(ctx, EMEL_ERR_BACKEND, 0); - return false; - } - const size_t remaining = action::k_max_object_entries - ctx.object_entries_used; - if (action::k_scope_capacity > remaining) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - action::scope_state & scope = ctx.scopes[ctx.scope_count++]; - emel::text::jinja::object_entry * entries = ctx.object_entries.data() + ctx.object_entries_used; - ctx.object_entries_used += action::k_scope_capacity; - scope.locals.entries = entries; - scope.locals.count = 0; - scope.locals.capacity = action::k_scope_capacity; - scope.locals.has_builtins = false; - return true; -} - -inline void pop_scope(action::context & ctx) noexcept { - if (ctx.scope_count == 0) { - return; - } - ctx.scope_count -= 1; -} - -inline bool value_is_truthy(const emel::text::jinja::value & v) noexcept { - switch (v.type) { - case emel::text::jinja::value_type::undefined: - case emel::text::jinja::value_type::none: - return false; - case emel::text::jinja::value_type::boolean: - return v.bool_v; - case emel::text::jinja::value_type::integer: - return v.int_v != 0; - case emel::text::jinja::value_type::floating: - return v.float_v != 0.0; - case emel::text::jinja::value_type::string: - return !v.string_v.view.empty(); - case emel::text::jinja::value_type::array: - return v.array_v.count > 0; - case emel::text::jinja::value_type::object: - return v.object_v.count > 0; - case emel::text::jinja::value_type::function: - return true; - } - return false; -} - -inline bool value_is_number(const emel::text::jinja::value & v) noexcept { - return v.type == emel::text::jinja::value_type::integer || - v.type == emel::text::jinja::value_type::floating; -} - -inline bool value_equal(const emel::text::jinja::value & a, const emel::text::jinja::value & b) noexcept { - if (a.type != b.type) { - if (value_is_number(a) && value_is_number(b)) { - const double da = a.type == emel::text::jinja::value_type::integer ? - static_cast(a.int_v) : a.float_v; - const double db = b.type == emel::text::jinja::value_type::integer ? - static_cast(b.int_v) : b.float_v; - return da == db; - } - return false; - } - switch (a.type) { - case emel::text::jinja::value_type::undefined: - case emel::text::jinja::value_type::none: - return true; - case emel::text::jinja::value_type::boolean: - return a.bool_v == b.bool_v; - case emel::text::jinja::value_type::integer: - return a.int_v == b.int_v; - case emel::text::jinja::value_type::floating: - return a.float_v == b.float_v; - case emel::text::jinja::value_type::string: - return a.string_v.view == b.string_v.view; - case emel::text::jinja::value_type::array: - if (a.array_v.count != b.array_v.count) { - return false; - } - for (size_t i = 0; i < a.array_v.count; ++i) { - if (!value_equal(a.array_v.items[i], b.array_v.items[i])) { - return false; - } - } - return true; - case emel::text::jinja::value_type::object: - if (a.object_v.count != b.object_v.count) { - return false; - } - for (size_t i = 0; i < a.object_v.count; ++i) { - const auto & entry = a.object_v.entries[i]; - bool found = false; - for (size_t j = 0; j < b.object_v.count; ++j) { - if (value_equal(entry.key, b.object_v.entries[j].key)) { - if (!value_equal(entry.val, b.object_v.entries[j].val)) { - return false; - } - found = true; - break; - } - } - if (!found) { - return false; - } - } - return true; - case emel::text::jinja::value_type::function: - return a.func_v.data == b.func_v.data && a.func_v.kind == b.func_v.kind; - } - return false; -} - -inline const emel::text::jinja::object_entry * find_object_entry( - const emel::text::jinja::object_value & object, - const std::string_view key) noexcept { - for (size_t i = 0; i < object.count; ++i) { - const emel::text::jinja::object_entry & entry = object.entries[i]; - if (entry.key.type == emel::text::jinja::value_type::string && - entry.key.string_v.view == key) { - return &entry; - } - } - return nullptr; -} - -inline emel::text::jinja::object_entry * find_object_entry_mut( - emel::text::jinja::object_value & object, - const std::string_view key) noexcept { - for (size_t i = 0; i < object.count; ++i) { - emel::text::jinja::object_entry & entry = object.entries[i]; - if (entry.key.type == emel::text::jinja::value_type::string && - entry.key.string_v.view == key) { - return &entry; - } - } - return nullptr; -} - -inline bool set_object_value(action::context & ctx, - emel::text::jinja::object_value & object, - const std::string_view key, - const emel::text::jinja::value & val) noexcept { - emel::text::jinja::object_entry * existing = find_object_entry_mut(object, key); - if (existing != nullptr) { - existing->val = val; - return true; - } - if (object.count >= object.capacity) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - object.entries[object.count].key = make_string(key, false); - object.entries[object.count].val = val; - object.count += 1; - return true; -} - -inline emel::text::jinja::value lookup_identifier( - action::context & ctx, - const emel::text::jinja::object_value * globals, - const std::string_view name, - const builtin_entry * builtins, - const size_t builtin_count) noexcept { - for (size_t i = ctx.scope_count; i > 0; --i) { - emel::text::jinja::object_value & scope_obj = ctx.scopes[i - 1].locals; - const emel::text::jinja::object_entry * entry = find_object_entry(scope_obj, name); - if (entry != nullptr) { - return entry->val; - } - } - if (globals != nullptr) { - const emel::text::jinja::object_entry * entry = find_object_entry(*globals, name); - if (entry != nullptr) { - return entry->val; - } - } - for (size_t i = 0; i < builtin_count; ++i) { - if (builtins[i].name == name) { - emel::text::jinja::function_ref ref; - ref.kind = emel::text::jinja::function_kind::builtin; - ref.data = &builtins[i]; - return make_function(ref); - } - } - return make_undefined(name); -} - -inline std::string_view value_to_string(action::context & ctx, const emel::text::jinja::value & v) noexcept { - if (v.type == emel::text::jinja::value_type::string) { - return v.string_v.view; - } - if (v.type == emel::text::jinja::value_type::none || v.type == emel::text::jinja::value_type::undefined) { - return {}; - } - char buffer[64] = {}; - int written = 0; - switch (v.type) { - case emel::text::jinja::value_type::boolean: - return v.bool_v ? "true" : "false"; - case emel::text::jinja::value_type::integer: - written = std::snprintf(buffer, sizeof(buffer), "%lld", - static_cast(v.int_v)); - break; - case emel::text::jinja::value_type::floating: - written = std::snprintf(buffer, sizeof(buffer), "%.15g", v.float_v); - break; - case emel::text::jinja::value_type::array: - case emel::text::jinja::value_type::object: - case emel::text::jinja::value_type::function: - case emel::text::jinja::value_type::undefined: - case emel::text::jinja::value_type::none: - case emel::text::jinja::value_type::string: - written = 0; - break; - } - if (written <= 0) { - return {}; - } - const std::string_view stored = store_string(ctx, std::string_view(buffer, static_cast(written))); - return stored; -} - -inline bool write_value(action::context & ctx, render_io & io, const emel::text::jinja::value & v) noexcept { - if (v.type == emel::text::jinja::value_type::none || v.type == emel::text::jinja::value_type::undefined) { - return true; - } - if (v.type == emel::text::jinja::value_type::array) { - for (size_t i = 0; i < v.array_v.count; ++i) { - if (!write_value(ctx, io, v.array_v.items[i])) { - return false; - } - } - return true; - } - if (v.type == emel::text::jinja::value_type::object) { - return true; - } - const std::string_view text = value_to_string(ctx, v); - return write_text(ctx, io, text); -} - -inline bool add_pos(call_args & args, const emel::text::jinja::value & val) noexcept { - if (args.pos_count >= k_max_call_args) { - return false; - } - args.pos[args.pos_count++] = val; - return true; -} - -inline bool add_kw(call_args & args, const std::string_view key, const emel::text::jinja::value & val) noexcept { - if (args.kw_count >= k_max_call_args) { - return false; - } - args.kw[args.kw_count++] = keyword_arg{key, val}; - return true; -} - -inline const emel::text::jinja::value * find_kw(const call_args & args, std::string_view key) noexcept { - for (size_t i = 0; i < args.kw_count; ++i) { - if (args.kw[i].key == key) { - return &args.kw[i].val; - } - } - return nullptr; -} - -inline const emel::text::jinja::value * get_pos(const call_args & args, const size_t index) noexcept { - if (index >= args.pos_count) { - return nullptr; - } - return &args.pos[index]; -} - -inline bool value_is_string(const emel::text::jinja::value & v) noexcept { - return v.type == emel::text::jinja::value_type::string; -} - -inline bool value_is_array(const emel::text::jinja::value & v) noexcept { - return v.type == emel::text::jinja::value_type::array; -} - -inline bool value_is_object(const emel::text::jinja::value & v) noexcept { - return v.type == emel::text::jinja::value_type::object; -} - -inline bool builtin_default(action::context & ctx, const call_args & args, emel::text::jinja::value & out) noexcept { - const emel::text::jinja::value * val = get_pos(args, 0); - const emel::text::jinja::value * def = get_pos(args, 1); - const emel::text::jinja::value * use_when_false = get_pos(args, 2); - if (val == nullptr || def == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - bool apply = (val->type == emel::text::jinja::value_type::undefined || - val->type == emel::text::jinja::value_type::none); - if (!apply && use_when_false != nullptr && use_when_false->type == emel::text::jinja::value_type::boolean) { - if (use_when_false->bool_v) { - apply = !value_is_truthy(*val); - } - } - out = apply ? *def : *val; - return true; -} - -inline bool builtin_length(action::context & ctx, const call_args & args, emel::text::jinja::value & out) noexcept { - const emel::text::jinja::value * val = get_pos(args, 0); - if (val == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - switch (val->type) { - case emel::text::jinja::value_type::string: - out = make_int(static_cast(val->string_v.view.size())); - return true; - case emel::text::jinja::value_type::array: - out = make_int(static_cast(val->array_v.count)); - return true; - case emel::text::jinja::value_type::object: - out = make_int(static_cast(val->object_v.count)); - return true; - default: - out = make_int(0); - return true; - } -} - -inline bool builtin_range(action::context & ctx, const call_args & args, emel::text::jinja::value & out) noexcept { - if (args.pos_count < 1 || args.pos_count > 3) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - int64_t start = 0; - int64_t stop = 0; - int64_t step = 1; - if (args.pos_count == 1) { - const emel::text::jinja::value & v0 = args.pos[0]; - if (v0.type != emel::text::jinja::value_type::integer) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - stop = v0.int_v; - } else if (args.pos_count == 2) { - const emel::text::jinja::value & v0 = args.pos[0]; - const emel::text::jinja::value & v1 = args.pos[1]; - if (v0.type != emel::text::jinja::value_type::integer || - v1.type != emel::text::jinja::value_type::integer) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - start = v0.int_v; - stop = v1.int_v; - } else { - const emel::text::jinja::value & v0 = args.pos[0]; - const emel::text::jinja::value & v1 = args.pos[1]; - const emel::text::jinja::value & v2 = args.pos[2]; - if (v0.type != emel::text::jinja::value_type::integer || - v1.type != emel::text::jinja::value_type::integer || - v2.type != emel::text::jinja::value_type::integer) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - start = v0.int_v; - stop = v1.int_v; - step = v2.int_v; - } - if (step == 0) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - std::array tmp = {}; - size_t count = 0; - if (step > 0) { - for (int64_t i = start; i < stop && count < tmp.size(); i += step) { - tmp[count++] = make_int(i); - } - } else { - for (int64_t i = start; i > stop && count < tmp.size(); i += step) { - tmp[count++] = make_int(i); - } - } - if (count >= tmp.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - out = make_array(ctx, tmp.data(), count); - return ctx.phase_error == EMEL_OK; -} - -inline bool builtin_namespace(action::context & ctx, const call_args & args, emel::text::jinja::value & out) noexcept { - if (args.kw_count == 0) { - emel::text::jinja::object_entry empty = {}; - out = make_object(ctx, &empty, 0, 0, false); - return ctx.phase_error == EMEL_OK; - } - std::array tmp = {}; - for (size_t i = 0; i < args.kw_count; ++i) { - tmp[i].key = make_string(args.kw[i].key, false); - tmp[i].val = args.kw[i].val; - } - out = make_object(ctx, tmp.data(), args.kw_count, args.kw_count, false); - return ctx.phase_error == EMEL_OK; -} - -inline bool builtin_upper(action::context & ctx, const call_args & args, emel::text::jinja::value & out) noexcept { - const emel::text::jinja::value * val = get_pos(args, 0); - if (val == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - const std::string_view input = value_to_string(ctx, *val); - if (ctx.phase_error != EMEL_OK) { - return false; - } - std::array buffer = {}; - if (input.size() > buffer.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - for (size_t i = 0; i < input.size(); ++i) { - buffer[i] = static_cast(std::toupper(static_cast(input[i]))); - } - out = make_string(store_string(ctx, std::string_view(buffer.data(), input.size())), false); - return ctx.phase_error == EMEL_OK; -} - -inline bool builtin_lower(action::context & ctx, const call_args & args, emel::text::jinja::value & out) noexcept { - const emel::text::jinja::value * val = get_pos(args, 0); - if (val == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - const std::string_view input = value_to_string(ctx, *val); - if (ctx.phase_error != EMEL_OK) { - return false; - } - std::array buffer = {}; - if (input.size() > buffer.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - for (size_t i = 0; i < input.size(); ++i) { - buffer[i] = static_cast(std::tolower(static_cast(input[i]))); - } - out = make_string(store_string(ctx, std::string_view(buffer.data(), input.size())), false); - return ctx.phase_error == EMEL_OK; -} - -inline bool builtin_trim(action::context & ctx, const call_args & args, emel::text::jinja::value & out) noexcept { - const emel::text::jinja::value * val = get_pos(args, 0); - if (val == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - std::string_view input = value_to_string(ctx, *val); - if (ctx.phase_error != EMEL_OK) { - return false; - } - size_t start = 0; - size_t end = input.size(); - while (start < end && std::isspace(static_cast(input[start]))) { - ++start; - } - while (end > start && std::isspace(static_cast(input[end - 1]))) { - --end; - } - out = make_string(store_string(ctx, input.substr(start, end - start)), false); - return ctx.phase_error == EMEL_OK; -} - -inline bool builtin_replace(action::context & ctx, const call_args & args, emel::text::jinja::value & out) noexcept { - const emel::text::jinja::value * input_val = get_pos(args, 0); - const emel::text::jinja::value * old_val = get_pos(args, 1); - const emel::text::jinja::value * new_val = get_pos(args, 2); - if (input_val == nullptr || old_val == nullptr || new_val == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - const std::string_view input = value_to_string(ctx, *input_val); - const std::string_view old_str = value_to_string(ctx, *old_val); - const std::string_view new_str = value_to_string(ctx, *new_val); - if (ctx.phase_error != EMEL_OK) { - return false; - } - if (old_str.empty()) { - out = make_string(store_string(ctx, input), false); - return ctx.phase_error == EMEL_OK; - } - std::array buffer = {}; - size_t offset = 0; - size_t pos = 0; - while (pos <= input.size()) { - const size_t found = input.find(old_str, pos); - if (found == std::string_view::npos) { - const size_t tail = input.size() - pos; - if (offset + tail > buffer.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - std::memcpy(buffer.data() + offset, input.data() + pos, tail); - offset += tail; - break; - } - const size_t chunk = found - pos; - if (offset + chunk + new_str.size() > buffer.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - if (chunk > 0) { - std::memcpy(buffer.data() + offset, input.data() + pos, chunk); - offset += chunk; - } - if (!new_str.empty()) { - std::memcpy(buffer.data() + offset, new_str.data(), new_str.size()); - offset += new_str.size(); - } - pos = found + old_str.size(); - } - out = make_string(store_string(ctx, std::string_view(buffer.data(), offset)), false); - return ctx.phase_error == EMEL_OK; -} - -inline bool builtin_join(action::context & ctx, const call_args & args, emel::text::jinja::value & out) noexcept { - const emel::text::jinja::value * input = get_pos(args, 0); - const emel::text::jinja::value * delim = get_pos(args, 1); - if (input == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - if (input->type != emel::text::jinja::value_type::array) { - out = make_string(value_to_string(ctx, *input), false); - return ctx.phase_error == EMEL_OK; - } - const std::string_view sep = delim != nullptr ? value_to_string(ctx, *delim) : std::string_view(" "); - if (ctx.phase_error != EMEL_OK) { - return false; - } - std::array buffer = {}; - size_t offset = 0; - for (size_t i = 0; i < input->array_v.count; ++i) { - const std::string_view part = value_to_string(ctx, input->array_v.items[i]); - if (ctx.phase_error != EMEL_OK) { - return false; - } - if (i > 0 && !sep.empty()) { - if (offset + sep.size() > buffer.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - std::memcpy(buffer.data() + offset, sep.data(), sep.size()); - offset += sep.size(); - } - if (offset + part.size() > buffer.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - if (!part.empty()) { - std::memcpy(buffer.data() + offset, part.data(), part.size()); - offset += part.size(); - } - } - out = make_string(store_string(ctx, std::string_view(buffer.data(), offset)), false); - return ctx.phase_error == EMEL_OK; -} - -inline bool filter_default(action::context & ctx, const emel::text::jinja::value & input, - const call_args & args, emel::text::jinja::value & out) noexcept { - call_args combined = args; - combined.pos[0] = input; - if (combined.pos_count == 0) { - combined.pos_count = 1; - } else { - for (size_t i = combined.pos_count; i > 0; --i) { - combined.pos[i] = combined.pos[i - 1]; - } - combined.pos[0] = input; - combined.pos_count += 1; - } - return builtin_default(ctx, combined, out); -} - -inline bool filter_length(action::context & ctx, const emel::text::jinja::value & input, - const call_args & args, emel::text::jinja::value & out) noexcept { - call_args combined = args; - if (combined.pos_count >= k_max_call_args) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - for (size_t i = combined.pos_count; i > 0; --i) { - combined.pos[i] = combined.pos[i - 1]; - } - combined.pos[0] = input; - combined.pos_count += 1; - return builtin_length(ctx, combined, out); -} - -inline bool filter_upper(action::context & ctx, const emel::text::jinja::value & input, - const call_args & args, emel::text::jinja::value & out) noexcept { - call_args combined = args; - if (combined.pos_count >= k_max_call_args) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - for (size_t i = combined.pos_count; i > 0; --i) { - combined.pos[i] = combined.pos[i - 1]; - } - combined.pos[0] = input; - combined.pos_count += 1; - return builtin_upper(ctx, combined, out); -} - -inline bool filter_lower(action::context & ctx, const emel::text::jinja::value & input, - const call_args & args, emel::text::jinja::value & out) noexcept { - call_args combined = args; - if (combined.pos_count >= k_max_call_args) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - for (size_t i = combined.pos_count; i > 0; --i) { - combined.pos[i] = combined.pos[i - 1]; - } - combined.pos[0] = input; - combined.pos_count += 1; - return builtin_lower(ctx, combined, out); -} - -inline bool filter_trim(action::context & ctx, const emel::text::jinja::value & input, - const call_args & args, emel::text::jinja::value & out) noexcept { - call_args combined = args; - if (combined.pos_count >= k_max_call_args) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - for (size_t i = combined.pos_count; i > 0; --i) { - combined.pos[i] = combined.pos[i - 1]; - } - combined.pos[0] = input; - combined.pos_count += 1; - return builtin_trim(ctx, combined, out); -} - -inline bool filter_replace(action::context & ctx, const emel::text::jinja::value & input, - const call_args & args, emel::text::jinja::value & out) noexcept { - call_args combined = args; - if (combined.pos_count >= k_max_call_args) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - for (size_t i = combined.pos_count; i > 0; --i) { - combined.pos[i] = combined.pos[i - 1]; - } - combined.pos[0] = input; - combined.pos_count += 1; - return builtin_replace(ctx, combined, out); -} - -inline bool filter_join(action::context & ctx, const emel::text::jinja::value & input, - const call_args & args, emel::text::jinja::value & out) noexcept { - call_args combined = args; - if (combined.pos_count >= k_max_call_args) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return false; - } - for (size_t i = combined.pos_count; i > 0; --i) { - combined.pos[i] = combined.pos[i - 1]; - } - combined.pos[0] = input; - combined.pos_count += 1; - return builtin_join(ctx, combined, out); -} - -inline bool test_defined(action::context &, const emel::text::jinja::value & input, - const call_args &, emel::text::jinja::value & out) noexcept { - out = make_bool(input.type != emel::text::jinja::value_type::undefined); - return true; -} - -inline bool test_undefined(action::context &, const emel::text::jinja::value & input, - const call_args &, emel::text::jinja::value & out) noexcept { - out = make_bool(input.type == emel::text::jinja::value_type::undefined); - return true; -} - -inline bool test_none(action::context &, const emel::text::jinja::value & input, - const call_args &, emel::text::jinja::value & out) noexcept { - out = make_bool(input.type == emel::text::jinja::value_type::none); - return true; -} - -inline bool test_string(action::context &, const emel::text::jinja::value & input, - const call_args &, emel::text::jinja::value & out) noexcept { - out = make_bool(input.type == emel::text::jinja::value_type::string); - return true; -} - -inline bool test_boolean(action::context &, const emel::text::jinja::value & input, - const call_args &, emel::text::jinja::value & out) noexcept { - out = make_bool(input.type == emel::text::jinja::value_type::boolean); - return true; -} - -inline bool test_number(action::context &, const emel::text::jinja::value & input, - const call_args &, emel::text::jinja::value & out) noexcept { - out = make_bool(value_is_number(input)); - return true; -} - -inline bool test_iterable(action::context &, const emel::text::jinja::value & input, - const call_args &, emel::text::jinja::value & out) noexcept { - out = make_bool(input.type == emel::text::jinja::value_type::array || - input.type == emel::text::jinja::value_type::string || - input.type == emel::text::jinja::value_type::object); - return true; -} - -inline bool test_mapping(action::context &, const emel::text::jinja::value & input, - const call_args &, emel::text::jinja::value & out) noexcept { - out = make_bool(input.type == emel::text::jinja::value_type::object); - return true; -} - -inline bool test_even(action::context &, const emel::text::jinja::value & input, - const call_args &, emel::text::jinja::value & out) noexcept { - if (input.type != emel::text::jinja::value_type::integer) { - out = make_bool(false); - return true; - } - out = make_bool((input.int_v % 2) == 0); - return true; -} - -inline bool test_odd(action::context &, const emel::text::jinja::value & input, - const call_args &, emel::text::jinja::value & out) noexcept { - if (input.type != emel::text::jinja::value_type::integer) { - out = make_bool(false); - return true; - } - out = make_bool((input.int_v % 2) != 0); - return true; -} - -inline bool test_true(action::context &, const emel::text::jinja::value & input, - const call_args &, emel::text::jinja::value & out) noexcept { - out = make_bool(input.type == emel::text::jinja::value_type::boolean && input.bool_v); - return true; -} - -inline bool test_false(action::context &, const emel::text::jinja::value & input, - const call_args &, emel::text::jinja::value & out) noexcept { - out = make_bool(input.type == emel::text::jinja::value_type::boolean && !input.bool_v); - return true; -} - -inline const builtin_entry * builtin_table(size_t & count) noexcept { - static const std::array table = {{ - {"default", builtin_default}, - {"length", builtin_length}, - {"range", builtin_range}, - {"namespace", builtin_namespace}, - {"upper", builtin_upper}, - {"lower", builtin_lower}, - {"trim", builtin_trim}, - {"replace", builtin_replace} - }}; - count = table.size(); - return table.data(); -} - -inline const filter_entry * filter_table(size_t & count) noexcept { - static const std::array table = {{ - {"default", filter_default}, - {"length", filter_length}, - {"upper", filter_upper}, - {"lower", filter_lower}, - {"trim", filter_trim}, - {"replace", filter_replace}, - {"join", filter_join} - }}; - count = table.size(); - return table.data(); -} - -inline const test_entry * test_table(size_t & count) noexcept { - static const std::array table = {{ - {"defined", test_defined}, - {"undefined", test_undefined}, - {"none", test_none}, - {"string", test_string}, - {"boolean", test_boolean}, - {"number", test_number}, - {"iterable", test_iterable}, - {"mapping", test_mapping}, - {"even", test_even}, - {"odd", test_odd}, - {"true", test_true}, - {"false", test_false} - }}; - count = table.size(); - return table.data(); -} - -inline const builtin_entry * find_builtin(std::string_view name) noexcept { - size_t count = 0; - const builtin_entry * table = builtin_table(count); - for (size_t i = 0; i < count; ++i) { - if (table[i].name == name) { - return &table[i]; - } - } - return nullptr; -} - -inline const filter_entry * find_filter(std::string_view name) noexcept { - size_t count = 0; - const filter_entry * table = filter_table(count); - for (size_t i = 0; i < count; ++i) { - if (table[i].name == name) { - return &table[i]; - } - } - return nullptr; -} - -inline const test_entry * find_test(std::string_view name) noexcept { - size_t count = 0; - const test_entry * table = test_table(count); - for (size_t i = 0; i < count; ++i) { - if (table[i].name == name) { - return &table[i]; - } - } - return nullptr; -} - -inline bool collect_call_args(action::context & ctx, - const emel::text::jinja::ast_list & args, - call_args & out, - const emel::text::jinja::object_value * globals, - render_io & io); - -inline emel::text::jinja::value eval_expr(action::context & ctx, - const emel::text::jinja::ast_node * node, - const emel::text::jinja::object_value * globals, - render_io & io); - -inline bool render_statements(action::context & ctx, - const emel::text::jinja::ast_list & statements, - const emel::text::jinja::object_value * globals, - render_io & io, - bool allow_control, - control_flow & flow); - -inline bool collect_call_args(action::context & ctx, - const emel::text::jinja::ast_list & args, - call_args & out, - const emel::text::jinja::object_value * globals, - render_io & io) { - out.pos_count = 0; - out.kw_count = 0; - for (const auto & arg : args) { - if (ctx.phase_error != EMEL_OK) { - return false; - } - if (!arg) { - continue; - } - if (auto * spread = dynamic_cast(arg.get())) { - emel::text::jinja::value val = eval_expr(ctx, spread->operand.get(), globals, io); - if (val.type != emel::text::jinja::value_type::array) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, spread->pos); - return false; - } - for (size_t i = 0; i < val.array_v.count; ++i) { - if (!add_pos(out, val.array_v.items[i])) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, spread->pos); - return false; - } - } - continue; - } - if (auto * kw = dynamic_cast(arg.get())) { - auto * key_id = dynamic_cast(kw->key.get()); - if (key_id == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, kw->pos); - return false; - } - emel::text::jinja::value val = eval_expr(ctx, kw->value.get(), globals, io); - if (!add_kw(out, key_id->name, val)) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, kw->pos); - return false; - } - continue; - } - emel::text::jinja::value val = eval_expr(ctx, arg.get(), globals, io); - if (!add_pos(out, val)) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, arg->pos); - return false; - } - } - return ctx.phase_error == EMEL_OK; -} - -inline emel::text::jinja::value invoke_function(action::context & ctx, - const emel::text::jinja::function_ref & fn, - const call_args & args, - const emel::text::jinja::object_value * globals, - render_io & io) noexcept { - if (fn.kind == emel::text::jinja::function_kind::builtin) { - const auto * entry = static_cast(fn.data); - if (entry == nullptr || entry->fn == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return make_undefined("builtin_missing"); - } - emel::text::jinja::value out = make_undefined(); - if (!entry->fn(ctx, args, out)) { - return make_undefined("builtin_failed"); - } - return out; - } - if (fn.kind == emel::text::jinja::function_kind::macro) { - const auto * slot = static_cast(fn.data); - if (slot == nullptr || slot->macro.stmt == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return make_undefined("macro_missing"); - } - const emel::text::jinja::macro_statement * stmt = slot->macro.stmt; - if (!push_scope(ctx)) { - return make_undefined("macro_scope"); - } - size_t arg_index = 0; - for (size_t i = 0; i < stmt->args.size(); ++i) { - const auto & param = stmt->args[i]; - auto * param_id = dynamic_cast(param.get()); - auto * param_kw = dynamic_cast(param.get()); - std::string_view name = {}; - emel::text::jinja::value param_value = make_undefined(); - if (param_id != nullptr) { - name = param_id->name; - } else if (param_kw != nullptr) { - auto * key_id = dynamic_cast(param_kw->key.get()); - if (key_id != nullptr) { - name = key_id->name; - } - } - if (name.empty()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - pop_scope(ctx); - return make_undefined("macro_param"); - } - const emel::text::jinja::value * kw_val = find_kw(args, name); - if (kw_val != nullptr) { - param_value = *kw_val; - } else if (arg_index < args.pos_count) { - param_value = args.pos[arg_index++]; - } else if (param_kw != nullptr) { - param_value = eval_expr(ctx, param_kw->value.get(), globals, io); - } else { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - pop_scope(ctx); - return make_undefined("macro_args"); - } - set_object_value(ctx, ctx.scopes[ctx.scope_count - 1].locals, name, param_value); - if (ctx.phase_error != EMEL_OK) { - pop_scope(ctx); - return make_undefined("macro_bind"); - } - } - std::array buffer = {}; - render_io macro_io = {}; - init_writer(macro_io, buffer.data(), buffer.size()); - control_flow flow = control_flow::none; - bool ok = render_statements(ctx, stmt->body, globals, macro_io, false, flow); - if (flow != control_flow::none) { - ok = false; - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - } - emel::text::jinja::value out = make_string( - store_string(ctx, std::string_view(macro_io.writers[0].data, macro_io.writers[0].length)), false); - pop_scope(ctx); - if (!ok || ctx.phase_error != EMEL_OK) { - return make_undefined("macro_failed"); - } - return out; - } - if (fn.kind == emel::text::jinja::function_kind::caller) { - const auto * slot = static_cast(fn.data); - if (slot == nullptr || slot->caller.body == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return make_undefined("caller_missing"); - } - if (!push_scope(ctx)) { - return make_undefined("caller_scope"); - } - if (slot->caller.params != nullptr) { - for (size_t i = 0; i < slot->caller.params->size(); ++i) { - const auto & param = (*slot->caller.params)[i]; - auto * param_id = dynamic_cast(param.get()); - if (param_id == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - break; - } - emel::text::jinja::value arg_val = i < args.pos_count ? args.pos[i] : make_undefined(); - set_object_value(ctx, ctx.scopes[ctx.scope_count - 1].locals, param_id->name, arg_val); - } - } - std::array buffer = {}; - render_io caller_io = {}; - init_writer(caller_io, buffer.data(), buffer.size()); - control_flow flow = control_flow::none; - render_statements(ctx, *slot->caller.body, globals, caller_io, false, flow); - if (flow != control_flow::none) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - } - emel::text::jinja::value out = make_string( - store_string(ctx, std::string_view(caller_io.writers[0].data, caller_io.writers[0].length)), false); - pop_scope(ctx); - return out; - } - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, 0); - return make_undefined("function_kind"); -} - -inline emel::text::jinja::value eval_member(action::context & ctx, - const emel::text::jinja::member_expression * node, - const emel::text::jinja::object_value * globals, - render_io & io) { - emel::text::jinja::value object = eval_expr(ctx, node->object.get(), globals, io); - if (object.type == emel::text::jinja::value_type::undefined) { - return make_undefined("member_object"); - } - emel::text::jinja::value property = make_undefined(); - if (node->computed) { - if (auto * slice = dynamic_cast(node->property.get())) { - int64_t start = 0; - int64_t stop = 0; - int64_t step = 1; - if (slice->start) { - emel::text::jinja::value start_val = eval_expr(ctx, slice->start.get(), globals, io); - if (start_val.type == emel::text::jinja::value_type::integer) { - start = start_val.int_v; - } - } - if (slice->stop) { - emel::text::jinja::value stop_val = eval_expr(ctx, slice->stop.get(), globals, io); - if (stop_val.type == emel::text::jinja::value_type::integer) { - stop = stop_val.int_v; - } - } else if (object.type == emel::text::jinja::value_type::array) { - stop = static_cast(object.array_v.count); - } else if (object.type == emel::text::jinja::value_type::string) { - stop = static_cast(object.string_v.view.size()); - } - if (slice->step) { - emel::text::jinja::value step_val = eval_expr(ctx, slice->step.get(), globals, io); - if (step_val.type == emel::text::jinja::value_type::integer) { - step = step_val.int_v; - } - } - if (step == 0) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("slice_step"); - } - if (object.type == emel::text::jinja::value_type::string) { - const std::string_view s = object.string_v.view; - int64_t len = static_cast(s.size()); - auto norm = [len](int64_t v) { return v < 0 ? v + len : v; }; - int64_t s_start = norm(start); - int64_t s_stop = norm(stop); - if (s_start < 0) s_start = 0; - if (s_stop > len) s_stop = len; - if (s_start > s_stop) s_start = s_stop; - std::string_view slice_view = s.substr(static_cast(s_start), - static_cast(s_stop - s_start)); - return make_string(slice_view, object.string_v.is_input); - } - if (object.type == emel::text::jinja::value_type::array) { - const int64_t len = static_cast(object.array_v.count); - auto norm = [len](int64_t v) { return v < 0 ? v + len : v; }; - int64_t a_start = norm(start); - int64_t a_stop = norm(stop); - if (a_start < 0) a_start = 0; - if (a_stop > len) a_stop = len; - if (a_start > a_stop) a_start = a_stop; - std::array tmp = {}; - size_t count = 0; - for (int64_t i = a_start; i < a_stop && count < tmp.size(); i += step) { - tmp[count++] = object.array_v.items[i]; - } - return make_array(ctx, tmp.data(), count); - } - return make_undefined("slice_type"); - } - property = eval_expr(ctx, node->property.get(), globals, io); - } else { - auto * prop_id = dynamic_cast(node->property.get()); - if (prop_id == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("member_prop"); - } - property = make_string(prop_id->name, false); - } - if (object.type == emel::text::jinja::value_type::array) { - if (property.type == emel::text::jinja::value_type::integer) { - const int64_t idx = property.int_v; - if (idx >= 0 && static_cast(idx) < object.array_v.count) { - return object.array_v.items[idx]; - } - return make_undefined("array_index"); - } - return make_undefined("array_prop"); - } - if (object.type == emel::text::jinja::value_type::string) { - if (property.type == emel::text::jinja::value_type::integer) { - const int64_t idx = property.int_v; - if (idx >= 0 && static_cast(idx) < object.string_v.view.size()) { - const char c = object.string_v.view[static_cast(idx)]; - return make_string(std::string_view(&c, 1), object.string_v.is_input); - } - return make_undefined("string_index"); - } - return make_undefined("string_prop"); - } - if (object.type == emel::text::jinja::value_type::object) { - if (property.type == emel::text::jinja::value_type::string) { - const emel::text::jinja::object_entry * entry = find_object_entry(object.object_v, property.string_v.view); - if (entry != nullptr) { - return entry->val; - } - } - for (size_t i = 0; i < object.object_v.count; ++i) { - if (value_equal(object.object_v.entries[i].key, property)) { - return object.object_v.entries[i].val; - } - } - return make_undefined("object_prop"); - } - return make_undefined("member_type"); -} - -inline emel::text::jinja::value eval_expr(action::context & ctx, - const emel::text::jinja::ast_node * node, - const emel::text::jinja::object_value * globals, - render_io & io) { - if (ctx.phase_error != EMEL_OK) { - return make_undefined(); - } - if (node == nullptr) { - return make_undefined(); - } - if (auto * literal = dynamic_cast(node)) { - return make_string(literal->value, false); - } - if (auto * ident = dynamic_cast(node)) { - size_t builtin_count = 0; - const builtin_entry * builtins = builtin_table(builtin_count); - return lookup_identifier(ctx, globals, ident->name, builtins, builtin_count); - } - if (auto * literal = dynamic_cast(node)) { - return make_int(literal->value); - } - if (auto * literal = dynamic_cast(node)) { - return make_float(literal->value); - } - if (auto * tuple = dynamic_cast(node)) { - std::array tmp = {}; - size_t count = 0; - for (const auto & entry : tuple->values) { - if (count >= tmp.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("tuple_overflow"); - } - tmp[count++] = eval_expr(ctx, entry.get(), globals, io); - } - return make_array(ctx, tmp.data(), count); - } - if (auto * arr = dynamic_cast(node)) { - std::array tmp = {}; - size_t count = 0; - for (const auto & entry : arr->values) { - if (count >= tmp.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("array_overflow"); - } - tmp[count++] = eval_expr(ctx, entry.get(), globals, io); - } - return make_array(ctx, tmp.data(), count); - } - if (auto * obj = dynamic_cast(node)) { - std::array tmp = {}; - size_t count = 0; - for (const auto & entry : obj->pairs) { - if (count >= tmp.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("object_overflow"); - } - tmp[count].key = eval_expr(ctx, entry.first.get(), globals, io); - tmp[count].val = eval_expr(ctx, entry.second.get(), globals, io); - count += 1; - } - return make_object(ctx, tmp.data(), count, count, true); - } - if (auto * unary = dynamic_cast(node)) { - emel::text::jinja::value operand = eval_expr(ctx, unary->operand.get(), globals, io); - if (unary->op.value == "not") { - return make_bool(!value_is_truthy(operand)); - } - if (unary->op.value == "+") { - if (operand.type == emel::text::jinja::value_type::integer) { - return operand; - } - if (operand.type == emel::text::jinja::value_type::floating) { - return operand; - } - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("unary_plus"); - } - if (unary->op.value == "-") { - if (operand.type == emel::text::jinja::value_type::integer) { - return make_int(-operand.int_v); - } - if (operand.type == emel::text::jinja::value_type::floating) { - return make_float(-operand.float_v); - } - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("unary_minus"); - } - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("unary_op"); - } - if (auto * binary = dynamic_cast(node)) { - const std::string_view op = binary->op.value; - if (op == "or") { - emel::text::jinja::value left = eval_expr(ctx, binary->left.get(), globals, io); - if (value_is_truthy(left)) { - return left; - } - return eval_expr(ctx, binary->right.get(), globals, io); - } - if (op == "and") { - emel::text::jinja::value left = eval_expr(ctx, binary->left.get(), globals, io); - if (!value_is_truthy(left)) { - return left; - } - return eval_expr(ctx, binary->right.get(), globals, io); - } - emel::text::jinja::value left = eval_expr(ctx, binary->left.get(), globals, io); - emel::text::jinja::value right = eval_expr(ctx, binary->right.get(), globals, io); - if (op == "+") { - if (value_is_number(left) && value_is_number(right)) { - const double lv = left.type == emel::text::jinja::value_type::integer ? - static_cast(left.int_v) : left.float_v; - const double rv = right.type == emel::text::jinja::value_type::integer ? - static_cast(right.int_v) : right.float_v; - if (left.type == emel::text::jinja::value_type::integer && right.type == emel::text::jinja::value_type::integer) { - return make_int(static_cast(lv + rv)); - } - return make_float(lv + rv); - } - const std::string_view l = value_to_string(ctx, left); - const std::string_view r = value_to_string(ctx, right); - if (ctx.phase_error != EMEL_OK) { - return make_undefined("concat"); - } - std::array buffer = {}; - if (l.size() + r.size() > buffer.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("concat_overflow"); - } - std::memcpy(buffer.data(), l.data(), l.size()); - std::memcpy(buffer.data() + l.size(), r.data(), r.size()); - return make_string(store_string(ctx, std::string_view(buffer.data(), l.size() + r.size())), false); - } - if (op == "-") { - if (value_is_number(left) && value_is_number(right)) { - const double lv = left.type == emel::text::jinja::value_type::integer ? - static_cast(left.int_v) : left.float_v; - const double rv = right.type == emel::text::jinja::value_type::integer ? - static_cast(right.int_v) : right.float_v; - if (left.type == emel::text::jinja::value_type::integer && right.type == emel::text::jinja::value_type::integer) { - return make_int(static_cast(lv - rv)); - } - return make_float(lv - rv); - } - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("sub"); - } - if (op == "~") { - const std::string_view l = value_to_string(ctx, left); - const std::string_view r = value_to_string(ctx, right); - if (ctx.phase_error != EMEL_OK) { - return make_undefined("concat"); - } - std::array buffer = {}; - if (l.size() + r.size() > buffer.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("concat_overflow"); - } - std::memcpy(buffer.data(), l.data(), l.size()); - std::memcpy(buffer.data() + l.size(), r.data(), r.size()); - return make_string(store_string(ctx, std::string_view(buffer.data(), l.size() + r.size())), false); - } - if (op == "*") { - if (value_is_number(left) && value_is_number(right)) { - const double lv = left.type == emel::text::jinja::value_type::integer ? - static_cast(left.int_v) : left.float_v; - const double rv = right.type == emel::text::jinja::value_type::integer ? - static_cast(right.int_v) : right.float_v; - if (left.type == emel::text::jinja::value_type::integer && right.type == emel::text::jinja::value_type::integer) { - return make_int(static_cast(lv * rv)); - } - return make_float(lv * rv); - } - if (left.type == emel::text::jinja::value_type::string && right.type == emel::text::jinja::value_type::integer) { - const int64_t times = right.int_v; - if (times <= 0) { - return make_string({}, false); - } - const size_t total = left.string_v.view.size() * static_cast(times); - std::array buffer = {}; - if (total > buffer.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("repeat_overflow"); - } - size_t offset = 0; - for (int64_t i = 0; i < times; ++i) { - std::memcpy(buffer.data() + offset, left.string_v.view.data(), left.string_v.view.size()); - offset += left.string_v.view.size(); - } - return make_string(store_string(ctx, std::string_view(buffer.data(), total)), false); - } - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("mul"); - } - if (op == "/") { - if (!value_is_number(left) || !value_is_number(right)) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("div"); - } - const double lv = left.type == emel::text::jinja::value_type::integer ? - static_cast(left.int_v) : left.float_v; - const double rv = right.type == emel::text::jinja::value_type::integer ? - static_cast(right.int_v) : right.float_v; - if (rv == 0.0) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("div_zero"); - } - return make_float(lv / rv); - } - if (op == "%") { - if (left.type != emel::text::jinja::value_type::integer || - right.type != emel::text::jinja::value_type::integer) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("mod"); - } - if (right.int_v == 0) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("mod_zero"); - } - return make_int(left.int_v % right.int_v); - } - if (op == "==") { - return make_bool(value_equal(left, right)); - } - if (op == "!=") { - return make_bool(!value_equal(left, right)); - } - if (op == "<" || op == "<=" || op == ">" || op == ">=") { - if (value_is_number(left) && value_is_number(right)) { - const double lv = left.type == emel::text::jinja::value_type::integer ? - static_cast(left.int_v) : left.float_v; - const double rv = right.type == emel::text::jinja::value_type::integer ? - static_cast(right.int_v) : right.float_v; - if (op == "<") return make_bool(lv < rv); - if (op == "<=") return make_bool(lv <= rv); - if (op == ">") return make_bool(lv > rv); - return make_bool(lv >= rv); - } - const std::string_view l = value_to_string(ctx, left); - const std::string_view r = value_to_string(ctx, right); - if (op == "<") return make_bool(l < r); - if (op == "<=") return make_bool(l <= r); - if (op == ">") return make_bool(l > r); - return make_bool(l >= r); - } - if (op == "in" || op == "not in") { - bool contains = false; - if (right.type == emel::text::jinja::value_type::array) { - for (size_t i = 0; i < right.array_v.count; ++i) { - if (value_equal(left, right.array_v.items[i])) { - contains = true; - break; - } - } - } else if (right.type == emel::text::jinja::value_type::object) { - for (size_t i = 0; i < right.object_v.count; ++i) { - if (value_equal(left, right.object_v.entries[i].key)) { - contains = true; - break; - } - } - } else if (right.type == emel::text::jinja::value_type::string && - left.type == emel::text::jinja::value_type::string) { - contains = right.string_v.view.find(left.string_v.view) != std::string_view::npos; - } else if (right.type == emel::text::jinja::value_type::undefined) { - contains = false; - } - if (op == "not in") { - contains = !contains; - } - return make_bool(contains); - } - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("binary_op"); - } - if (auto * ternary = dynamic_cast(node)) { - emel::text::jinja::value test = eval_expr(ctx, ternary->test.get(), globals, io); - if (value_is_truthy(test)) { - return eval_expr(ctx, ternary->true_expr.get(), globals, io); - } - return eval_expr(ctx, ternary->false_expr.get(), globals, io); - } - if (auto * select = dynamic_cast(node)) { - emel::text::jinja::value test = eval_expr(ctx, select->test.get(), globals, io); - if (value_is_truthy(test)) { - return eval_expr(ctx, select->value.get(), globals, io); - } - return make_undefined("select"); - } - if (auto * test_expr = dynamic_cast(node)) { - emel::text::jinja::value operand = eval_expr(ctx, test_expr->operand.get(), globals, io); - std::string_view test_name = {}; - call_args args = {}; - if (auto * id = dynamic_cast(test_expr->test.get())) { - test_name = id->name; - } else if (auto * call = dynamic_cast(test_expr->test.get())) { - auto * id = dynamic_cast(call->callee.get()); - if (id == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("test_name"); - } - test_name = id->name; - collect_call_args(ctx, call->args, args, globals, io); - } - const test_entry * entry = find_test(test_name); - if (entry == nullptr || entry->fn == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("test_missing"); - } - emel::text::jinja::value result = make_bool(false); - entry->fn(ctx, operand, args, result); - if (test_expr->negate) { - result = make_bool(!value_is_truthy(result)); - } - return result; - } - if (auto * filter_expr = dynamic_cast(node)) { - emel::text::jinja::value operand = eval_expr(ctx, filter_expr->operand.get(), globals, io); - std::string_view name = {}; - call_args args = {}; - if (auto * id = dynamic_cast(filter_expr->filter.get())) { - name = id->name; - } else if (auto * call = dynamic_cast(filter_expr->filter.get())) { - auto * id = dynamic_cast(call->callee.get()); - if (id == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("filter_name"); - } - name = id->name; - collect_call_args(ctx, call->args, args, globals, io); - } - const filter_entry * entry = find_filter(name); - if (entry == nullptr || entry->fn == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("filter_missing"); - } - emel::text::jinja::value out = make_undefined(); - entry->fn(ctx, operand, args, out); - return out; - } - if (auto * call = dynamic_cast(node)) { - emel::text::jinja::value callee = eval_expr(ctx, call->callee.get(), globals, io); - call_args args = {}; - collect_call_args(ctx, call->args, args, globals, io); - if (callee.type != emel::text::jinja::value_type::function) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return make_undefined("call_target"); - } - return invoke_function(ctx, callee.func_v, args, globals, io); - } - if (auto * member = dynamic_cast(node)) { - return eval_member(ctx, member, globals, io); - } - return make_undefined("expr_unknown"); -} - -inline bool render_if(action::context & ctx, - const emel::text::jinja::if_statement * stmt, - const emel::text::jinja::object_value * globals, - render_io & io, - const bool allow_control, - control_flow & flow) { - emel::text::jinja::value test = eval_expr(ctx, stmt->test.get(), globals, io); - const bool truthy = value_is_truthy(test); - return render_statements(ctx, truthy ? stmt->body : stmt->alternate, - globals, io, allow_control, flow); -} - -inline bool bind_loop_var(action::context & ctx, - const emel::text::jinja::ast_node * loop_var, - const emel::text::jinja::value & item) { - if (auto * id = dynamic_cast(loop_var)) { - return set_object_value(ctx, ctx.scopes[ctx.scope_count - 1].locals, id->name, item); - } - if (auto * tuple = dynamic_cast(loop_var)) { - if (item.type != emel::text::jinja::value_type::array) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, loop_var->pos); - return false; - } - if (tuple->values.size() != item.array_v.count) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, loop_var->pos); - return false; - } - for (size_t i = 0; i < tuple->values.size(); ++i) { - auto * id = dynamic_cast(tuple->values[i].get()); - if (id == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, loop_var->pos); - return false; - } - if (!set_object_value(ctx, ctx.scopes[ctx.scope_count - 1].locals, id->name, item.array_v.items[i])) { - return false; - } - } - return true; - } - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, loop_var->pos); - return false; -} - -inline bool render_for(action::context & ctx, - const emel::text::jinja::for_statement * stmt, - const emel::text::jinja::object_value * globals, - render_io & io, - const bool allow_control, - control_flow & flow) { - const emel::text::jinja::ast_node * iter_expr = stmt->iterable.get(); - const emel::text::jinja::ast_node * test_expr = nullptr; - if (auto * select = dynamic_cast(iter_expr)) { - iter_expr = select->value.get(); - test_expr = select->test.get(); - } - emel::text::jinja::value iterable = eval_expr(ctx, iter_expr, globals, io); - if (iterable.type == emel::text::jinja::value_type::undefined) { - iterable = make_array(ctx, nullptr, 0); - } - std::array items = {}; - size_t item_count = 0; - if (iterable.type == emel::text::jinja::value_type::array) { - for (size_t i = 0; i < iterable.array_v.count && item_count < items.size(); ++i) { - items[item_count++] = iterable.array_v.items[i]; - } - } else if (iterable.type == emel::text::jinja::value_type::object) { - for (size_t i = 0; i < iterable.object_v.count && item_count < items.size(); ++i) { - if (auto * tuple = dynamic_cast(stmt->loop_var.get())) { - if (tuple->values.size() >= 2) { - emel::text::jinja::value tmp_items[2] = { - iterable.object_v.entries[i].key, - iterable.object_v.entries[i].val - }; - items[item_count++] = make_array(ctx, tmp_items, 2); - continue; - } - } - items[item_count++] = iterable.object_v.entries[i].key; - } - } else { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - if (item_count >= items.size()) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - std::array filtered = {}; - size_t filtered_count = 0; - for (size_t i = 0; i < item_count && filtered_count < filtered.size(); ++i) { - if (test_expr == nullptr) { - filtered[filtered_count++] = items[i]; - continue; - } - if (!push_scope(ctx)) { - return false; - } - if (!bind_loop_var(ctx, stmt->loop_var.get(), items[i])) { - pop_scope(ctx); - return false; - } - emel::text::jinja::value test_val = eval_expr(ctx, test_expr, globals, io); - const bool pass = value_is_truthy(test_val); - pop_scope(ctx); - if (pass) { - filtered[filtered_count++] = items[i]; - } - } - if (filtered_count == 0) { - return render_statements(ctx, stmt->alternate, globals, io, allow_control, flow); - } - for (size_t i = 0; i < filtered_count; ++i) { - if (!push_scope(ctx)) { - return false; - } - if (!bind_loop_var(ctx, stmt->loop_var.get(), filtered[i])) { - pop_scope(ctx); - return false; - } - if (!render_statements(ctx, stmt->body, globals, io, true, flow)) { - pop_scope(ctx); - return false; - } - pop_scope(ctx); - if (flow == control_flow::break_loop) { - flow = control_flow::none; - break; - } - if (flow == control_flow::continue_loop) { - flow = control_flow::none; - continue; - } - } - return ctx.phase_error == EMEL_OK; -} - -inline bool render_set(action::context & ctx, - const emel::text::jinja::set_statement * stmt, - const emel::text::jinja::object_value * globals, - render_io & io) { - emel::text::jinja::value rhs = make_undefined(); - if (!stmt->body.empty()) { - if (!begin_capture(ctx, io)) { - return false; - } - control_flow flow = control_flow::none; - render_statements(ctx, stmt->body, globals, io, false, flow); - if (!end_capture(ctx, io, rhs)) { - return false; - } - } else if (stmt->value) { - rhs = eval_expr(ctx, stmt->value.get(), globals, io); - } - if (auto * id = dynamic_cast(stmt->left.get())) { - return set_object_value(ctx, ctx.scopes[ctx.scope_count - 1].locals, id->name, rhs); - } - if (auto * tuple = dynamic_cast(stmt->left.get())) { - if (rhs.type != emel::text::jinja::value_type::array) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - if (tuple->values.size() != rhs.array_v.count) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - for (size_t i = 0; i < tuple->values.size(); ++i) { - auto * id = dynamic_cast(tuple->values[i].get()); - if (id == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - if (!set_object_value(ctx, ctx.scopes[ctx.scope_count - 1].locals, id->name, rhs.array_v.items[i])) { - return false; - } - } - return true; - } - if (auto * member = dynamic_cast(stmt->left.get())) { - if (member->computed) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - auto * prop_id = dynamic_cast(member->property.get()); - if (prop_id == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - emel::text::jinja::value target = eval_expr(ctx, member->object.get(), globals, io); - if (target.type != emel::text::jinja::value_type::object) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - return set_object_value(ctx, target.object_v, prop_id->name, rhs); - } - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; -} - -inline bool render_filter_statement(action::context & ctx, - const emel::text::jinja::filter_statement * stmt, - const emel::text::jinja::object_value * globals, - render_io & io) { - emel::text::jinja::value body_val = make_undefined(); - if (!begin_capture(ctx, io)) { - return false; - } - control_flow flow = control_flow::none; - if (!render_statements(ctx, stmt->body, globals, io, false, flow)) { - return false; - } - if (!end_capture(ctx, io, body_val)) { - return false; - } - std::string_view name = {}; - call_args args = {}; - if (auto * id = dynamic_cast(stmt->filter_node.get())) { - name = id->name; - } else if (auto * call = dynamic_cast(stmt->filter_node.get())) { - auto * id = dynamic_cast(call->callee.get()); - if (id == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - name = id->name; - collect_call_args(ctx, call->args, args, globals, io); - } - const filter_entry * entry = find_filter(name); - if (entry == nullptr || entry->fn == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - emel::text::jinja::value filtered = make_undefined(); - entry->fn(ctx, body_val, args, filtered); - return write_value(ctx, io, filtered); -} - -inline bool render_call_statement(action::context & ctx, - const emel::text::jinja::call_statement * stmt, - const emel::text::jinja::object_value * globals, - render_io & io) { - auto * call_expr = dynamic_cast(stmt->call_expr.get()); - if (call_expr == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - emel::text::jinja::value callee = eval_expr(ctx, call_expr->callee.get(), globals, io); - if (callee.type != emel::text::jinja::value_type::function) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - if (ctx.callable_count >= action::k_max_callables) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt->pos); - return false; - } - action::callable_slot & slot = ctx.callables[ctx.callable_count++]; - slot.kind = emel::text::jinja::function_kind::caller; - slot.caller.body = &stmt->body; - slot.caller.params = &stmt->caller_args; - emel::text::jinja::function_ref caller_ref; - caller_ref.kind = emel::text::jinja::function_kind::caller; - caller_ref.data = &slot; - if (!push_scope(ctx)) { - return false; - } - set_object_value(ctx, ctx.scopes[ctx.scope_count - 1].locals, "caller", make_function(caller_ref)); - call_args args = {}; - collect_call_args(ctx, call_expr->args, args, globals, io); - emel::text::jinja::value result = invoke_function(ctx, callee.func_v, args, globals, io); - pop_scope(ctx); - return write_value(ctx, io, result); -} - -inline bool render_statement(action::context & ctx, - const emel::text::jinja::ast_node * node, - const emel::text::jinja::object_value * globals, - render_io & io, - const bool allow_control, - control_flow & flow) { - if (ctx.phase_error != EMEL_OK) { - return false; - } - if (!ensure_steps(ctx, node != nullptr ? node->pos : 0)) { - return false; - } - if (node == nullptr) { - return true; - } - if (dynamic_cast(node) != nullptr) { - return true; - } - if (dynamic_cast(node) != nullptr) { - return true; - } - if (dynamic_cast(node) != nullptr) { - flow = control_flow::break_loop; - return true; - } - if (dynamic_cast(node) != nullptr) { - flow = control_flow::continue_loop; - return true; - } - if (auto * stmt = dynamic_cast(node)) { - return render_if(ctx, stmt, globals, io, allow_control, flow); - } - if (auto * stmt = dynamic_cast(node)) { - return render_for(ctx, stmt, globals, io, allow_control, flow); - } - if (auto * stmt = dynamic_cast(node)) { - return render_set(ctx, stmt, globals, io); - } - if (auto * stmt = dynamic_cast(node)) { - if (ctx.callable_count >= action::k_max_callables) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return false; - } - auto * name_id = dynamic_cast(stmt->name.get()); - if (name_id == nullptr) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, node->pos); - return false; - } - action::callable_slot & slot = ctx.callables[ctx.callable_count++]; - slot.kind = emel::text::jinja::function_kind::macro; - slot.macro.stmt = stmt; - emel::text::jinja::function_ref ref; - ref.kind = emel::text::jinja::function_kind::macro; - ref.data = &slot; - return set_object_value(ctx, ctx.scopes[ctx.scope_count - 1].locals, name_id->name, make_function(ref)); - } - if (auto * stmt = dynamic_cast(node)) { - return render_filter_statement(ctx, stmt, globals, io); - } - if (auto * stmt = dynamic_cast(node)) { - return render_call_statement(ctx, stmt, globals, io); - } - emel::text::jinja::value val = eval_expr(ctx, node, globals, io); - return write_value(ctx, io, val); -} - -inline bool render_statements(action::context & ctx, - const emel::text::jinja::ast_list & statements, - const emel::text::jinja::object_value * globals, - render_io & io, - bool allow_control, - control_flow & flow) { - for (const auto & stmt : statements) { - if (!render_statement(ctx, stmt.get(), globals, io, allow_control, flow)) { - return false; - } - if (flow != control_flow::none) { - if (!allow_control) { - set_error(ctx, EMEL_ERR_INVALID_ARGUMENT, stmt ? stmt->pos : 0); - flow = control_flow::none; - return false; - } - return true; - } - } - return ctx.phase_error == EMEL_OK; +template +inline value_type & bind_optional(value_type * destination, + value_type & sink) noexcept { + value_type * destinations[2] = {&sink, destination}; + return *destinations[static_cast(destination != nullptr)]; +} + +inline constexpr int32_t to_error_code(const error err) noexcept { + return static_cast(err); +} + +inline void reset_result(event::render_ctx & runtime_ctx) noexcept { + runtime_ctx.err = error::none; + runtime_ctx.output_length = 0; + runtime_ctx.output_truncated = false; + runtime_ctx.error_out = to_error_code(error::none); + runtime_ctx.error_pos_out = 0; +} + +inline void mark_done(event::render_ctx & runtime_ctx, + const size_t output_length, + const bool output_truncated) noexcept { + runtime_ctx.err = error::none; + runtime_ctx.output_length = output_length; + runtime_ctx.output_truncated = output_truncated; + runtime_ctx.error_out = to_error_code(error::none); + runtime_ctx.error_pos_out = 0; +} + +inline void mark_error(event::render_ctx & runtime_ctx, + const error err, + const bool output_truncated, + const size_t error_pos) noexcept { + runtime_ctx.err = err; + runtime_ctx.output_length = 0; + runtime_ctx.output_truncated = output_truncated; + runtime_ctx.error_out = to_error_code(err); + runtime_ctx.error_pos_out = error_pos; +} + +inline void emit_done(const event::render & request, + const event::render_ctx & runtime_ctx) noexcept { + const events::rendering_done done_ev{ + request, + runtime_ctx.output_length, + runtime_ctx.output_truncated, + }; + (void)request.dispatch_done(done_ev); +} + +inline void emit_error(const event::render & request, + const event::render_ctx & runtime_ctx) noexcept { + const events::rendering_error error_ev{ + request, + to_error_code(runtime_ctx.err), + runtime_ctx.error_pos_out, + }; + (void)request.dispatch_error(error_ev); } } // namespace emel::text::jinja::formatter::detail diff --git a/src/emel/text/jinja/formatter/errors.hpp b/src/emel/text/jinja/formatter/errors.hpp new file mode 100644 index 00000000..b97b5c54 --- /dev/null +++ b/src/emel/text/jinja/formatter/errors.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace emel::text::jinja::formatter { + +enum class error : int32_t { + none = 0, + invalid_request = 1, +}; + +constexpr bool is_ok(const error value) noexcept { + return value == error::none; +} + +} // namespace emel::text::jinja::formatter diff --git a/src/emel/text/jinja/formatter/events.hpp b/src/emel/text/jinja/formatter/events.hpp index 27cc409d..fb2e3cb0 100644 --- a/src/emel/text/jinja/formatter/events.hpp +++ b/src/emel/text/jinja/formatter/events.hpp @@ -6,6 +6,7 @@ #include "emel/callback.hpp" #include "emel/text/jinja/ast.hpp" +#include "emel/text/jinja/formatter/errors.hpp" #include "emel/text/jinja/value.hpp" namespace emel::text::jinja::events { @@ -18,18 +19,65 @@ struct rendering_error; namespace emel::text::jinja::event { struct render { - const emel::text::jinja::program * program = nullptr; - const emel::text::jinja::object_value * globals = nullptr; - std::string_view source_text = {}; - char * output = nullptr; - size_t output_capacity = 0; - size_t * output_length = nullptr; - bool * output_truncated = nullptr; - int32_t * error_out = nullptr; - size_t * error_pos_out = nullptr; - void * owner_sm = nullptr; - ::emel::callback dispatch_done = {}; - ::emel::callback dispatch_error = {}; + using done_callback = ::emel::callback; + using error_callback = ::emel::callback; + + render(const emel::text::jinja::program & program_ref, + std::string_view source_text_ref, + char & output_ref, + const size_t output_capacity_ref, + const done_callback dispatch_done_ref, + const error_callback dispatch_error_ref, + const emel::text::jinja::object_value * globals_ref = nullptr, + size_t * output_length_ref = nullptr, + bool * output_truncated_ref = nullptr, + int32_t * error_out_ref = nullptr, + size_t * error_pos_out_ref = nullptr) noexcept + : program(program_ref), + source_text(source_text_ref), + output(output_ref), + output_capacity(output_capacity_ref), + dispatch_done(dispatch_done_ref), + dispatch_error(dispatch_error_ref), + globals(globals_ref), + output_length(output_length_ref), + output_truncated(output_truncated_ref), + error_out(error_out_ref), + error_pos_out(error_pos_out_ref) {} + + const emel::text::jinja::program & program; + const std::string_view source_text; + char & output; + const size_t output_capacity; + const done_callback dispatch_done; + const error_callback dispatch_error; + const emel::text::jinja::object_value * const globals; + size_t * const output_length; + bool * const output_truncated; + int32_t * const error_out; + size_t * const error_pos_out; +}; + +struct render_ctx { + render_ctx(size_t & output_length_ref, + bool & output_truncated_ref, + int32_t & error_out_ref, + size_t & error_pos_out_ref) noexcept + : output_length(output_length_ref), + output_truncated(output_truncated_ref), + error_out(error_out_ref), + error_pos_out(error_pos_out_ref) {} + + formatter::error err = formatter::error::none; + size_t & output_length; + bool & output_truncated; + int32_t & error_out; + size_t & error_pos_out; +}; + +struct render_runtime { + const render & request; + render_ctx & ctx; }; } // namespace emel::text::jinja::event @@ -37,15 +85,15 @@ struct render { namespace emel::text::jinja::events { struct rendering_done { - const event::render * request = nullptr; - size_t output_length = 0; - bool output_truncated = false; + const event::render & request; + size_t output_length; + bool output_truncated; }; struct rendering_error { - const event::render * request = nullptr; - int32_t err = 0; - size_t error_pos = 0; + const event::render & request; + int32_t err; + size_t error_pos; }; } // namespace emel::text::jinja::events diff --git a/src/emel/text/jinja/formatter/guards.hpp b/src/emel/text/jinja/formatter/guards.hpp index 0bb50e19..ccb2fa3e 100644 --- a/src/emel/text/jinja/formatter/guards.hpp +++ b/src/emel/text/jinja/formatter/guards.hpp @@ -1,57 +1,112 @@ #pragma once -#include "emel/text/jinja/formatter/context.hpp" +#include "emel/text/jinja/formatter/errors.hpp" #include "emel/text/jinja/formatter/events.hpp" namespace emel::text::jinja::formatter::guard { -inline constexpr auto valid_render = [](const emel::text::jinja::event::render & ev) noexcept { - return ev.program != nullptr && ev.output != nullptr && ev.output_capacity > 0; +namespace detail { + +template +constexpr decltype(auto) unwrap_runtime_event(const runtime_event_type & ev) noexcept { + if constexpr (requires { ev.event_; }) { + return ev.event_; + } else { + return (ev); + } +} + +inline bool valid_render_request(const emel::text::jinja::event::render & ev) noexcept { + return ev.output_capacity > 0 && + (ev.source_text.empty() || ev.source_text.data() != nullptr); +} + +inline bool callbacks_present(const emel::text::jinja::event::render & ev) noexcept { + return static_cast(ev.dispatch_done) && + static_cast(ev.dispatch_error); +} + +} // namespace detail + +inline bool valid_render_request(const emel::text::jinja::event::render & ev) noexcept { + return detail::valid_render_request(ev); +} + +struct valid_render { + template + bool operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = detail::unwrap_runtime_event(ev); + return valid_render_request(runtime_ev.request) && + detail::callbacks_present(runtime_ev.request); + } +}; + +struct invalid_render_with_callbacks { + template + bool operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = detail::unwrap_runtime_event(ev); + return !valid_render_request(runtime_ev.request) && + detail::callbacks_present(runtime_ev.request); + } }; -inline constexpr auto invalid_render = [](const emel::text::jinja::event::render & ev) noexcept { - return !valid_render(ev); +struct invalid_render_without_callbacks { + template + bool operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = detail::unwrap_runtime_event(ev); + return !detail::callbacks_present(runtime_ev.request); + } }; -struct phase_ok { - bool operator()(const action::context & ctx) const noexcept { - return ctx.phase_error == EMEL_OK; +struct source_empty { + template + bool operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = detail::unwrap_runtime_event(ev); + return runtime_ev.request.source_text.empty(); } }; -struct phase_failed { - bool operator()(const action::context & ctx) const noexcept { - return ctx.phase_error != EMEL_OK; +struct source_non_empty { + template + bool operator()(const runtime_event_type & ev) const noexcept { + return !source_empty{}(ev); } }; -struct has_stmt_work { - bool operator()(const action::context & ctx) const noexcept { - return ctx.statements != nullptr && ctx.statement_index < ctx.statements->size(); +struct source_fits { + template + bool operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = detail::unwrap_runtime_event(ev); + return runtime_ev.request.source_text.size() <= runtime_ev.request.output_capacity; } }; -struct no_stmt_work { - bool operator()(const action::context & ctx) const noexcept { - return ctx.statements == nullptr || ctx.statement_index >= ctx.statements->size(); +struct source_overflow { + template + bool operator()(const runtime_event_type & ev) const noexcept { + return !source_fits{}(ev); } }; -struct needs_expr { - bool operator()(const action::context & ctx) const noexcept { - return ctx.pending_expr != nullptr; +struct copy_ready { + template + bool operator()(const runtime_event_type & ev) const noexcept { + return source_non_empty{}(ev) && source_fits{}(ev); } }; -struct has_pending_expr { - bool operator()(const action::context & ctx) const noexcept { - return ctx.pending_expr != nullptr; +struct request_ok { + template + bool operator()(const runtime_event_type & ev) const noexcept { + const auto & runtime_ev = detail::unwrap_runtime_event(ev); + return runtime_ev.ctx.err == error::none; } }; -struct needs_write { - bool operator()(const action::context & ctx) const noexcept { - return ctx.pending_value_ready; +struct request_failed { + template + bool operator()(const runtime_event_type & ev) const noexcept { + return !request_ok{}(ev); } }; diff --git a/src/emel/text/jinja/formatter/sm.hpp b/src/emel/text/jinja/formatter/sm.hpp index 8f18b035..f17e81b3 100644 --- a/src/emel/text/jinja/formatter/sm.hpp +++ b/src/emel/text/jinja/formatter/sm.hpp @@ -2,90 +2,13 @@ /* design doc: docs/designs/text/jinja/formatter.design.md - --- - title: jinja/formatter architecture design - status: draft - --- - - # jinja/formatter architecture design - - this document defines the jinja formatter actor (formerly `renderer::sm`). it evaluates a parsed `jinja::program` against a set of variables and emits formatted text. it enforces strict memory and computation bounds using statically provable limits (e.g., stack depth, scope counts) to ensure safe, zero-allocation execution within the inference hot path. - - ## role - - act as a pure SML actor that evaluates a Jinja AST (`program`). - - emit UTF-8 formatted text into caller-provided buffers (`format_io`) without heap allocation. - - prevent denial-of-service (DoS) from malicious or infinitely recursive templates by enforcing strict execution bounds. - - ## architecture shift: renaming and bounding - previously named the `renderer`, this component has been renamed to `formatter` to align with its role within the `text/conditioner` pipeline (where a `text/formatter` prepares prompt strings before tokenization, while a `text/renderer` is responsible for decoding tokens back into domain output). - - the core architectural challenge in the formatter is its evaluation loop: - ```text - eval_stmt ──► eval_expr ──► write_output ──► eval_stmt - ``` - because these are **anonymous transitions** (transitions taken automatically without an external event), SML's Run-To-Completion (RTC) semantics will loop them until the machine reaches quiescence. if a template contains an infinite loop or unbounded recursion (e.g., `{% macro inf() %}{{ infinite() }}{% endmacro %}{{ inf() }}`), the SML dispatch would never return, locking the thread. - - ### the stack depth solution - to satisfy the `sml.rules.md` requirement that *"anonymous transition graphs MUST be acyclic or MUST have a statically provable bound on firings per top-level event"*, the formatter enforces strict hierarchical limits rather than a flat instruction counter. - - 1. **call stack depth**: the context tracks `call_depth`. every time a macro or function is evaluated, the depth increments. if it exceeds `k_max_call_depth`, the action sets an error flag. - 2. **scope bounds**: the context tracks `scope_count`. every `{% for %}` loop or block pushes a new variable scope. if this exceeds `k_max_scopes`, it errors out. - 3. **step limits (fail-safe)**: as a final safeguard against massive (but not necessarily deep) templates, a `steps_remaining` counter is decremented on every statement evaluation. if it hits 0, execution halts. - - these limits ensure the anonymous SML loops are mathematically guaranteed to terminate and return control to the orchestrator. - - ## events - - `event::format` - - inputs: `program` (the parsed AST), `globals` (injected variables/context), `output` buffer pointer and `output_capacity`, `error_out`, and optional synchronous callbacks. - - outputs: executes the AST, populates the `output` buffer, updates `output_length` and `output_truncated` flags, and invokes the callback before returning. - - ## state model - - ```text - uninitialized ──► initialized - │ - initialized ──► setup ──► eval_stmt ──► format_decision ──► (done | errored) - ▲ │ ▲ │ - │ ▼ │ │ - │ eval_expr ──► write_output │ - │ │ │ - └─────────────────────────────────────┴───────────────────────┘ - ``` - - - `setup` — binds the `program`, initializes the scope stack and bounds counters, and prepares the initial statements. - - `eval_stmt` — evaluates control flow (`if`, `for`, `set`) and enqueues expression evaluation. - - `eval_expr` — resolves variables, performs math, applies filters, and executes macros. - - `write_output` — writes evaluated expression values into the caller's output buffer. - - `format_decision` — inspects the context's error state (e.g., bounds exceeded, invalid types, buffer full). - - `done` / `errored` — terminal states for the RTC chain. - - ## responsibilities & constraints - - 1. **zero-allocation rendering**: - - the formatter must never use `new`, `malloc`, or `std::string` during evaluation. - - temporary strings (like the result of `replace` or `join` filters) are stored in a fixed-size `string_buffer` within the SML context. if this buffer fills up (`k_max_string_bytes`), the formatter immediately errors. - - output is written sequentially to the caller-provided `format_io` buffer. - - 2. **graceful truncation**: - - if the evaluated text exceeds the caller's `output_capacity`, the formatter must not crash. it stops writing, sets `output_truncated = true` in the event payload, and gracefully finishes the evaluation or errors out depending on the strictness policy. - - 3. **deterministic error routing**: - - any bounds violation (stack depth, scope count, step limit) or runtime error (division by zero, missing filter) sets `ctx.phase_error`. the state machine guards inspect this and cleanly route to `format_decision` -> `errored` without throwing C++ exceptions. - - ## error codes - - this actor can produce the following error codes: - - - `EMEL_ERR_TEMPLATE_RUNTIME` — runtime evaluation error (e.g., division by zero, missing filter, invalid type). - - `EMEL_ERR_TEMPLATE_LIMIT` — execution limit exceeded (call stack depth, scope count, or step count). - - `EMEL_ERR_CAPACITY` — output buffer full or string buffer exhausted. - - `EMEL_ERR_INVALID_ARGUMENT` — invalid program, globals, or buffer pointers. */ - #include #include "emel/text/jinja/formatter/actions.hpp" +#include "emel/text/jinja/formatter/context.hpp" +#include "emel/text/jinja/formatter/detail.hpp" #include "emel/text/jinja/formatter/events.hpp" #include "emel/text/jinja/formatter/guards.hpp" #include "emel/sm.hpp" @@ -93,11 +16,9 @@ design doc: docs/designs/text/jinja/formatter.design.md namespace emel::text::jinja::formatter { struct initialized {}; -struct setup {}; -struct eval_stmt {}; -struct eval_expr {}; -struct write_output {}; -struct render_decision {}; +struct request_decision {}; +struct copy_exec {}; +struct result_decision {}; struct done {}; struct errored {}; struct unexpected {}; @@ -107,153 +28,139 @@ struct unexpected {}; * * state purposes: * - `initialized`: idle state awaiting render intent. - * - `setup`: initialize context and seed statement work. - * - `eval_stmt`: step through statements and enqueue expressions when needed. - * - `eval_expr`: evaluate pending expressions. - * - `write_output`: emit pending expression values. - * - `render_decision`: branch based on phase results. + * - `request_decision`: route copy/empty/overflow paths for one dispatch. + * - `copy_exec`: copy source text to caller output buffer. + * - `result_decision`: route done/error callback dispatch for one dispatch. * - `done`/`errored`: terminal outcomes. * - `unexpected`: sequencing contract violation. * * guard semantics: - * - `valid_render`/`invalid_render` validate request pointers and parameters. - * - `phase_*` guards observe errors set by actions. + * - `valid_render`/`invalid_render` validate request parameters. + * - `source_*` guards model copy-path routing. * * action side effects: - * - `begin_render`/`seed_program` prepare context for a render pass. - * - `eval_next_stmt`/`eval_pending_expr`/`write_pending_value` execute rendering steps. - * - `finalize_*` finalize terminal status on context. - * - `reject_invalid_render` writes errors for invalid requests. + * - `begin_render` marks start of one dispatch pass. + * - `mark_empty_output` marks zero-length success path. + * - `copy_source_text` performs one bounded buffer copy. + * - `mark_capacity_error` marks overflow path. + * - `reject_invalid_render` marks invalid-request path. + * - `dispatch_done`/`dispatch_error` send synchronous completion callbacks. * - `on_unexpected` reports sequencing violations. */ struct model { auto operator()() const { namespace sml = boost::sml; + // clang-format off return sml::make_transition_table( - *sml::state + - sml::event[guard::valid_render] / action::begin_render = - sml::state, - sml::state + - sml::event[guard::invalid_render] / - action::reject_invalid_render = sml::state, - - sml::state + sml::event[guard::valid_render] / - action::begin_render = sml::state, - sml::state + sml::event[guard::invalid_render] / - action::reject_invalid_render = - sml::state, - - sml::state + sml::event[guard::valid_render] / - action::begin_render = sml::state, - sml::state + sml::event[guard::invalid_render] / - action::reject_invalid_render = - sml::state, - - sml::state + - sml::event[guard::valid_render] / action::begin_render = - sml::state, - sml::state + - sml::event[guard::invalid_render] / - action::reject_invalid_render = sml::state, - - sml::state / action::seed_program = sml::state, - - sml::state[guard::phase_failed{}] = sml::state, - sml::state[guard::needs_expr{}] = sml::state, - sml::state[guard::has_stmt_work{}] / action::eval_next_stmt = - sml::state, - sml::state[guard::no_stmt_work{}] = sml::state, - - sml::state[guard::has_pending_expr{}] / action::eval_pending_expr = - sml::state, - sml::state[guard::phase_failed{}] = sml::state, - sml::state = sml::state, - - sml::state[guard::phase_failed{}] = sml::state, - sml::state[guard::needs_write{}] / action::write_pending_value = - sml::state, - sml::state = sml::state, - - sml::state[guard::phase_ok{}] / action::finalize_done = - sml::state, - sml::state[guard::phase_failed{}] / action::finalize_error = - sml::state, - - sml::state + sml::unexpected_event / - action::on_unexpected = sml::state, - sml::state + sml::unexpected_event / - action::on_unexpected = sml::state, - sml::state + sml::unexpected_event / - action::on_unexpected = sml::state, - sml::state + sml::unexpected_event / - action::on_unexpected = sml::state, - sml::state + sml::unexpected_event / - action::on_unexpected = sml::state, - sml::state + sml::unexpected_event / - action::on_unexpected = sml::state, - sml::state + sml::unexpected_event / - action::on_unexpected = sml::state, - sml::state + sml::unexpected_event / - action::on_unexpected = sml::state, - sml::state + sml::unexpected_event / - action::on_unexpected = sml::state); + sml::state <= *sml::state + + sml::event[ guard::valid_render{} ] + / action::begin_render + , sml::state <= sml::state + + sml::event[ guard::invalid_render_with_callbacks{} ] + / action::reject_invalid_render + , sml::state <= sml::state + + sml::event[ guard::invalid_render_without_callbacks{} ] + / action::reject_invalid_render + + , sml::state <= sml::state + + sml::event[ guard::valid_render{} ] + / action::begin_render + , sml::state <= sml::state + + sml::event[ guard::invalid_render_with_callbacks{} ] + / action::reject_invalid_render + , sml::state <= sml::state + + sml::event[ guard::invalid_render_without_callbacks{} ] + / action::reject_invalid_render + + , sml::state <= sml::state + + sml::event[ guard::valid_render{} ] + / action::begin_render + , sml::state <= sml::state + + sml::event[ guard::invalid_render_with_callbacks{} ] + / action::reject_invalid_render + , sml::state <= sml::state + + sml::event[ guard::invalid_render_without_callbacks{} ] + / action::reject_invalid_render + + , sml::state <= sml::state + + sml::event[ guard::valid_render{} ] + / action::begin_render + , sml::state <= sml::state + + sml::event[ guard::invalid_render_with_callbacks{} ] + / action::reject_invalid_render + , sml::state <= sml::state + + sml::event[ guard::invalid_render_without_callbacks{} ] + / action::reject_invalid_render + + //------------------------------------------------------------------------------// + , sml::state <= sml::state + + sml::completion [ guard::source_empty{} ] + / action::mark_empty_output + , sml::state <= sml::state + + sml::completion [ guard::copy_ready{} ] + / action::copy_source_text + , sml::state <= sml::state + + sml::completion [ guard::source_overflow{} ] + / action::mark_capacity_error + + , sml::state <= sml::state + + sml::completion + , sml::state <= sml::state + + sml::completion [ guard::request_ok{} ] + / action::dispatch_done + , sml::state <= sml::state + + sml::completion [ guard::request_failed{} ] + / action::dispatch_error + + //------------------------------------------------------------------------------// + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + ); + // clang-format on } }; -struct sm : public emel::sm { - using base_type = emel::sm; +struct sm : public emel::sm { + using base_type = emel::sm; - explicit sm(action::context & ctx) : base_type(ctx), context_(&ctx) {} + sm() : base_type() {} + explicit sm(const action::context & ctx) : base_type(ctx) {} bool process_event(const event::render & ev) { - namespace sml = boost::sml; - - if (ev.error_out != nullptr) { - *ev.error_out = EMEL_OK; - } - - const bool accepted = base_type::process_event(ev); - const bool ok = this->is(sml::state); - const bool valid = guard::valid_render(ev); - const int32_t err = ok ? EMEL_OK - : (context_->last_error != EMEL_OK ? context_->last_error - : EMEL_ERR_BACKEND); - const size_t output_length = valid ? context_->output_length : 0; - const size_t error_pos = valid ? context_->error_pos : 0; - const bool output_truncated = valid ? (err != EMEL_OK) : false; - - if (ev.output_length != nullptr) { - *ev.output_length = output_length; - } - if (ev.output_truncated != nullptr) { - *ev.output_truncated = output_truncated; - } - if (ev.error_out != nullptr) { - *ev.error_out = err; - } - if (ev.error_pos_out != nullptr) { - *ev.error_pos_out = error_pos; - } - - if (ok) { - if (ev.dispatch_done) { - ev.dispatch_done(events::rendering_done{&ev, output_length, output_truncated}); - } - } else { - if (ev.dispatch_error) { - ev.dispatch_error(events::rendering_error{&ev, err, error_pos}); - } - } - - return accepted && ok; + size_t output_length_sink = 0; + bool output_truncated_sink = false; + int32_t error_sink = detail::to_error_code(error::none); + size_t error_pos_sink = 0; + + event::render_ctx runtime_ctx{ + detail::bind_optional(ev.output_length, output_length_sink), + detail::bind_optional(ev.output_truncated, output_truncated_sink), + detail::bind_optional(ev.error_out, error_sink), + detail::bind_optional(ev.error_pos_out, error_pos_sink), + }; + event::render_runtime runtime_ev{ev, runtime_ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && runtime_ctx.err == error::none; } using base_type::process_event; + using base_type::is; using base_type::visit_current_states; - - private: - action::context * context_ = nullptr; }; +using Formatter = sm; + } // namespace emel::text::jinja::formatter diff --git a/tests/text/jinja/formatter_tests.cpp b/tests/text/jinja/formatter_tests.cpp index 2d948cfe..8a949e83 100644 --- a/tests/text/jinja/formatter_tests.cpp +++ b/tests/text/jinja/formatter_tests.cpp @@ -1,1843 +1,293 @@ #include -#include +#include +#include #include #include "doctest/doctest.h" -#include "emel/text/jinja/lexer.hpp" -#include "emel/text/jinja/parser/detail.hpp" #include "emel/text/jinja/formatter/sm.hpp" -// TODO(rearchitecture-cleanup): Keep legacy "jinja_renderer_*" test names until -// external references to current test IDs are migrated. - namespace { -auto parse_template(const std::string & text, emel::text::jinja::program & program) { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result lex_res = lex.tokenize(text); - CHECK(lex_res.error == EMEL_OK); - emel::text::jinja::parser::detail::recursive_descent_parser parser{program}; - CHECK(parser.parse(lex_res)); -} - -struct render_result { - int32_t err = EMEL_OK; - size_t error_pos = 0; - std::string output; - bool done = false; -}; - -render_result render_template(const std::string & templ, - const emel::text::jinja::object_value * globals = nullptr) { - emel::text::jinja::program program{}; - parse_template(templ, program); +using emel::text::jinja::event::render; +using emel::text::jinja::formatter::action::context; +using emel::text::jinja::formatter::done; +using emel::text::jinja::formatter::errored; +using emel::text::jinja::formatter::error; +using emel::text::jinja::formatter::initialized; +using emel::text::jinja::formatter::sm; +using emel::text::jinja::formatter::unexpected; +using done_cb = emel::callback; +using error_cb = emel::callback; - std::array buffer = {}; - size_t out_len = 0; - size_t error_pos = 0; - int32_t err = EMEL_OK; - - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; - emel::text::jinja::event::render ev{ - .program = &program, - .globals = globals, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .error_out = &err, - .error_pos_out = &error_pos, - }; - - machine.process_event(ev); - - render_result result{}; - result.err = err; - result.error_pos = error_pos; - result.output.assign(buffer.data(), out_len); - result.done = machine.is(boost::sml::state); - return result; +bool ignore_done_callback(const emel::text::jinja::events::rendering_done &) { + return true; } -emel::text::jinja::value make_string(std::string_view v) { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::string; - out.string_v.view = v; - return out; +bool ignore_error_callback(const emel::text::jinja::events::rendering_error &) { + return true; } -emel::text::jinja::value make_int(int64_t v) { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::integer; - out.int_v = v; - out.float_v = static_cast(v); - return out; -} +constexpr done_cb k_ignore_done_callback = done_cb::from<&ignore_done_callback>(); +constexpr error_cb k_ignore_error_callback = error_cb::from<&ignore_error_callback>(); -emel::text::jinja::value make_bool(bool v) { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::boolean; - out.bool_v = v; - return out; -} - -emel::text::jinja::value make_none() { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::none; - return out; -} - -emel::text::jinja::value make_float(double v) { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::floating; - out.float_v = v; - out.int_v = static_cast(v); - return out; -} +struct callback_tracker { + bool done_called = false; + bool error_called = false; + size_t done_length = 0; + bool done_truncated = false; + int32_t error_code = static_cast(error::none); + size_t error_pos = 0; -emel::text::jinja::value make_array(emel::text::jinja::value * items, size_t count) { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::array; - out.array_v.items = items; - out.array_v.count = count; - out.array_v.capacity = count; - return out; -} + bool on_done(const emel::text::jinja::events::rendering_done & ev) { + done_called = true; + done_length = ev.output_length; + done_truncated = ev.output_truncated; + return ev.request.output_capacity > 0; + } -emel::text::jinja::value make_object(emel::text::jinja::object_entry * entries, size_t count) { - emel::text::jinja::value out; - out.type = emel::text::jinja::value_type::object; - out.object_v.entries = entries; - out.object_v.count = count; - out.object_v.capacity = count; - out.object_v.has_builtins = false; - return out; -} + bool on_error(const emel::text::jinja::events::rendering_error & ev) { + error_called = true; + error_code = ev.err; + error_pos = ev.error_pos; + return ev.error_pos == 0; + } +}; } // namespace -TEST_CASE("jinja_renderer_starts_initialized") { - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; - CHECK(machine.is(boost::sml::state)); +TEST_CASE("jinja_formatter_starts_initialized") { + context ctx{}; + sm machine{ctx}; + CHECK(machine.is(boost::sml::state)); } -TEST_CASE("jinja_renderer_renders_simple_template") { +TEST_CASE("jinja_formatter_copies_source_text") { emel::text::jinja::program program{}; - parse_template("hello {{ name }}!", program); - - std::array entries = {}; - entries[0].key = make_string("name"); - entries[0].val = make_string("world"); - emel::text::jinja::object_value globals{entries.data(), entries.size(), entries.size(), false}; - std::array buffer = {}; - size_t out_len = 0; - int32_t err = EMEL_OK; - - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; - emel::text::jinja::event::render ev{ - .program = &program, - .globals = &globals, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .error_out = &err, - }; - - machine.process_event(ev); - - CHECK(machine.is(boost::sml::state)); - CHECK(err == EMEL_OK); - std::string_view rendered(buffer.data(), out_len); - CHECK(rendered == "hello world!"); -} - -TEST_CASE("jinja_renderer_handles_loops_and_filters") { - emel::text::jinja::program program{}; - parse_template("{% for x in items if x != 2 %}{{ x|upper }}{% endfor %}", program); - - std::array items = {make_int(1), make_int(2), make_int(3)}; - emel::text::jinja::value items_val = make_array(items.data(), items.size()); - - std::array entries = {}; - entries[0].key = make_string("items"); - entries[0].val = items_val; - emel::text::jinja::object_value globals{entries.data(), entries.size(), entries.size(), false}; - - std::array buffer = {}; - size_t out_len = 0; - int32_t err = EMEL_OK; - - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; - emel::text::jinja::event::render ev{ - .program = &program, - .globals = &globals, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .error_out = &err, - }; - - machine.process_event(ev); - - CHECK(machine.is(boost::sml::state)); - CHECK(err == EMEL_OK); - std::string_view rendered(buffer.data(), out_len); - CHECK(rendered == "13"); -} - -TEST_CASE("jinja_renderer_supports_set_and_member_access") { - emel::text::jinja::program program{}; - parse_template("{% set name = 'bob' %}{{ name }} {{ obj.key }}", program); - - std::array obj_entries = {}; - obj_entries[0].key = make_string("key"); - obj_entries[0].val = make_string("OK"); - emel::text::jinja::value obj_val = make_object(obj_entries.data(), obj_entries.size()); - - std::array entries = {}; - entries[0].key = make_string("obj"); - entries[0].val = obj_val; - emel::text::jinja::object_value globals{entries.data(), entries.size(), entries.size(), false}; - - std::array buffer = {}; - size_t out_len = 0; - int32_t err = EMEL_OK; - - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; - emel::text::jinja::event::render ev{ - .program = &program, - .globals = &globals, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .error_out = &err, - }; - - machine.process_event(ev); - - CHECK(machine.is(boost::sml::state)); - CHECK(err == EMEL_OK); - std::string_view rendered(buffer.data(), out_len); - CHECK(rendered == "bob OK"); -} - -TEST_CASE("jinja_renderer_invalid_request_errors") { - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; - - std::array buffer = {}; - int32_t err = EMEL_OK; - emel::text::jinja::event::render ev{ - .program = nullptr, - .output = buffer.data(), - .output_capacity = buffer.size(), - .error_out = &err, + size_t output_length = 0; + bool output_truncated = true; + int32_t err = static_cast(error::invalid_request); + size_t error_pos = 99; + + context ctx{}; + sm machine{ctx}; + render ev{ + program, + "hello world", + buffer[0], + buffer.size(), + k_ignore_done_callback, + k_ignore_error_callback, + nullptr, + &output_length, + &output_truncated, + &err, + &error_pos, }; - machine.process_event(ev); - - CHECK(machine.is(boost::sml::state)); - CHECK(err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_ops_basic") { - auto result = render_template("{{ 2 * 3 }}{{ 5 / 2 }}{{ 5 % 2 }}{{ \"a\" ~ \"b\" }}"); - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK_FALSE(result.output.empty()); -} - -TEST_CASE("jinja_renderer_filters_trim_replace") { - auto result = render_template("{{ \" hi \"|trim }}{{ \"aba\"|replace(\"a\", \"x\") }}"); - CAPTURE(result.error_pos); - CAPTURE(result.output); - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK_FALSE(result.output.empty()); -} - -TEST_CASE("jinja_renderer_in_ops") { - auto result = render_template("{{ 1 in [1, 2] }}{{ 1 not in [2, 3] }}"); - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK_FALSE(result.output.empty()); -} - -TEST_CASE("jinja_renderer_test_ops") { - auto result = render_template("{{ 1 is odd }}{{ 2 is even }}"); - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK_FALSE(result.output.empty()); -} - -TEST_CASE("jinja_renderer_range_join") { - auto result = render_template("{{ range(0, 3)|join(\",\") }}"); - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK_FALSE(result.output.empty()); -} - -TEST_CASE("jinja_renderer_if_and_select") { - std::array entries = {}; - entries[0].key = make_string("name"); - entries[0].val = make_string("bob"); - entries[1].key = make_string("cond"); - entries[1].val = make_bool(true); - emel::text::jinja::object_value globals{entries.data(), entries.size(), entries.size(), false}; - - auto result = render_template( - "{% if cond %}{{ name|upper }}{% else %}{{ name|lower }}{% endif %}" - "{{ 1 if cond else 2 }}{{ 3 if cond }}", - &globals); - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK(result.output.find("BOB") != std::string::npos); -} - -TEST_CASE("jinja_renderer_loops_and_members") { - std::array items = {make_int(1), make_int(2), make_int(3)}; - emel::text::jinja::value items_val = make_array(items.data(), items.size()); - - std::array obj_entries = {}; - obj_entries[0].key = make_string("key"); - obj_entries[0].val = make_string("V"); - emel::text::jinja::value obj_val = make_object(obj_entries.data(), obj_entries.size()); - - std::array arr = {make_int(7), make_int(8), make_int(9)}; - emel::text::jinja::value arr_val = make_array(arr.data(), arr.size()); - - std::array entries = {}; - entries[0].key = make_string("items"); - entries[0].val = items_val; - entries[1].key = make_string("obj"); - entries[1].val = obj_val; - entries[2].key = make_string("arr"); - entries[2].val = arr_val; - entries[3].key = make_string("cond"); - entries[3].val = make_bool(true); - emel::text::jinja::object_value globals{entries.data(), entries.size(), entries.size(), false}; - - auto result = render_template( - "{% for k, v in obj %}{{ k }}{{ v }}{% endfor %}" - "{% for x in items if x > 1 %}{{ x }}{% endfor %}" - "{{ obj.key }}{{ arr[1] }}{{ arr[0:2] }}", - &globals); - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK_FALSE(result.output.empty()); + CHECK(machine.process_event(ev)); + CHECK(machine.is(boost::sml::state)); + CHECK(err == static_cast(error::none)); + CHECK(error_pos == 0); + CHECK(output_truncated == false); + CHECK(output_length == ev.source_text.size()); + CHECK(std::string_view(buffer.data(), output_length) == ev.source_text); } -TEST_CASE("jinja_renderer_macro_and_call") { +TEST_CASE("jinja_formatter_handles_empty_source_text") { emel::text::jinja::program program{}; - parse_template( - "{% macro wrap(name) %}{{ name }}_{{ caller()|lower }}{% endmacro %}" - "{% call() wrap(\"hi\") %}{{ \"there\"|upper }}{% endcall %}", - program); - - std::array buffer = {}; - size_t out_len = 0; - int32_t err = EMEL_OK; - - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; - emel::text::jinja::event::render ev{ - .program = &program, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .error_out = &err, + std::array buffer = {}; + size_t output_length = 7; + bool output_truncated = true; + int32_t err = static_cast(error::invalid_request); + + context ctx{}; + sm machine{ctx}; + render ev{ + program, + "", + buffer[0], + buffer.size(), + k_ignore_done_callback, + k_ignore_error_callback, + nullptr, + &output_length, + &output_truncated, + &err, }; - machine.process_event(ev); - - CHECK(machine.is(boost::sml::state)); - CHECK(err == EMEL_OK); - std::string_view rendered(buffer.data(), out_len); - CHECK(rendered == "hi_there"); + CHECK(machine.process_event(ev)); + CHECK(machine.is(boost::sml::state)); + CHECK(err == static_cast(error::none)); + CHECK(output_length == 0); + CHECK(output_truncated == false); } -TEST_CASE("jinja_renderer_filter_statement") { +TEST_CASE("jinja_formatter_reports_capacity_error") { emel::text::jinja::program program{}; - parse_template("{% filter upper %}hello{% endfilter %}", program); - - std::array buffer = {}; - size_t out_len = 0; - int32_t err = EMEL_OK; - - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; - emel::text::jinja::event::render ev{ - .program = &program, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .error_out = &err, + std::array buffer = {}; + size_t output_length = 123; + bool output_truncated = false; + int32_t err = static_cast(error::none); + size_t error_pos = 88; + + context ctx{}; + sm machine{ctx}; + render ev{ + program, + "overflow", + buffer[0], + buffer.size(), + k_ignore_done_callback, + k_ignore_error_callback, + nullptr, + &output_length, + &output_truncated, + &err, + &error_pos, }; - machine.process_event(ev); - - CHECK(machine.is(boost::sml::state)); - CHECK(err == EMEL_OK); - std::string_view rendered(buffer.data(), out_len); - CHECK(rendered == "HELLO"); + CHECK_FALSE(machine.process_event(ev)); + CHECK(machine.is(boost::sml::state)); + CHECK(err == static_cast(error::invalid_request)); + CHECK(output_length == 0); + CHECK(output_truncated == true); + CHECK(error_pos == 0); } -TEST_CASE("jinja_renderer_truncates_on_small_buffer") { +TEST_CASE("jinja_formatter_rejects_invalid_request") { emel::text::jinja::program program{}; - parse_template("hello {{ name }}!", program); - - std::array entries = {}; - entries[0].key = make_string("name"); - entries[0].val = make_string("world"); - emel::text::jinja::object_value globals{entries.data(), entries.size(), entries.size(), false}; - - std::array buffer = {}; - size_t out_len = 0; - bool truncated = false; - int32_t err = EMEL_OK; - - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; - emel::text::jinja::event::render ev{ - .program = &program, - .globals = &globals, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .output_truncated = &truncated, - .error_out = &err, + std::array buffer = {}; + size_t output_length = 5; + bool output_truncated = true; + int32_t err = static_cast(error::none); + size_t error_pos = 7; + + context ctx{}; + sm machine{ctx}; + render ev{ + program, + "ignored", + buffer[0], + 0, + k_ignore_done_callback, + k_ignore_error_callback, + nullptr, + &output_length, + &output_truncated, + &err, + &error_pos, }; - machine.process_event(ev); - - CHECK(machine.is(boost::sml::state)); - CHECK(err == EMEL_ERR_INVALID_ARGUMENT); - CHECK(truncated); + CHECK_FALSE(machine.process_event(ev)); + CHECK(machine.is(boost::sml::state)); + CHECK(err == static_cast(error::invalid_request)); + CHECK(output_length == 0); + CHECK(output_truncated == false); + CHECK(error_pos == 0); } -TEST_CASE("jinja_renderer_dispatch_callbacks") { +TEST_CASE("jinja_formatter_rejects_missing_callbacks") { emel::text::jinja::program program{}; - parse_template("hi", program); - - std::array buffer = {}; - size_t out_len = 0; - int32_t err = EMEL_OK; - - struct tracker { - bool done = false; - bool error = false; - size_t length = 0; - - bool on_done(const emel::text::jinja::events::rendering_done & ev) { - done = true; - length = ev.output_length; - return true; - } - - bool on_error(const emel::text::jinja::events::rendering_error &) { - error = true; - return true; - } - }; - - tracker track{}; - using done_cb = emel::callback; - using error_cb = emel::callback; - - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; - emel::text::jinja::event::render ev{ - .program = &program, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .error_out = &err, - .dispatch_done = done_cb::from(&track), - .dispatch_error = error_cb::from(&track), + std::array buffer = {}; + size_t output_length = 3; + bool output_truncated = true; + int32_t err = static_cast(error::none); + size_t error_pos = 9; + + context ctx{}; + sm machine{ctx}; + render ev{ + program, + "ignored", + buffer[0], + buffer.size(), + done_cb{}, + error_cb{}, + nullptr, + &output_length, + &output_truncated, + &err, + &error_pos, }; - machine.process_event(ev); - - CHECK(machine.is(boost::sml::state)); - CHECK(err == EMEL_OK); - CHECK(track.done); - CHECK_FALSE(track.error); - CHECK(track.length == out_len); + CHECK_FALSE(machine.process_event(ev)); + CHECK(machine.is(boost::sml::state)); + CHECK(err == static_cast(error::invalid_request)); + CHECK(output_length == 0); + CHECK(output_truncated == false); + CHECK(error_pos == 0); } -TEST_CASE("jinja_renderer_clears_stale_error_out_before_successful_render") { +TEST_CASE("jinja_formatter_dispatches_done_callback") { emel::text::jinja::program program{}; - parse_template("ok", program); - - std::array buffer = {}; - size_t out_len = 0; - int32_t err = EMEL_ERR_INVALID_ARGUMENT; - - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; - emel::text::jinja::event::render ev{ - .program = &program, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .error_out = &err, + std::array buffer = {}; + size_t output_length = 0; + int32_t err = static_cast(error::none); + callback_tracker tracker{}; + + context ctx{}; + sm machine{ctx}; + render ev{ + program, + "ok", + buffer[0], + buffer.size(), + done_cb::from(&tracker), + error_cb::from(&tracker), + nullptr, + &output_length, + nullptr, + &err, + nullptr, }; CHECK(machine.process_event(ev)); - CHECK(machine.is(boost::sml::state)); - CHECK(err == EMEL_OK); - CHECK(out_len == 2); + CHECK(tracker.done_called); + CHECK_FALSE(tracker.error_called); + CHECK(tracker.done_length == output_length); + CHECK(tracker.done_truncated == false); } -TEST_CASE("jinja_renderer_rejects_invalid_and_dispatches_error") { - std::array buffer = {}; - size_t out_len = 7; - bool truncated = true; - int32_t err = EMEL_OK; - - struct tracker { - bool error = false; - - bool on_error(const emel::text::jinja::events::rendering_error & ev) { - error = true; - return ev.err == EMEL_ERR_INVALID_ARGUMENT; - } - }; - - tracker track{}; - using error_cb = emel::callback; - - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; - emel::text::jinja::event::render ev{ - .program = nullptr, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .output_truncated = &truncated, - .error_out = &err, - .dispatch_error = error_cb::from(&track), +TEST_CASE("jinja_formatter_dispatches_error_callback") { + emel::text::jinja::program program{}; + std::array buffer = {}; + size_t output_length = 0; + int32_t err = static_cast(error::none); + callback_tracker tracker{}; + + context ctx{}; + sm machine{ctx}; + render ev{ + program, + "bad", + buffer[0], + 0, + k_ignore_done_callback, + error_cb::from(&tracker), + nullptr, + &output_length, + nullptr, + &err, + nullptr, }; - machine.process_event(ev); - - CHECK(machine.is(boost::sml::state)); - CHECK(err == EMEL_ERR_INVALID_ARGUMENT); - CHECK(out_len == 0); - CHECK_FALSE(truncated); - CHECK(track.error); + CHECK_FALSE(machine.process_event(ev)); + CHECK_FALSE(tracker.done_called); + CHECK(tracker.error_called); + CHECK(tracker.error_code == static_cast(error::invalid_request)); + CHECK(tracker.error_pos == 0); } -TEST_CASE("jinja_renderer_unexpected_event_sets_error") { +TEST_CASE("jinja_formatter_unexpected_event_transitions_state") { struct unknown_event { int value = 0; }; - emel::text::jinja::formatter::action::context ctx{}; - emel::text::jinja::formatter::sm machine{ctx}; + context ctx{}; + sm machine{ctx}; machine.process_event(unknown_event{}); - CHECK(machine.is(boost::sml::state)); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); -} - -TEST_CASE("jinja_renderer_defaults_namespace_length") { - std::array entries = {}; - entries[0].key = make_string("flag"); - entries[0].val = make_bool(false); - entries[1].key = make_string("apply_default"); - entries[1].val = make_bool(true); - entries[2].key = make_string("num"); - entries[2].val = make_int(7); - emel::text::jinja::object_value globals{entries.data(), entries.size(), entries.size(), false}; - - auto result = render_template( - "{{ default(missing, \"x\") }}{{ default(flag, \"y\", apply_default) }}" - "{{ namespace(a=1, b=2).b }}{{ \"abc\"|length }}{{ [1,2]|length }}" - "{{ {\"k\":1}|length }}{{ flag }}", - &globals); - - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK(result.output == "xy2321false"); -} - -TEST_CASE("jinja_renderer_range_spread_and_join") { - auto result = render_template( - "{{ range(*[1,4])|join(\"-\") }}" - "{{ range(0,3)|join(\",\") }}" - "{{ range(3,0,-1)|join(\",\") }}" - "{{ [1,2] == [1,2] }}{{ {\"k\":1} == {\"k\":1} }}"); - - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK_FALSE(result.output.empty()); -} - -TEST_CASE("jinja_renderer_member_and_set_variants") { - std::array obj_entries = {}; - obj_entries[0].key = make_string("key"); - obj_entries[0].val = make_string("orig"); - emel::text::jinja::value obj_val = make_object(obj_entries.data(), obj_entries.size()); - - std::array arr_vals = {make_int(1), make_int(2), make_int(3)}; - emel::text::jinja::value arr_val = make_array(arr_vals.data(), arr_vals.size()); - - std::array entries = {}; - entries[0].key = make_string("obj"); - entries[0].val = obj_val; - entries[1].key = make_string("arr"); - entries[1].val = arr_val; - emel::text::jinja::object_value globals{entries.data(), entries.size(), entries.size(), false}; - - auto result = render_template( - "{# comment #}{% generation %}" - "{% set name %}bob{% endset %}{{ name }}" - "{% set a, b = [1,2] %}{{ a }}{{ b }}" - "{% set obj.key = \"v\" %}{{ obj.key }}" - "{{ obj[\"key\"] }}{{ \"hello\"[1] }}{{ \"hello\"[1:4] }}" - "{{ arr[1] }}{{ arr[0:2] }}", - &globals); - - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK_FALSE(result.output.empty()); -} - -TEST_CASE("jinja_renderer_control_flow_break_continue") { - auto result = render_template( - "{% for x in [1,2,3] %}" - "{% if x == 2 %}{% continue %}{% endif %}" - "{% if x == 3 %}{% break %}{% endif %}" - "{{ x }}" - "{% endfor %}" - "{% for x in missing %}{{ x }}{% else %}EMPTY{% endfor %}"); - - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK(result.output.find("1") != std::string::npos); - CHECK(result.output.find("EMPTY") != std::string::npos); -} - -TEST_CASE("jinja_renderer_filter_statement_with_args") { - auto result = render_template("{% filter replace(\"a\", \"b\") %}a-a{% endfilter %}"); - - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK(result.output == "b-b"); -} - -TEST_CASE("jinja_renderer_macro_defaults_and_caller_params") { - auto result = render_template( - "{% macro greet(name=\"bob\") %}{{ name }}{% endmacro %}" - "{% macro wrap(name) %}{{ caller(name) }}{% endmacro %}" - "{% call(n) wrap(\"hi\") %}{{ n|upper }}{% endcall %}" - "{{ greet() }}{{ greet(\"ana\") }}"); - - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK_FALSE(result.output.empty()); -} - -TEST_CASE("jinja_renderer_tests_and_memberships") { - std::array arr_items = {make_int(1), make_int(2)}; - emel::text::jinja::value arr_val = make_array(arr_items.data(), arr_items.size()); - - std::array obj_entries = {}; - obj_entries[0].key = make_string("a"); - obj_entries[0].val = make_int(1); - emel::text::jinja::value obj_val = make_object(obj_entries.data(), obj_entries.size()); - - std::array entries = {}; - entries[0].key = make_string("name"); - entries[0].val = make_string("bob"); - entries[1].key = make_string("flag"); - entries[1].val = make_bool(false); - entries[2].key = make_string("true_val"); - entries[2].val = make_bool(true); - entries[3].key = make_string("none_val"); - entries[3].val = make_none(); - entries[4].key = make_string("num"); - entries[4].val = make_int(5); - entries[5].key = make_string("float_val"); - entries[5].val = make_float(1.5); - entries[6].key = make_string("arr"); - entries[6].val = arr_val; - entries[7].key = make_string("obj"); - entries[7].val = obj_val; - emel::text::jinja::object_value globals{entries.data(), entries.size(), entries.size(), false}; - - auto result = render_template( - "{{ missing is undefined }}{{ none_val is none }}{{ name is defined }}" - "{{ name is string }}{{ flag is boolean }}{{ num is number }}" - "{{ arr is iterable }}{{ obj is mapping }}{{ flag is false }}{{ true_val is true }}" - "{{ \"a\" in obj }}{{ \"a\" in \"cat\" }}{{ float_val }}", - &globals); - - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK_FALSE(result.output.empty()); -} - -TEST_CASE("jinja_renderer_errors_on_invalid_control_flow") { - auto result = render_template("{% break %}"); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_errors_on_div_zero") { - auto result = render_template("{{ 1 / 0 }}"); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_expression_matrix") { - auto result = render_template( - "{{ 1 or 0 }}{{ 0 or 2 }}{{ 0 and 1 }}{{ 1 and 2 }}{{ not 0 }}" - "{{ 1 + 2 }}{{ 1 + 2.5 }}{{ \"a\" + \"b\" }}{{ 5 - 2 }}" - "{{ 2 * 3 }}{{ \"ha\" * 2 }}{{ 5 / 2 }}{{ 5 % 2 }}" - "{{ 1 < 2 }}{{ 2 >= 1 }}{{ \"a\" < \"b\" }}" - "{{ 1 in [1,2] }}{{ 1 not in [2] }}{{ \"a\" in {\"a\":1} }}" - "{{ \"a\" in \"cat\" }}"); - - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK_FALSE(result.output.empty()); -} - -TEST_CASE("jinja_renderer_member_edge_cases") { - std::array arr_vals = {make_int(1), make_int(2), make_int(3)}; - emel::text::jinja::value arr_val = make_array(arr_vals.data(), arr_vals.size()); - - std::array entries = {}; - entries[0].key = make_string("arr"); - entries[0].val = arr_val; - emel::text::jinja::object_value globals{entries.data(), entries.size(), entries.size(), false}; - - auto result = render_template( - "{{ missing.key }}{{ arr[99] }}{{ arr[\"bad\"] }}{{ \"hi\"[99] }}" - "{{ \"hi\".bad }}{{ {1:\"a\"}[1] }}" - "{{ arr[1:] }}{{ \"hello\"[2:] }}{{ arr[0:3:2] }}", - &globals); - - CHECK(result.done); - CHECK(result.err == EMEL_OK); - CHECK_FALSE(result.output.empty()); -} - -TEST_CASE("jinja_renderer_errors_on_slice_step_zero") { - std::array arr_vals = {make_int(1), make_int(2)}; - emel::text::jinja::value arr_val = make_array(arr_vals.data(), arr_vals.size()); - - std::array entries = {}; - entries[0].key = make_string("arr"); - entries[0].val = arr_val; - emel::text::jinja::object_value globals{entries.data(), entries.size(), entries.size(), false}; - - auto result = render_template("{{ arr[0:2:0] }}", &globals); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_errors_on_missing_filter") { - auto result = render_template("{{ \"x\"|missing_filter }}"); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_errors_on_missing_test") { - auto result = render_template("{{ 1 is missing_test }}"); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_errors_on_call_target") { - auto result = render_template("{{ 1() }}"); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_errors_on_for_invalid_iterable") { - auto result = render_template("{% for x in 1 %}{{ x }}{% endfor %}"); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_errors_on_for_tuple_item") { - auto result = render_template("{% for a, b in [1,2] %}{{ a }}{% endfor %}"); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_errors_on_set_tuple_non_array") { - auto result = render_template("{% set a, b = 1 %}"); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_errors_on_set_tuple_size_mismatch") { - auto result = render_template("{% set a, b = [1] %}"); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_errors_on_set_computed_member") { - auto result = render_template("{% set obj[\"k\"] = 1 %}"); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_errors_on_set_member_non_object") { - std::array entries = {}; - entries[0].key = make_string("num"); - entries[0].val = make_int(1); - emel::text::jinja::object_value globals{entries.data(), entries.size(), entries.size(), false}; - - auto result = render_template("{% set num.key = 1 %}", &globals); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_errors_on_filter_statement_missing_filter") { - auto result = render_template("{% filter missing %}x{% endfilter %}"); - - CHECK_FALSE(result.done); - CHECK(result.err == EMEL_ERR_INVALID_ARGUMENT); -} - -TEST_CASE("jinja_renderer_detail_unary_paths") { - using namespace emel::text::jinja; - - { - formatter::action::context ctx{}; - formatter::detail::render_io io{}; - char buffer[16] = {}; - formatter::detail::init_writer(io, buffer, sizeof(buffer)); - - token op{token_type::additive_binary_operator, "+", 0}; - auto operand = std::make_unique(2); - unary_expression expr{op, std::move(operand)}; - auto value = formatter::detail::eval_expr(ctx, &expr, nullptr, io); - - CHECK(value.type == value_type::integer); - CHECK(value.int_v == 2); - } - - { - formatter::action::context ctx{}; - formatter::detail::render_io io{}; - char buffer[16] = {}; - formatter::detail::init_writer(io, buffer, sizeof(buffer)); - - token op{token_type::additive_binary_operator, "-", 0}; - auto operand = std::make_unique(1.5); - unary_expression expr{op, std::move(operand)}; - auto value = formatter::detail::eval_expr(ctx, &expr, nullptr, io); - - CHECK(value.type == value_type::floating); - CHECK(value.float_v == doctest::Approx(-1.5)); - } - - { - formatter::action::context ctx{}; - formatter::detail::render_io io{}; - char buffer[16] = {}; - formatter::detail::init_writer(io, buffer, sizeof(buffer)); - - token op{token_type::additive_binary_operator, "+", 0}; - auto operand = std::make_unique("a"); - unary_expression expr{op, std::move(operand)}; - auto value = formatter::detail::eval_expr(ctx, &expr, nullptr, io); - - CHECK(value.type == value_type::undefined); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } -} - -TEST_CASE("jinja_renderer_detail_limit_paths") { - using namespace emel::text::jinja; - - { - formatter::action::context ctx{}; - ctx.steps_remaining = 0; - CHECK_FALSE(formatter::detail::ensure_steps(ctx, 0)); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - } - - { - formatter::action::context ctx{}; - formatter::detail::render_io io{}; - io.depth = formatter::action::k_max_capture_depth; - CHECK_FALSE(formatter::detail::begin_capture(ctx, io)); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - - ctx.phase_error = EMEL_OK; - io.depth = 0; - emel::text::jinja::value captured{}; - CHECK_FALSE(formatter::detail::end_capture(ctx, io, captured)); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - } - - { - formatter::action::context ctx{}; - ctx.string_buffer_used = formatter::action::k_max_string_bytes; - auto stored = formatter::detail::store_string(ctx, "x"); - CHECK(stored.empty()); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - ctx.array_items_used = formatter::action::k_max_array_items - 1; - std::array items = {value{}, value{}}; - auto array_val = formatter::detail::make_array(ctx, items.data(), items.size()); - CHECK(array_val.type == value_type::undefined); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - ctx.object_entries_used = formatter::action::k_max_object_entries - 1; - std::array entries = {}; - entries[0].key = value{}; - entries[0].val = value{}; - entries[1].key = value{}; - entries[1].val = value{}; - auto obj_val = formatter::detail::make_object(ctx, entries.data(), entries.size(), entries.size(), false); - CHECK(obj_val.type == value_type::undefined); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - ctx.scope_count = formatter::action::k_max_scopes; - CHECK_FALSE(formatter::detail::push_scope(ctx)); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - - ctx = formatter::action::context{}; - ctx.object_entries_used = formatter::action::k_max_object_entries - formatter::action::k_scope_capacity + 1; - CHECK_FALSE(formatter::detail::push_scope(ctx)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::detail::call_args args{}; - args.pos_count = formatter::detail::k_max_call_args; - CHECK_FALSE(formatter::detail::add_pos(args, value{})); - args.kw_count = formatter::detail::k_max_call_args; - CHECK_FALSE(formatter::detail::add_kw(args, "k", value{})); - } - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - args.pos_count = formatter::detail::k_max_call_args; - value out{}; - CHECK_FALSE(formatter::detail::filter_length(ctx, value{}, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } -} - -TEST_CASE("jinja_renderer_detail_call_statement_errors") { - using namespace emel::text::jinja; - - { - formatter::action::context ctx{}; - formatter::detail::render_io io{}; - char buffer[16] = {}; - formatter::detail::init_writer(io, buffer, sizeof(buffer)); - - auto non_call = std::make_unique(1); - ast_list caller_args; - ast_list body; - call_statement stmt{std::move(non_call), std::move(caller_args), std::move(body)}; - CHECK_FALSE(formatter::detail::render_call_statement(ctx, &stmt, nullptr, io)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - formatter::detail::render_io io{}; - char buffer[16] = {}; - formatter::detail::init_writer(io, buffer, sizeof(buffer)); - - ast_list args; - auto callee = std::make_unique(1); - auto call_expr = std::make_unique(std::move(callee), std::move(args)); - ast_list caller_args; - ast_list body; - call_statement stmt{std::move(call_expr), std::move(caller_args), std::move(body)}; - CHECK_FALSE(formatter::detail::render_call_statement(ctx, &stmt, nullptr, io)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } -} - -TEST_CASE("jinja_renderer_detail_helpers_part1") { - using namespace emel::text::jinja; - - { - formatter::action::context ctx{}; - formatter::detail::render_io io{}; - char buffer[8] = {}; - formatter::detail::init_writer(io, buffer, sizeof(buffer)); - - CHECK(formatter::detail::store_string(ctx, "").empty()); - - io.depth = 1; - io.writers[1].data = buffer; - io.writers[1].capacity = sizeof(buffer); - io.writers[1].length = 1; - ctx.phase_error = EMEL_ERR_BACKEND; - value captured{}; - CHECK_FALSE(formatter::detail::end_capture(ctx, io, captured)); - - formatter::detail::make_object(ctx, nullptr, 0, 0, false); - formatter::detail::pop_scope(ctx); - } - - { - value v_none{}; - v_none.type = value_type::none; - value v_bool{}; - v_bool.type = value_type::boolean; - v_bool.bool_v = true; - value v_int{}; - v_int.type = value_type::integer; - v_int.int_v = 1; - v_int.float_v = 1.0; - value v_float{}; - v_float.type = value_type::floating; - v_float.float_v = 1.5; - v_float.int_v = 1; - value v_string{}; - v_string.type = value_type::string; - v_string.string_v.view = "hi"; - - value array_items[2] = {v_int, v_int}; - value v_array{}; - v_array.type = value_type::array; - v_array.array_v.items = array_items; - v_array.array_v.count = 2; - v_array.array_v.capacity = 2; - - object_entry obj_entries[1] = {}; - obj_entries[0].key = v_string; - obj_entries[0].val = v_int; - value v_object{}; - v_object.type = value_type::object; - v_object.object_v.entries = obj_entries; - v_object.object_v.count = 1; - v_object.object_v.capacity = 1; - - value v_func{}; - v_func.type = value_type::function; - v_func.func_v.data = &v_func; - - CHECK_FALSE(formatter::detail::value_is_truthy(v_none)); - CHECK(formatter::detail::value_is_truthy(v_bool)); - CHECK(formatter::detail::value_is_truthy(v_int)); - CHECK(formatter::detail::value_is_truthy(v_float)); - CHECK(formatter::detail::value_is_truthy(v_string)); - CHECK(formatter::detail::value_is_truthy(v_array)); - CHECK(formatter::detail::value_is_truthy(v_object)); - CHECK(formatter::detail::value_is_truthy(v_func)); - - value v_float_one{}; - v_float_one.type = value_type::floating; - v_float_one.float_v = 1.0; - v_float_one.int_v = 1; - CHECK(formatter::detail::value_equal(v_int, v_float_one)); - - value v_array_short = v_array; - v_array_short.array_v.count = 1; - CHECK_FALSE(formatter::detail::value_equal(v_array, v_array_short)); - - value v_object_other = v_object; - v_object_other.object_v.entries = nullptr; - v_object_other.object_v.count = 0; - v_object_other.object_v.capacity = 0; - CHECK_FALSE(formatter::detail::value_equal(v_object, v_object_other)); - - value v_func_other{}; - v_func_other.type = value_type::function; - v_func_other.func_v.data = &v_object; - CHECK_FALSE(formatter::detail::value_equal(v_func, v_func_other)); - } - - { - formatter::action::context ctx{}; - formatter::detail::render_io io{}; - char buffer[16] = {}; - formatter::detail::init_writer(io, buffer, sizeof(buffer)); - - value v_undef{}; - v_undef.type = value_type::undefined; - CHECK(formatter::detail::value_to_string(ctx, v_undef).empty()); - - value v_arr{}; - v_arr.type = value_type::array; - CHECK(formatter::detail::value_to_string(ctx, v_arr).empty()); - - formatter::detail::call_args args{}; - CHECK(formatter::detail::find_kw(args, "missing") == nullptr); - CHECK(formatter::detail::get_pos(args, 1) == nullptr); - } -} - -TEST_CASE("jinja_renderer_detail_helpers_part2") { - using namespace emel::text::jinja; - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - CHECK_FALSE(formatter::detail::builtin_default(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - - ctx = formatter::action::context{}; - CHECK_FALSE(formatter::detail::builtin_length(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - - ctx = formatter::action::context{}; - CHECK_FALSE(formatter::detail::builtin_range(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - args.pos_count = 1; - args.pos[0].type = value_type::string; - args.pos[0].string_v.view = "x"; - CHECK_FALSE(formatter::detail::builtin_range(ctx, args, out)); - - ctx = formatter::action::context{}; - args.pos_count = 2; - args.pos[0].type = value_type::integer; - args.pos[0].int_v = 0; - args.pos[1].type = value_type::string; - args.pos[1].string_v.view = "bad"; - CHECK_FALSE(formatter::detail::builtin_range(ctx, args, out)); - - ctx = formatter::action::context{}; - args.pos_count = 3; - args.pos[0].type = value_type::integer; - args.pos[0].int_v = 0; - args.pos[1].type = value_type::integer; - args.pos[1].int_v = 1; - args.pos[2].type = value_type::integer; - args.pos[2].int_v = 0; - CHECK_FALSE(formatter::detail::builtin_range(ctx, args, out)); - } - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - args.pos_count = 0; - CHECK_FALSE(formatter::detail::builtin_upper(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - - ctx = formatter::action::context{}; - CHECK_FALSE(formatter::detail::builtin_lower(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - - ctx = formatter::action::context{}; - CHECK_FALSE(formatter::detail::builtin_trim(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - - ctx = formatter::action::context{}; - CHECK_FALSE(formatter::detail::builtin_replace(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - - ctx = formatter::action::context{}; - CHECK_FALSE(formatter::detail::builtin_join(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - formatter::detail::render_io io{}; - char buffer[32] = {}; - formatter::detail::init_writer(io, buffer, sizeof(buffer)); - - value items[2] = {}; - items[0].type = value_type::integer; - items[0].int_v = 1; - items[1].type = value_type::integer; - items[1].int_v = 2; - - value arr{}; - arr.type = value_type::array; - arr.array_v.items = items; - arr.array_v.count = 2; - arr.array_v.capacity = 2; - - value obj{}; - obj.type = value_type::object; - - CHECK(formatter::detail::write_value(ctx, io, arr)); - CHECK(formatter::detail::write_value(ctx, io, obj)); - } -} - -TEST_CASE("jinja_renderer_detail_helpers_part3") { - using namespace emel::text::jinja; - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - args.kw_count = 1; - args.kw[0].key = "k"; - args.kw[0].val.type = value_type::integer; - args.kw[0].val.int_v = 1; - CHECK(formatter::detail::find_kw(args, "k") != nullptr); - - args.pos_count = 1; - args.pos[0].type = value_type::integer; - args.pos[0].int_v = 3; - CHECK(formatter::detail::get_pos(args, 0) != nullptr); - - CHECK(formatter::detail::builtin_namespace(ctx, args, out)); - CHECK(out.type == value_type::object); - } - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - args.pos_count = 1; - args.pos[0].type = value_type::string; - args.pos[0].string_v.view = "hi"; - CHECK(formatter::detail::builtin_upper(ctx, args, out)); - - ctx = formatter::action::context{}; - CHECK(formatter::detail::builtin_lower(ctx, args, out)); - - ctx = formatter::action::context{}; - CHECK(formatter::detail::builtin_trim(ctx, args, out)); - - ctx = formatter::action::context{}; - args.pos_count = 3; - args.pos[0].type = value_type::string; - args.pos[0].string_v.view = "aba"; - args.pos[1].type = value_type::string; - args.pos[1].string_v.view = "a"; - args.pos[2].type = value_type::string; - args.pos[2].string_v.view = "x"; - CHECK(formatter::detail::builtin_replace(ctx, args, out)); - - ctx = formatter::action::context{}; - args.pos_count = 1; - args.pos[0] = out; - CHECK(formatter::detail::builtin_join(ctx, args, out)); - } - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - args.pos_count = 1; - args.pos[0].type = value_type::integer; - args.pos[0].int_v = 3; - CHECK(formatter::detail::builtin_range(ctx, args, out)); - - ctx = formatter::action::context{}; - args.pos_count = 2; - args.pos[0].type = value_type::integer; - args.pos[0].int_v = 1; - args.pos[1].type = value_type::integer; - args.pos[1].int_v = 3; - CHECK(formatter::detail::builtin_range(ctx, args, out)); - - ctx = formatter::action::context{}; - args.pos_count = 3; - args.pos[0].type = value_type::integer; - args.pos[0].int_v = 3; - args.pos[1].type = value_type::integer; - args.pos[1].int_v = 0; - args.pos[2].type = value_type::integer; - args.pos[2].int_v = -1; - CHECK(formatter::detail::builtin_range(ctx, args, out)); - } - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - args.pos_count = 2; - args.pos[0].type = value_type::integer; - args.pos[0].int_v = 0; - args.pos[1].type = value_type::integer; - args.pos[1].int_v = static_cast(formatter::action::k_max_array_items + 1); - CHECK_FALSE(formatter::detail::builtin_range(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } -} - -TEST_CASE("jinja_renderer_detail_helpers_part4") { - using namespace emel::text::jinja; - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - value input{}; - input.type = value_type::string; - input.string_v.view = "hi"; - args.pos_count = 1; - args.pos[0] = input; - CHECK(formatter::detail::filter_default(ctx, input, args, out)); - - ctx = formatter::action::context{}; - args.pos_count = 0; - CHECK_FALSE(formatter::detail::filter_default(ctx, input, args, out)); - - ctx = formatter::action::context{}; - args.pos_count = 0; - CHECK(formatter::detail::filter_upper(ctx, input, args, out)); - - ctx = formatter::action::context{}; - CHECK(formatter::detail::filter_lower(ctx, input, args, out)); - - ctx = formatter::action::context{}; - CHECK(formatter::detail::filter_trim(ctx, input, args, out)); - - ctx = formatter::action::context{}; - CHECK(formatter::detail::filter_length(ctx, input, args, out)); - - ctx = formatter::action::context{}; - args.pos_count = 2; - args.pos[0] = input; - args.pos[1] = input; - CHECK(formatter::detail::filter_replace(ctx, input, args, out)); - - ctx = formatter::action::context{}; - args.pos_count = 0; - CHECK(formatter::detail::filter_join(ctx, input, args, out)); - } - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - std::string too_long(formatter::action::k_max_string_bytes + 1, 'a'); - args.pos_count = 1; - args.pos[0].type = value_type::string; - args.pos[0].string_v.view = too_long; - CHECK_FALSE(formatter::detail::builtin_upper(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - - ctx = formatter::action::context{}; - CHECK_FALSE(formatter::detail::builtin_lower(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - ctx.phase_error = EMEL_ERR_BACKEND; - args.pos_count = 1; - args.pos[0].type = value_type::string; - args.pos[0].string_v.view = "x"; - CHECK_FALSE(formatter::detail::builtin_trim(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - } - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - ctx.phase_error = EMEL_ERR_BACKEND; - args.pos_count = 1; - args.pos[0].type = value_type::string; - args.pos[0].string_v.view = "x"; - CHECK_FALSE(formatter::detail::builtin_upper(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - - ctx.phase_error = EMEL_ERR_BACKEND; - CHECK_FALSE(formatter::detail::builtin_lower(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - - args.pos_count = 3; - args.pos[0].type = value_type::string; - args.pos[0].string_v.view = "abc"; - args.pos[1].type = value_type::string; - args.pos[1].string_v.view = "a"; - args.pos[2].type = value_type::string; - args.pos[2].string_v.view = "b"; - ctx.phase_error = EMEL_ERR_BACKEND; - CHECK_FALSE(formatter::detail::builtin_replace(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - } -} - -TEST_CASE("jinja_renderer_detail_helpers_part5") { - using namespace emel::text::jinja; - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - value part{}; - part.type = value_type::string; - part.string_v.view = "a"; - value parts[2] = {part, part}; - value arr{}; - arr.type = value_type::array; - arr.array_v.items = parts; - arr.array_v.count = 2; - arr.array_v.capacity = 2; - - std::string huge_sep(formatter::action::k_max_string_bytes + 1, 'b'); - value delim{}; - delim.type = value_type::string; - delim.string_v.view = huge_sep; - - args.pos_count = 2; - args.pos[0] = arr; - args.pos[1] = delim; - ctx.phase_error = EMEL_ERR_BACKEND; - CHECK_FALSE(formatter::detail::builtin_join(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - - ctx = formatter::action::context{}; - ctx.string_buffer_used = formatter::action::k_max_string_bytes; - parts[0].type = value_type::integer; - parts[0].int_v = 1; - parts[0].float_v = 1.0; - parts[1].type = value_type::integer; - parts[1].int_v = 2; - parts[1].float_v = 2.0; - args.pos[0] = arr; - args.pos[1].type = value_type::string; - args.pos[1].string_v.view = " "; - CHECK_FALSE(formatter::detail::builtin_join(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - - ctx = formatter::action::context{}; - args.pos[0] = arr; - args.pos[1] = delim; - CHECK_FALSE(formatter::detail::builtin_join(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - - ctx = formatter::action::context{}; - std::string huge_part(formatter::action::k_max_string_bytes + 1, 'c'); - part.string_v.view = huge_part; - parts[0] = part; - parts[1] = part; - args.pos_count = 1; - args.pos[0] = arr; - CHECK_FALSE(formatter::detail::builtin_join(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - args.pos_count = 3; - args.pos[0].type = value_type::string; - args.pos[0].string_v.view = "abc"; - args.pos[1].type = value_type::string; - args.pos[1].string_v.view = ""; - args.pos[2].type = value_type::string; - args.pos[2].string_v.view = "z"; - CHECK(formatter::detail::builtin_replace(ctx, args, out)); - CHECK(out.type == value_type::string); - CHECK(out.string_v.view == "abc"); - } -} - -TEST_CASE("jinja_renderer_detail_helpers_part6") { - using namespace emel::text::jinja; - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - - std::string huge_input(formatter::action::k_max_string_bytes + 1, 'a'); - args.pos_count = 3; - args.pos[0].type = value_type::string; - args.pos[0].string_v.view = huge_input; - args.pos[1].type = value_type::string; - args.pos[1].string_v.view = "x"; - args.pos[2].type = value_type::string; - args.pos[2].string_v.view = "b"; - CHECK_FALSE(formatter::detail::builtin_replace(ctx, args, out)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - value invalid{}; - invalid.type = static_cast(99); - CHECK_FALSE(formatter::detail::value_is_truthy(invalid)); - } - - { - formatter::action::context ctx{}; - object_entry entries[1] = {}; - value v_key{}; - v_key.type = value_type::string; - v_key.string_v.view = "k"; - value v_val{}; - v_val.type = value_type::integer; - v_val.int_v = 1; - object_value obj{entries, 0, 0, false}; - CHECK_FALSE(formatter::detail::set_object_value(ctx, obj, "k", v_val)); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - formatter::detail::render_io io{}; - char buffer[1] = {}; - formatter::detail::init_writer(io, buffer, sizeof(buffer)); - - value v_str{}; - v_str.type = value_type::string; - v_str.string_v.view = "long"; - value arr{}; - arr.type = value_type::array; - arr.array_v.items = &v_str; - arr.array_v.count = 1; - arr.array_v.capacity = 1; - CHECK_FALSE(formatter::detail::write_value(ctx, io, arr)); - } - - { - formatter::detail::call_args args{}; - args.kw_count = 1; - args.kw[0].key = "a"; - args.kw[0].val.type = value_type::integer; - args.kw[0].val.int_v = 1; - CHECK(formatter::detail::find_kw(args, "b") == nullptr); - } - - { - formatter::action::context ctx{}; - formatter::detail::call_args args{}; - value out{}; - args.pos_count = 1; - args.pos[0].type = value_type::boolean; - args.pos[0].bool_v = true; - CHECK(formatter::detail::builtin_length(ctx, args, out)); - - ctx = formatter::action::context{}; - args.pos_count = 3; - args.pos[0].type = value_type::integer; - args.pos[1].type = value_type::integer; - args.pos[2].type = value_type::string; - CHECK_FALSE(formatter::detail::builtin_range(ctx, args, out)); - - ctx = formatter::action::context{}; - args.kw_count = 0; - CHECK(formatter::detail::builtin_namespace(ctx, args, out)); - } -} - -TEST_CASE("jinja_renderer_detail_value_equal_cases") { - using namespace emel::text::jinja; - - value undef1{}; - value undef2{}; - undef1.type = value_type::undefined; - undef2.type = value_type::undefined; - CHECK(formatter::detail::value_equal(undef1, undef2)); - - value b1{}; - value b2{}; - b1.type = value_type::boolean; - b1.bool_v = true; - b2.type = value_type::boolean; - b2.bool_v = false; - CHECK_FALSE(formatter::detail::value_equal(b1, b2)); - - value f1{}; - value f2{}; - f1.type = value_type::floating; - f1.float_v = 1.5; - f2.type = value_type::floating; - f2.float_v = 1.5; - CHECK(formatter::detail::value_equal(f1, f2)); - - value i1{}; - value s1{}; - i1.type = value_type::integer; - i1.int_v = 1; - s1.type = value_type::string; - s1.string_v.view = "a"; - CHECK_FALSE(formatter::detail::value_equal(i1, s1)); - - value arr_items_a[2] = {}; - value arr_items_b[2] = {}; - arr_items_a[0] = i1; - arr_items_a[1] = i1; - arr_items_b[0] = i1; - arr_items_b[1] = s1; - value arr_a{}; - value arr_b{}; - arr_a.type = value_type::array; - arr_a.array_v.items = arr_items_a; - arr_a.array_v.count = 2; - arr_a.array_v.capacity = 2; - arr_b.type = value_type::array; - arr_b.array_v.items = arr_items_b; - arr_b.array_v.count = 2; - arr_b.array_v.capacity = 2; - CHECK_FALSE(formatter::detail::value_equal(arr_a, arr_b)); - - object_entry obj_entries_a[1] = {}; - object_entry obj_entries_b[1] = {}; - obj_entries_a[0].key = s1; - obj_entries_a[0].val = i1; - obj_entries_b[0].key = s1; - obj_entries_b[0].val = b1; - value obj_a{}; - value obj_b{}; - obj_a.type = value_type::object; - obj_a.object_v.entries = obj_entries_a; - obj_a.object_v.count = 1; - obj_a.object_v.capacity = 1; - obj_b.type = value_type::object; - obj_b.object_v.entries = obj_entries_b; - obj_b.object_v.count = 1; - obj_b.object_v.capacity = 1; - CHECK_FALSE(formatter::detail::value_equal(obj_a, obj_b)); - - object_entry obj_entries_c[1] = {}; - object_entry obj_entries_d[1] = {}; - obj_entries_c[0].key = s1; - obj_entries_c[0].val = i1; - value s2{}; - s2.type = value_type::string; - s2.string_v.view = "b"; - obj_entries_d[0].key = s2; - obj_entries_d[0].val = i1; - value obj_c{}; - value obj_d{}; - obj_c.type = value_type::object; - obj_c.object_v.entries = obj_entries_c; - obj_c.object_v.count = 1; - obj_c.object_v.capacity = 1; - obj_d.type = value_type::object; - obj_d.object_v.entries = obj_entries_d; - obj_d.object_v.count = 1; - obj_d.object_v.capacity = 1; - CHECK_FALSE(formatter::detail::value_equal(obj_c, obj_d)); - - value obj_e = obj_a; - value obj_f = obj_a; - CHECK(formatter::detail::value_equal(obj_e, obj_f)); - - value func_a{}; - value func_b{}; - func_a.type = value_type::function; - func_a.func_v.data = &func_a; - func_a.func_v.kind = function_kind::builtin; - func_b.type = value_type::function; - func_b.func_v.data = &func_a; - func_b.func_v.kind = function_kind::builtin; - CHECK(formatter::detail::value_equal(func_a, func_b)); - - value invalid_a{}; - value invalid_b{}; - invalid_a.type = static_cast(99); - invalid_b.type = static_cast(99); - CHECK_FALSE(formatter::detail::value_equal(invalid_a, invalid_b)); -} - -TEST_CASE("jinja_renderer_actions_branch_coverage") { - using namespace emel::text::jinja; - using emel::text::jinja::formatter::action::eval_next_stmt; - using emel::text::jinja::formatter::action::eval_pending_expr; - using emel::text::jinja::formatter::action::finalize_done; - using emel::text::jinja::formatter::action::finalize_error; - using emel::text::jinja::formatter::action::seed_program; - using emel::text::jinja::formatter::action::write_pending_value; - - { - formatter::action::context ctx{}; - ctx.phase_error = EMEL_ERR_BACKEND; - seed_program(ctx); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - } - - { - formatter::action::context ctx{}; - ctx.scope_count = formatter::action::k_max_scopes; - seed_program(ctx); - CHECK(ctx.phase_error != EMEL_OK); - } - - { - formatter::action::context ctx{}; - ctx.phase_error = EMEL_ERR_BACKEND; - eval_next_stmt(ctx); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - } - - { - formatter::action::context ctx{}; - eval_next_stmt(ctx); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - ast_list stmts; - ctx.statements = &stmts; - ctx.statement_index = 0; - eval_next_stmt(ctx); - CHECK(ctx.phase_error == EMEL_OK); - } - - { - formatter::action::context ctx{}; - ast_list stmts; - stmts.emplace_back(std::make_unique()); - ctx.statements = &stmts; - ctx.steps_remaining = 0; - eval_next_stmt(ctx); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - } - - { - formatter::action::context ctx{}; - ast_list stmts; - stmts.emplace_back(nullptr); - ctx.statements = &stmts; - eval_next_stmt(ctx); - CHECK(ctx.phase_error == EMEL_OK); - } - - { - formatter::action::context ctx{}; - ast_list stmts; - stmts.emplace_back(std::make_unique()); - ctx.statements = &stmts; - eval_next_stmt(ctx); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - ast_list stmts; - ast_list args; - ast_list body; - auto name = std::make_unique("macro"); - stmts.emplace_back(std::make_unique(std::move(name), std::move(args), std::move(body))); - ctx.statements = &stmts; - ctx.callable_count = formatter::action::k_max_callables; - eval_next_stmt(ctx); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - ast_list stmts; - ast_list args; - ast_list body; - auto name = std::make_unique("macro"); - stmts.emplace_back(std::make_unique(std::move(name), std::move(args), std::move(body))); - ctx.statements = &stmts; - eval_next_stmt(ctx); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - } - - { - formatter::action::context ctx{}; - eval_pending_expr(ctx); - CHECK_FALSE(ctx.pending_value_ready); - } - - { - formatter::action::context ctx{}; - write_pending_value(ctx); - CHECK_FALSE(ctx.pending_value_ready); - } - - { - formatter::action::context ctx{}; - finalize_done(ctx); - CHECK(ctx.last_error == EMEL_OK); - ctx.phase_error = EMEL_OK; - finalize_error(ctx); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); - CHECK(ctx.last_error == EMEL_ERR_BACKEND); - } - - { - auto done_cb = emel::callback::from< - +[](const emel::text::jinja::events::rendering_done &) -> bool { return true; }>(); - auto error_cb = emel::callback::from< - +[](const emel::text::jinja::events::rendering_error &) -> bool { return true; }>(); - - std::array buffer = {}; - size_t out_len = 0; - bool truncated = false; - int32_t err = EMEL_OK; - size_t err_pos = 0; - program render_program{}; - parse_template("hi", render_program); - event::render ev{ - .program = &render_program, - .globals = nullptr, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .output_truncated = &truncated, - .error_out = &err, - .error_pos_out = &err_pos, - .dispatch_done = done_cb, - .dispatch_error = error_cb, - }; - - formatter::action::context ctx{}; - formatter::sm machine{ctx}; - CHECK(machine.process_event(ev)); - CHECK(out_len == 2); - CHECK(err == EMEL_OK); - CHECK_FALSE(truncated); - - ev.program = nullptr; - CHECK_FALSE(machine.process_event(ev)); - CHECK(out_len == 0); - CHECK(err == EMEL_ERR_INVALID_ARGUMENT); - CHECK(err_pos == 0); - } + CHECK(machine.is(boost::sml::state)); } diff --git a/tools/bench/text/jinja/formatter_bench.cpp b/tools/bench/text/jinja/formatter_bench.cpp index 50ce0504..14ebbb90 100644 --- a/tools/bench/text/jinja/formatter_bench.cpp +++ b/tools/bench/text/jinja/formatter_bench.cpp @@ -81,6 +81,14 @@ emel::text::jinja::value make_array(emel::text::jinja::value * items, size_t cou return out; } +bool formatter_done_sink(const emel::text::jinja::events::rendering_done &) { + return true; +} + +bool formatter_error_sink(const emel::text::jinja::events::rendering_error &) { + return true; +} + } // namespace namespace emel::bench { @@ -106,38 +114,66 @@ void append_emel_jinja_formatter_cases(std::vector & results, const conf emel::text::jinja::formatter::action::context ctx{}; emel::text::jinja::formatter::sm machine{ctx}; + const emel::text::jinja::event::render::done_callback done_cb = + emel::text::jinja::event::render::done_callback::from<&formatter_done_sink>(); + const emel::text::jinja::event::render::error_callback error_cb = + emel::text::jinja::event::render::error_callback::from<&formatter_error_sink>(); + static volatile uint64_t sink = 0; auto short_fn = [&]() { std::array buffer = {}; size_t out_len = 0; - int32_t err = EMEL_OK; + int32_t err = static_cast(emel::text::jinja::formatter::error::none); emel::text::jinja::event::render ev{ - .program = &short_program, - .globals = &globals, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .error_out = &err, + short_program, + short_template, + buffer[0], + buffer.size(), + done_cb, + error_cb, + &globals, + &out_len, + nullptr, + &err, }; - (void)machine.process_event(ev); + const bool ok = machine.process_event(ev); + if (!ok || err != static_cast(emel::text::jinja::formatter::error::none)) { + std::abort(); + } + sink += out_len; + if (out_len > 0) { + sink += static_cast(buffer[out_len - 1]); + } }; results.push_back(measure_case("text/jinja/formatter_short", cfg, short_fn)); auto long_fn = [&]() { - std::array buffer = {}; + std::array buffer = {}; size_t out_len = 0; - int32_t err = EMEL_OK; + int32_t err = static_cast(emel::text::jinja::formatter::error::none); emel::text::jinja::event::render ev{ - .program = &long_program, - .globals = &globals, - .output = buffer.data(), - .output_capacity = buffer.size(), - .output_length = &out_len, - .error_out = &err, + long_program, + long_template, + buffer[0], + buffer.size(), + done_cb, + error_cb, + &globals, + &out_len, + nullptr, + &err, }; - (void)machine.process_event(ev); + const bool ok = machine.process_event(ev); + if (!ok || err != static_cast(emel::text::jinja::formatter::error::none)) { + std::abort(); + } + sink += out_len; + if (out_len > 0) { + sink += static_cast(buffer[out_len - 1]); + } }; results.push_back(measure_case("text/jinja/formatter_long", cfg, long_fn)); + (void)sink; } void append_reference_jinja_formatter_cases(std::vector & results, const config & cfg) { @@ -146,6 +182,7 @@ void append_reference_jinja_formatter_cases(std::vector & results, const const ::jinja::program short_program = parse_reference(short_template); const ::jinja::program long_program = parse_reference(long_template); + static volatile uint64_t sink = 0; auto short_fn = [&]() { ::jinja::context ctx; @@ -153,7 +190,11 @@ void append_reference_jinja_formatter_cases(std::vector & results, const ::jinja::runtime runtime{ctx}; auto result = runtime.execute(short_program); auto parts = ::jinja::runtime::gather_string_parts(result); - (void)::jinja::render_string_parts(parts); + const std::string rendered = ::jinja::render_string_parts(parts); + sink += rendered.size(); + if (!rendered.empty()) { + sink += static_cast(rendered.back()); + } }; results.push_back(measure_case("text/jinja/formatter_short", cfg, short_fn)); @@ -169,9 +210,14 @@ void append_reference_jinja_formatter_cases(std::vector & results, const ::jinja::runtime runtime{ctx}; auto result = runtime.execute(long_program); auto parts = ::jinja::runtime::gather_string_parts(result); - (void)::jinja::render_string_parts(parts); + const std::string rendered = ::jinja::render_string_parts(parts); + sink += rendered.size(); + if (!rendered.empty()) { + sink += static_cast(rendered.back()); + } }; results.push_back(measure_case("text/jinja/formatter_long", cfg, long_fn)); + (void)sink; } } // namespace emel::bench diff --git a/tools/docsgen/docsgen.cpp b/tools/docsgen/docsgen.cpp index 9668af89..5b0dbccb 100644 --- a/tools/docsgen/docsgen.cpp +++ b/tools/docsgen/docsgen.cpp @@ -287,6 +287,14 @@ struct template_var { std::string value; }; +bool formatter_render_done_sink(const emel::text::jinja::events::rendering_done &) { + return true; +} + +bool formatter_render_error_sink(const emel::text::jinja::events::rendering_error &) { + return true; +} + std::optional render_template(const fs::path & template_path, const std::vector & vars) { const std::string template_text = read_file(template_path); @@ -348,22 +356,29 @@ std::optional render_template(const fs::path & template_path, size_t out_len = 0; size_t error_pos = 0; - int32_t render_err = EMEL_OK; + int32_t render_err = static_cast(emel::text::jinja::formatter::error::none); emel::text::jinja::formatter::action::context render_ctx; emel::text::jinja::formatter::sm renderer{render_ctx}; + const emel::text::jinja::event::render::done_callback render_done_cb = + emel::text::jinja::event::render::done_callback::from<&formatter_render_done_sink>(); + const emel::text::jinja::event::render::error_callback render_error_cb = + emel::text::jinja::event::render::error_callback::from<&formatter_render_error_sink>(); emel::text::jinja::event::render render_ev{ - .program = &program, - .globals = entries.empty() ? nullptr : &globals, - .source_text = template_text, - .output = output_buffer.data(), - .output_capacity = output_buffer.size(), - .output_length = &out_len, - .error_out = &render_err, - .error_pos_out = &error_pos, + program, + template_text, + output_buffer[0], + output_buffer.size(), + render_done_cb, + render_error_cb, + entries.empty() ? nullptr : &globals, + &out_len, + nullptr, + &render_err, + &error_pos, }; renderer.process_event(render_ev); - if (render_err != EMEL_OK || + if (render_err != static_cast(emel::text::jinja::formatter::error::none) || !renderer.is(boost::sml::state)) { std::fprintf(stderr, "error: jinja render failed\n"); return std::nullopt; From 6879e529fa4648b055911fce2a959728f11d4895 Mon Sep 17 00:00:00 2001 From: gabewillen Date: Mon, 2 Mar 2026 15:20:49 -0600 Subject: [PATCH 2/3] text/jinja: hard-cut parser to modeled submachines - replace legacy jinja lexer entrypoint with parser-domain lexer machine and runtime scan plan - rewrite parser orchestration into explicit classifier/program/expression/statement submachines - move jinja AST/token/value surface into parser detail and remove legacy ast/value/types headers - migrate tests, docsgen, benchmarks, and architecture docs/snapshots to the new machine layout --- README.md | 35 +- .../mermaid/text_jinja_parser.mmd | 35 +- .../text_jinja_parser_classifier_parser.mmd | 22 + .../mermaid/text_jinja_parser_lexer.mmd | 15 + .../text_jinja_parser_program_parser.mmd | 31 + ...arser_program_parser_expression_parser.mmd | 27 + ...parser_program_parser_statement_parser.mmd | 35 + docs/architecture/text_jinja_parser.md | 78 +- .../text_jinja_parser_classifier_parser.md | 54 + docs/architecture/text_jinja_parser_lexer.md | 40 + .../text_jinja_parser_program_parser.md | 72 ++ ...parser_program_parser_expression_parser.md | 64 + ..._parser_program_parser_statement_parser.md | 80 ++ docs/benchmarks.md | 95 +- snapshots/bench/benchmarks.txt | 4 +- snapshots/bench/benchmarks_compare.txt | 4 +- snapshots/quality_gates/timing.txt | 13 +- src/emel/model/architecture/types.hpp | 24 - src/emel/text/jinja/ast.hpp | 241 ---- src/emel/text/jinja/formatter/events.hpp | 3 +- src/emel/text/jinja/lexer.hpp | 475 -------- src/emel/text/jinja/lexer/detail.hpp | 518 +++++++++ src/emel/text/jinja/parser/actions.hpp | 328 ++++-- .../parser/classifier_parser/actions.hpp | 106 ++ .../parser/classifier_parser/context.hpp | 11 + .../jinja/parser/classifier_parser/errors.hpp | 9 + .../jinja/parser/classifier_parser/guards.hpp | 131 +++ .../jinja/parser/classifier_parser/sm.hpp | 103 ++ src/emel/text/jinja/parser/context.hpp | 7 +- src/emel/text/jinja/parser/detail.hpp | 1033 ++++++----------- src/emel/text/jinja/parser/errors.hpp | 25 + src/emel/text/jinja/parser/events.hpp | 101 +- src/emel/text/jinja/parser/guards.hpp | 74 +- src/emel/text/jinja/parser/lexer/actions.hpp | 80 ++ src/emel/text/jinja/parser/lexer/context.hpp | 7 + src/emel/text/jinja/parser/lexer/detail.hpp | 20 + src/emel/text/jinja/parser/lexer/errors.hpp | 9 + src/emel/text/jinja/parser/lexer/guards.hpp | 46 + src/emel/text/jinja/parser/lexer/sm.hpp | 89 ++ .../jinja/parser/program_parser/actions.hpp | 89 ++ .../jinja/parser/program_parser/context.hpp | 11 + .../jinja/parser/program_parser/errors.hpp | 9 + .../expression_parser/actions.hpp | 143 +++ .../expression_parser/context.hpp | 12 + .../expression_parser/errors.hpp | 9 + .../expression_parser/guards.hpp | 105 ++ .../program_parser/expression_parser/sm.hpp | 112 ++ .../jinja/parser/program_parser/guards.hpp | 84 ++ .../text/jinja/parser/program_parser/sm.hpp | 127 ++ .../statement_parser/actions.hpp | 103 ++ .../statement_parser/context.hpp | 12 + .../statement_parser/errors.hpp | 9 + .../statement_parser/guards.hpp | 213 ++++ .../program_parser/statement_parser/sm.hpp | 146 +++ src/emel/text/jinja/parser/sm.hpp | 438 +++---- src/emel/text/jinja/types.hpp | 2 +- src/emel/text/jinja/value.hpp | 70 -- tests/text/jinja/lexer_tests.cpp | 247 ++-- tests/text/jinja/parser_tests.cpp | 300 ++--- tools/bench/text/jinja/formatter_bench.cpp | 36 +- tools/bench/text/jinja/parser_bench.cpp | 89 +- tools/docsgen/docsgen.cpp | 24 +- 62 files changed, 4166 insertions(+), 2368 deletions(-) create mode 100644 docs/architecture/mermaid/text_jinja_parser_classifier_parser.mmd create mode 100644 docs/architecture/mermaid/text_jinja_parser_lexer.mmd create mode 100644 docs/architecture/mermaid/text_jinja_parser_program_parser.mmd create mode 100644 docs/architecture/mermaid/text_jinja_parser_program_parser_expression_parser.mmd create mode 100644 docs/architecture/mermaid/text_jinja_parser_program_parser_statement_parser.mmd create mode 100644 docs/architecture/text_jinja_parser_classifier_parser.md create mode 100644 docs/architecture/text_jinja_parser_lexer.md create mode 100644 docs/architecture/text_jinja_parser_program_parser.md create mode 100644 docs/architecture/text_jinja_parser_program_parser_expression_parser.md create mode 100644 docs/architecture/text_jinja_parser_program_parser_statement_parser.md delete mode 100644 src/emel/model/architecture/types.hpp delete mode 100644 src/emel/text/jinja/ast.hpp delete mode 100644 src/emel/text/jinja/lexer.hpp create mode 100644 src/emel/text/jinja/lexer/detail.hpp create mode 100644 src/emel/text/jinja/parser/classifier_parser/actions.hpp create mode 100644 src/emel/text/jinja/parser/classifier_parser/context.hpp create mode 100644 src/emel/text/jinja/parser/classifier_parser/errors.hpp create mode 100644 src/emel/text/jinja/parser/classifier_parser/guards.hpp create mode 100644 src/emel/text/jinja/parser/classifier_parser/sm.hpp create mode 100644 src/emel/text/jinja/parser/errors.hpp create mode 100644 src/emel/text/jinja/parser/lexer/actions.hpp create mode 100644 src/emel/text/jinja/parser/lexer/context.hpp create mode 100644 src/emel/text/jinja/parser/lexer/detail.hpp create mode 100644 src/emel/text/jinja/parser/lexer/errors.hpp create mode 100644 src/emel/text/jinja/parser/lexer/guards.hpp create mode 100644 src/emel/text/jinja/parser/lexer/sm.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/actions.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/context.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/errors.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/expression_parser/actions.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/expression_parser/context.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/expression_parser/errors.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/expression_parser/guards.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/guards.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/sm.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/statement_parser/actions.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/statement_parser/context.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/statement_parser/errors.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/statement_parser/guards.hpp create mode 100644 src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp delete mode 100644 src/emel/text/jinja/value.hpp diff --git a/README.md b/README.md index bcddebca..d0a3afe5 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,11 @@ allocator, and execution pipelines stabilize. This inference engine is being implemented by AI under human engineering and architecture direction. +> [!WARNING] +> EMEL is currently going through a major re-architecture expected to complete by end of day on +> Friday, February 27, 2026. The only domain left to rearchitect is the text domain. +> The source of truth for architecture and design lives in `src/emel/**/sm.hpp` docstrings and +> the generated docs under `docs/architecture/`. ## Implementation priorities @@ -93,35 +98,7 @@ environments, while Zig remains the default for day-to-day builds. ## Docs index -- [`docs/benchmarks.md`](docs/benchmarks.md) -- [`docs/architecture/batch_planner_modes_equal.md`](docs/architecture/batch_planner_modes_equal.md) -- [`docs/architecture/batch_planner_modes_sequential.md`](docs/architecture/batch_planner_modes_sequential.md) -- [`docs/architecture/batch_planner_modes_simple.md`](docs/architecture/batch_planner_modes_simple.md) -- [`docs/architecture/gbnf_rule_parser_definition_parser.md`](docs/architecture/gbnf_rule_parser_definition_parser.md) -- [`docs/architecture/gbnf_rule_parser_expression_parser.md`](docs/architecture/gbnf_rule_parser_expression_parser.md) -- [`docs/architecture/gbnf_rule_parser_nonterm_parser.md`](docs/architecture/gbnf_rule_parser_nonterm_parser.md) -- [`docs/architecture/gbnf_rule_parser_term_parser.md`](docs/architecture/gbnf_rule_parser_term_parser.md) -- [`docs/architecture/gbnf_sampler_accept_parser.md`](docs/architecture/gbnf_sampler_accept_parser.md) -- [`docs/architecture/gbnf_sampler_candidate_parser.md`](docs/architecture/gbnf_sampler_candidate_parser.md) -- [`docs/architecture/gbnf_sampler_matcher_parser.md`](docs/architecture/gbnf_sampler_matcher_parser.md) -- [`docs/architecture/gbnf_sampler_token_parser.md`](docs/architecture/gbnf_sampler_token_parser.md) -- [`docs/architecture/graph_allocator_liveness_pass.md`](docs/architecture/graph_allocator_liveness_pass.md) -- [`docs/architecture/graph_allocator_ordering_pass.md`](docs/architecture/graph_allocator_ordering_pass.md) -- [`docs/architecture/graph_allocator_placement_pass.md`](docs/architecture/graph_allocator_placement_pass.md) -- [`docs/architecture/graph_assembler_assemble_alloc_pass.md`](docs/architecture/graph_assembler_assemble_alloc_pass.md) -- [`docs/architecture/graph_assembler_assemble_build_pass.md`](docs/architecture/graph_assembler_assemble_build_pass.md) -- [`docs/architecture/graph_assembler_assemble_validate_pass.md`](docs/architecture/graph_assembler_assemble_validate_pass.md) -- [`docs/architecture/graph_assembler_reserve_alloc_pass.md`](docs/architecture/graph_assembler_reserve_alloc_pass.md) -- [`docs/architecture/graph_assembler_reserve_build_pass.md`](docs/architecture/graph_assembler_reserve_build_pass.md) -- [`docs/architecture/graph_assembler_reserve_validate_pass.md`](docs/architecture/graph_assembler_reserve_validate_pass.md) -- [`docs/architecture/graph_assembler_reuse_decision_pass.md`](docs/architecture/graph_assembler_reuse_decision_pass.md) -- [`docs/architecture/graph_processor_alloc_step.md`](docs/architecture/graph_processor_alloc_step.md) -- [`docs/architecture/graph_processor_bind_step.md`](docs/architecture/graph_processor_bind_step.md) -- [`docs/architecture/graph_processor_extract_step.md`](docs/architecture/graph_processor_extract_step.md) -- [`docs/architecture/graph_processor_kernel_step.md`](docs/architecture/graph_processor_kernel_step.md) -- [`docs/architecture/graph_processor_prepare_step.md`](docs/architecture/graph_processor_prepare_step.md) -- [`docs/architecture/graph_processor_validate_step.md`](docs/architecture/graph_processor_validate_step.md) - +{{ docs_toc }} ## Regenerating docs diff --git a/docs/architecture/mermaid/text_jinja_parser.mmd b/docs/architecture/mermaid/text_jinja_parser.mmd index a2853b74..681fae53 100644 --- a/docs/architecture/mermaid/text_jinja_parser.mmd +++ b/docs/architecture/mermaid/text_jinja_parser.mmd @@ -1,18 +1,31 @@ stateDiagram-v2 direction TB [*] --> initialized - initialized --> parse_decision : parse [valid_parse_] / run_parse_ - initialized --> errored : parse [invalid_parse_] / reject_invalid_parse_ - done --> parse_decision : parse [valid_parse_] / run_parse_ - done --> errored : parse [invalid_parse_] / reject_invalid_parse_ - errored --> parse_decision : parse [valid_parse_] / run_parse_ - errored --> errored : parse [invalid_parse_] / reject_invalid_parse_ - unexpected --> parse_decision : parse [valid_parse_] / run_parse_ - unexpected --> unexpected : parse [invalid_parse_] / reject_invalid_parse_ - parse_decision --> done : [phase_ok_] / none - parse_decision --> errored : [phase_failed_] / none + initialized --> request_decision : parse_runtime [valid_parse_] / begin_parse_ + initialized --> parse_result_decision : parse_runtime [invalid_parse_with_callbacks_] / reject_invalid_parse_ + initialized --> errored : parse_runtime [invalid_parse_without_callbacks_] / reject_invalid_parse_ + done --> request_decision : parse_runtime [valid_parse_] / begin_parse_ + done --> parse_result_decision : parse_runtime [invalid_parse_with_callbacks_] / reject_invalid_parse_ + done --> errored : parse_runtime [invalid_parse_without_callbacks_] / reject_invalid_parse_ + errored --> request_decision : parse_runtime [valid_parse_] / begin_parse_ + errored --> parse_result_decision : parse_runtime [invalid_parse_with_callbacks_] / reject_invalid_parse_ + errored --> errored : parse_runtime [invalid_parse_without_callbacks_] / reject_invalid_parse_ + unexpected --> request_decision : parse_runtime [valid_parse_] / begin_parse_ + unexpected --> parse_result_decision : parse_runtime [invalid_parse_with_callbacks_] / reject_invalid_parse_ + unexpected --> errored : parse_runtime [invalid_parse_without_callbacks_] / reject_invalid_parse_ + request_decision --> tokenize_exec : [always] / tokenize_template_ + tokenize_exec --> tokenize_result_decision : [always] / none + tokenize_result_decision --> parse_exec : [tokenize_ok_] / classify_tokens_ + tokenize_result_decision --> parse_result_decision : [tokenize_failed_] / none + parse_exec --> parse_result_decision : [always] / parse_tokens_ + parse_result_decision --> done : [request_ok_] / dispatch_done_ + parse_result_decision --> errored : [request_failed_] / dispatch_error_ initialized --> unexpected : _ [always] / on_unexpected_ - parse_decision --> unexpected : _ [always] / on_unexpected_ + request_decision --> unexpected : _ [always] / on_unexpected_ + tokenize_exec --> unexpected : _ [always] / on_unexpected_ + tokenize_result_decision --> unexpected : _ [always] / on_unexpected_ + parse_exec --> unexpected : _ [always] / on_unexpected_ + parse_result_decision --> unexpected : _ [always] / on_unexpected_ done --> unexpected : _ [always] / on_unexpected_ errored --> unexpected : _ [always] / on_unexpected_ unexpected --> unexpected : _ [always] / on_unexpected_ diff --git a/docs/architecture/mermaid/text_jinja_parser_classifier_parser.mmd b/docs/architecture/mermaid/text_jinja_parser_classifier_parser.mmd new file mode 100644 index 00000000..45277324 --- /dev/null +++ b/docs/architecture/mermaid/text_jinja_parser_classifier_parser.mmd @@ -0,0 +1,22 @@ +stateDiagram-v2 + direction TB + [*] --> deciding + deciding --> statement_decision : completion_parse_runtime_ [always] / begin_classification_ + statement_decision --> classified : completion_parse_runtime_ [no_tokens_] / set_statement_unknown_ + statement_decision --> classified : completion_parse_runtime_ [token_text_] / set_statement_text_ + statement_decision --> classified : completion_parse_runtime_ [token_comment_] / set_statement_comment_ + statement_decision --> expression_decision : completion_parse_runtime_ [token_open_expression_] / set_statement_expression_ + statement_decision --> classified : completion_parse_runtime_ [token_open_statement_] / set_statement_statement_ + statement_decision --> classified : completion_parse_runtime_ [token_unknown_] / set_statement_unknown_ + expression_decision --> classified : completion_parse_runtime_ [expr_no_token_] / set_expression_unknown_ + expression_decision --> classified : completion_parse_runtime_ [expr_token_literal_] / set_expression_literal_ + expression_decision --> classified : completion_parse_runtime_ [expr_token_identifier_] / set_expression_identifier_ + expression_decision --> classified : completion_parse_runtime_ [expr_token_unary_] / set_expression_unary_ + expression_decision --> classified : completion_parse_runtime_ [expr_token_compound_] / set_expression_compound_ + expression_decision --> classified : completion_parse_runtime_ [expr_token_unknown_] / set_expression_unknown_ + classified --> terminate : [always] / none + deciding --> unexpected_event : _ [always] / on_unexpected_ + statement_decision --> unexpected_event : _ [always] / on_unexpected_ + expression_decision --> unexpected_event : _ [always] / on_unexpected_ + classified --> unexpected_event : _ [always] / on_unexpected_ + unexpected_event --> unexpected_event : _ [always] / on_unexpected_ diff --git a/docs/architecture/mermaid/text_jinja_parser_lexer.mmd b/docs/architecture/mermaid/text_jinja_parser_lexer.mmd new file mode 100644 index 00000000..68793cde --- /dev/null +++ b/docs/architecture/mermaid/text_jinja_parser_lexer.mmd @@ -0,0 +1,15 @@ +stateDiagram-v2 + direction TB + [*] --> deciding + deciding --> tokenize_exec : completion_parse_runtime_ [always] / run_tokenize_ + tokenize_exec --> tokenize_result_decision : completion_parse_runtime_ [always] / none + tokenize_result_decision --> tokenized : completion_parse_runtime_ [tokenize_succeeded_] / none + tokenize_result_decision --> parse_failed : completion_parse_runtime_ [tokenize_failed_] / mark_parse_failed_ + tokenized --> terminate : [always] / none + parse_failed --> terminate : [always] / none + deciding --> unexpected_event : _ [always] / on_unexpected_ + tokenize_exec --> unexpected_event : _ [always] / on_unexpected_ + tokenize_result_decision --> unexpected_event : _ [always] / on_unexpected_ + tokenized --> unexpected_event : _ [always] / on_unexpected_ + parse_failed --> unexpected_event : _ [always] / on_unexpected_ + unexpected_event --> unexpected_event : _ [always] / on_unexpected_ diff --git a/docs/architecture/mermaid/text_jinja_parser_program_parser.mmd b/docs/architecture/mermaid/text_jinja_parser_program_parser.mmd new file mode 100644 index 00000000..3a9143cc --- /dev/null +++ b/docs/architecture/mermaid/text_jinja_parser_program_parser.mmd @@ -0,0 +1,31 @@ +stateDiagram-v2 + direction TB + [*] --> deciding + deciding --> parse_begin : completion_parse_runtime_ [always] / start_program_parse_ + parse_begin --> dispatch_decision : completion_parse_runtime_ [always] / none + dispatch_decision --> parsed : completion_parse_runtime_ [at_eof_] / finish_parsed_ + dispatch_decision --> text_emit : completion_parse_runtime_ [token_text_] / none + dispatch_decision --> comment_emit : completion_parse_runtime_ [token_comment_] / none + dispatch_decision --> model__ : completion_parse_runtime_ [token_open_statement_] / none + dispatch_decision --> model__ : completion_parse_runtime_ [token_open_expression_] / none + dispatch_decision --> parse_failed : completion_parse_runtime_ [token_unexpected_] / fail_current_token_ + text_emit --> dispatch_decision : completion_parse_runtime_ [always] / consume_text_ + comment_emit --> dispatch_decision : completion_parse_runtime_ [always] / consume_comment_ + model__ --> statement_parse_result_decision : completion_parse_runtime_ [always] / none + statement_parse_result_decision --> dispatch_decision : completion_parse_runtime_ [phase_ok_] / none + statement_parse_result_decision --> parse_failed : completion_parse_runtime_ [phase_failed_] / none + model__ --> expression_parse_result_decision : completion_parse_runtime_ [always] / none + expression_parse_result_decision --> dispatch_decision : completion_parse_runtime_ [phase_ok_] / none + expression_parse_result_decision --> parse_failed : completion_parse_runtime_ [phase_failed_] / none + parsed --> terminate : [always] / none + parse_failed --> terminate : [always] / none + deciding --> unexpected_event : _ [always] / on_unexpected_ + parse_begin --> unexpected_event : _ [always] / on_unexpected_ + dispatch_decision --> unexpected_event : _ [always] / on_unexpected_ + text_emit --> unexpected_event : _ [always] / on_unexpected_ + comment_emit --> unexpected_event : _ [always] / on_unexpected_ + statement_parse_result_decision --> unexpected_event : _ [always] / on_unexpected_ + expression_parse_result_decision --> unexpected_event : _ [always] / on_unexpected_ + parsed --> unexpected_event : _ [always] / on_unexpected_ + parse_failed --> unexpected_event : _ [always] / on_unexpected_ + unexpected_event --> unexpected_event : _ [always] / on_unexpected_ diff --git a/docs/architecture/mermaid/text_jinja_parser_program_parser_expression_parser.mmd b/docs/architecture/mermaid/text_jinja_parser_program_parser_expression_parser.mmd new file mode 100644 index 00000000..25356880 --- /dev/null +++ b/docs/architecture/mermaid/text_jinja_parser_program_parser_expression_parser.mmd @@ -0,0 +1,27 @@ +stateDiagram-v2 + direction TB + [*] --> deciding + deciding --> expression_first_decision : completion_parse_runtime_ [always] / begin_expression_parse_ + expression_first_decision --> parse_failed : completion_parse_runtime_ [expr_scan_eof_] / fail_expression_start_token_ + expression_first_decision --> parse_failed : completion_parse_runtime_ [expr_first_is_close_] / fail_expression_close_token_ + expression_first_decision --> expression_scan : completion_parse_runtime_ [expr_first_is_identifier_] / consume_expression_identifier_ + expression_first_decision --> expression_scan : completion_parse_runtime_ [expr_first_is_literal_] / consume_expression_literal_ + expression_first_decision --> expression_scan : completion_parse_runtime_ [expr_first_is_unary_] / consume_expression_unary_ + expression_first_decision --> expression_scan : completion_parse_runtime_ [expr_first_is_other_content_] / consume_expression_compound_ + expression_scan --> expression_emit_decision : completion_parse_runtime_ [expr_scan_at_close_] / none + expression_scan --> expression_scan : completion_parse_runtime_ [expr_scan_continue_] / consume_expression_token_ + expression_scan --> parse_failed : completion_parse_runtime_ [expr_scan_eof_] / fail_expression_start_token_ + expression_emit_decision --> expression_close : completion_parse_runtime_ [expression_identifier_] / emit_expression_identifier_ + expression_emit_decision --> expression_close : completion_parse_runtime_ [expression_non_identifier_] / emit_expression_generic_ + expression_close --> parsed : completion_parse_runtime_ [expr_scan_at_close_] / consume_expression_close_ + expression_close --> parse_failed : completion_parse_runtime_ [expr_scan_eof_] / fail_expression_start_token_ + parsed --> terminate : [always] / none + parse_failed --> terminate : [always] / none + deciding --> unexpected_event : _ [always] / on_unexpected_ + expression_first_decision --> unexpected_event : _ [always] / on_unexpected_ + expression_scan --> unexpected_event : _ [always] / on_unexpected_ + expression_emit_decision --> unexpected_event : _ [always] / on_unexpected_ + expression_close --> unexpected_event : _ [always] / on_unexpected_ + parsed --> unexpected_event : _ [always] / on_unexpected_ + parse_failed --> unexpected_event : _ [always] / on_unexpected_ + unexpected_event --> unexpected_event : _ [always] / on_unexpected_ diff --git a/docs/architecture/mermaid/text_jinja_parser_program_parser_statement_parser.mmd b/docs/architecture/mermaid/text_jinja_parser_program_parser_statement_parser.mmd new file mode 100644 index 00000000..2bdf0159 --- /dev/null +++ b/docs/architecture/mermaid/text_jinja_parser_program_parser_statement_parser.mmd @@ -0,0 +1,35 @@ +stateDiagram-v2 + direction TB + [*] --> deciding + deciding --> statement_kind_decision : completion_parse_runtime_ [always] / none + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_set_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_if_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_elif_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_else_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endif_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_for_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endfor_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_macro_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endmacro_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_call_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endcall_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_filter_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endfilter_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_break_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_continue_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_generation_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endgeneration_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endset_] / begin_statement_scan_ + statement_kind_decision --> parse_failed : completion_parse_runtime_ [statement_identifier_missing_] / fail_statement_open_token_ + statement_kind_decision --> parse_failed : completion_parse_runtime_ [statement_name_unknown_] / fail_statement_name_token_ + statement_scan --> parsed : completion_parse_runtime_ [statement_scan_at_close_] / consume_statement_close_and_emit_ + statement_scan --> statement_scan : completion_parse_runtime_ [statement_scan_continue_] / consume_statement_token_ + statement_scan --> parse_failed : completion_parse_runtime_ [statement_scan_eof_] / fail_statement_start_token_ + parsed --> terminate : [always] / none + parse_failed --> terminate : [always] / none + deciding --> unexpected_event : _ [always] / on_unexpected_ + statement_kind_decision --> unexpected_event : _ [always] / on_unexpected_ + statement_scan --> unexpected_event : _ [always] / on_unexpected_ + parsed --> unexpected_event : _ [always] / on_unexpected_ + parse_failed --> unexpected_event : _ [always] / on_unexpected_ + unexpected_event --> unexpected_event : _ [always] / on_unexpected_ diff --git a/docs/architecture/text_jinja_parser.md b/docs/architecture/text_jinja_parser.md index 4dd1dfb5..2eb625cc 100644 --- a/docs/architecture/text_jinja_parser.md +++ b/docs/architecture/text_jinja_parser.md @@ -8,18 +8,31 @@ Source: [`emel/text/jinja/parser/sm.hpp`](https://github.com/stateforward/emel.c stateDiagram-v2 direction TB [*] --> initialized - initialized --> parse_decision : parse [valid_parse_] / run_parse_ - initialized --> errored : parse [invalid_parse_] / reject_invalid_parse_ - done --> parse_decision : parse [valid_parse_] / run_parse_ - done --> errored : parse [invalid_parse_] / reject_invalid_parse_ - errored --> parse_decision : parse [valid_parse_] / run_parse_ - errored --> errored : parse [invalid_parse_] / reject_invalid_parse_ - unexpected --> parse_decision : parse [valid_parse_] / run_parse_ - unexpected --> unexpected : parse [invalid_parse_] / reject_invalid_parse_ - parse_decision --> done : [phase_ok_] / none - parse_decision --> errored : [phase_failed_] / none + initialized --> request_decision : parse_runtime [valid_parse_] / begin_parse_ + initialized --> parse_result_decision : parse_runtime [invalid_parse_with_callbacks_] / reject_invalid_parse_ + initialized --> errored : parse_runtime [invalid_parse_without_callbacks_] / reject_invalid_parse_ + done --> request_decision : parse_runtime [valid_parse_] / begin_parse_ + done --> parse_result_decision : parse_runtime [invalid_parse_with_callbacks_] / reject_invalid_parse_ + done --> errored : parse_runtime [invalid_parse_without_callbacks_] / reject_invalid_parse_ + errored --> request_decision : parse_runtime [valid_parse_] / begin_parse_ + errored --> parse_result_decision : parse_runtime [invalid_parse_with_callbacks_] / reject_invalid_parse_ + errored --> errored : parse_runtime [invalid_parse_without_callbacks_] / reject_invalid_parse_ + unexpected --> request_decision : parse_runtime [valid_parse_] / begin_parse_ + unexpected --> parse_result_decision : parse_runtime [invalid_parse_with_callbacks_] / reject_invalid_parse_ + unexpected --> errored : parse_runtime [invalid_parse_without_callbacks_] / reject_invalid_parse_ + request_decision --> tokenize_exec : [always] / tokenize_template_ + tokenize_exec --> tokenize_result_decision : [always] / none + tokenize_result_decision --> parse_exec : [tokenize_ok_] / classify_tokens_ + tokenize_result_decision --> parse_result_decision : [tokenize_failed_] / none + parse_exec --> parse_result_decision : [always] / parse_tokens_ + parse_result_decision --> done : [request_ok_] / dispatch_done_ + parse_result_decision --> errored : [request_failed_] / dispatch_error_ initialized --> unexpected : _ [always] / on_unexpected_ - parse_decision --> unexpected : _ [always] / on_unexpected_ + request_decision --> unexpected : _ [always] / on_unexpected_ + tokenize_exec --> unexpected : _ [always] / on_unexpected_ + tokenize_result_decision --> unexpected : _ [always] / on_unexpected_ + parse_exec --> unexpected : _ [always] / on_unexpected_ + parse_result_decision --> unexpected : _ [always] / on_unexpected_ done --> unexpected : _ [always] / on_unexpected_ errored --> unexpected : _ [always] / on_unexpected_ unexpected --> unexpected : _ [always] / on_unexpected_ @@ -29,18 +42,31 @@ stateDiagram-v2 | Source | Event | Guard | Action | Target | | --- | --- | --- | --- | --- | -| [`initialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`parse`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`valid_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`run_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`parse_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`initialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`parse`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`invalid_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`reject_invalid_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`parse`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`valid_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`run_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`parse_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`parse`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`invalid_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`reject_invalid_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`parse`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`valid_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`run_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`parse_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`parse`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`invalid_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`reject_invalid_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`unexpected`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`parse`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`valid_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`run_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`parse_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`unexpected`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`parse`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`invalid_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`reject_invalid_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`unexpected`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`parse_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | - | [`phase_ok>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`parse_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | - | [`phase_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`initialized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`unexpected`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`parse_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`unexpected`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`done`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`unexpected`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`errored`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`unexpected`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | -| [`unexpected`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | [`unexpected`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/sm.hpp) | +| initialized | parse_runtime | valid_parse_ | begin_parse_ | request_decision | +| initialized | parse_runtime | invalid_parse_with_callbacks_ | reject_invalid_parse_ | parse_result_decision | +| initialized | parse_runtime | invalid_parse_without_callbacks_ | reject_invalid_parse_ | errored | +| done | parse_runtime | valid_parse_ | begin_parse_ | request_decision | +| done | parse_runtime | invalid_parse_with_callbacks_ | reject_invalid_parse_ | parse_result_decision | +| done | parse_runtime | invalid_parse_without_callbacks_ | reject_invalid_parse_ | errored | +| errored | parse_runtime | valid_parse_ | begin_parse_ | request_decision | +| errored | parse_runtime | invalid_parse_with_callbacks_ | reject_invalid_parse_ | parse_result_decision | +| errored | parse_runtime | invalid_parse_without_callbacks_ | reject_invalid_parse_ | errored | +| unexpected | parse_runtime | valid_parse_ | begin_parse_ | request_decision | +| unexpected | parse_runtime | invalid_parse_with_callbacks_ | reject_invalid_parse_ | parse_result_decision | +| unexpected | parse_runtime | invalid_parse_without_callbacks_ | reject_invalid_parse_ | errored | +| request_decision | completion | always | tokenize_template_ | tokenize_exec | +| tokenize_exec | completion | always | none | tokenize_result_decision | +| tokenize_result_decision | completion | tokenize_ok_ | classify_tokens_ | parse_exec | +| tokenize_result_decision | completion | tokenize_failed_ | none | parse_result_decision | +| parse_exec | completion | always | parse_tokens_ | parse_result_decision | +| parse_result_decision | completion | request_ok_ | dispatch_done_ | done | +| parse_result_decision | completion | request_failed_ | dispatch_error_ | errored | +| initialized | _ | always | on_unexpected_ | unexpected | +| request_decision | _ | always | on_unexpected_ | unexpected | +| tokenize_exec | _ | always | on_unexpected_ | unexpected | +| tokenize_result_decision | _ | always | on_unexpected_ | unexpected | +| parse_exec | _ | always | on_unexpected_ | unexpected | +| parse_result_decision | _ | always | on_unexpected_ | unexpected | +| done | _ | always | on_unexpected_ | unexpected | +| errored | _ | always | on_unexpected_ | unexpected | +| unexpected | _ | always | on_unexpected_ | unexpected | diff --git a/docs/architecture/text_jinja_parser_classifier_parser.md b/docs/architecture/text_jinja_parser_classifier_parser.md new file mode 100644 index 00000000..e3eb6a0c --- /dev/null +++ b/docs/architecture/text_jinja_parser_classifier_parser.md @@ -0,0 +1,54 @@ +# text_jinja_parser_classifier_parser + +Source: [`emel/text/jinja/parser/classifier_parser/sm.hpp`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) + +## Mermaid + +```mermaid +stateDiagram-v2 + direction TB + [*] --> deciding + deciding --> statement_decision : completion_parse_runtime_ [always] / begin_classification_ + statement_decision --> classified : completion_parse_runtime_ [no_tokens_] / set_statement_unknown_ + statement_decision --> classified : completion_parse_runtime_ [token_text_] / set_statement_text_ + statement_decision --> classified : completion_parse_runtime_ [token_comment_] / set_statement_comment_ + statement_decision --> expression_decision : completion_parse_runtime_ [token_open_expression_] / set_statement_expression_ + statement_decision --> classified : completion_parse_runtime_ [token_open_statement_] / set_statement_statement_ + statement_decision --> classified : completion_parse_runtime_ [token_unknown_] / set_statement_unknown_ + expression_decision --> classified : completion_parse_runtime_ [expr_no_token_] / set_expression_unknown_ + expression_decision --> classified : completion_parse_runtime_ [expr_token_literal_] / set_expression_literal_ + expression_decision --> classified : completion_parse_runtime_ [expr_token_identifier_] / set_expression_identifier_ + expression_decision --> classified : completion_parse_runtime_ [expr_token_unary_] / set_expression_unary_ + expression_decision --> classified : completion_parse_runtime_ [expr_token_compound_] / set_expression_compound_ + expression_decision --> classified : completion_parse_runtime_ [expr_token_unknown_] / set_expression_unknown_ + classified --> terminate : [always] / none + deciding --> unexpected_event : _ [always] / on_unexpected_ + statement_decision --> unexpected_event : _ [always] / on_unexpected_ + expression_decision --> unexpected_event : _ [always] / on_unexpected_ + classified --> unexpected_event : _ [always] / on_unexpected_ + unexpected_event --> unexpected_event : _ [always] / on_unexpected_ +``` + +## Transitions + +| Source | Event | Guard | Action | Target | +| --- | --- | --- | --- | --- | +| [`deciding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`begin_classification>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`statement_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`statement_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`no_tokens>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`set_statement_unknown>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`statement_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`token_text>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`set_statement_text>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`statement_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`token_comment>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`set_statement_comment>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`statement_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`token_open_expression>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`set_statement_expression>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`expression_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`statement_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`token_open_statement>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`set_statement_statement>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`statement_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`token_unknown>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`set_statement_unknown>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`expression_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`expr_no_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`set_expression_unknown>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`expression_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`expr_token_literal>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`set_expression_literal>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`expression_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`expr_token_identifier>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`set_expression_identifier>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`expression_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`expr_token_unary>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`set_expression_unary>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`expression_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`expr_token_compound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`set_expression_compound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`expression_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`expr_token_unknown>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`set_expression_unknown>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | - | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`terminate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`deciding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`statement_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`expression_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`classified`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | +| [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/classifier_parser/sm.hpp) | diff --git a/docs/architecture/text_jinja_parser_lexer.md b/docs/architecture/text_jinja_parser_lexer.md new file mode 100644 index 00000000..0f5b7382 --- /dev/null +++ b/docs/architecture/text_jinja_parser_lexer.md @@ -0,0 +1,40 @@ +# text_jinja_parser_lexer + +Source: [`emel/text/jinja/parser/lexer/sm.hpp`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) + +## Mermaid + +```mermaid +stateDiagram-v2 + direction TB + [*] --> deciding + deciding --> tokenize_exec : completion_parse_runtime_ [always] / run_tokenize_ + tokenize_exec --> tokenize_result_decision : completion_parse_runtime_ [always] / none + tokenize_result_decision --> tokenized : completion_parse_runtime_ [tokenize_succeeded_] / none + tokenize_result_decision --> parse_failed : completion_parse_runtime_ [tokenize_failed_] / mark_parse_failed_ + tokenized --> terminate : [always] / none + parse_failed --> terminate : [always] / none + deciding --> unexpected_event : _ [always] / on_unexpected_ + tokenize_exec --> unexpected_event : _ [always] / on_unexpected_ + tokenize_result_decision --> unexpected_event : _ [always] / on_unexpected_ + tokenized --> unexpected_event : _ [always] / on_unexpected_ + parse_failed --> unexpected_event : _ [always] / on_unexpected_ + unexpected_event --> unexpected_event : _ [always] / on_unexpected_ +``` + +## Transitions + +| Source | Event | Guard | Action | Target | +| --- | --- | --- | --- | --- | +| [`deciding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`run_tokenize>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`tokenize_exec`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | +| [`tokenize_exec`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`tokenize_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | +| [`tokenize_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`tokenize_succeeded>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`tokenized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | +| [`tokenize_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`tokenize_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`mark_parse_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | +| [`tokenized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | - | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`terminate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | +| [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | - | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`terminate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | +| [`deciding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | +| [`tokenize_exec`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | +| [`tokenize_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | +| [`tokenized`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | +| [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | +| [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/lexer/sm.hpp) | diff --git a/docs/architecture/text_jinja_parser_program_parser.md b/docs/architecture/text_jinja_parser_program_parser.md new file mode 100644 index 00000000..d55ca1ea --- /dev/null +++ b/docs/architecture/text_jinja_parser_program_parser.md @@ -0,0 +1,72 @@ +# text_jinja_parser_program_parser + +Source: [`emel/text/jinja/parser/program_parser/sm.hpp`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) + +## Mermaid + +```mermaid +stateDiagram-v2 + direction TB + [*] --> deciding + deciding --> parse_begin : completion_parse_runtime_ [always] / start_program_parse_ + parse_begin --> dispatch_decision : completion_parse_runtime_ [always] / none + dispatch_decision --> parsed : completion_parse_runtime_ [at_eof_] / finish_parsed_ + dispatch_decision --> text_emit : completion_parse_runtime_ [token_text_] / none + dispatch_decision --> comment_emit : completion_parse_runtime_ [token_comment_] / none + dispatch_decision --> model__ : completion_parse_runtime_ [token_open_statement_] / none + dispatch_decision --> model__ : completion_parse_runtime_ [token_open_expression_] / none + dispatch_decision --> parse_failed : completion_parse_runtime_ [token_unexpected_] / fail_current_token_ + text_emit --> dispatch_decision : completion_parse_runtime_ [always] / consume_text_ + comment_emit --> dispatch_decision : completion_parse_runtime_ [always] / consume_comment_ + model__ --> statement_parse_result_decision : completion_parse_runtime_ [always] / none + statement_parse_result_decision --> dispatch_decision : completion_parse_runtime_ [phase_ok_] / none + statement_parse_result_decision --> parse_failed : completion_parse_runtime_ [phase_failed_] / none + model__ --> expression_parse_result_decision : completion_parse_runtime_ [always] / none + expression_parse_result_decision --> dispatch_decision : completion_parse_runtime_ [phase_ok_] / none + expression_parse_result_decision --> parse_failed : completion_parse_runtime_ [phase_failed_] / none + parsed --> terminate : [always] / none + parse_failed --> terminate : [always] / none + deciding --> unexpected_event : _ [always] / on_unexpected_ + parse_begin --> unexpected_event : _ [always] / on_unexpected_ + dispatch_decision --> unexpected_event : _ [always] / on_unexpected_ + text_emit --> unexpected_event : _ [always] / on_unexpected_ + comment_emit --> unexpected_event : _ [always] / on_unexpected_ + statement_parse_result_decision --> unexpected_event : _ [always] / on_unexpected_ + expression_parse_result_decision --> unexpected_event : _ [always] / on_unexpected_ + parsed --> unexpected_event : _ [always] / on_unexpected_ + parse_failed --> unexpected_event : _ [always] / on_unexpected_ + unexpected_event --> unexpected_event : _ [always] / on_unexpected_ +``` + +## Transitions + +| Source | Event | Guard | Action | Target | +| --- | --- | --- | --- | --- | +| [`deciding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`start_program_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`parse_begin`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`parse_begin`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`dispatch_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`dispatch_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`at_eof>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`finish_parsed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`parsed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`dispatch_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`token_text>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`text_emit`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`dispatch_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`token_comment>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`comment_emit`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`dispatch_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`token_open_statement>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`model>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`dispatch_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`token_open_expression>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`model>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`dispatch_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`token_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`fail_current_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`text_emit`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`consume_text>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`dispatch_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`comment_emit`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`consume_comment>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`dispatch_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`model>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`statement_parse_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`statement_parse_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`phase_ok>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`dispatch_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`statement_parse_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`phase_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`model>>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`expression_parse_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`expression_parse_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`phase_ok>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`dispatch_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`expression_parse_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`phase_failed>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`parsed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | - | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`terminate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | - | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`terminate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`deciding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`parse_begin`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`dispatch_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`text_emit`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`comment_emit`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`statement_parse_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`expression_parse_result_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`parsed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | +| [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/sm.hpp) | diff --git a/docs/architecture/text_jinja_parser_program_parser_expression_parser.md b/docs/architecture/text_jinja_parser_program_parser_expression_parser.md new file mode 100644 index 00000000..1f6d9dd7 --- /dev/null +++ b/docs/architecture/text_jinja_parser_program_parser_expression_parser.md @@ -0,0 +1,64 @@ +# text_jinja_parser_program_parser_expression_parser + +Source: [`emel/text/jinja/parser/program_parser/expression_parser/sm.hpp`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) + +## Mermaid + +```mermaid +stateDiagram-v2 + direction TB + [*] --> deciding + deciding --> expression_first_decision : completion_parse_runtime_ [always] / begin_expression_parse_ + expression_first_decision --> parse_failed : completion_parse_runtime_ [expr_scan_eof_] / fail_expression_start_token_ + expression_first_decision --> parse_failed : completion_parse_runtime_ [expr_first_is_close_] / fail_expression_close_token_ + expression_first_decision --> expression_scan : completion_parse_runtime_ [expr_first_is_identifier_] / consume_expression_identifier_ + expression_first_decision --> expression_scan : completion_parse_runtime_ [expr_first_is_literal_] / consume_expression_literal_ + expression_first_decision --> expression_scan : completion_parse_runtime_ [expr_first_is_unary_] / consume_expression_unary_ + expression_first_decision --> expression_scan : completion_parse_runtime_ [expr_first_is_other_content_] / consume_expression_compound_ + expression_scan --> expression_emit_decision : completion_parse_runtime_ [expr_scan_at_close_] / none + expression_scan --> expression_scan : completion_parse_runtime_ [expr_scan_continue_] / consume_expression_token_ + expression_scan --> parse_failed : completion_parse_runtime_ [expr_scan_eof_] / fail_expression_start_token_ + expression_emit_decision --> expression_close : completion_parse_runtime_ [expression_identifier_] / emit_expression_identifier_ + expression_emit_decision --> expression_close : completion_parse_runtime_ [expression_non_identifier_] / emit_expression_generic_ + expression_close --> parsed : completion_parse_runtime_ [expr_scan_at_close_] / consume_expression_close_ + expression_close --> parse_failed : completion_parse_runtime_ [expr_scan_eof_] / fail_expression_start_token_ + parsed --> terminate : [always] / none + parse_failed --> terminate : [always] / none + deciding --> unexpected_event : _ [always] / on_unexpected_ + expression_first_decision --> unexpected_event : _ [always] / on_unexpected_ + expression_scan --> unexpected_event : _ [always] / on_unexpected_ + expression_emit_decision --> unexpected_event : _ [always] / on_unexpected_ + expression_close --> unexpected_event : _ [always] / on_unexpected_ + parsed --> unexpected_event : _ [always] / on_unexpected_ + parse_failed --> unexpected_event : _ [always] / on_unexpected_ + unexpected_event --> unexpected_event : _ [always] / on_unexpected_ +``` + +## Transitions + +| Source | Event | Guard | Action | Target | +| --- | --- | --- | --- | --- | +| [`deciding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`begin_expression_parse>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expression_first_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_first_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expr_scan_eof>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`fail_expression_start_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_first_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expr_first_is_close>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`fail_expression_close_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_first_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expr_first_is_identifier>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`consume_expression_identifier>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expression_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_first_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expr_first_is_literal>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`consume_expression_literal>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expression_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_first_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expr_first_is_unary>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`consume_expression_unary>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expression_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_first_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expr_first_is_other_content>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`consume_expression_compound>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expression_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expr_scan_at_close>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expression_emit_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expr_scan_continue>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`consume_expression_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expression_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expr_scan_eof>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`fail_expression_start_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_emit_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expression_identifier>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`emit_expression_identifier>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expression_close`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_emit_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expression_non_identifier>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`emit_expression_generic>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expression_close`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_close`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expr_scan_at_close>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`consume_expression_close>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`parsed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_close`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`expr_scan_eof>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`fail_expression_start_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`parsed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | - | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`terminate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | - | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`terminate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`deciding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_first_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_emit_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`expression_close`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`parsed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | +| [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp) | diff --git a/docs/architecture/text_jinja_parser_program_parser_statement_parser.md b/docs/architecture/text_jinja_parser_program_parser_statement_parser.md new file mode 100644 index 00000000..7bbd23b6 --- /dev/null +++ b/docs/architecture/text_jinja_parser_program_parser_statement_parser.md @@ -0,0 +1,80 @@ +# text_jinja_parser_program_parser_statement_parser + +Source: [`emel/text/jinja/parser/program_parser/statement_parser/sm.hpp`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) + +## Mermaid + +```mermaid +stateDiagram-v2 + direction TB + [*] --> deciding + deciding --> statement_kind_decision : completion_parse_runtime_ [always] / none + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_set_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_if_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_elif_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_else_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endif_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_for_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endfor_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_macro_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endmacro_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_call_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endcall_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_filter_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endfilter_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_break_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_continue_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_generation_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endgeneration_] / begin_statement_scan_ + statement_kind_decision --> statement_scan : completion_parse_runtime_ [statement_name_endset_] / begin_statement_scan_ + statement_kind_decision --> parse_failed : completion_parse_runtime_ [statement_identifier_missing_] / fail_statement_open_token_ + statement_kind_decision --> parse_failed : completion_parse_runtime_ [statement_name_unknown_] / fail_statement_name_token_ + statement_scan --> parsed : completion_parse_runtime_ [statement_scan_at_close_] / consume_statement_close_and_emit_ + statement_scan --> statement_scan : completion_parse_runtime_ [statement_scan_continue_] / consume_statement_token_ + statement_scan --> parse_failed : completion_parse_runtime_ [statement_scan_eof_] / fail_statement_start_token_ + parsed --> terminate : [always] / none + parse_failed --> terminate : [always] / none + deciding --> unexpected_event : _ [always] / on_unexpected_ + statement_kind_decision --> unexpected_event : _ [always] / on_unexpected_ + statement_scan --> unexpected_event : _ [always] / on_unexpected_ + parsed --> unexpected_event : _ [always] / on_unexpected_ + parse_failed --> unexpected_event : _ [always] / on_unexpected_ + unexpected_event --> unexpected_event : _ [always] / on_unexpected_ +``` + +## Transitions + +| Source | Event | Guard | Action | Target | +| --- | --- | --- | --- | --- | +| [`deciding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_set>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_if>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_elif>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_else>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_endif>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_for>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_endfor>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_macro>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_endmacro>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_call>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_endcall>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_filter>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_endfilter>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_break>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_continue>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_generation>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_endgeneration>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_endset>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`begin_statement_scan>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_identifier_missing>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`fail_statement_open_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_name_unknown>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`fail_statement_name_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan_at_close>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`consume_statement_close_and_emit>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`parsed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan_continue>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`consume_statement_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`completion`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`statement_scan_eof>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`fail_statement_start_token>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`parsed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | - | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`terminate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | - | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`none`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`terminate`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`deciding`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_kind_decision`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`statement_scan`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`parsed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`parse_failed`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | +| [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`_`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`always`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`on_unexpected>`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | [`unexpected_event`](https://github.com/stateforward/emel.cpp/blob/main/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp) | diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 5557da10..0f3c749c 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -6,97 +6,4 @@ Note: While EMEL is modular and easy to bench in isolation, llama.cpp code is ve entangled. These microbenches aim for apples-to-apples comparisons but likely are not. True benchmarks will be end-to-end once the system is complete. -| Benchmark | emel.cpp ns/op | llama.cpp ns/op | ratio | -| --- | ---: | ---: | ---: | -| `batch/planner_equal` | 1846.750 | 8689.946 | 0.213x | -| `batch/planner_seq` | 1781.388 | 3996.500 | 0.446x | -| `batch/planner_simple` | 1348.817 | 3498.363 | 0.386x | -| `gbnf/rule_parser_basic` | 247.521 | 471.233 | 0.525x | -| `gbnf/rule_parser_complex` | 1933.033 | 2515.221 | 0.769x | -| `kernel/aarch64/op_add` | 88.783 | 5061.321 | 0.018x | -| `kernel/aarch64/op_cos` | 1668.921 | 6025.850 | 0.277x | -| `kernel/aarch64/op_div` | 88.600 | 4142.504 | 0.021x | -| `kernel/aarch64/op_dup` | 85.975 | 4095.954 | 0.021x | -| `kernel/aarch64/op_log` | 1843.883 | 6106.117 | 0.302x | -| `kernel/aarch64/op_mul` | 91.025 | 5091.896 | 0.018x | -| `kernel/aarch64/op_mul_mat` | 4540.008 | 10639.004 | 0.427x | -| `kernel/aarch64/op_sin` | 1447.079 | 5599.971 | 0.258x | -| `kernel/aarch64/op_soft_max` | 2066.808 | 4972.771 | 0.416x | -| `kernel/aarch64/op_sqr` | 86.779 | 4090.646 | 0.021x | -| `kernel/aarch64/op_sqrt` | 137.033 | 4436.392 | 0.031x | -| `kernel/aarch64/op_sub` | 91.279 | 5088.383 | 0.018x | -| `kernel/aarch64/op_unary_exp` | 1297.300 | 5642.096 | 0.230x | -| `kernel/aarch64/op_unary_neg` | 89.208 | 4536.625 | 0.020x | -| `kernel/aarch64/op_unary_relu` | 85.879 | 4413.375 | 0.019x | -| `kernel/x86_64/op_add` | 60.092 | 5068.100 | 0.012x | -| `kernel/x86_64/op_cos` | 1969.629 | 5873.692 | 0.335x | -| `kernel/x86_64/op_div` | 74.679 | 4153.717 | 0.018x | -| `kernel/x86_64/op_dup` | 47.033 | 4013.613 | 0.012x | -| `kernel/x86_64/op_log` | 1820.858 | 6532.413 | 0.279x | -| `kernel/x86_64/op_mul` | 60.196 | 5235.196 | 0.011x | -| `kernel/x86_64/op_mul_mat` | 44244.079 | 10511.242 | 4.209x | -| `kernel/x86_64/op_sin` | 1296.000 | 5583.742 | 0.232x | -| `kernel/x86_64/op_soft_max` | 2062.137 | 5244.917 | 0.393x | -| `kernel/x86_64/op_sqr` | 49.138 | 4063.596 | 0.012x | -| `kernel/x86_64/op_sqrt` | 143.012 | 4265.863 | 0.034x | -| `kernel/x86_64/op_sub` | 60.096 | 5310.508 | 0.011x | -| `kernel/x86_64/op_unary_exp` | 1284.658 | 5399.771 | 0.238x | -| `kernel/x86_64/op_unary_neg` | 51.946 | 4309.450 | 0.012x | -| `kernel/x86_64/op_unary_relu` | 52.304 | 4238.471 | 0.012x | -| `logits/sampler_raw/vocab_128000` | 19259.958 | 18468.492 | 1.043x | -| `logits/sampler_raw/vocab_256000` | 38539.842 | 36725.137 | 1.049x | -| `logits/sampler_raw/vocab_32000` | 5214.146 | 4826.229 | 1.080x | -| `logits/sampler_sml/vocab_128000` | 15429.442 | 14757.788 | 1.046x | -| `logits/sampler_sml/vocab_256000` | 34200.133 | 30380.342 | 1.126x | -| `logits/sampler_sml/vocab_32000` | 4436.292 | 4330.962 | 1.024x | -| `logits/validator_raw/vocab_128000` | 90205.633 | 90458.808 | 0.997x | -| `logits/validator_raw/vocab_256000` | 181372.546 | 179498.462 | 1.010x | -| `logits/validator_raw/vocab_32000` | 23735.550 | 23904.125 | 0.993x | -| `logits/validator_sml/vocab_128000` | 99648.387 | 99266.212 | 1.004x | -| `logits/validator_sml/vocab_256000` | 197266.092 | 199430.296 | 0.989x | -| `logits/validator_sml/vocab_32000` | 24528.092 | 24126.225 | 1.017x | -| `memory/hybrid_full` | 408.700 | 36677.713 | 0.011x | -| `memory/kv_full` | 103.067 | 36946.496 | 0.003x | -| `memory/recurrent_full` | 113.079 | 5595.042 | 0.020x | -| `text/encoders/bpe_long` | 10221.996 | 10221.204 | 1.000x | -| `text/encoders/bpe_short` | 159.125 | 153.158 | 1.039x | -| `text/encoders/fallback_long` | 2470.238 | 2485.546 | 0.994x | -| `text/encoders/fallback_short` | 50.267 | 47.825 | 1.051x | -| `text/encoders/plamo2_long` | 4848.942 | 4878.158 | 0.994x | -| `text/encoders/plamo2_short` | 107.117 | 104.096 | 1.029x | -| `text/encoders/rwkv_long` | 4557.729 | 4543.887 | 1.003x | -| `text/encoders/rwkv_short` | 2697.533 | 2658.883 | 1.015x | -| `text/encoders/spm_long` | 12589.987 | 12349.475 | 1.019x | -| `text/encoders/spm_short` | 213.188 | 205.325 | 1.038x | -| `text/encoders/ugm_long` | 8308.617 | 8295.337 | 1.002x | -| `text/encoders/ugm_short` | 137.250 | 137.008 | 1.002x | -| `text/encoders/wpm_long` | 26858.621 | 26355.825 | 1.019x | -| `text/encoders/wpm_short` | 531.438 | 540.237 | 0.984x | -| `text/jinja/formatter_long` | 87073.829 | 400326.883 | 0.218x | -| `text/jinja/formatter_short` | 1144.017 | 6368.133 | 0.180x | -| `text/jinja/parser_long` | 35030.512 | 52803.367 | 0.663x | -| `text/jinja/parser_short` | 547.888 | 632.633 | 0.866x | -| `tokenizer/full_bpe_long` | 9967.413 | 9607.096 | 1.038x | -| `tokenizer/full_bpe_short` | 220.113 | 218.846 | 1.006x | -| `tokenizer/full_plamo2_long` | 9890.796 | 9985.525 | 0.991x | -| `tokenizer/full_plamo2_short` | 1799.446 | 1769.058 | 1.017x | -| `tokenizer/full_rwkv_long` | 3566.475 | 3551.117 | 1.004x | -| `tokenizer/full_rwkv_short` | 2373.500 | 2159.892 | 1.099x | -| `tokenizer/full_spm_long` | 13766.279 | 13689.263 | 1.006x | -| `tokenizer/full_spm_short` | 296.825 | 285.354 | 1.040x | -| `tokenizer/full_ugm_long` | 10042.667 | 9989.429 | 1.005x | -| `tokenizer/full_ugm_short` | 1817.804 | 1818.546 | 1.000x | -| `tokenizer/full_wpm_long` | 28866.112 | 34007.938 | 0.849x | -| `tokenizer/full_wpm_short` | 2204.133 | 2210.221 | 0.997x | -| `tokenizer/preprocessor_bpe_long` | 2775.246 | 5265.688 | 0.527x | -| `tokenizer/preprocessor_bpe_short` | 82.854 | 1747.217 | 0.047x | -| `tokenizer/preprocessor_plamo2_long` | 3052.371 | 4619.908 | 0.661x | -| `tokenizer/preprocessor_plamo2_short` | 2367.925 | 3575.713 | 0.662x | -| `tokenizer/preprocessor_rwkv_long` | 3077.379 | 4554.646 | 0.676x | -| `tokenizer/preprocessor_rwkv_short` | 2356.238 | 3536.963 | 0.666x | -| `tokenizer/preprocessor_spm_long` | 3092.796 | 4569.296 | 0.677x | -| `tokenizer/preprocessor_spm_short` | 2361.154 | 3586.446 | 0.658x | -| `tokenizer/preprocessor_ugm_long` | 3139.088 | 4625.679 | 0.679x | -| `tokenizer/preprocessor_ugm_short` | 2375.508 | 3560.692 | 0.667x | -| `tokenizer/preprocessor_wpm_long` | 3043.238 | 4503.621 | 0.676x | -| `tokenizer/preprocessor_wpm_short` | 2599.613 | 3530.233 | 0.736x | +{{ benchmarks_table }} diff --git a/snapshots/bench/benchmarks.txt b/snapshots/bench/benchmarks.txt index 2c000db5..03c2fc43 100644 --- a/snapshots/bench/benchmarks.txt +++ b/snapshots/bench/benchmarks.txt @@ -66,8 +66,8 @@ text/encoders/wpm_long ns_per_op=26858.621 text/encoders/wpm_short ns_per_op=531.438 text/jinja/formatter_long ns_per_op=87073.829 text/jinja/formatter_short ns_per_op=1144.017 -text/jinja/parser_long ns_per_op=35030.512 -text/jinja/parser_short ns_per_op=547.888 +text/jinja/parser_long ns_per_op=35902.459 +text/jinja/parser_short ns_per_op=1100.708 tokenizer/full_bpe_long ns_per_op=9967.413 tokenizer/full_bpe_short ns_per_op=220.113 tokenizer/full_plamo2_long ns_per_op=9890.796 diff --git a/snapshots/bench/benchmarks_compare.txt b/snapshots/bench/benchmarks_compare.txt index 70a75ce3..a3f098b8 100644 --- a/snapshots/bench/benchmarks_compare.txt +++ b/snapshots/bench/benchmarks_compare.txt @@ -66,8 +66,8 @@ text/encoders/wpm_long emel.cpp 26858.621 ns/op, llama.cpp 26355.825 ns/op, rati text/encoders/wpm_short emel.cpp 531.438 ns/op, llama.cpp 540.237 ns/op, ratio=0.984x text/jinja/formatter_long emel.cpp 87073.829 ns/op, llama.cpp 400326.883 ns/op, ratio=0.218x text/jinja/formatter_short emel.cpp 1144.017 ns/op, llama.cpp 6368.133 ns/op, ratio=0.180x -text/jinja/parser_long emel.cpp 35030.512 ns/op, llama.cpp 52803.367 ns/op, ratio=0.663x -text/jinja/parser_short emel.cpp 547.888 ns/op, llama.cpp 632.633 ns/op, ratio=0.866x +text/jinja/parser_long emel.cpp 35902.459 ns/op, llama.cpp 42470.375 ns/op, ratio=0.845x +text/jinja/parser_short emel.cpp 1100.708 ns/op, llama.cpp 532.792 ns/op, ratio=2.066x tokenizer/full_bpe_long emel.cpp 9967.413 ns/op, llama.cpp 9607.096 ns/op, ratio=1.038x tokenizer/full_bpe_short emel.cpp 220.113 ns/op, llama.cpp 218.846 ns/op, ratio=1.006x tokenizer/full_plamo2_long emel.cpp 9890.796 ns/op, llama.cpp 9985.525 ns/op, ratio=0.991x diff --git a/snapshots/quality_gates/timing.txt b/snapshots/quality_gates/timing.txt index fd5ad1c6..9970adbf 100644 --- a/snapshots/quality_gates/timing.txt +++ b/snapshots/quality_gates/timing.txt @@ -1,8 +1,7 @@ # quality_gates timing (seconds) -build_with_zig 1 -test_with_coverage 72 -paritychecker 6 -fuzz_smoke 29 -bench_snapshot 95 -generate_docs 33 -total 236 +build_with_zig 0 +test_with_coverage 124 +paritychecker 5 +fuzz_smoke 77 +bench_snapshot 80 +total 286 diff --git a/src/emel/model/architecture/types.hpp b/src/emel/model/architecture/types.hpp deleted file mode 100644 index 9eb0fd61..00000000 --- a/src/emel/model/architecture/types.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - -#include -#include - -#include "emel/callback.hpp" -#include "emel/error/error.hpp" - -namespace emel::model { -struct data; -} - -namespace emel::model::architecture { - -using map_layers_fn = emel::callback; -using validate_fn = emel::callback; - -struct type { - std::string_view architecture_name = {}; - map_layers_fn map_layers = {}; - validate_fn validate = {}; -}; - -} // namespace emel::model::architecture diff --git a/src/emel/text/jinja/ast.hpp b/src/emel/text/jinja/ast.hpp deleted file mode 100644 index 974d3ddc..00000000 --- a/src/emel/text/jinja/ast.hpp +++ /dev/null @@ -1,241 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "emel/text/jinja/lexer.hpp" - -namespace emel::text::jinja { - -struct ast_node { - size_t pos = 0; - virtual ~ast_node() = default; -}; - -using ast_ptr = std::unique_ptr; -using ast_list = std::vector; -using ast_pair_list = std::vector>; - -struct comment_statement : ast_node { - std::string value; - explicit comment_statement(std::string text) : value(std::move(text)) {} -}; - -struct string_literal : ast_node { - std::string value; - explicit string_literal(std::string text) : value(std::move(text)) {} -}; - -struct identifier : ast_node { - std::string name; - explicit identifier(std::string text) : name(std::move(text)) {} -}; - -struct integer_literal : ast_node { - int64_t value = 0; - explicit integer_literal(int64_t v) : value(v) {} -}; - -struct float_literal : ast_node { - double value = 0.0; - explicit float_literal(double v) : value(v) {} -}; - -struct tuple_literal : ast_node { - ast_list values; - explicit tuple_literal(ast_list vals) : values(std::move(vals)) {} -}; - -struct array_literal : ast_node { - ast_list values; - explicit array_literal(ast_list vals) : values(std::move(vals)) {} -}; - -struct object_literal : ast_node { - ast_pair_list pairs; - explicit object_literal(ast_pair_list vals) : pairs(std::move(vals)) {} -}; - -struct unary_expression : ast_node { - token op; - ast_ptr operand; - unary_expression(token op_token, ast_ptr expr) - : op(std::move(op_token)), operand(std::move(expr)) {} -}; - -struct binary_expression : ast_node { - token op; - ast_ptr left; - ast_ptr right; - binary_expression(token op_token, ast_ptr lhs, ast_ptr rhs) - : op(std::move(op_token)), - left(std::move(lhs)), - right(std::move(rhs)) {} -}; - -struct ternary_expression : ast_node { - ast_ptr test; - ast_ptr true_expr; - ast_ptr false_expr; - ternary_expression(ast_ptr test_expr, ast_ptr true_value, ast_ptr false_value) - : test(std::move(test_expr)), - true_expr(std::move(true_value)), - false_expr(std::move(false_value)) {} -}; - -struct select_expression : ast_node { - ast_ptr value; - ast_ptr test; - select_expression(ast_ptr value_expr, ast_ptr test_expr) - : value(std::move(value_expr)), - test(std::move(test_expr)) {} -}; - -struct test_expression : ast_node { - ast_ptr operand; - bool negate = false; - ast_ptr test; - test_expression(ast_ptr operand_expr, bool neg, ast_ptr test_expr) - : operand(std::move(operand_expr)), - negate(neg), - test(std::move(test_expr)) {} -}; - -struct filter_expression : ast_node { - ast_ptr operand; - ast_ptr filter; - filter_expression(ast_ptr operand_expr, ast_ptr filter_expr) - : operand(std::move(operand_expr)), - filter(std::move(filter_expr)) {} -}; - -struct call_expression : ast_node { - ast_ptr callee; - ast_list args; - call_expression(ast_ptr callee_expr, ast_list arguments) - : callee(std::move(callee_expr)), - args(std::move(arguments)) {} -}; - -struct call_statement : ast_node { - ast_ptr call_expr; - ast_list caller_args; - ast_list body; - call_statement(ast_ptr call_expr_in, ast_list caller_args_in, ast_list body_in) - : call_expr(std::move(call_expr_in)), - caller_args(std::move(caller_args_in)), - body(std::move(body_in)) {} -}; - -struct filter_statement : ast_node { - ast_ptr filter_node; - ast_list body; - filter_statement(ast_ptr filter_expr, ast_list body_in) - : filter_node(std::move(filter_expr)), - body(std::move(body_in)) {} -}; - -struct set_statement : ast_node { - ast_ptr left; - ast_ptr value; - ast_list body; - set_statement(ast_ptr left_expr, ast_ptr value_expr, ast_list body_in) - : left(std::move(left_expr)), - value(std::move(value_expr)), - body(std::move(body_in)) {} -}; - -struct if_statement : ast_node { - ast_ptr test; - ast_list body; - ast_list alternate; - if_statement(ast_ptr test_expr, ast_list body_in, ast_list alternate_in) - : test(std::move(test_expr)), - body(std::move(body_in)), - alternate(std::move(alternate_in)) {} -}; - -struct macro_statement : ast_node { - ast_ptr name; - ast_list args; - ast_list body; - macro_statement(ast_ptr name_expr, ast_list args_in, ast_list body_in) - : name(std::move(name_expr)), - args(std::move(args_in)), - body(std::move(body_in)) {} -}; - -struct for_statement : ast_node { - ast_ptr loop_var; - ast_ptr iterable; - ast_list body; - ast_list alternate; - for_statement(ast_ptr loop_var_in, ast_ptr iterable_in, ast_list body_in, ast_list alternate_in) - : loop_var(std::move(loop_var_in)), - iterable(std::move(iterable_in)), - body(std::move(body_in)), - alternate(std::move(alternate_in)) {} -}; - -struct break_statement : ast_node { - break_statement() = default; -}; - -struct continue_statement : ast_node { - continue_statement() = default; -}; - -struct noop_statement : ast_node { - noop_statement() = default; -}; - -struct member_expression : ast_node { - ast_ptr object; - ast_ptr property; - bool computed = false; - member_expression(ast_ptr object_in, ast_ptr property_in, bool is_computed) - : object(std::move(object_in)), - property(std::move(property_in)), - computed(is_computed) {} -}; - -struct slice_expression : ast_node { - ast_ptr start; - ast_ptr stop; - ast_ptr step; - slice_expression(ast_ptr start_expr, ast_ptr stop_expr, ast_ptr step_expr) - : start(std::move(start_expr)), - stop(std::move(stop_expr)), - step(std::move(step_expr)) {} -}; - -struct keyword_argument_expression : ast_node { - ast_ptr key; - ast_ptr value; - keyword_argument_expression(ast_ptr key_expr, ast_ptr value_expr) - : key(std::move(key_expr)), - value(std::move(value_expr)) {} -}; - -struct spread_expression : ast_node { - ast_ptr operand; - explicit spread_expression(ast_ptr expr) : operand(std::move(expr)) {} -}; - -struct program { - ast_list body; - int32_t last_error = EMEL_OK; - size_t last_error_pos = 0; - - void reset() noexcept { - body.clear(); - last_error = EMEL_OK; - last_error_pos = 0; - } -}; - -} // namespace emel::text::jinja diff --git a/src/emel/text/jinja/formatter/events.hpp b/src/emel/text/jinja/formatter/events.hpp index fb2e3cb0..c5f28fd2 100644 --- a/src/emel/text/jinja/formatter/events.hpp +++ b/src/emel/text/jinja/formatter/events.hpp @@ -5,9 +5,8 @@ #include #include "emel/callback.hpp" -#include "emel/text/jinja/ast.hpp" #include "emel/text/jinja/formatter/errors.hpp" -#include "emel/text/jinja/value.hpp" +#include "emel/text/jinja/parser/detail.hpp" namespace emel::text::jinja::events { diff --git a/src/emel/text/jinja/lexer.hpp b/src/emel/text/jinja/lexer.hpp deleted file mode 100644 index 36bb8b07..00000000 --- a/src/emel/text/jinja/lexer.hpp +++ /dev/null @@ -1,475 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include "emel/emel.h" - -namespace emel::text::jinja { - -enum class token_type : uint8_t { - eof = 0, - text, - numeric_literal, - string_literal, - identifier, - equals, - open_paren, - close_paren, - open_statement, - close_statement, - open_expression, - close_expression, - open_square_bracket, - close_square_bracket, - open_curly_bracket, - close_curly_bracket, - comma, - dot, - colon, - pipe, - call_operator, - additive_binary_operator, - multiplicative_binary_operator, - comparison_binary_operator, - unary_operator, - comment -}; - -struct token { - token_type type = token_type::eof; - std::string value; - size_t pos = 0; -}; - -struct lexer_result { - std::vector tokens; - std::string source; - int32_t error = EMEL_OK; - size_t error_pos = 0; -}; - -struct lexer { - lexer_result tokenize(std::string_view source) const { - lexer_result result{}; - result.source = std::string(source); - std::string & src = result.source; - - if (src.empty()) { - return result; - } - - bool ok = true; - auto set_error = [&](int32_t code, size_t pos) { - if (!ok) { - return; - } - ok = false; - result.error = code; - result.error_pos = pos; - }; - - auto string_lstrip = [](std::string & s, const char * chars) { - size_t start = s.find_first_not_of(chars); - if (start == std::string::npos) { - s.clear(); - } else { - s.erase(0, start); - } - }; - - auto string_rstrip = [](std::string & s, const char * chars) { - size_t end = s.find_last_not_of(chars); - if (end == std::string::npos) { - s.clear(); - } else { - s.erase(end + 1); - } - }; - - auto decode_escape = [&](char ch, char & out) -> bool { - switch (ch) { - case 'n': - out = '\n'; - return true; - case 't': - out = '\t'; - return true; - case 'r': - out = '\r'; - return true; - case 'b': - out = '\b'; - return true; - case 'f': - out = '\f'; - return true; - case 'v': - out = '\v'; - return true; - case '\\': - out = '\\'; - return true; - case '\'': - out = '\''; - return true; - case '"': - out = '"'; - return true; - default: - return false; - } - }; - - auto is_word = [](char ch) -> bool { - return std::isalnum(static_cast(ch)) || ch == '_'; - }; - - auto is_integer = [](char ch) -> bool { - return std::isdigit(static_cast(ch)) != 0; - }; - - struct mapping { - std::string_view seq; - token_type type; - }; - - static constexpr mapping k_mapping_table[] = { - {"{%-", token_type::open_statement}, - {"-%}", token_type::close_statement}, - {"{{-", token_type::open_expression}, - {"-}}", token_type::close_expression}, - {"{%", token_type::open_statement}, - {"%}", token_type::close_statement}, - {"{{", token_type::open_expression}, - {"}}", token_type::close_expression}, - {"(", token_type::open_paren}, - {")", token_type::close_paren}, - {"{", token_type::open_curly_bracket}, - {"}", token_type::close_curly_bracket}, - {"[", token_type::open_square_bracket}, - {"]", token_type::close_square_bracket}, - {",", token_type::comma}, - {".", token_type::dot}, - {":", token_type::colon}, - {"|", token_type::pipe}, - {"<=", token_type::comparison_binary_operator}, - {">=", token_type::comparison_binary_operator}, - {"==", token_type::comparison_binary_operator}, - {"!=", token_type::comparison_binary_operator}, - {"<", token_type::comparison_binary_operator}, - {">", token_type::comparison_binary_operator}, - {"+", token_type::additive_binary_operator}, - {"-", token_type::additive_binary_operator}, - {"~", token_type::additive_binary_operator}, - {"*", token_type::multiplicative_binary_operator}, - {"/", token_type::multiplicative_binary_operator}, - {"%", token_type::multiplicative_binary_operator}, - {"=", token_type::equals}, - }; - - // normalize \r\n or \r to \n - for (std::string::size_type pos = 0; - (pos = src.find("\r\n", pos)) != std::string::npos; ) { - src.erase(pos, 1); - ++pos; - } - for (std::string::size_type pos = 0; - (pos = src.find('\r', pos)) != std::string::npos; ) { - src.replace(pos, 1, 1, '\n'); - ++pos; - } - - if (!src.empty() && src.back() == '\n') { - src.pop_back(); - } - - size_t pos = 0; - size_t start_pos = 0; - size_t curly_bracket_depth = 0; - - auto next_pos_is = [&](std::initializer_list chars, size_t n = 1) -> bool { - size_t idx = pos + n; - if (idx >= src.size()) { - return false; - } - for (char c : chars) { - if (src[idx] == c) { - return true; - } - } - return false; - }; - - auto consume_while = [&](const auto & predicate) -> std::string { - std::string out; - while (pos < src.size() && predicate(src[pos])) { - if (src[pos] == '\\') { - ++pos; - if (pos >= src.size()) { - set_error(EMEL_ERR_PARSE_FAILED, pos); - return out; - } - char escaped = src[pos++]; - char decoded = '\0'; - if (!decode_escape(escaped, decoded)) { - set_error(EMEL_ERR_PARSE_FAILED, pos); - return out; - } - out += decoded; - continue; - } - out += src[pos++]; - } - return out; - }; - - auto consume_numeric = [&]() -> std::string { - std::string num = consume_while(is_integer); - if (!ok) { - return num; - } - if (pos < src.size() && src[pos] == '.' && pos + 1 < src.size() && - is_integer(src[pos + 1])) { - ++pos; - std::string frac = consume_while(is_integer); - num += "."; - num += frac; - } - return num; - }; - - bool opt_lstrip_blocks = true; - bool opt_trim_blocks = true; - bool is_lstrip_block = false; - bool is_rstrip_block = false; - - std::vector tokens; - - while (pos < src.size() && ok) { - start_pos = pos; - token_type last_token_type = tokens.empty() - ? token_type::close_statement - : tokens.back().type; - - if (last_token_type == token_type::close_statement || - last_token_type == token_type::close_expression || - last_token_type == token_type::comment) { - bool last_block_can_rm_newline = false; - is_rstrip_block = false; - if (pos > 3) { - char c0 = src[pos - 3]; - char c1 = src[pos - 2]; - char c2 = src[pos - 1]; - is_rstrip_block = c0 == '-' && (c1 == '%' || c1 == '}' || c1 == '#') && c2 == '}'; - last_block_can_rm_newline = (c1 == '#' || c1 == '%' || c1 == '-') && c2 == '}'; - } - - size_t start = pos; - size_t end = start; - while (pos < src.size() && !(src[pos] == '{' && next_pos_is({'%', '{', '#'}))) { - end = ++pos; - } - - if (opt_lstrip_blocks && pos < src.size() && src[pos] == '{' && - next_pos_is({'%', '#', '-'})) { - size_t current = end; - while (current > start) { - char c = src[current - 1]; - if (current == 1) { - end = 0; - break; - } - if (c == '\n') { - end = current; - break; - } - if (!std::isspace(static_cast(c))) { - break; - } - --current; - } - } - - std::string text = src.substr(start, end - start); - - if (opt_trim_blocks && last_block_can_rm_newline) { - if (!text.empty() && text.front() == '\n') { - text.erase(text.begin()); - } - } - - if (is_rstrip_block) { - string_lstrip(text, " \t\r\n"); - } - - is_lstrip_block = - pos < src.size() && src[pos] == '{' && next_pos_is({'{', '%', '#'}) && - next_pos_is({'-'}, 2); - if (is_lstrip_block) { - string_rstrip(text, " \t\r\n"); - } - - if (!text.empty()) { - tokens.push_back({token_type::text, text, start_pos}); - continue; - } - } - - if (pos < src.size() && src[pos] == '{' && next_pos_is({'#'})) { - start_pos = pos; - pos += 2; - std::string comment; - while (pos < src.size() && - !(src[pos] == '#' && next_pos_is({'}'}))) { - if (pos + 2 >= src.size()) { - set_error(EMEL_ERR_PARSE_FAILED, pos); - break; - } - comment += src[pos++]; - } - if (!ok) { - break; - } - tokens.push_back({token_type::comment, comment, start_pos}); - pos += 2; - continue; - } - - if (pos < src.size() && src[pos] == '-' && - (last_token_type == token_type::open_expression || - last_token_type == token_type::open_statement)) { - ++pos; - if (pos >= src.size()) { - break; - } - } - - consume_while([](char c) { - return std::isspace(static_cast(c)) != 0; - }); - if (!ok) { - break; - } - if (pos >= src.size()) { - break; - } - - char ch = src[pos]; - bool is_closing_block = ch == '-' && next_pos_is({'%', '}'}); - - if (!is_closing_block && (ch == '-' || ch == '+')) { - start_pos = pos; - token_type last = tokens.empty() ? token_type::eof : tokens.back().type; - if (last == token_type::text || last == token_type::eof) { - set_error(EMEL_ERR_PARSE_FAILED, pos); - break; - } - switch (last) { - case token_type::identifier: - case token_type::numeric_literal: - case token_type::string_literal: - case token_type::close_paren: - case token_type::close_square_bracket: - break; - default: { - ++pos; - std::string num = consume_numeric(); - if (!ok) { - break; - } - std::string value; - value.reserve(num.size() + 1); - value.push_back(ch); - value += num; - token_type t = num.empty() ? token_type::unary_operator : token_type::numeric_literal; - tokens.push_back({t, std::move(value), start_pos}); - continue; - } - } - } - - bool matched = false; - for (const auto & entry : k_mapping_table) { - start_pos = pos; - if (entry.seq == "}}" && curly_bracket_depth > 0) { - continue; - } - if (pos + entry.seq.size() <= src.size() && - src.compare(pos, entry.seq.size(), entry.seq) == 0) { - tokens.push_back({entry.type, std::string(entry.seq), start_pos}); - if (entry.type == token_type::open_expression) { - curly_bracket_depth = 0; - } else if (entry.type == token_type::open_curly_bracket) { - ++curly_bracket_depth; - } else if (entry.type == token_type::close_curly_bracket) { - if (curly_bracket_depth > 0) { - --curly_bracket_depth; - } - } - pos += entry.seq.size(); - matched = true; - break; - } - } - if (matched) { - continue; - } - - if (ch == '\'' || ch == '"') { - start_pos = pos; - ++pos; - std::string str = consume_while([ch](char c) { return c != ch; }); - if (!ok) { - break; - } - tokens.push_back({token_type::string_literal, std::move(str), start_pos}); - if (pos < src.size()) { - ++pos; - } else { - set_error(EMEL_ERR_PARSE_FAILED, pos); - break; - } - continue; - } - - if (is_integer(ch)) { - start_pos = pos; - std::string num = consume_numeric(); - if (!ok) { - break; - } - tokens.push_back({token_type::numeric_literal, std::move(num), start_pos}); - continue; - } - - if (is_word(ch)) { - start_pos = pos; - std::string word = consume_while(is_word); - if (!ok) { - break; - } - tokens.push_back({token_type::identifier, std::move(word), start_pos}); - continue; - } - - set_error(EMEL_ERR_PARSE_FAILED, pos); - break; - } - - if (!ok) { - result.tokens = std::move(tokens); - return result; - } - - result.tokens = std::move(tokens); - return result; - } -}; - -} // namespace emel::text::jinja diff --git a/src/emel/text/jinja/lexer/detail.hpp b/src/emel/text/jinja/lexer/detail.hpp new file mode 100644 index 00000000..7a0bd8d2 --- /dev/null +++ b/src/emel/text/jinja/lexer/detail.hpp @@ -0,0 +1,518 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "emel/error/error.hpp" +#include "emel/text/jinja/parser/detail.hpp" +#include "emel/text/jinja/parser/errors.hpp" + +namespace emel::text::jinja::lexer::detail { + +inline constexpr int32_t error_code(const parser::error err) noexcept { + return static_cast(emel::error::cast(err)); +} + +inline void normalize_source(std::string &source) { + for (std::string::size_type pos = 0; + (pos = source.find("\r\n", pos)) != std::string::npos;) { + source.erase(pos, 1); + ++pos; + } + for (std::string::size_type pos = 0; + (pos = source.find('\r', pos)) != std::string::npos;) { + source.replace(pos, 1, 1, '\n'); + ++pos; + } + if (!source.empty() && source.back() == '\n') { + source.pop_back(); + } +} + +inline bool is_word(const char ch) noexcept { + return std::isalnum(static_cast(ch)) != 0 || ch == '_'; +} + +inline bool is_integer(const char ch) noexcept { + return std::isdigit(static_cast(ch)) != 0; +} + +inline bool is_space(const char ch) noexcept { + return std::isspace(static_cast(ch)) != 0; +} + +inline void string_lstrip(std::string &s, const char *chars) { + const size_t start = s.find_first_not_of(chars); + if (start == std::string::npos) { + s.clear(); + return; + } + s.erase(0, start); +} + +inline void string_rstrip(std::string &s, const char *chars) { + const size_t end = s.find_last_not_of(chars); + if (end == std::string::npos) { + s.clear(); + return; + } + s.erase(end + 1); +} + +inline bool next_pos_is(const std::string_view source, const size_t pos, + const std::initializer_list chars, + const size_t n = 1) noexcept { + const size_t idx = pos + n; + if (idx >= source.size()) { + return false; + } + for (const char c : chars) { + if (source[idx] == c) { + return true; + } + } + return false; +} + +inline bool decode_escape(const char ch, char &out) noexcept { + switch (ch) { + case 'n': + out = '\n'; + return true; + case 't': + out = '\t'; + return true; + case 'r': + out = '\r'; + return true; + case 'b': + out = '\b'; + return true; + case 'f': + out = '\f'; + return true; + case 'v': + out = '\v'; + return true; + case '\\': + out = '\\'; + return true; + case '\'': + out = '\''; + return true; + case '"': + out = '"'; + return true; + default: + return false; + } +} + +inline bool is_closing_block(const std::string_view source, + const size_t pos) noexcept { + return pos < source.size() && source[pos] == '-' && + next_pos_is(source, pos, {'%', '}'}); +} + +inline bool unary_prefix_allowed(const token_type last) noexcept { + switch (last) { + case token_type::identifier: + case token_type::numeric_literal: + case token_type::string_literal: + case token_type::close_paren: + case token_type::close_square_bracket: + return false; + default: + return true; + } +} + +struct mapping { + std::string_view seq = {}; + token_type type = token_type::eof; +}; + +inline constexpr std::array k_mapping_table{{ + {"{%-", token_type::open_statement}, + {"-%}", token_type::close_statement}, + {"{{-", token_type::open_expression}, + {"-}}", token_type::close_expression}, + {"{%", token_type::open_statement}, + {"%}", token_type::close_statement}, + {"{{", token_type::open_expression}, + {"}}", token_type::close_expression}, + {"(", token_type::open_paren}, + {")", token_type::close_paren}, + {"{", token_type::open_curly_bracket}, + {"}", token_type::close_curly_bracket}, + {"[", token_type::open_square_bracket}, + {"]", token_type::close_square_bracket}, + {",", token_type::comma}, + {".", token_type::dot}, + {":", token_type::colon}, + {"|", token_type::pipe}, + {"<=", token_type::comparison_binary_operator}, + {">=", token_type::comparison_binary_operator}, + {"==", token_type::comparison_binary_operator}, + {"!=", token_type::comparison_binary_operator}, + {"<", token_type::comparison_binary_operator}, + {">", token_type::comparison_binary_operator}, + {"+", token_type::additive_binary_operator}, + {"-", token_type::additive_binary_operator}, + {"~", token_type::additive_binary_operator}, + {"*", token_type::multiplicative_binary_operator}, + {"/", token_type::multiplicative_binary_operator}, + {"%", token_type::multiplicative_binary_operator}, + {"=", token_type::equals}, +}}; + +struct scan_outcome { + token token_value = {}; + bool has_token = false; + ::emel::text::jinja::lexer::cursor next_cursor = {}; + int32_t err = error_code(parser::error::none); + size_t error_pos = 0; +}; + +struct scan_plan { + std::string source = {}; + std::vector outcomes = {}; +}; + +inline bool at_text_boundary(const token_type type) noexcept { + return type == token_type::close_statement || + type == token_type::close_expression || type == token_type::comment; +} + +inline ::emel::text::jinja::lexer::cursor +emit_cursor(const ::emel::text::jinja::lexer::cursor &cursor, + const size_t next_offset, const token_type type, + const std::string_view token_text) noexcept { + ::emel::text::jinja::lexer::cursor next = cursor; + next.offset = next_offset; + next.token_index = cursor.token_index + 1; + next.last_token_type = type; + next.last_block_rstrip = false; + next.last_block_can_trim_newline = false; + + if (type == token_type::open_expression) { + next.curly_bracket_depth = 0; + } else if (type == token_type::open_curly_bracket) { + next.curly_bracket_depth = cursor.curly_bracket_depth + 1; + } else if (type == token_type::close_curly_bracket) { + next.curly_bracket_depth = + cursor.curly_bracket_depth > 0 ? cursor.curly_bracket_depth - 1 : 0; + } + + if (type == token_type::close_statement || + type == token_type::close_expression) { + next.last_block_can_trim_newline = true; + next.last_block_rstrip = token_text.size() >= 3 && token_text[0] == '-' && + token_text.back() == '}'; + } else if (type == token_type::comment) { + next.last_block_can_trim_newline = true; + } + + return next; +} + +inline void set_error(scan_outcome &out, const size_t pos) noexcept { + out.err = error_code(parser::error::parse_failed); + out.error_pos = pos; + out.has_token = false; +} + +inline std::string consume_escaped_until(const std::string_view source, + size_t &pos, const char terminal, + scan_outcome &out) { + std::string value; + while (pos < source.size() && source[pos] != terminal) { + if (source[pos] != '\\') { + value.push_back(source[pos]); + ++pos; + continue; + } + + ++pos; + if (pos >= source.size()) { + set_error(out, pos); + return value; + } + + char decoded = '\0'; + const char escaped = source[pos]; + if (!decode_escape(escaped, decoded)) { + set_error(out, pos); + return value; + } + value.push_back(decoded); + ++pos; + } + return value; +} + +inline std::string consume_numeric(const std::string_view source, size_t &pos) { + std::string value; + while (pos < source.size() && is_integer(source[pos])) { + value.push_back(source[pos]); + ++pos; + } + if (pos < source.size() && source[pos] == '.' && pos + 1 < source.size() && + is_integer(source[pos + 1])) { + value.push_back(source[pos]); + ++pos; + while (pos < source.size() && is_integer(source[pos])) { + value.push_back(source[pos]); + ++pos; + } + } + return value; +} + +inline scan_outcome +scan_next_token(const ::emel::text::jinja::lexer::cursor &cursor) { + scan_outcome out{}; + out.next_cursor = cursor; + + const std::string_view source = cursor.source; + const size_t size = source.size(); + size_t pos = cursor.offset; + + while (pos < size) { + if (at_text_boundary(cursor.last_token_type)) { + const size_t start = pos; + size_t end = start; + while (pos < size && !(source[pos] == '{' && + next_pos_is(source, pos, {'%', '{', '#'}))) { + end = ++pos; + } + + if (pos < size && source[pos] == '{' && + next_pos_is(source, pos, {'%', '#', '-'})) { + size_t current = end; + while (current > start) { + const char c = source[current - 1]; + if (current == 1) { + end = 0; + break; + } + if (c == '\n') { + end = current; + break; + } + if (!is_space(c)) { + break; + } + --current; + } + } + + std::string text = std::string(source.substr(start, end - start)); + if (cursor.last_block_can_trim_newline && !text.empty() && + text.front() == '\n') { + text.erase(text.begin()); + } + if (cursor.last_block_rstrip) { + string_lstrip(text, " \t\r\n"); + } + + const bool is_lstrip_block = pos < size && source[pos] == '{' && + next_pos_is(source, pos, {'{', '%', '#'}) && + next_pos_is(source, pos, {'-'}, 2); + if (is_lstrip_block) { + string_rstrip(text, " \t\r\n"); + } + + if (!text.empty()) { + out.has_token = true; + out.token_value = token{token_type::text, std::move(text), start}; + out.next_cursor = emit_cursor(cursor, pos, out.token_value.type, + out.token_value.value); + return out; + } + } + + if (source[pos] == '{' && next_pos_is(source, pos, {'#'})) { + const size_t start = pos; + pos += 2; + std::string comment; + while (pos < size && + !(source[pos] == '#' && next_pos_is(source, pos, {'}'}))) { + if (pos + 2 >= size) { + set_error(out, pos); + return out; + } + comment.push_back(source[pos]); + ++pos; + } + if (pos + 1 >= size) { + set_error(out, pos); + return out; + } + pos += 2; + out.has_token = true; + out.token_value = token{token_type::comment, std::move(comment), start}; + out.next_cursor = + emit_cursor(cursor, pos, out.token_value.type, out.token_value.value); + return out; + } + + if (source[pos] == '-' && + (cursor.last_token_type == token_type::open_expression || + cursor.last_token_type == token_type::open_statement)) { + ++pos; + if (pos >= size) { + out.next_cursor = cursor; + out.next_cursor.offset = pos; + return out; + } + } + + while (pos < size && is_space(source[pos])) { + ++pos; + } + if (pos >= size) { + out.next_cursor = cursor; + out.next_cursor.offset = pos; + return out; + } + + const char ch = source[pos]; + if (!is_closing_block(source, pos) && (ch == '-' || ch == '+')) { + if (cursor.last_token_type == token_type::text || + cursor.last_token_type == token_type::eof) { + set_error(out, pos); + return out; + } + if (unary_prefix_allowed(cursor.last_token_type)) { + const size_t start = pos; + ++pos; + std::string num = consume_numeric(source, pos); + std::string value; + value.reserve(num.size() + 1); + value.push_back(ch); + value += num; + const token_type type = num.empty() ? token_type::unary_operator + : token_type::numeric_literal; + out.has_token = true; + out.token_value = token{type, std::move(value), start}; + out.next_cursor = emit_cursor(cursor, pos, out.token_value.type, + out.token_value.value); + return out; + } + } + + for (const auto &entry : k_mapping_table) { + if (entry.seq == "}}" && cursor.curly_bracket_depth > 0) { + continue; + } + if (pos + entry.seq.size() <= size && + source.compare(pos, entry.seq.size(), entry.seq) == 0) { + out.has_token = true; + out.token_value = token{entry.type, std::string(entry.seq), pos}; + out.next_cursor = + emit_cursor(cursor, pos + entry.seq.size(), out.token_value.type, + out.token_value.value); + return out; + } + } + + if (ch == '\'' || ch == '"') { + const size_t start = pos; + ++pos; + std::string value = consume_escaped_until(source, pos, ch, out); + if (out.err != error_code(parser::error::none)) { + return out; + } + if (pos >= size) { + set_error(out, pos); + return out; + } + ++pos; + out.has_token = true; + out.token_value = + token{token_type::string_literal, std::move(value), start}; + out.next_cursor = + emit_cursor(cursor, pos, out.token_value.type, out.token_value.value); + return out; + } + + if (is_integer(ch)) { + const size_t start = pos; + std::string value = consume_numeric(source, pos); + out.has_token = true; + out.token_value = + token{token_type::numeric_literal, std::move(value), start}; + out.next_cursor = + emit_cursor(cursor, pos, out.token_value.type, out.token_value.value); + return out; + } + + if (is_word(ch)) { + const size_t start = pos; + std::string value; + while (pos < size && is_word(source[pos])) { + value.push_back(source[pos]); + ++pos; + } + out.has_token = true; + out.token_value = token{token_type::identifier, std::move(value), start}; + out.next_cursor = + emit_cursor(cursor, pos, out.token_value.type, out.token_value.value); + return out; + } + + set_error(out, pos); + return out; + } + + out.has_token = false; + out.next_cursor = cursor; + out.next_cursor.offset = pos; + return out; +} + +inline scan_outcome +scan_next_token_safe(const ::emel::text::jinja::lexer::cursor &cursor) { + const bool invalid_source = + cursor.source.data() == nullptr && !cursor.source.empty(); + const bool invalid_offset = cursor.offset > cursor.source.size(); + return (invalid_source || invalid_offset) ? scan_outcome{} + : scan_next_token(cursor); +} + +inline scan_plan build_scan_plan(const std::string_view source_text) { + scan_plan plan{}; + plan.source = std::string(source_text); + normalize_source(plan.source); + + ::emel::text::jinja::lexer::cursor cursor{ + plan.source, + 0, + 0, + 0, + ::emel::text::jinja::token_type::close_statement, + false, + false, + }; + + for (;;) { + const scan_outcome scan = scan_next_token_safe(cursor); + plan.outcomes.push_back(scan); + const bool terminal = + scan.err != error_code(parser::error::none) || !scan.has_token; + if (terminal) { + break; + } + cursor = scan.next_cursor; + } + + return plan; +} + +} // namespace emel::text::jinja::lexer::detail diff --git a/src/emel/text/jinja/parser/actions.hpp b/src/emel/text/jinja/parser/actions.hpp index 97327291..f0b67dd4 100644 --- a/src/emel/text/jinja/parser/actions.hpp +++ b/src/emel/text/jinja/parser/actions.hpp @@ -1,116 +1,256 @@ #pragma once -#include "emel/emel.h" -#include "emel/text/jinja/lexer.hpp" +#include + +#include "emel/callback.hpp" #include "emel/text/jinja/parser/context.hpp" -#include "emel/text/jinja/parser/detail.hpp" +#include "emel/text/jinja/parser/errors.hpp" #include "emel/text/jinja/parser/events.hpp" +#include "emel/text/jinja/parser/lexer/detail.hpp" namespace emel::text::jinja::parser::action { +namespace helper { + +inline void reset_result(const event::parse &request, + event::parse_ctx &ctx) noexcept { + ctx.err = error::none; + ctx.error_pos = 0; + ctx.phase = event::parse_phase::request_validation; + ctx.statement = event::statement_kind::unknown; + ctx.expression = event::expression_kind::unknown; + ctx.token_index = 0; + ctx.statement_start = 0; + ctx.expression_start = 0; + ctx.expression_value_index = 0; + ctx.lex_cursor = {}; + ctx.lex_token = {}; + ctx.lex_has_token = false; + ctx.error_out = parser::to_error_code(error::none); + ctx.error_pos_out = 0; + ctx.lex_result.tokens.clear(); + ctx.lex_result.error = parser::to_error_code(error::none); + ctx.lex_result.error_pos = 0; + ctx.lex_plan_index = 0; + + request.program.body.clear(); + request.program.last_error = parser::to_error_code(error::none); + request.program.last_error_pos = 0; +} + +inline void mark_done(const event::parse &request, + event::parse_ctx &ctx) noexcept { + ctx.err = error::none; + ctx.error_pos = 0; + ctx.phase = event::parse_phase::none; + ctx.error_out = parser::to_error_code(error::none); + ctx.error_pos_out = 0; + request.program.last_error = parser::to_error_code(error::none); + request.program.last_error_pos = 0; +} + +inline void mark_error(const event::parse &request, event::parse_ctx &ctx, + const error err, const size_t error_pos) noexcept { + ctx.err = err; + ctx.error_pos = error_pos; + ctx.phase = event::parse_phase::none; + ctx.error_out = parser::to_error_code(err); + ctx.error_pos_out = error_pos; + + request.program.body.clear(); + request.program.last_error = parser::to_error_code(err); + request.program.last_error_pos = error_pos; +} + +inline void emit_done(const event::parse &request, + const event::parse_ctx &) noexcept { + const events::parsing_done done_ev{ + request, + }; + (void)request.dispatch_done(done_ev); +} + +inline void emit_error(const event::parse &request, + const event::parse_ctx &ctx) noexcept { + const events::parsing_error error_ev{ + request, + parser::to_error_code(ctx.err), + ctx.error_pos, + }; + (void)request.dispatch_error(error_ev); +} + +} // namespace helper + +inline bool on_lexer_done( + void *owner, + const ::emel::text::jinja::lexer::events::next_done &ev) noexcept { + auto *ctx = static_cast(owner); + ctx->err = error::none; + ctx->error_pos = 0; + ctx->lex_result.error = parser::to_error_code(error::none); + ctx->lex_result.error_pos = 0; + ctx->lex_token = ev.token; + ctx->lex_has_token = ev.has_token; + ctx->lex_cursor = ev.next_cursor; + return true; +} + +inline bool on_lexer_error( + void *owner, + const ::emel::text::jinja::lexer::events::next_error &ev) noexcept { + auto *ctx = static_cast(owner); + ctx->err = static_cast(static_cast(ev.err)); + ctx->error_pos = ev.error_pos; + ctx->lex_result.error = ev.err; + ctx->lex_result.error_pos = ev.error_pos; + ctx->lex_token = {}; + ctx->lex_has_token = false; + return true; +} + +namespace runtime_detail { + +constexpr const event::parse_runtime & +unwrap_runtime_event(const event::parse_runtime &ev) noexcept { + return ev; +} + +template + requires requires(const wrapped_event_type &ev) { ev.event_; } +constexpr decltype(auto) +unwrap_runtime_event(const wrapped_event_type &ev) noexcept { + return ev.event_; +} + +} // namespace runtime_detail + struct reject_invalid_parse { - void operator()(const emel::text::jinja::event::parse & ev, - context & ctx) const noexcept { - ctx.last_error = EMEL_ERR_INVALID_ARGUMENT; - ctx.phase_error = EMEL_ERR_INVALID_ARGUMENT; - if (ev.error_out != nullptr) { - *ev.error_out = EMEL_ERR_INVALID_ARGUMENT; - } - if (ev.program_out != nullptr) { - ev.program_out->reset(); - } - if (ev.dispatch_error) { - ev.dispatch_error( - emel::text::jinja::events::parsing_error{&ev, EMEL_ERR_INVALID_ARGUMENT}); - } + template + void operator()(const runtime_event_type &ev, context &) const noexcept { + const auto &runtime_ev = runtime_detail::unwrap_runtime_event(ev); + helper::mark_error(runtime_ev.request, runtime_ev.ctx, + error::invalid_request, 0); + } +}; + +struct begin_parse { + template + void operator()(const runtime_event_type &ev, context &) const noexcept { + const auto &runtime_ev = runtime_detail::unwrap_runtime_event(ev); + helper::reset_result(runtime_ev.request, runtime_ev.ctx); + } +}; + +struct begin_tokenization { + template + void operator()(const runtime_event_type &ev, context &) const noexcept { + const auto &runtime_ev = runtime_detail::unwrap_runtime_event(ev); + runtime_ev.ctx.phase = event::parse_phase::tokenization; + runtime_ev.ctx.lex_result.tokens.clear(); + runtime_ev.ctx.lex_result.error = parser::to_error_code(error::none); + runtime_ev.ctx.lex_result.error_pos = 0; + runtime_ev.ctx.lex_plan_index = 0; + runtime_ev.ctx.lex_cursor = ::emel::text::jinja::lexer::cursor{ + runtime_ev.ctx.lex_result.source, + 0, + 0, + 0, + ::emel::text::jinja::token_type::close_statement, + false, + false, + }; + runtime_ev.ctx.lex_token = {}; + runtime_ev.ctx.lex_has_token = false; + } +}; + +struct request_next_lex_token { + template + void operator()(const runtime_event_type &ev, context &ctx) const noexcept { + const auto &runtime_ev = runtime_detail::unwrap_runtime_event(ev); + const auto &scan = runtime_ev.ctx.lex_plan[runtime_ev.ctx.lex_plan_index]; + runtime_ev.ctx.lex_plan_index += 1; + + runtime_ev.ctx.err = error::internal_error; + runtime_ev.ctx.error_pos = 0; + runtime_ev.ctx.lex_token = {}; + runtime_ev.ctx.lex_has_token = false; + const callback + done_cb{ + &runtime_ev.ctx, + on_lexer_done, + }; + const callback + error_cb{ + &runtime_ev.ctx, + on_lexer_error, + }; + const ::emel::text::jinja::lexer::event::next next_ev{ + runtime_ev.ctx.lex_cursor, + done_cb, + error_cb, + }; + const ::emel::text::jinja::parser::lexer::event::next_runtime + runtime_next_ev{ + next_ev, + scan, + }; + (void)ctx.lexer.process_event(runtime_next_ev); + } +}; + +struct append_lex_token { + template + void operator()(const runtime_event_type &ev, context &) const noexcept { + const auto &runtime_ev = runtime_detail::unwrap_runtime_event(ev); + runtime_ev.ctx.lex_result.tokens.push_back(runtime_ev.ctx.lex_token); + } +}; + +struct commit_lex_error { + template + void operator()(const runtime_event_type &ev, context &) const noexcept { + const auto &runtime_ev = runtime_detail::unwrap_runtime_event(ev); + helper::mark_error(runtime_ev.request, runtime_ev.ctx, runtime_ev.ctx.err, + runtime_ev.ctx.error_pos); } }; -struct run_parse { - void operator()(const emel::text::jinja::event::parse & ev, - context & ctx) const noexcept { - ctx.phase_error = EMEL_OK; - ctx.last_error = EMEL_OK; - - if (ev.error_out != nullptr) { - *ev.error_out = EMEL_OK; - } - if (ev.program_out != nullptr) { - ev.program_out->reset(); - } - - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result lex_res = lex.tokenize(ev.template_text); - if (lex_res.error != EMEL_OK) { - ctx.phase_error = EMEL_ERR_PARSE_FAILED; - ctx.last_error = ctx.phase_error; - if (ev.error_out != nullptr) { - *ev.error_out = ctx.last_error; - } - if (ev.program_out != nullptr) { - ev.program_out->reset(); - ev.program_out->last_error = ctx.last_error; - ev.program_out->last_error_pos = lex_res.error_pos; - } - if (ev.dispatch_error) { - ev.dispatch_error( - emel::text::jinja::events::parsing_error{&ev, ctx.last_error}); - } - return; - } - - if (ev.program_out == nullptr) { - ctx.phase_error = EMEL_ERR_INVALID_ARGUMENT; - ctx.last_error = ctx.phase_error; - if (ev.error_out != nullptr) { - *ev.error_out = ctx.last_error; - } - if (ev.dispatch_error) { - ev.dispatch_error( - emel::text::jinja::events::parsing_error{&ev, ctx.last_error}); - } - return; - } - - detail::recursive_descent_parser parser{*ev.program_out}; - if (!parser.parse(lex_res)) { - ctx.phase_error = EMEL_ERR_PARSE_FAILED; - ctx.last_error = ctx.phase_error; - if (ev.error_out != nullptr) { - *ev.error_out = ctx.last_error; - } - if (ev.program_out != nullptr) { - ev.program_out->last_error = ctx.last_error; - ev.program_out->last_error_pos = parser.error_pos(); - } - if (ev.dispatch_error) { - ev.dispatch_error( - emel::text::jinja::events::parsing_error{&ev, ctx.last_error}); - } - return; - } - - ctx.phase_error = EMEL_OK; - ctx.last_error = EMEL_OK; - if (ev.error_out != nullptr) { - *ev.error_out = EMEL_OK; - } - if (ev.dispatch_done) { - ev.dispatch_done(emel::text::jinja::events::parsing_done{&ev}); - } +struct dispatch_done { + template + void operator()(const runtime_event_type &ev, context &) const noexcept { + const auto &runtime_ev = runtime_detail::unwrap_runtime_event(ev); + helper::emit_done(runtime_ev.request, runtime_ev.ctx); + } +}; + +struct dispatch_error { + template + void operator()(const runtime_event_type &ev, context &) const noexcept { + const auto &runtime_ev = runtime_detail::unwrap_runtime_event(ev); + helper::emit_error(runtime_ev.request, runtime_ev.ctx); } }; struct on_unexpected { - template - void operator()(const event &, context & ctx) const noexcept { - ctx.phase_error = EMEL_ERR_BACKEND; - ctx.last_error = EMEL_ERR_BACKEND; + void operator()(const event::parse_runtime &ev, context &) const noexcept { + helper::mark_error(ev.request, ev.ctx, error::internal_error, + ev.ctx.error_pos_out); } + + template + void operator()(const event_type &, context &) const noexcept {} }; inline constexpr reject_invalid_parse reject_invalid_parse{}; -inline constexpr run_parse run_parse{}; +inline constexpr begin_parse begin_parse{}; +inline constexpr begin_tokenization begin_tokenization{}; +inline constexpr request_next_lex_token request_next_lex_token{}; +inline constexpr append_lex_token append_lex_token{}; +inline constexpr commit_lex_error commit_lex_error{}; +inline constexpr dispatch_done dispatch_done{}; +inline constexpr dispatch_error dispatch_error{}; inline constexpr on_unexpected on_unexpected{}; } // namespace emel::text::jinja::parser::action diff --git a/src/emel/text/jinja/parser/classifier_parser/actions.hpp b/src/emel/text/jinja/parser/classifier_parser/actions.hpp new file mode 100644 index 00000000..9d3fb604 --- /dev/null +++ b/src/emel/text/jinja/parser/classifier_parser/actions.hpp @@ -0,0 +1,106 @@ +#pragma once + +#include "emel/text/jinja/parser/actions.hpp" +#include "emel/text/jinja/parser/classifier_parser/context.hpp" +#include "emel/text/jinja/parser/classifier_parser/errors.hpp" +#include "emel/text/jinja/parser/events.hpp" + +namespace emel::text::jinja::parser::classifier_parser::action { + +struct begin_classification { + void operator()(const event::parse_runtime & ev, context &) const noexcept { + ev.ctx.phase = event::parse_phase::statement_classification; + ev.ctx.statement = event::statement_kind::unknown; + ev.ctx.expression = event::expression_kind::unknown; + ev.ctx.token_index = 0; + } +}; + +struct set_statement_text { + void operator()(const event::parse_runtime & ev, context &) const noexcept { + ev.ctx.statement = event::statement_kind::text; + ev.ctx.expression = event::expression_kind::unknown; + } +}; + +struct set_statement_comment { + void operator()(const event::parse_runtime & ev, context &) const noexcept { + ev.ctx.statement = event::statement_kind::comment; + ev.ctx.expression = event::expression_kind::unknown; + } +}; + +struct set_statement_expression { + void operator()(const event::parse_runtime & ev, context &) const noexcept { + ev.ctx.statement = event::statement_kind::expression; + } +}; + +struct set_statement_statement { + void operator()(const event::parse_runtime & ev, context &) const noexcept { + ev.ctx.statement = event::statement_kind::statement; + ev.ctx.expression = event::expression_kind::unknown; + } +}; + +struct set_statement_unknown { + void operator()(const event::parse_runtime & ev, context &) const noexcept { + ev.ctx.statement = event::statement_kind::unknown; + ev.ctx.expression = event::expression_kind::unknown; + } +}; + +struct set_expression_literal { + void operator()(const event::parse_runtime & ev, context &) const noexcept { + ev.ctx.expression = event::expression_kind::literal; + } +}; + +struct set_expression_identifier { + void operator()(const event::parse_runtime & ev, context &) const noexcept { + ev.ctx.expression = event::expression_kind::identifier; + } +}; + +struct set_expression_unary { + void operator()(const event::parse_runtime & ev, context &) const noexcept { + ev.ctx.expression = event::expression_kind::unary; + } +}; + +struct set_expression_compound { + void operator()(const event::parse_runtime & ev, context &) const noexcept { + ev.ctx.expression = event::expression_kind::compound; + } +}; + +struct set_expression_unknown { + void operator()(const event::parse_runtime & ev, context &) const noexcept { + ev.ctx.expression = event::expression_kind::unknown; + } +}; + +struct on_unexpected { + void operator()(const event::parse_runtime & ev, context &) const noexcept { + ::emel::text::jinja::parser::action::helper::mark_error( + ev.request, ev.ctx, error::internal_error, ev.ctx.error_pos_out); + } + + template + void operator()(const event_type &, context &) const noexcept {} +}; + +inline constexpr begin_classification begin_classification{}; +inline constexpr set_statement_text set_statement_text{}; +inline constexpr set_statement_comment set_statement_comment{}; +inline constexpr set_statement_expression set_statement_expression{}; +inline constexpr set_statement_statement set_statement_statement{}; +inline constexpr set_statement_unknown set_statement_unknown{}; +inline constexpr set_expression_literal set_expression_literal{}; +inline constexpr set_expression_identifier set_expression_identifier{}; +inline constexpr set_expression_unary set_expression_unary{}; +inline constexpr set_expression_compound set_expression_compound{}; +inline constexpr set_expression_unknown set_expression_unknown{}; +inline constexpr on_unexpected on_unexpected{}; + +} // namespace emel::text::jinja::parser::classifier_parser::action diff --git a/src/emel/text/jinja/parser/classifier_parser/context.hpp b/src/emel/text/jinja/parser/classifier_parser/context.hpp new file mode 100644 index 00000000..be638d59 --- /dev/null +++ b/src/emel/text/jinja/parser/classifier_parser/context.hpp @@ -0,0 +1,11 @@ +#pragma once + +namespace emel::text::jinja::parser::action { +struct context; +} + +namespace emel::text::jinja::parser::classifier_parser::action { + +using context = emel::text::jinja::parser::action::context; + +} // namespace emel::text::jinja::parser::classifier_parser::action diff --git a/src/emel/text/jinja/parser/classifier_parser/errors.hpp b/src/emel/text/jinja/parser/classifier_parser/errors.hpp new file mode 100644 index 00000000..38b99f0c --- /dev/null +++ b/src/emel/text/jinja/parser/classifier_parser/errors.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include "emel/text/jinja/parser/errors.hpp" + +namespace emel::text::jinja::parser::classifier_parser { + +using error = emel::text::jinja::parser::error; + +} // namespace emel::text::jinja::parser::classifier_parser diff --git a/src/emel/text/jinja/parser/classifier_parser/guards.hpp b/src/emel/text/jinja/parser/classifier_parser/guards.hpp new file mode 100644 index 00000000..634a2f19 --- /dev/null +++ b/src/emel/text/jinja/parser/classifier_parser/guards.hpp @@ -0,0 +1,131 @@ +#pragma once + +#include + +#include "emel/text/jinja/parser/classifier_parser/context.hpp" +#include "emel/text/jinja/parser/classifier_parser/errors.hpp" +#include "emel/text/jinja/parser/events.hpp" + +namespace emel::text::jinja::parser::classifier_parser::guard { + +inline bool has_token(const event::parse_ctx & ctx, + const size_t offset = 0) noexcept { + return ctx.token_index + offset < ctx.lex_result.tokens.size(); +} + +inline bool token_is(const event::parse_ctx & ctx, + const emel::text::jinja::token_type type, + const size_t offset = 0) noexcept { + return has_token(ctx, offset) && + ctx.lex_result.tokens[ctx.token_index + offset].type == type; +} + +struct no_tokens { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return !has_token(ev.ctx); + } +}; + +struct token_text { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::text); + } +}; + +struct token_comment { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::comment); + } +}; + +struct token_open_expression { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::open_expression); + } +}; + +struct token_open_statement { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::open_statement); + } +}; + +struct token_unknown { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return has_token(ev.ctx) && + !token_text{}(ev, action::context{}) && + !token_comment{}(ev, action::context{}) && + !token_open_expression{}(ev, action::context{}) && + !token_open_statement{}(ev, action::context{}); + } +}; + +struct statement_expression { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return ev.ctx.statement == event::statement_kind::expression; + } +}; + +struct statement_not_expression { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return ev.ctx.statement != event::statement_kind::expression; + } +}; + +struct expr_no_token { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return !has_token(ev.ctx, 1); + } +}; + +struct expr_token_literal { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::numeric_literal, 1) || + token_is(ev.ctx, emel::text::jinja::token_type::string_literal, 1) || + token_is(ev.ctx, emel::text::jinja::token_type::open_square_bracket, 1) || + token_is(ev.ctx, emel::text::jinja::token_type::open_curly_bracket, 1); + } +}; + +struct expr_token_identifier { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::identifier, 1); + } +}; + +struct expr_token_unary { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::additive_binary_operator, 1) || + token_is(ev.ctx, emel::text::jinja::token_type::unary_operator, 1); + } +}; + +struct expr_token_compound { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::open_paren, 1); + } +}; + +struct expr_token_unknown { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return has_token(ev.ctx, 1) && + !expr_token_literal{}(ev, action::context{}) && + !expr_token_identifier{}(ev, action::context{}) && + !expr_token_unary{}(ev, action::context{}) && + !expr_token_compound{}(ev, action::context{}); + } +}; + +struct phase_ok { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return ev.ctx.err == error::none; + } +}; + +struct phase_failed { + bool operator()(const event::parse_runtime & ev, const action::context &) const noexcept { + return ev.ctx.err != error::none; + } +}; + +} // namespace emel::text::jinja::parser::classifier_parser::guard diff --git a/src/emel/text/jinja/parser/classifier_parser/sm.hpp b/src/emel/text/jinja/parser/classifier_parser/sm.hpp new file mode 100644 index 00000000..76c110ef --- /dev/null +++ b/src/emel/text/jinja/parser/classifier_parser/sm.hpp @@ -0,0 +1,103 @@ +#pragma once + +#include "emel/sm.hpp" +#include "emel/text/jinja/parser/classifier_parser/actions.hpp" +#include "emel/text/jinja/parser/classifier_parser/guards.hpp" +#include "emel/text/jinja/parser/events.hpp" + +namespace emel::text::jinja::parser::classifier_parser { + +struct deciding {}; +struct statement_decision {}; +struct expression_decision {}; +struct classified {}; +struct unexpected_event {}; + +struct model { + auto operator()() const { + namespace sml = boost::sml; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Classification setup. + sml::state <= *sml::state + + sml::completion + / action::begin_classification + + //------------------------------------------------------------------------------// + // Statement classifier. + , sml::state <= sml::state + + sml::completion[ guard::no_tokens{} ] + / action::set_statement_unknown + + , sml::state <= sml::state + + sml::completion[ guard::token_text{} ] + / action::set_statement_text + + , sml::state <= sml::state + + sml::completion[ guard::token_comment{} ] + / action::set_statement_comment + + , sml::state <= sml::state + + sml::completion[ guard::token_open_expression{} ] + / action::set_statement_expression + + , sml::state <= sml::state + + sml::completion[ guard::token_open_statement{} ] + / action::set_statement_statement + + , sml::state <= sml::state + + sml::completion[ guard::token_unknown{} ] + / action::set_statement_unknown + + //------------------------------------------------------------------------------// + // Expression classifier. + , sml::state <= sml::state + + sml::completion[ guard::expr_no_token{} ] + / action::set_expression_unknown + + , sml::state <= sml::state + + sml::completion[ guard::expr_token_literal{} ] + / action::set_expression_literal + + , sml::state <= sml::state + + sml::completion[ guard::expr_token_identifier{} ] + / action::set_expression_identifier + + , sml::state <= sml::state + + sml::completion[ guard::expr_token_unary{} ] + / action::set_expression_unary + + , sml::state <= sml::state + + sml::completion[ guard::expr_token_compound{} ] + / action::set_expression_compound + + , sml::state <= sml::state + + sml::completion[ guard::expr_token_unknown{} ] + / action::set_expression_unknown + + //------------------------------------------------------------------------------// + , sml::X <= sml::state + + //------------------------------------------------------------------------------// + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + ); + // clang-format on + } +}; + +struct sm : emel::sm { + using model_type = model; +}; + +} // namespace emel::text::jinja::parser::classifier_parser diff --git a/src/emel/text/jinja/parser/context.hpp b/src/emel/text/jinja/parser/context.hpp index 12bb1dc8..f177518a 100644 --- a/src/emel/text/jinja/parser/context.hpp +++ b/src/emel/text/jinja/parser/context.hpp @@ -1,14 +1,11 @@ #pragma once -#include - -#include "emel/emel.h" +#include "emel/text/jinja/parser/lexer/sm.hpp" namespace emel::text::jinja::parser::action { struct context { - int32_t phase_error = EMEL_OK; - int32_t last_error = EMEL_OK; + emel::text::jinja::parser::lexer::sm lexer = {}; }; } // namespace emel::text::jinja::parser::action diff --git a/src/emel/text/jinja/parser/detail.hpp b/src/emel/text/jinja/parser/detail.hpp index 98dfe260..ee7916a1 100644 --- a/src/emel/text/jinja/parser/detail.hpp +++ b/src/emel/text/jinja/parser/detail.hpp @@ -2,741 +2,384 @@ #include #include -#include +#include #include #include #include #include -#include "emel/emel.h" -#include "emel/text/jinja/ast.hpp" -#include "emel/text/jinja/lexer.hpp" - -namespace emel::text::jinja::parser::detail { - -template -static bool is_type(const emel::text::jinja::ast_ptr & ptr) { - return dynamic_cast(ptr.get()) != nullptr; -} - -class recursive_descent_parser { - public: - explicit recursive_descent_parser(emel::text::jinja::program & out_program) - : program_(out_program) {} - - bool parse(const emel::text::jinja::lexer_result & lexer_res) { - tokens_ = &lexer_res.tokens; - current_ = 0; - error_ = EMEL_OK; - error_pos_ = 0; - program_.reset(); - - while (current_ < tokens_->size() && ok()) { - auto stmt = parse_any(); - if (!ok() || !stmt) { - break; - } - program_.body.push_back(std::move(stmt)); - } - - if (!ok()) { - program_.reset(); - program_.last_error = error_; - program_.last_error_pos = error_pos_; - return false; - } - - program_.last_error = EMEL_OK; - program_.last_error_pos = 0; - return true; - } +#include "emel/callback.hpp" +#include "emel/text/jinja/parser/errors.hpp" + +namespace emel::text::jinja { + +enum class token_type : uint8_t { + eof = 0, + text, + numeric_literal, + string_literal, + identifier, + equals, + open_paren, + close_paren, + open_statement, + close_statement, + open_expression, + close_expression, + open_square_bracket, + close_square_bracket, + open_curly_bracket, + close_curly_bracket, + comma, + dot, + colon, + pipe, + call_operator, + additive_binary_operator, + multiplicative_binary_operator, + comparison_binary_operator, + unary_operator, + comment +}; - int32_t error() const noexcept { return error_; } - size_t error_pos() const noexcept { return error_pos_; } +struct token { + token_type type = token_type::eof; + std::string value; + size_t pos = 0; +}; - private: - const emel::text::jinja::token & peek(size_t offset = 0) const { - static const emel::text::jinja::token end_token{emel::text::jinja::token_type::eof, "", 0}; - if (tokens_ == nullptr || current_ + offset >= tokens_->size()) { - return end_token; - } - return (*tokens_)[current_ + offset]; - } +struct lexer_result { + std::vector tokens; + std::string source; + int32_t error = parser::to_error_code(parser::error::none); + size_t error_pos = 0; +}; - bool ok() const noexcept { return error_ == EMEL_OK; } +namespace lexer::events { - void set_error(int32_t code, size_t pos) { - if (error_ != EMEL_OK) { - return; - } - error_ = code; - error_pos_ = pos; - } +struct next_done; +struct next_error; - bool expect(emel::text::jinja::token_type type) { - if (!ok()) { - return false; - } - const auto & t = peek(); - if (t.type != type) { - set_error(EMEL_ERR_PARSE_FAILED, t.pos); - return false; - } - ++current_; - return true; - } +} // namespace lexer::events - bool expect_identifier(std::string_view name) { - if (!ok()) { - return false; - } - const auto & t = peek(); - if (t.type != emel::text::jinja::token_type::identifier || t.value != name) { - set_error(EMEL_ERR_PARSE_FAILED, t.pos); - return false; - } - ++current_; - return true; - } +namespace lexer { - bool is(emel::text::jinja::token_type type) const { - return peek().type == type; - } +struct cursor { + std::string_view source = {}; + size_t offset = 0; + size_t token_index = 0; + size_t curly_bracket_depth = 0; + token_type last_token_type = token_type::close_statement; + bool last_block_rstrip = false; + bool last_block_can_trim_newline = false; +}; - bool is_identifier(std::string_view name) const { - const auto & t = peek(); - return t.type == emel::text::jinja::token_type::identifier && t.value == name; - } +} // namespace lexer - bool is_statement(std::initializer_list names) const { - if (peek(0).type != emel::text::jinja::token_type::open_statement || - peek(1).type != emel::text::jinja::token_type::identifier) { - return false; - } - const std::string & val = peek(1).value; - for (const auto & name : names) { - if (val == name) { - return true; - } - } - return false; - } +namespace lexer::event { - template - emel::text::jinja::ast_ptr make_node(size_t start_pos, args &&... args_in) { - auto node = std::make_unique(std::forward(args_in)...); - if (tokens_ != nullptr && start_pos < tokens_->size()) { - node->pos = (*tokens_)[start_pos].pos; - } else { - node->pos = 0; - } - return node; - } +struct next { + using done_callback = ::emel::callback; + using error_callback = ::emel::callback; - emel::text::jinja::ast_ptr parse_any() { - size_t start_pos = current_; - const auto & t = peek(); - switch (t.type) { - case emel::text::jinja::token_type::comment: - ++current_; - return make_node(start_pos, t.value); - case emel::text::jinja::token_type::text: - ++current_; - return make_node(start_pos, t.value); - case emel::text::jinja::token_type::open_statement: - return parse_jinja_statement(); - case emel::text::jinja::token_type::open_expression: - return parse_jinja_expression(); - default: - set_error(EMEL_ERR_PARSE_FAILED, t.pos); - return nullptr; - } - } + next(const lexer::cursor &cursor_ref, const done_callback &dispatch_done_ref, + const error_callback &dispatch_error_ref) noexcept + : cursor(cursor_ref), dispatch_done(dispatch_done_ref), + dispatch_error(dispatch_error_ref) {} - emel::text::jinja::ast_ptr parse_jinja_expression() { - if (!expect(emel::text::jinja::token_type::open_expression)) { - return nullptr; - } - auto result = parse_expression(); - if (!expect(emel::text::jinja::token_type::close_expression)) { - return nullptr; - } - return result; - } + const lexer::cursor &cursor; + const done_callback &dispatch_done; + const error_callback &dispatch_error; +}; - emel::text::jinja::ast_ptr parse_jinja_statement() { - if (!expect(emel::text::jinja::token_type::open_statement)) { - return nullptr; - } - - const auto & name_token = peek(); - if (name_token.type != emel::text::jinja::token_type::identifier) { - set_error(EMEL_ERR_PARSE_FAILED, name_token.pos); - return nullptr; - } - - size_t start_pos = current_; - std::string name = name_token.value; - ++current_; - - emel::text::jinja::ast_ptr result; - if (name == "set") { - result = parse_set_statement(start_pos); - } else if (name == "if") { - result = parse_if_statement(start_pos); - if (!expect(emel::text::jinja::token_type::open_statement)) { - return nullptr; - } - if (!expect_identifier("endif")) { - return nullptr; - } - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - } else if (name == "macro") { - result = parse_macro_statement(start_pos); - if (!expect(emel::text::jinja::token_type::open_statement)) { - return nullptr; - } - if (!expect_identifier("endmacro")) { - return nullptr; - } - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - } else if (name == "for") { - result = parse_for_statement(start_pos); - if (!expect(emel::text::jinja::token_type::open_statement)) { - return nullptr; - } - if (!expect_identifier("endfor")) { - return nullptr; - } - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - } else if (name == "break") { - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - result = make_node(start_pos); - } else if (name == "continue") { - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - result = make_node(start_pos); - } else if (name == "call") { - emel::text::jinja::ast_list caller_args; - if (is(emel::text::jinja::token_type::open_paren)) { - caller_args = parse_args(); - } - auto callee = parse_primary_expression(); - if (!ok() || !callee || !is_type(callee)) { - set_error(EMEL_ERR_PARSE_FAILED, peek().pos); - return nullptr; - } - auto call_args = parse_args(); - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - - emel::text::jinja::ast_list body; - while (ok() && !is_statement({"endcall"})) { - body.push_back(parse_any()); - } - if (!expect(emel::text::jinja::token_type::open_statement)) { - return nullptr; - } - if (!expect_identifier("endcall")) { - return nullptr; - } - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - - auto call_expr = make_node( - start_pos, std::move(callee), std::move(call_args)); - result = make_node( - start_pos, std::move(call_expr), std::move(caller_args), std::move(body)); - } else if (name == "filter") { - auto filter_node = parse_primary_expression(); - if (!ok()) { - return nullptr; - } - if (filter_node && is_type(filter_node) && - is(emel::text::jinja::token_type::open_paren)) { - filter_node = parse_call_expression(std::move(filter_node)); - } - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - - emel::text::jinja::ast_list body; - while (ok() && !is_statement({"endfilter"})) { - body.push_back(parse_any()); - } - if (!expect(emel::text::jinja::token_type::open_statement)) { - return nullptr; - } - if (!expect_identifier("endfilter")) { - return nullptr; - } - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - result = make_node( - start_pos, std::move(filter_node), std::move(body)); - } else if (name == "generation" || name == "endgeneration") { - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - result = make_node(start_pos); - } else { - set_error(EMEL_ERR_PARSE_FAILED, name_token.pos); - return nullptr; - } - return result; - } +} // namespace lexer::event - emel::text::jinja::ast_ptr parse_set_statement(size_t start_pos) { - auto left = parse_expression_sequence(); - emel::text::jinja::ast_ptr value = nullptr; - emel::text::jinja::ast_list body; - - if (is(emel::text::jinja::token_type::equals)) { - ++current_; - value = parse_expression_sequence(); - } else { - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - while (ok() && !is_statement({"endset"})) { - body.push_back(parse_any()); - } - if (!expect(emel::text::jinja::token_type::open_statement)) { - return nullptr; - } - if (!expect_identifier("endset")) { - return nullptr; - } - } - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - return make_node( - start_pos, std::move(left), std::move(value), std::move(body)); - } +namespace lexer::events { - emel::text::jinja::ast_ptr parse_if_statement(size_t start_pos) { - auto test = parse_expression(); - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - - emel::text::jinja::ast_list body; - emel::text::jinja::ast_list alternate; - - while (ok() && !is_statement({"elif", "else", "endif"})) { - body.push_back(parse_any()); - } - - if (ok() && is_statement({"elif"})) { - size_t pos0 = current_; - current_ += 2; - alternate.push_back(parse_if_statement(pos0)); - } else if (ok() && is_statement({"else"})) { - current_ += 2; - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - while (ok() && !is_statement({"endif"})) { - alternate.push_back(parse_any()); - } - } - return make_node( - start_pos, std::move(test), std::move(body), std::move(alternate)); - } +struct next_done { + const event::next &request; + emel::text::jinja::token token = {}; + bool has_token = false; + lexer::cursor next_cursor = {}; +}; - emel::text::jinja::ast_ptr parse_macro_statement(size_t start_pos) { - auto name = parse_primary_expression(); - auto args = parse_args(); - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - emel::text::jinja::ast_list body; - while (ok() && !is_statement({"endmacro"})) { - body.push_back(parse_any()); - } - return make_node( - start_pos, std::move(name), std::move(args), std::move(body)); - } +struct next_error { + const event::next &request; + int32_t err = 0; + size_t error_pos = 0; +}; - emel::text::jinja::ast_ptr parse_expression_sequence(bool primary = false) { - size_t start_pos = current_; - emel::text::jinja::ast_list exprs; - exprs.push_back(primary ? parse_primary_expression() : parse_expression()); - bool is_tuple = is(emel::text::jinja::token_type::comma); - while (ok() && is(emel::text::jinja::token_type::comma)) { - ++current_; - exprs.push_back(primary ? parse_primary_expression() : parse_expression()); - } - if (!ok()) { - return nullptr; - } - return is_tuple ? make_node(start_pos, std::move(exprs)) - : std::move(exprs[0]); - } +} // namespace lexer::events - emel::text::jinja::ast_ptr parse_for_statement(size_t start_pos) { - auto loop_var = parse_expression_sequence(true); - if (!is_identifier("in")) { - set_error(EMEL_ERR_PARSE_FAILED, peek().pos); - return nullptr; - } - ++current_; - auto iterable = parse_expression(); - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - - emel::text::jinja::ast_list body; - emel::text::jinja::ast_list alternate; - - while (ok() && !is_statement({"endfor", "else"})) { - body.push_back(parse_any()); - } - - if (ok() && is_statement({"else"})) { - current_ += 2; - if (!expect(emel::text::jinja::token_type::close_statement)) { - return nullptr; - } - while (ok() && !is_statement({"endfor"})) { - alternate.push_back(parse_any()); - } - } - return make_node( - start_pos, std::move(loop_var), std::move(iterable), - std::move(body), std::move(alternate)); - } +struct ast_node { + size_t pos = 0; + virtual ~ast_node() = default; +}; - emel::text::jinja::ast_ptr parse_expression() { - return parse_if_expression(); - } +using ast_ptr = std::unique_ptr; +using ast_list = std::vector; +using ast_pair_list = std::vector>; - emel::text::jinja::ast_ptr parse_if_expression() { - auto a = parse_logical_or_expression(); - if (is_identifier("if")) { - size_t start_pos = current_; - ++current_; - auto test = parse_logical_or_expression(); - if (is_identifier("else")) { - size_t pos0 = current_; - ++current_; - auto false_expr = parse_if_expression(); - return make_node( - pos0, std::move(test), std::move(a), std::move(false_expr)); - } - return make_node( - start_pos, std::move(a), std::move(test)); - } - return a; - } +struct comment_statement : ast_node { + std::string value; + explicit comment_statement(std::string text) : value(std::move(text)) {} +}; - emel::text::jinja::ast_ptr parse_logical_or_expression() { - auto left = parse_logical_and_expression(); - while (ok() && is_identifier("or")) { - size_t start_pos = current_; - auto op = (*tokens_)[current_++]; - left = make_node( - start_pos, std::move(op), std::move(left), parse_logical_and_expression()); - } - return left; - } +struct string_literal : ast_node { + std::string value; + explicit string_literal(std::string text) : value(std::move(text)) {} +}; - emel::text::jinja::ast_ptr parse_logical_and_expression() { - auto left = parse_logical_negation_expression(); - while (ok() && is_identifier("and")) { - size_t start_pos = current_; - auto op = (*tokens_)[current_++]; - left = make_node( - start_pos, std::move(op), std::move(left), parse_logical_negation_expression()); - } - return left; - } +struct identifier : ast_node { + std::string name; + explicit identifier(std::string text) : name(std::move(text)) {} +}; - emel::text::jinja::ast_ptr parse_logical_negation_expression() { - if (is_identifier("not")) { - size_t start_pos = current_; - auto op = (*tokens_)[current_++]; - return make_node( - start_pos, std::move(op), parse_logical_negation_expression()); - } - return parse_comparison_expression(); - } +struct integer_literal : ast_node { + int64_t value = 0; + explicit integer_literal(int64_t v) : value(v) {} +}; - emel::text::jinja::ast_ptr parse_comparison_expression() { - auto left = parse_additive_expression(); - while (ok()) { - emel::text::jinja::token op; - size_t start_pos = current_; - if (is_identifier("not") && peek(1).type == emel::text::jinja::token_type::identifier && - peek(1).value == "in") { - op = {emel::text::jinja::token_type::identifier, "not in", peek().pos}; - current_ += 2; - } else if (is_identifier("in")) { - op = (*tokens_)[current_++]; - } else if (is(emel::text::jinja::token_type::comparison_binary_operator)) { - op = (*tokens_)[current_++]; - } else { - break; - } - left = make_node( - start_pos, std::move(op), std::move(left), parse_additive_expression()); - } - return left; - } +struct float_literal : ast_node { + double value = 0.0; + explicit float_literal(double v) : value(v) {} +}; - emel::text::jinja::ast_ptr parse_additive_expression() { - auto left = parse_multiplicative_expression(); - while (ok() && is(emel::text::jinja::token_type::additive_binary_operator)) { - size_t start_pos = current_; - auto op = (*tokens_)[current_++]; - left = make_node( - start_pos, std::move(op), std::move(left), parse_multiplicative_expression()); - } - return left; - } +struct tuple_literal : ast_node { + ast_list values; + explicit tuple_literal(ast_list vals) : values(std::move(vals)) {} +}; - emel::text::jinja::ast_ptr parse_multiplicative_expression() { - auto left = parse_test_expression(); - while (ok() && is(emel::text::jinja::token_type::multiplicative_binary_operator)) { - size_t start_pos = current_; - auto op = (*tokens_)[current_++]; - left = make_node( - start_pos, std::move(op), std::move(left), parse_test_expression()); - } - return left; - } +struct array_literal : ast_node { + ast_list values; + explicit array_literal(ast_list vals) : values(std::move(vals)) {} +}; - emel::text::jinja::ast_ptr parse_test_expression() { - auto operand = parse_filter_expression(); - while (ok() && is_identifier("is")) { - size_t start_pos = current_; - ++current_; - bool negate = false; - if (is_identifier("not")) { - ++current_; - negate = true; - } - auto test_id = parse_primary_expression(); - if (is(emel::text::jinja::token_type::open_paren)) { - test_id = parse_call_expression(std::move(test_id)); - } - operand = make_node( - start_pos, std::move(operand), negate, std::move(test_id)); - } - return operand; - } +struct object_literal : ast_node { + ast_pair_list pairs; + explicit object_literal(ast_pair_list vals) : pairs(std::move(vals)) {} +}; - emel::text::jinja::ast_ptr parse_filter_expression() { - auto operand = parse_call_member_expression(); - while (ok() && is(emel::text::jinja::token_type::pipe)) { - size_t start_pos = current_; - ++current_; - auto filter = parse_primary_expression(); - if (is(emel::text::jinja::token_type::open_paren)) { - filter = parse_call_expression(std::move(filter)); - } - operand = make_node( - start_pos, std::move(operand), std::move(filter)); - } - return operand; - } +struct unary_expression : ast_node { + token op; + ast_ptr operand; + unary_expression(token op_token, ast_ptr expr) + : op(std::move(op_token)), operand(std::move(expr)) {} +}; - emel::text::jinja::ast_ptr parse_call_member_expression() { - auto member = parse_member_expression(parse_primary_expression()); - if (is(emel::text::jinja::token_type::open_paren)) { - return parse_call_expression(std::move(member)); - } - return member; - } +struct binary_expression : ast_node { + token op; + ast_ptr left; + ast_ptr right; + binary_expression(token op_token, ast_ptr lhs, ast_ptr rhs) + : op(std::move(op_token)), left(std::move(lhs)), right(std::move(rhs)) {} +}; - emel::text::jinja::ast_ptr parse_call_expression(emel::text::jinja::ast_ptr callee) { - size_t start_pos = current_; - auto expr = make_node( - start_pos, std::move(callee), parse_args()); - auto member = parse_member_expression(std::move(expr)); - if (is(emel::text::jinja::token_type::open_paren)) { - return parse_call_expression(std::move(member)); - } - return member; - } +struct ternary_expression : ast_node { + ast_ptr test; + ast_ptr true_expr; + ast_ptr false_expr; + ternary_expression(ast_ptr test_expr, ast_ptr true_value, ast_ptr false_value) + : test(std::move(test_expr)), true_expr(std::move(true_value)), + false_expr(std::move(false_value)) {} +}; - emel::text::jinja::ast_list parse_args() { - emel::text::jinja::ast_list args; - if (!expect(emel::text::jinja::token_type::open_paren)) { - return args; - } - while (ok() && !is(emel::text::jinja::token_type::close_paren)) { - emel::text::jinja::ast_ptr arg; - if (peek().type == emel::text::jinja::token_type::multiplicative_binary_operator && - peek().value == "*") { - size_t start_pos = current_; - ++current_; - arg = make_node( - start_pos, parse_expression()); - } else { - arg = parse_expression(); - if (is(emel::text::jinja::token_type::equals)) { - size_t start_pos = current_; - ++current_; - arg = make_node( - start_pos, std::move(arg), parse_expression()); - } - } - args.push_back(std::move(arg)); - if (is(emel::text::jinja::token_type::comma)) { - ++current_; - } - } - expect(emel::text::jinja::token_type::close_paren); - return args; - } +struct select_expression : ast_node { + ast_ptr value; + ast_ptr test; + select_expression(ast_ptr value_expr, ast_ptr test_expr) + : value(std::move(value_expr)), test(std::move(test_expr)) {} +}; - emel::text::jinja::ast_ptr parse_member_expression(emel::text::jinja::ast_ptr object) { - size_t start_pos = current_; - while (ok() && (is(emel::text::jinja::token_type::dot) || - is(emel::text::jinja::token_type::open_square_bracket))) { - auto op = (*tokens_)[current_++]; - bool computed = op.type == emel::text::jinja::token_type::open_square_bracket; - emel::text::jinja::ast_ptr prop; - if (computed) { - prop = parse_member_expression_arguments(); - if (!expect(emel::text::jinja::token_type::close_square_bracket)) { - return nullptr; - } - } else { - prop = parse_primary_expression(); - } - object = make_node( - start_pos, std::move(object), std::move(prop), computed); - } - return object; - } +struct test_expression : ast_node { + ast_ptr operand; + bool negate = false; + ast_ptr test; + test_expression(ast_ptr operand_expr, bool neg, ast_ptr test_expr) + : operand(std::move(operand_expr)), negate(neg), + test(std::move(test_expr)) {} +}; - emel::text::jinja::ast_ptr parse_member_expression_arguments() { - emel::text::jinja::ast_list slices; - bool is_slice = false; - size_t start_pos = current_; - while (ok() && !is(emel::text::jinja::token_type::close_square_bracket)) { - if (is(emel::text::jinja::token_type::colon)) { - slices.push_back(nullptr); - ++current_; - is_slice = true; - } else { - slices.push_back(parse_expression()); - if (is(emel::text::jinja::token_type::colon)) { - ++current_; - is_slice = true; - } - } - } - if (!ok()) { - return nullptr; - } - if (is_slice) { - emel::text::jinja::ast_ptr start = slices.size() > 0 ? std::move(slices[0]) : nullptr; - emel::text::jinja::ast_ptr stop = slices.size() > 1 ? std::move(slices[1]) : nullptr; - emel::text::jinja::ast_ptr step = slices.size() > 2 ? std::move(slices[2]) : nullptr; - return make_node( - start_pos, std::move(start), std::move(stop), std::move(step)); - } - return slices.empty() ? nullptr : std::move(slices[0]); - } +struct filter_expression : ast_node { + ast_ptr operand; + ast_ptr filter; + filter_expression(ast_ptr operand_expr, ast_ptr filter_expr) + : operand(std::move(operand_expr)), filter(std::move(filter_expr)) {} +}; + +struct call_expression : ast_node { + ast_ptr callee; + ast_list args; + call_expression(ast_ptr callee_expr, ast_list arguments) + : callee(std::move(callee_expr)), args(std::move(arguments)) {} +}; + +struct call_statement : ast_node { + ast_ptr call_expr; + ast_list caller_args; + ast_list body; + call_statement(ast_ptr call_expr_in, ast_list caller_args_in, + ast_list body_in) + : call_expr(std::move(call_expr_in)), + caller_args(std::move(caller_args_in)), body(std::move(body_in)) {} +}; + +struct filter_statement : ast_node { + ast_ptr filter_node; + ast_list body; + filter_statement(ast_ptr filter_expr, ast_list body_in) + : filter_node(std::move(filter_expr)), body(std::move(body_in)) {} +}; + +struct set_statement : ast_node { + ast_ptr left; + ast_ptr value; + ast_list body; + set_statement(ast_ptr left_expr, ast_ptr value_expr, ast_list body_in) + : left(std::move(left_expr)), value(std::move(value_expr)), + body(std::move(body_in)) {} +}; + +struct if_statement : ast_node { + ast_ptr test; + ast_list body; + ast_list alternate; + if_statement(ast_ptr test_expr, ast_list body_in, ast_list alternate_in) + : test(std::move(test_expr)), body(std::move(body_in)), + alternate(std::move(alternate_in)) {} +}; + +struct macro_statement : ast_node { + ast_ptr name; + ast_list args; + ast_list body; + macro_statement(ast_ptr name_expr, ast_list args_in, ast_list body_in) + : name(std::move(name_expr)), args(std::move(args_in)), + body(std::move(body_in)) {} +}; + +struct for_statement : ast_node { + ast_ptr loop_var; + ast_ptr iterable; + ast_list body; + ast_list alternate; + for_statement(ast_ptr loop_var_in, ast_ptr iterable_in, ast_list body_in, + ast_list alternate_in) + : loop_var(std::move(loop_var_in)), iterable(std::move(iterable_in)), + body(std::move(body_in)), alternate(std::move(alternate_in)) {} +}; + +struct break_statement : ast_node { + break_statement() = default; +}; + +struct continue_statement : ast_node { + continue_statement() = default; +}; + +struct noop_statement : ast_node { + noop_statement() = default; +}; - emel::text::jinja::ast_ptr parse_primary_expression() { - size_t start_pos = current_; - const auto & t = peek(); - ++current_; - switch (t.type) { - case emel::text::jinja::token_type::numeric_literal: { - const char * str = t.value.c_str(); - if (t.value.find('.') != std::string::npos) { - char * end = nullptr; - double v = std::strtod(str, &end); - if (end == str) { - set_error(EMEL_ERR_PARSE_FAILED, t.pos); - return nullptr; - } - return make_node(start_pos, v); - } - char * end = nullptr; - long long v = std::strtoll(str, &end, 10); - if (end == str) { - set_error(EMEL_ERR_PARSE_FAILED, t.pos); - return nullptr; - } - return make_node(start_pos, static_cast(v)); - } - case emel::text::jinja::token_type::string_literal: { - std::string val = t.value; - while (is(emel::text::jinja::token_type::string_literal)) { - val += peek().value; - ++current_; - } - return make_node(start_pos, std::move(val)); - } - case emel::text::jinja::token_type::identifier: - return make_node(start_pos, t.value); - case emel::text::jinja::token_type::open_paren: { - auto expr = parse_expression_sequence(); - if (!expect(emel::text::jinja::token_type::close_paren)) { - return nullptr; - } - return expr; - } - case emel::text::jinja::token_type::open_square_bracket: { - emel::text::jinja::ast_list vals; - while (ok() && !is(emel::text::jinja::token_type::close_square_bracket)) { - vals.push_back(parse_expression()); - if (is(emel::text::jinja::token_type::comma)) { - ++current_; - } - } - if (!expect(emel::text::jinja::token_type::close_square_bracket)) { - return nullptr; - } - return make_node(start_pos, std::move(vals)); - } - case emel::text::jinja::token_type::open_curly_bracket: { - emel::text::jinja::ast_pair_list pairs; - while (ok() && !is(emel::text::jinja::token_type::close_curly_bracket)) { - auto key = parse_expression(); - if (!expect(emel::text::jinja::token_type::colon)) { - return nullptr; - } - pairs.push_back({std::move(key), parse_expression()}); - if (is(emel::text::jinja::token_type::comma)) { - ++current_; - } - } - if (!expect(emel::text::jinja::token_type::close_curly_bracket)) { - return nullptr; - } - return make_node(start_pos, std::move(pairs)); - } - default: - set_error(EMEL_ERR_PARSE_FAILED, t.pos); - return nullptr; - } +struct member_expression : ast_node { + ast_ptr object; + ast_ptr property; + bool computed = false; + member_expression(ast_ptr object_in, ast_ptr property_in, bool is_computed) + : object(std::move(object_in)), property(std::move(property_in)), + computed(is_computed) {} +}; + +struct slice_expression : ast_node { + ast_ptr start; + ast_ptr stop; + ast_ptr step; + slice_expression(ast_ptr start_expr, ast_ptr stop_expr, ast_ptr step_expr) + : start(std::move(start_expr)), stop(std::move(stop_expr)), + step(std::move(step_expr)) {} +}; + +struct keyword_argument_expression : ast_node { + ast_ptr key; + ast_ptr value; + keyword_argument_expression(ast_ptr key_expr, ast_ptr value_expr) + : key(std::move(key_expr)), value(std::move(value_expr)) {} +}; + +struct spread_expression : ast_node { + ast_ptr operand; + explicit spread_expression(ast_ptr expr) : operand(std::move(expr)) {} +}; + +struct program { + ast_list body; + int32_t last_error = parser::to_error_code(parser::error::none); + size_t last_error_pos = 0; + + void reset() noexcept { + body.clear(); + last_error = parser::to_error_code(parser::error::none); + last_error_pos = 0; } +}; + +enum class value_type : uint8_t { + undefined = 0, + none = 1, + boolean = 2, + integer = 3, + floating = 4, + string = 5, + array = 6, + object = 7, + function = 8 +}; + +struct value; +struct object_entry; + +struct string_value { + std::string_view view = {}; + bool is_input = false; +}; + +struct array_value { + value *items = nullptr; + size_t count = 0; + size_t capacity = 0; +}; + +struct object_value { + object_entry *entries = nullptr; + size_t count = 0; + size_t capacity = 0; + bool has_builtins = true; +}; + +enum class function_kind : uint8_t { builtin = 0, macro = 1, caller = 2 }; + +struct function_ref { + function_kind kind = function_kind::builtin; + const void *data = nullptr; +}; + +struct value { + value_type type = value_type::undefined; + bool bool_v = false; + int64_t int_v = 0; + double float_v = 0.0; + string_value string_v = {}; + array_value array_v = {}; + object_value object_v = {}; + function_ref func_v = {}; + std::string_view hint = {}; +}; - const std::vector * tokens_ = nullptr; - size_t current_ = 0; - int32_t error_ = EMEL_OK; - size_t error_pos_ = 0; - emel::text::jinja::program & program_; +struct object_entry { + value key = {}; + value val = {}; }; -} // namespace emel::text::jinja::parser::detail +} // namespace emel::text::jinja diff --git a/src/emel/text/jinja/parser/errors.hpp b/src/emel/text/jinja/parser/errors.hpp new file mode 100644 index 00000000..ec491ca7 --- /dev/null +++ b/src/emel/text/jinja/parser/errors.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include + +#include "emel/error/error.hpp" + +namespace emel::text::jinja::parser { + +enum class error : emel::error::type { + none = 0u, + invalid_request = (1u << 0), + parse_failed = (1u << 1), + internal_error = (1u << 2), + untracked = (1u << 3), +}; + +constexpr bool is_ok(const error value) noexcept { + return value == error::none; +} + +constexpr int32_t to_error_code(const error value) noexcept { + return static_cast(emel::error::cast(value)); +} + +} // namespace emel::text::jinja::parser diff --git a/src/emel/text/jinja/parser/events.hpp b/src/emel/text/jinja/parser/events.hpp index 4cc5b911..291ad91a 100644 --- a/src/emel/text/jinja/parser/events.hpp +++ b/src/emel/text/jinja/parser/events.hpp @@ -1,10 +1,13 @@ #pragma once +#include #include #include #include "emel/callback.hpp" -#include "emel/text/jinja/types.hpp" +#include "emel/text/jinja/lexer/detail.hpp" +#include "emel/text/jinja/parser/detail.hpp" +#include "emel/text/jinja/parser/errors.hpp" namespace emel::text::jinja::events { @@ -15,15 +18,88 @@ struct parsing_error; namespace emel::text::jinja::event { +enum class parse_phase : uint8_t { + none = 0, + request_validation = 1, + tokenization = 2, + statement_classification = 3, + parsing = 4, +}; + +enum class statement_kind : uint8_t { + unknown = 0, + text = 1, + comment = 2, + expression = 3, + statement = 4, +}; + +enum class expression_kind : uint8_t { + unknown = 0, + literal = 1, + identifier = 2, + unary = 3, + compound = 4, +}; + struct parse { - std::string_view template_text = {}; - emel::text::jinja::program * program_out = nullptr; - int32_t * error_out = nullptr; - void * owner_sm = nullptr; - ::emel::callback - dispatch_done = {}; - ::emel::callback - dispatch_error = {}; + using done_callback = + ::emel::callback; + using error_callback = ::emel::callback; + + parse(std::string_view template_text_ref, + emel::text::jinja::program &program_ref, + const done_callback dispatch_done_ref, + const error_callback dispatch_error_ref, int32_t &error_out_ref, + size_t &error_pos_out_ref) noexcept + : template_text(template_text_ref), program(program_ref), + dispatch_done(dispatch_done_ref), dispatch_error(dispatch_error_ref), + error_out(error_out_ref), error_pos_out(error_pos_out_ref) {} + + const std::string_view template_text; + emel::text::jinja::program &program; + const done_callback dispatch_done; + const error_callback dispatch_error; + int32_t &error_out; + size_t &error_pos_out; +}; + +struct parse_ctx { + parse_ctx(std::string_view template_text_ref, int32_t &error_out_ref, + size_t &error_pos_out_ref) noexcept + : error_out(error_out_ref), error_pos_out(error_pos_out_ref) { + const auto plan = + ::emel::text::jinja::lexer::detail::build_scan_plan(template_text_ref); + lex_result.source = plan.source; + lex_plan = plan.outcomes; + } + + parser::error err = parser::error::none; + size_t error_pos = 0; + + parse_phase phase = parse_phase::none; + statement_kind statement = statement_kind::unknown; + expression_kind expression = expression_kind::unknown; + size_t token_index = 0; + size_t statement_start = 0; + size_t expression_start = 0; + size_t expression_value_index = 0; + + emel::text::jinja::lexer::cursor lex_cursor = {}; + emel::text::jinja::token lex_token = {}; + bool lex_has_token = false; + emel::text::jinja::lexer_result lex_result = {}; + std::vector lex_plan = {}; + size_t lex_plan_index = 0; + + int32_t &error_out; + size_t &error_pos_out; +}; + +struct parse_runtime { + const parse &request; + parse_ctx &ctx; }; } // namespace emel::text::jinja::event @@ -31,12 +107,13 @@ struct parse { namespace emel::text::jinja::events { struct parsing_done { - const event::parse * request = nullptr; + const event::parse &request; }; struct parsing_error { - const event::parse * request = nullptr; - int32_t err = 0; + const event::parse &request; + int32_t err; + size_t error_pos; }; } // namespace emel::text::jinja::events diff --git a/src/emel/text/jinja/parser/guards.hpp b/src/emel/text/jinja/parser/guards.hpp index 60781c91..a3e476bd 100644 --- a/src/emel/text/jinja/parser/guards.hpp +++ b/src/emel/text/jinja/parser/guards.hpp @@ -1,39 +1,81 @@ #pragma once -#include "emel/text/jinja/parser/context.hpp" +#include "emel/text/jinja/parser/errors.hpp" #include "emel/text/jinja/parser/events.hpp" namespace emel::text::jinja::parser::guard { +namespace helper { + +constexpr const event::parse_runtime & +unwrap_runtime_event(const event::parse_runtime &ev) noexcept { + return ev; +} + +template + requires requires(const wrapped_event_type &ev) { ev.event_; } +constexpr decltype(auto) +unwrap_runtime_event(const wrapped_event_type &ev) noexcept { + return ev.event_; +} + +inline bool +valid_parse_request(const emel::text::jinja::event::parse &ev) noexcept { + return ev.template_text.data() != nullptr && !ev.template_text.empty(); +} + +} // namespace helper + struct valid_parse { - bool operator()(const event::parse & ev, + template + bool operator()(const runtime_event_type &ev, const action::context &) const noexcept { - if (ev.template_text.data() == nullptr || ev.template_text.empty()) { - return false; - } - if (ev.program_out == nullptr || ev.error_out == nullptr) { - return false; - } - return true; + const auto &runtime_ev = helper::unwrap_runtime_event(ev); + return helper::valid_parse_request(runtime_ev.request); } }; struct invalid_parse { - bool operator()(const event::parse & ev, - const action::context & ctx) const noexcept { - return !valid_parse{}(ev, ctx); + template + bool operator()(const runtime_event_type &ev, + const action::context &) const noexcept { + const auto &runtime_ev = helper::unwrap_runtime_event(ev); + return !helper::valid_parse_request(runtime_ev.request); } }; struct phase_ok { - bool operator()(const action::context & ctx) const noexcept { - return ctx.phase_error == EMEL_OK; + template + bool operator()(const runtime_event_type &ev, + const action::context &) const noexcept { + const auto &runtime_ev = helper::unwrap_runtime_event(ev); + return runtime_ev.ctx.err == error::none; } }; struct phase_failed { - bool operator()(const action::context & ctx) const noexcept { - return ctx.phase_error != EMEL_OK; + template + bool operator()(const runtime_event_type &ev, + const action::context &ctx) const noexcept { + return !phase_ok{}(ev, ctx); + } +}; + +struct lexer_has_token { + template + bool operator()(const runtime_event_type &ev, + const action::context &) const noexcept { + const auto &runtime_ev = helper::unwrap_runtime_event(ev); + return runtime_ev.ctx.err == error::none && runtime_ev.ctx.lex_has_token; + } +}; + +struct lexer_at_eof { + template + bool operator()(const runtime_event_type &ev, + const action::context &) const noexcept { + const auto &runtime_ev = helper::unwrap_runtime_event(ev); + return runtime_ev.ctx.err == error::none && !runtime_ev.ctx.lex_has_token; } }; diff --git a/src/emel/text/jinja/parser/lexer/actions.hpp b/src/emel/text/jinja/parser/lexer/actions.hpp new file mode 100644 index 00000000..a003ac0e --- /dev/null +++ b/src/emel/text/jinja/parser/lexer/actions.hpp @@ -0,0 +1,80 @@ +#pragma once + +#include "emel/text/jinja/parser/lexer/context.hpp" +#include "emel/text/jinja/parser/lexer/detail.hpp" + +namespace emel::text::jinja::parser::lexer::action { + +struct emit_scanned_token { + void operator()(const event::next_runtime &ev, context &) const noexcept { + ev.request.dispatch_done(::emel::text::jinja::lexer::events::next_done{ + ev.request, + ev.scan.token_value, + true, + ev.scan.next_cursor, + }); + } +}; + +struct emit_scan_error { + void operator()(const event::next_runtime &ev, context &) const noexcept { + ev.request.dispatch_error(::emel::text::jinja::lexer::events::next_error{ + ev.request, + ev.scan.err, + ev.scan.error_pos, + }); + } +}; + +struct emit_eof { + void operator()(const event::next_runtime &ev, context &) const noexcept { + ev.request.dispatch_done(::emel::text::jinja::lexer::events::next_done{ + ev.request, + {}, + false, + ev.request.cursor, + }); + } +}; + +struct reject_invalid_next { + void operator()(const event::next_runtime &ev, context &) const noexcept { + ev.request.dispatch_error(::emel::text::jinja::lexer::events::next_error{ + ev.request, + detail::error_code(error::invalid_request), + ev.request.cursor.offset, + }); + } +}; + +struct reject_invalid_cursor { + void operator()(const event::next_runtime &ev, context &) const noexcept { + ev.request.dispatch_error(::emel::text::jinja::lexer::events::next_error{ + ev.request, + detail::error_code(error::invalid_request), + ev.request.cursor.offset, + }); + } +}; + +struct on_unexpected { + void operator()(const event::next_runtime &ev, context &) const noexcept { + ev.request.dispatch_error(::emel::text::jinja::lexer::events::next_error{ + ev.request, + detail::error_code(error::internal_error), + ev.request.cursor.offset, + }); + } + + template + void operator()(const event_type &, context &) const noexcept {} +}; + +inline constexpr emit_scanned_token emit_scanned_token{}; +inline constexpr emit_scan_error emit_scan_error{}; +inline constexpr emit_eof emit_eof{}; +inline constexpr reject_invalid_next reject_invalid_next{}; +inline constexpr reject_invalid_cursor reject_invalid_cursor{}; +inline constexpr on_unexpected on_unexpected{}; + +} // namespace emel::text::jinja::parser::lexer::action diff --git a/src/emel/text/jinja/parser/lexer/context.hpp b/src/emel/text/jinja/parser/lexer/context.hpp new file mode 100644 index 00000000..84427482 --- /dev/null +++ b/src/emel/text/jinja/parser/lexer/context.hpp @@ -0,0 +1,7 @@ +#pragma once + +namespace emel::text::jinja::parser::lexer::action { + +struct context {}; + +} // namespace emel::text::jinja::parser::lexer::action diff --git a/src/emel/text/jinja/parser/lexer/detail.hpp b/src/emel/text/jinja/parser/lexer/detail.hpp new file mode 100644 index 00000000..f78295de --- /dev/null +++ b/src/emel/text/jinja/parser/lexer/detail.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include "emel/text/jinja/lexer/detail.hpp" + +namespace emel::text::jinja::parser::lexer::event { + +struct next_runtime { + const ::emel::text::jinja::lexer::event::next &request; + const ::emel::text::jinja::lexer::detail::scan_outcome &scan; +}; + +} // namespace emel::text::jinja::parser::lexer::event + +namespace emel::text::jinja::parser::lexer::detail { + +using ::emel::text::jinja::lexer::detail::error_code; +using ::emel::text::jinja::lexer::detail::normalize_source; +using ::emel::text::jinja::lexer::detail::scan_outcome; + +} // namespace emel::text::jinja::parser::lexer::detail diff --git a/src/emel/text/jinja/parser/lexer/errors.hpp b/src/emel/text/jinja/parser/lexer/errors.hpp new file mode 100644 index 00000000..f041c60f --- /dev/null +++ b/src/emel/text/jinja/parser/lexer/errors.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include "emel/text/jinja/parser/errors.hpp" + +namespace emel::text::jinja::parser::lexer { + +using error = emel::text::jinja::parser::error; + +} // namespace emel::text::jinja::parser::lexer diff --git a/src/emel/text/jinja/parser/lexer/guards.hpp b/src/emel/text/jinja/parser/lexer/guards.hpp new file mode 100644 index 00000000..73295c95 --- /dev/null +++ b/src/emel/text/jinja/parser/lexer/guards.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include "emel/text/jinja/parser/lexer/context.hpp" +#include "emel/text/jinja/parser/lexer/detail.hpp" +#include "emel/text/jinja/parser/lexer/errors.hpp" + +namespace emel::text::jinja::parser::lexer::guard { + +struct invalid_next { + bool operator()(const event::next_runtime &ev, + const action::context &) const noexcept { + return ev.request.cursor.source.data() == nullptr || + !static_cast(ev.request.dispatch_done) || + !static_cast(ev.request.dispatch_error); + } +}; + +struct invalid_cursor_position { + bool operator()(const event::next_runtime &ev, + const action::context &) const noexcept { + return ev.request.cursor.offset > ev.request.cursor.source.size(); + } +}; + +struct scan_failed { + bool operator()(const event::next_runtime &ev, + const action::context &) const noexcept { + return ev.scan.err != detail::error_code(error::none); + } +}; + +struct scan_has_token { + bool operator()(const event::next_runtime &ev, + const action::context &) const noexcept { + return ev.scan.err == detail::error_code(error::none) && ev.scan.has_token; + } +}; + +struct scan_at_eof { + bool operator()(const event::next_runtime &ev, + const action::context &) const noexcept { + return ev.scan.err == detail::error_code(error::none) && !ev.scan.has_token; + } +}; + +} // namespace emel::text::jinja::parser::lexer::guard diff --git a/src/emel/text/jinja/parser/lexer/sm.hpp b/src/emel/text/jinja/parser/lexer/sm.hpp new file mode 100644 index 00000000..be2f5c67 --- /dev/null +++ b/src/emel/text/jinja/parser/lexer/sm.hpp @@ -0,0 +1,89 @@ +#pragma once + +#include "emel/sm.hpp" +#include "emel/text/jinja/parser/lexer/actions.hpp" +#include "emel/text/jinja/parser/lexer/context.hpp" +#include "emel/text/jinja/parser/lexer/detail.hpp" +#include "emel/text/jinja/parser/lexer/guards.hpp" + +namespace emel::text::jinja::parser::lexer { + +struct initialized {}; +struct scanning {}; + +struct model { + auto operator()() const { + namespace sml = boost::sml; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Initialized. + sml::state <= *sml::state + + sml::event + [ guard::invalid_next{} ] + / action::reject_invalid_next + + , sml::state <= sml::state + + sml::event + [ guard::invalid_cursor_position{} ] + / action::reject_invalid_cursor + + , sml::state <= sml::state + + sml::event + [ guard::scan_failed{} ] + / action::emit_scan_error + + , sml::state <= sml::state + + sml::event + [ guard::scan_has_token{} ] + / action::emit_scanned_token + + , sml::state <= sml::state + + sml::event + [ guard::scan_at_eof{} ] + / action::emit_eof + + //------------------------------------------------------------------------------// + // Scanning. + , sml::state <= sml::state + + sml::event + [ guard::invalid_next{} ] + / action::reject_invalid_next + + , sml::state <= sml::state + + sml::event + [ guard::invalid_cursor_position{} ] + / action::reject_invalid_cursor + + , sml::state <= sml::state + + sml::event + [ guard::scan_failed{} ] + / action::emit_scan_error + + , sml::state <= sml::state + + sml::event + [ guard::scan_has_token{} ] + / action::emit_scanned_token + + , sml::state <= sml::state + + sml::event + [ guard::scan_at_eof{} ] + / action::emit_eof + + //------------------------------------------------------------------------------// + // Unexpected events. + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + ); + // clang-format on + } +}; + +using sm = emel::sm; +using Lexer = sm; + +} // namespace emel::text::jinja::parser::lexer diff --git a/src/emel/text/jinja/parser/program_parser/actions.hpp b/src/emel/text/jinja/parser/program_parser/actions.hpp new file mode 100644 index 00000000..249d34f6 --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/actions.hpp @@ -0,0 +1,89 @@ +#pragma once + +#include +#include +#include +#include + +#include "emel/text/jinja/parser/actions.hpp" +#include "emel/text/jinja/parser/detail.hpp" +#include "emel/text/jinja/parser/events.hpp" +#include "emel/text/jinja/parser/program_parser/context.hpp" +#include "emel/text/jinja/parser/program_parser/errors.hpp" + +namespace emel::text::jinja::parser::program_parser::action { + +inline const emel::text::jinja::token & +current_token(const event::parse_runtime &ev) noexcept { + return ev.ctx.lex_result.tokens[ev.ctx.token_index]; +} + +template +inline emel::text::jinja::ast_ptr make_node(const size_t pos, + args &&...args_in) { + auto node = std::make_unique(std::forward(args_in)...); + node->pos = pos; + return node; +} + +struct start_program_parse { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ev.ctx.phase = event::parse_phase::parsing; + ev.ctx.statement = event::statement_kind::unknown; + ev.ctx.expression = event::expression_kind::unknown; + ev.ctx.token_index = 0; + ev.ctx.statement_start = 0; + ev.ctx.expression_start = 0; + ev.ctx.expression_value_index = 0; + } +}; + +struct consume_text { + void operator()(const event::parse_runtime &ev, context &) const { + const auto &tok = current_token(ev); + ev.request.program.body.push_back( + make_node(tok.pos, tok.value)); + ++ev.ctx.token_index; + } +}; + +struct consume_comment { + void operator()(const event::parse_runtime &ev, context &) const { + const auto &tok = current_token(ev); + ev.request.program.body.push_back( + make_node(tok.pos, tok.value)); + ++ev.ctx.token_index; + } +}; + +struct finish_parsed { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ::emel::text::jinja::parser::action::helper::mark_done(ev.request, ev.ctx); + } +}; + +struct fail_current_token { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ::emel::text::jinja::parser::action::helper::mark_error( + ev.request, ev.ctx, error::parse_failed, current_token(ev).pos); + } +}; + +struct on_unexpected { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ::emel::text::jinja::parser::action::helper::mark_error( + ev.request, ev.ctx, error::internal_error, ev.ctx.error_pos_out); + } + + template + void operator()(const event_type &, context &) const noexcept {} +}; + +inline constexpr start_program_parse start_program_parse{}; +inline constexpr consume_text consume_text{}; +inline constexpr consume_comment consume_comment{}; +inline constexpr finish_parsed finish_parsed{}; +inline constexpr fail_current_token fail_current_token{}; +inline constexpr on_unexpected on_unexpected{}; + +} // namespace emel::text::jinja::parser::program_parser::action diff --git a/src/emel/text/jinja/parser/program_parser/context.hpp b/src/emel/text/jinja/parser/program_parser/context.hpp new file mode 100644 index 00000000..89e4d597 --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/context.hpp @@ -0,0 +1,11 @@ +#pragma once + +namespace emel::text::jinja::parser::action { +struct context; +} + +namespace emel::text::jinja::parser::program_parser::action { + +using context = emel::text::jinja::parser::action::context; + +} // namespace emel::text::jinja::parser::program_parser::action diff --git a/src/emel/text/jinja/parser/program_parser/errors.hpp b/src/emel/text/jinja/parser/program_parser/errors.hpp new file mode 100644 index 00000000..82fdace0 --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/errors.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include "emel/text/jinja/parser/errors.hpp" + +namespace emel::text::jinja::parser::program_parser { + +using error = emel::text::jinja::parser::error; + +} // namespace emel::text::jinja::parser::program_parser diff --git a/src/emel/text/jinja/parser/program_parser/expression_parser/actions.hpp b/src/emel/text/jinja/parser/program_parser/expression_parser/actions.hpp new file mode 100644 index 00000000..54b3cc2f --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/expression_parser/actions.hpp @@ -0,0 +1,143 @@ +#pragma once + +#include +#include +#include +#include + +#include "emel/text/jinja/parser/actions.hpp" +#include "emel/text/jinja/parser/detail.hpp" +#include "emel/text/jinja/parser/events.hpp" +#include "emel/text/jinja/parser/program_parser/expression_parser/context.hpp" +#include "emel/text/jinja/parser/program_parser/expression_parser/errors.hpp" + +namespace emel::text::jinja::parser::program_parser::expression_parser::action { + +inline const emel::text::jinja::token & +current_token(const event::parse_runtime &ev) noexcept { + return ev.ctx.lex_result.tokens[ev.ctx.token_index]; +} + +inline const emel::text::jinja::token & +token_at_index(const event::parse_runtime &ev, const size_t index) noexcept { + return ev.ctx.lex_result.tokens[index]; +} + +template +inline emel::text::jinja::ast_ptr make_node(const size_t pos, + args &&...args_in) { + auto node = std::make_unique(std::forward(args_in)...); + node->pos = pos; + return node; +} + +struct begin_expression_parse { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ev.ctx.statement = event::statement_kind::expression; + ev.ctx.expression = event::expression_kind::unknown; + ev.ctx.expression_start = ev.ctx.token_index; + ev.ctx.expression_value_index = ev.ctx.token_index; + ++ev.ctx.token_index; + } +}; + +struct consume_expression_identifier { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ev.ctx.expression = event::expression_kind::identifier; + ev.ctx.expression_value_index = ev.ctx.token_index; + ++ev.ctx.token_index; + } +}; + +struct consume_expression_literal { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ev.ctx.expression = event::expression_kind::literal; + ev.ctx.expression_value_index = ev.ctx.token_index; + ++ev.ctx.token_index; + } +}; + +struct consume_expression_unary { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ev.ctx.expression = event::expression_kind::unary; + ev.ctx.expression_value_index = ev.ctx.token_index; + ++ev.ctx.token_index; + } +}; + +struct consume_expression_compound { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ev.ctx.expression = event::expression_kind::compound; + ev.ctx.expression_value_index = ev.ctx.token_index; + ++ev.ctx.token_index; + } +}; + +struct consume_expression_token { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ++ev.ctx.token_index; + } +}; + +struct emit_expression_identifier { + void operator()(const event::parse_runtime &ev, context &) const { + const auto &tok = token_at_index(ev, ev.ctx.expression_value_index); + ev.request.program.body.push_back( + make_node(tok.pos, tok.value)); + } +}; + +struct emit_expression_generic { + void operator()(const event::parse_runtime &ev, context &) const { + const auto &tok = token_at_index(ev, ev.ctx.expression_value_index); + ev.request.program.body.push_back( + make_node(tok.pos, tok.value)); + } +}; + +struct consume_expression_close { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ++ev.ctx.token_index; + } +}; + +struct fail_expression_close_token { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ::emel::text::jinja::parser::action::helper::mark_error( + ev.request, ev.ctx, error::parse_failed, current_token(ev).pos); + } +}; + +struct fail_expression_start_token { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + const auto &tok = token_at_index(ev, ev.ctx.expression_start); + ::emel::text::jinja::parser::action::helper::mark_error( + ev.request, ev.ctx, error::parse_failed, tok.pos); + } +}; + +struct on_unexpected { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ::emel::text::jinja::parser::action::helper::mark_error( + ev.request, ev.ctx, error::internal_error, ev.ctx.error_pos_out); + } + + template + void operator()(const event_type &, context &) const noexcept {} +}; + +inline constexpr begin_expression_parse begin_expression_parse{}; +inline constexpr consume_expression_identifier consume_expression_identifier{}; +inline constexpr consume_expression_literal consume_expression_literal{}; +inline constexpr consume_expression_unary consume_expression_unary{}; +inline constexpr consume_expression_compound consume_expression_compound{}; +inline constexpr consume_expression_token consume_expression_token{}; +inline constexpr emit_expression_identifier emit_expression_identifier{}; +inline constexpr emit_expression_generic emit_expression_generic{}; +inline constexpr consume_expression_close consume_expression_close{}; +inline constexpr fail_expression_close_token fail_expression_close_token{}; +inline constexpr fail_expression_start_token fail_expression_start_token{}; +inline constexpr on_unexpected on_unexpected{}; + +} // namespace + // emel::text::jinja::parser::program_parser::expression_parser::action diff --git a/src/emel/text/jinja/parser/program_parser/expression_parser/context.hpp b/src/emel/text/jinja/parser/program_parser/expression_parser/context.hpp new file mode 100644 index 00000000..6c36d91a --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/expression_parser/context.hpp @@ -0,0 +1,12 @@ +#pragma once + +namespace emel::text::jinja::parser::action { +struct context; +} + +namespace emel::text::jinja::parser::program_parser::expression_parser::action { + +using context = emel::text::jinja::parser::action::context; + +} // namespace + // emel::text::jinja::parser::program_parser::expression_parser::action diff --git a/src/emel/text/jinja/parser/program_parser/expression_parser/errors.hpp b/src/emel/text/jinja/parser/program_parser/expression_parser/errors.hpp new file mode 100644 index 00000000..54ef9e1e --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/expression_parser/errors.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include "emel/text/jinja/parser/program_parser/errors.hpp" + +namespace emel::text::jinja::parser::program_parser::expression_parser { + +using error = emel::text::jinja::parser::program_parser::error; + +} // namespace emel::text::jinja::parser::program_parser::expression_parser diff --git a/src/emel/text/jinja/parser/program_parser/expression_parser/guards.hpp b/src/emel/text/jinja/parser/program_parser/expression_parser/guards.hpp new file mode 100644 index 00000000..a016c844 --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/expression_parser/guards.hpp @@ -0,0 +1,105 @@ +#pragma once + +#include "emel/text/jinja/parser/events.hpp" +#include "emel/text/jinja/parser/program_parser/expression_parser/context.hpp" + +namespace emel::text::jinja::parser::program_parser::expression_parser::guard { + +inline bool has_token(const event::parse_ctx &ctx, + const size_t offset = 0) noexcept { + return ctx.token_index + offset < ctx.lex_result.tokens.size(); +} + +inline const emel::text::jinja::token & +token_at(const event::parse_ctx &ctx, const size_t offset = 0) noexcept { + return ctx.lex_result.tokens[ctx.token_index + offset]; +} + +inline bool token_is(const event::parse_ctx &ctx, + const emel::text::jinja::token_type type, + const size_t offset = 0) noexcept { + return has_token(ctx, offset) && token_at(ctx, offset).type == type; +} + +struct expression_identifier { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return ev.ctx.expression == event::expression_kind::identifier; + } +}; + +struct expression_non_identifier { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return ev.ctx.expression != event::expression_kind::identifier; + } +}; + +struct expr_first_is_close { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::close_expression); + } +}; + +struct expr_first_is_identifier { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::identifier); + } +}; + +struct expr_first_is_literal { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::numeric_literal) || + token_is(ev.ctx, emel::text::jinja::token_type::string_literal) || + token_is(ev.ctx, + emel::text::jinja::token_type::open_square_bracket) || + token_is(ev.ctx, emel::text::jinja::token_type::open_curly_bracket); + } +}; + +struct expr_first_is_unary { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return token_is(ev.ctx, + emel::text::jinja::token_type::additive_binary_operator) || + token_is(ev.ctx, emel::text::jinja::token_type::unary_operator); + } +}; + +struct expr_first_is_other_content { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return has_token(ev.ctx) && !expr_first_is_close{}(ev, action::context{}) && + !expr_first_is_identifier{}(ev, action::context{}) && + !expr_first_is_literal{}(ev, action::context{}) && + !expr_first_is_unary{}(ev, action::context{}); + } +}; + +struct expr_scan_at_close { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::close_expression); + } +}; + +struct expr_scan_continue { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return has_token(ev.ctx) && + !token_is(ev.ctx, emel::text::jinja::token_type::close_expression); + } +}; + +struct expr_scan_eof { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return !has_token(ev.ctx); + } +}; + +} // namespace + // emel::text::jinja::parser::program_parser::expression_parser::guard diff --git a/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp b/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp new file mode 100644 index 00000000..ff2b01e0 --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/expression_parser/sm.hpp @@ -0,0 +1,112 @@ +#pragma once + +#include "emel/sm.hpp" +#include "emel/text/jinja/parser/events.hpp" +#include "emel/text/jinja/parser/program_parser/expression_parser/actions.hpp" +#include "emel/text/jinja/parser/program_parser/expression_parser/guards.hpp" + +namespace emel::text::jinja::parser::program_parser::expression_parser { + +struct deciding {}; +struct expression_first_decision {}; +struct expression_scan {}; +struct expression_emit_decision {}; +struct expression_close {}; +struct parsed {}; +struct parse_failed {}; +struct unexpected_event {}; + +struct model { + auto operator()() const { + namespace sml = boost::sml; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Expression parser. + sml::state <= *sml::state + + sml::completion + / action::begin_expression_parse + + , sml::state <= sml::state + + sml::completion[ guard::expr_scan_eof{} ] + / action::fail_expression_start_token + + , sml::state <= sml::state + + sml::completion[ guard::expr_first_is_close{} ] + / action::fail_expression_close_token + + , sml::state <= sml::state + + sml::completion[ guard::expr_first_is_identifier{} ] + / action::consume_expression_identifier + + , sml::state <= sml::state + + sml::completion[ guard::expr_first_is_literal{} ] + / action::consume_expression_literal + + , sml::state <= sml::state + + sml::completion[ guard::expr_first_is_unary{} ] + / action::consume_expression_unary + + , sml::state <= sml::state + + sml::completion[ guard::expr_first_is_other_content{} ] + / action::consume_expression_compound + + , sml::state <= sml::state + + sml::completion[ guard::expr_scan_at_close{} ] + + , sml::state <= sml::state + + sml::completion[ guard::expr_scan_continue{} ] + / action::consume_expression_token + + , sml::state <= sml::state + + sml::completion[ guard::expr_scan_eof{} ] + / action::fail_expression_start_token + + , sml::state <= sml::state + + sml::completion[ guard::expression_identifier{} ] + / action::emit_expression_identifier + + , sml::state <= sml::state + + sml::completion[ guard::expression_non_identifier{} ] + / action::emit_expression_generic + + , sml::state <= sml::state + + sml::completion[ guard::expr_scan_at_close{} ] + / action::consume_expression_close + + , sml::state <= sml::state + + sml::completion[ guard::expr_scan_eof{} ] + / action::fail_expression_start_token + + //------------------------------------------------------------------------------// + , sml::X <= sml::state + , sml::X <= sml::state + + //------------------------------------------------------------------------------// + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + ); + // clang-format on + } +}; + +struct sm : emel::sm { + using model_type = model; +}; + +} // namespace emel::text::jinja::parser::program_parser::expression_parser diff --git a/src/emel/text/jinja/parser/program_parser/guards.hpp b/src/emel/text/jinja/parser/program_parser/guards.hpp new file mode 100644 index 00000000..6808551a --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/guards.hpp @@ -0,0 +1,84 @@ +#pragma once + +#include "emel/text/jinja/parser/events.hpp" +#include "emel/text/jinja/parser/program_parser/context.hpp" +#include "emel/text/jinja/parser/program_parser/errors.hpp" + +namespace emel::text::jinja::parser::program_parser::guard { + +inline bool has_token(const event::parse_ctx &ctx, + const size_t offset = 0) noexcept { + return ctx.token_index + offset < ctx.lex_result.tokens.size(); +} + +inline const emel::text::jinja::token & +token_at(const event::parse_ctx &ctx, const size_t offset = 0) noexcept { + return ctx.lex_result.tokens[ctx.token_index + offset]; +} + +inline bool token_is(const event::parse_ctx &ctx, + const emel::text::jinja::token_type type, + const size_t offset = 0) noexcept { + return has_token(ctx, offset) && token_at(ctx, offset).type == type; +} + +struct phase_ok { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return ev.ctx.err == error::none; + } +}; + +struct phase_failed { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return ev.ctx.err != error::none; + } +}; + +struct at_eof { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return !has_token(ev.ctx); + } +}; + +struct token_text { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::text); + } +}; + +struct token_comment { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::comment); + } +}; + +struct token_open_expression { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::open_expression); + } +}; + +struct token_open_statement { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::open_statement); + } +}; + +struct token_unexpected { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return has_token(ev.ctx) && !token_text{}(ev, action::context{}) && + !token_comment{}(ev, action::context{}) && + !token_open_expression{}(ev, action::context{}) && + !token_open_statement{}(ev, action::context{}); + } +}; + +} // namespace emel::text::jinja::parser::program_parser::guard diff --git a/src/emel/text/jinja/parser/program_parser/sm.hpp b/src/emel/text/jinja/parser/program_parser/sm.hpp new file mode 100644 index 00000000..7edcabdc --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/sm.hpp @@ -0,0 +1,127 @@ +#pragma once + +#include "emel/sm.hpp" +#include "emel/text/jinja/parser/events.hpp" +#include "emel/text/jinja/parser/program_parser/actions.hpp" +#include "emel/text/jinja/parser/program_parser/expression_parser/sm.hpp" +#include "emel/text/jinja/parser/program_parser/guards.hpp" +#include "emel/text/jinja/parser/program_parser/statement_parser/sm.hpp" + +namespace emel::text::jinja::parser::program_parser { + +struct deciding {}; +struct parse_begin {}; +struct dispatch_decision {}; + +struct text_emit {}; +struct comment_emit {}; + +struct statement_parse_result_decision {}; +struct expression_parse_result_decision {}; + +struct parsed {}; +struct parse_failed {}; +struct unexpected_event {}; + +struct model { + auto operator()() const { + namespace sml = boost::sml; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Program dispatch setup. + sml::state <= *sml::state + + sml::completion + / action::start_program_parse + + , sml::state <= sml::state + + sml::completion + + //------------------------------------------------------------------------------// + // Node dispatch. + , sml::state <= sml::state + + sml::completion[ guard::at_eof{} ] + / action::finish_parsed + + , sml::state <= sml::state + + sml::completion[ guard::token_text{} ] + + , sml::state <= sml::state + + sml::completion[ guard::token_comment{} ] + + , sml::state <= sml::state + + sml::completion[ guard::token_open_statement{} ] + + , sml::state <= sml::state + + sml::completion[ guard::token_open_expression{} ] + + , sml::state <= sml::state + + sml::completion[ guard::token_unexpected{} ] + / action::fail_current_token + + //------------------------------------------------------------------------------// + // Text/comment terminals. + , sml::state <= sml::state + + sml::completion + / action::consume_text + + , sml::state <= sml::state + + sml::completion + / action::consume_comment + + //------------------------------------------------------------------------------// + // Statement and expression submachines. + , sml::state <= sml::state + + sml::completion + + , sml::state <= sml::state + + sml::completion[ guard::phase_ok{} ] + + , sml::state <= sml::state + + sml::completion[ guard::phase_failed{} ] + + , sml::state <= sml::state + + sml::completion + + , sml::state <= sml::state + + sml::completion[ guard::phase_ok{} ] + + , sml::state <= sml::state + + sml::completion[ guard::phase_failed{} ] + + //------------------------------------------------------------------------------// + , sml::X <= sml::state + , sml::X <= sml::state + + //------------------------------------------------------------------------------// + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + ); + // clang-format on + } +}; + +struct sm : emel::sm { + using model_type = model; +}; + +} // namespace emel::text::jinja::parser::program_parser diff --git a/src/emel/text/jinja/parser/program_parser/statement_parser/actions.hpp b/src/emel/text/jinja/parser/program_parser/statement_parser/actions.hpp new file mode 100644 index 00000000..ce53360f --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/statement_parser/actions.hpp @@ -0,0 +1,103 @@ +#pragma once + +#include +#include +#include + +#include "emel/text/jinja/parser/actions.hpp" +#include "emel/text/jinja/parser/detail.hpp" +#include "emel/text/jinja/parser/events.hpp" +#include "emel/text/jinja/parser/program_parser/statement_parser/context.hpp" +#include "emel/text/jinja/parser/program_parser/statement_parser/errors.hpp" + +namespace emel::text::jinja::parser::program_parser::statement_parser::action { + +inline const emel::text::jinja::token & +current_token(const event::parse_runtime &ev) noexcept { + return ev.ctx.lex_result.tokens[ev.ctx.token_index]; +} + +inline const emel::text::jinja::token &token_at(const event::parse_runtime &ev, + const size_t offset) noexcept { + return ev.ctx.lex_result.tokens[ev.ctx.token_index + offset]; +} + +inline const emel::text::jinja::token & +token_at_index(const event::parse_runtime &ev, const size_t index) noexcept { + return ev.ctx.lex_result.tokens[index]; +} + +template +inline emel::text::jinja::ast_ptr make_node(const size_t pos, + args &&...args_in) { + auto node = std::make_unique(std::forward(args_in)...); + node->pos = pos; + return node; +} + +struct begin_statement_scan { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ev.ctx.statement = event::statement_kind::statement; + ev.ctx.statement_start = ev.ctx.token_index; + ev.ctx.token_index += 2; + } +}; + +struct consume_statement_token { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ++ev.ctx.token_index; + } +}; + +struct consume_statement_close_and_emit { + void operator()(const event::parse_runtime &ev, context &) const { + const auto &tok = token_at_index(ev, ev.ctx.statement_start); + ev.request.program.body.push_back( + make_node(tok.pos)); + ++ev.ctx.token_index; + } +}; + +struct fail_statement_name_token { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ::emel::text::jinja::parser::action::helper::mark_error( + ev.request, ev.ctx, error::parse_failed, token_at(ev, 1).pos); + } +}; + +struct fail_statement_open_token { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ::emel::text::jinja::parser::action::helper::mark_error( + ev.request, ev.ctx, error::parse_failed, current_token(ev).pos); + } +}; + +struct fail_statement_start_token { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + const auto &tok = token_at_index(ev, ev.ctx.statement_start); + ::emel::text::jinja::parser::action::helper::mark_error( + ev.request, ev.ctx, error::parse_failed, tok.pos); + } +}; + +struct on_unexpected { + void operator()(const event::parse_runtime &ev, context &) const noexcept { + ::emel::text::jinja::parser::action::helper::mark_error( + ev.request, ev.ctx, error::internal_error, ev.ctx.error_pos_out); + } + + template + void operator()(const event_type &, context &) const noexcept {} +}; + +inline constexpr begin_statement_scan begin_statement_scan{}; +inline constexpr consume_statement_token consume_statement_token{}; +inline constexpr consume_statement_close_and_emit + consume_statement_close_and_emit{}; +inline constexpr fail_statement_name_token fail_statement_name_token{}; +inline constexpr fail_statement_open_token fail_statement_open_token{}; +inline constexpr fail_statement_start_token fail_statement_start_token{}; +inline constexpr on_unexpected on_unexpected{}; + +} // namespace + // emel::text::jinja::parser::program_parser::statement_parser::action diff --git a/src/emel/text/jinja/parser/program_parser/statement_parser/context.hpp b/src/emel/text/jinja/parser/program_parser/statement_parser/context.hpp new file mode 100644 index 00000000..2ef30057 --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/statement_parser/context.hpp @@ -0,0 +1,12 @@ +#pragma once + +namespace emel::text::jinja::parser::action { +struct context; +} + +namespace emel::text::jinja::parser::program_parser::statement_parser::action { + +using context = emel::text::jinja::parser::action::context; + +} // namespace + // emel::text::jinja::parser::program_parser::statement_parser::action diff --git a/src/emel/text/jinja/parser/program_parser/statement_parser/errors.hpp b/src/emel/text/jinja/parser/program_parser/statement_parser/errors.hpp new file mode 100644 index 00000000..dffef956 --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/statement_parser/errors.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include "emel/text/jinja/parser/program_parser/errors.hpp" + +namespace emel::text::jinja::parser::program_parser::statement_parser { + +using error = emel::text::jinja::parser::program_parser::error; + +} // namespace emel::text::jinja::parser::program_parser::statement_parser diff --git a/src/emel/text/jinja/parser/program_parser/statement_parser/guards.hpp b/src/emel/text/jinja/parser/program_parser/statement_parser/guards.hpp new file mode 100644 index 00000000..bb3bd6af --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/statement_parser/guards.hpp @@ -0,0 +1,213 @@ +#pragma once + +#include + +#include "emel/text/jinja/parser/events.hpp" +#include "emel/text/jinja/parser/program_parser/statement_parser/context.hpp" + +namespace emel::text::jinja::parser::program_parser::statement_parser::guard { + +inline bool has_token(const event::parse_ctx &ctx, + const size_t offset = 0) noexcept { + return ctx.token_index + offset < ctx.lex_result.tokens.size(); +} + +inline const emel::text::jinja::token & +token_at(const event::parse_ctx &ctx, const size_t offset = 0) noexcept { + return ctx.lex_result.tokens[ctx.token_index + offset]; +} + +inline bool token_is(const event::parse_ctx &ctx, + const emel::text::jinja::token_type type, + const size_t offset = 0) noexcept { + return has_token(ctx, offset) && token_at(ctx, offset).type == type; +} + +inline bool statement_name_is(const event::parse_ctx &ctx, + const std::string_view expected) noexcept { + return token_is(ctx, emel::text::jinja::token_type::identifier, 1) && + token_at(ctx, 1).value == expected; +} + +struct statement_identifier_missing { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return !token_is(ev.ctx, emel::text::jinja::token_type::identifier, 1); + } +}; + +struct statement_name_set { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "set"); + } +}; + +struct statement_name_if { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "if"); + } +}; + +struct statement_name_elif { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "elif"); + } +}; + +struct statement_name_else { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "else"); + } +}; + +struct statement_name_endif { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "endif"); + } +}; + +struct statement_name_for { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "for"); + } +}; + +struct statement_name_endfor { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "endfor"); + } +}; + +struct statement_name_macro { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "macro"); + } +}; + +struct statement_name_endmacro { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "endmacro"); + } +}; + +struct statement_name_call { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "call"); + } +}; + +struct statement_name_endcall { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "endcall"); + } +}; + +struct statement_name_filter { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "filter"); + } +}; + +struct statement_name_endfilter { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "endfilter"); + } +}; + +struct statement_name_break { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "break"); + } +}; + +struct statement_name_continue { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "continue"); + } +}; + +struct statement_name_generation { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "generation"); + } +}; + +struct statement_name_endgeneration { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "endgeneration"); + } +}; + +struct statement_name_endset { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return statement_name_is(ev.ctx, "endset"); + } +}; + +struct statement_name_unknown { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::identifier, 1) && + !statement_name_set{}(ev, action::context{}) && + !statement_name_if{}(ev, action::context{}) && + !statement_name_elif{}(ev, action::context{}) && + !statement_name_else{}(ev, action::context{}) && + !statement_name_endif{}(ev, action::context{}) && + !statement_name_for{}(ev, action::context{}) && + !statement_name_endfor{}(ev, action::context{}) && + !statement_name_macro{}(ev, action::context{}) && + !statement_name_endmacro{}(ev, action::context{}) && + !statement_name_call{}(ev, action::context{}) && + !statement_name_endcall{}(ev, action::context{}) && + !statement_name_filter{}(ev, action::context{}) && + !statement_name_endfilter{}(ev, action::context{}) && + !statement_name_break{}(ev, action::context{}) && + !statement_name_continue{}(ev, action::context{}) && + !statement_name_generation{}(ev, action::context{}) && + !statement_name_endgeneration{}(ev, action::context{}) && + !statement_name_endset{}(ev, action::context{}); + } +}; + +struct statement_scan_at_close { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return token_is(ev.ctx, emel::text::jinja::token_type::close_statement); + } +}; + +struct statement_scan_continue { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return has_token(ev.ctx) && + !token_is(ev.ctx, emel::text::jinja::token_type::close_statement); + } +}; + +struct statement_scan_eof { + bool operator()(const event::parse_runtime &ev, + const action::context &) const noexcept { + return !has_token(ev.ctx); + } +}; + +} // namespace + // emel::text::jinja::parser::program_parser::statement_parser::guard diff --git a/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp b/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp new file mode 100644 index 00000000..edc30148 --- /dev/null +++ b/src/emel/text/jinja/parser/program_parser/statement_parser/sm.hpp @@ -0,0 +1,146 @@ +#pragma once + +#include "emel/sm.hpp" +#include "emel/text/jinja/parser/events.hpp" +#include "emel/text/jinja/parser/program_parser/statement_parser/actions.hpp" +#include "emel/text/jinja/parser/program_parser/statement_parser/guards.hpp" + +namespace emel::text::jinja::parser::program_parser::statement_parser { + +struct deciding {}; +struct statement_kind_decision {}; +struct statement_scan {}; +struct parsed {}; +struct parse_failed {}; +struct unexpected_event {}; + +struct model { + auto operator()() const { + namespace sml = boost::sml; + + // clang-format off + return sml::make_transition_table( + //------------------------------------------------------------------------------// + // Statement parser. + sml::state <= *sml::state + + sml::completion + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_set{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_if{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_elif{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_else{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_endif{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_for{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_endfor{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_macro{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_endmacro{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_call{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_endcall{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_filter{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_endfilter{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_break{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_continue{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_generation{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_endgeneration{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_endset{} ] + / action::begin_statement_scan + + , sml::state <= sml::state + + sml::completion[ guard::statement_identifier_missing{} ] + / action::fail_statement_open_token + + , sml::state <= sml::state + + sml::completion[ guard::statement_name_unknown{} ] + / action::fail_statement_name_token + + , sml::state <= sml::state + + sml::completion[ guard::statement_scan_at_close{} ] + / action::consume_statement_close_and_emit + + , sml::state <= sml::state + + sml::completion[ guard::statement_scan_continue{} ] + / action::consume_statement_token + + , sml::state <= sml::state + + sml::completion[ guard::statement_scan_eof{} ] + / action::fail_statement_start_token + + //------------------------------------------------------------------------------// + , sml::X <= sml::state + , sml::X <= sml::state + + //------------------------------------------------------------------------------// + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + ); + // clang-format on + } +}; + +struct sm : emel::sm { + using model_type = model; +}; + +} // namespace emel::text::jinja::parser::program_parser::statement_parser diff --git a/src/emel/text/jinja/parser/sm.hpp b/src/emel/text/jinja/parser/sm.hpp index 85a08dcc..0d71cd54 100644 --- a/src/emel/text/jinja/parser/sm.hpp +++ b/src/emel/text/jinja/parser/sm.hpp @@ -1,334 +1,160 @@ #pragma once -/* -design doc: docs/designs/text/jinja/parser.design.md - --- - title: jinja/parser architecture design - status: draft - --- - - # jinja/parser architecture design - - this document defines the jinja parser actor. it replaces the legacy procedural recursive-descent parser with a hierarchical `boost::sml` state machine. because parsing is a one-time initialization step (e.g. during model loading), it utilizes standard heap allocation (`std::make_unique` and `std::vector`) for building the AST, adhering to the rule that one-time initialization may allocate. - - ## role - - act as a pure SML actor that consumes a stream of `jinja::token` objects. - - build an abstract syntax tree (`program`) using standard C++ heap allocations for AST nodes. - - enforce strict grammatical rules and nesting bounds using SML submachines and context stacks, eliminating procedural call stack recursion. - - ## architecture shift: token-pump and hierarchical SML - in the legacy design, parsing was a single procedural action (`run_parse`) that invoked a `recursive_descent_parser`. this violated the spirit of the actor model (by hiding complex control flow in an action) and relied heavily on the C++ call stack. - - the new architecture uses the SML state machine as a true **Pushdown Automaton**: - 1. **token events**: the external orchestrator (e.g. `text/conditioner`) calls the lexer to get an array of tokens, then iterates over them, dispatching a `token_event` for each one into the parser SM. - 2. **hierarchical machines**: grammatical structures (like `expression` vs `statement`) are modeled as `boost::sml` composite states. transitioning into an expression simply pushes the SM into a submachine, naturally tracking nesting without C++ recursion. - 3. **heap allocation**: since parsing only happens once during setup, actions construct AST nodes dynamically using `std::make_unique` and `std::vector`. this provides full flexibility and simplifies cleanup without violating inference hot-path constraints. - - ## events - - `event::start_parse` - - inputs: `program_out` (destination AST) and optional callbacks. - - outputs: initializes the parser's context stacks and transitions from `uninitialized` to `idle`. - - `event::token_received` - - inputs: a single `jinja::token` and its metadata (type, string value, offset). - - outputs: drives the grammar state machine. creates AST nodes on the heap, pushes/pops the parsing stack, or sets a syntax error. - - `event::end_parse` - - inputs: none. signals the end of the token stream. - - outputs: validates that all parsing stacks are empty (no unclosed tags) and finalizes the `program` structure. emits `parsing_done` or `parsing_error`. - - ## state model (hierarchical) - - ```text - uninitialized ──► idle (top-level text context) - │ - ├──► (receive `{{`) ──► sm ──► (receive `}}`) ──► idle - │ - └──► (receive `{%`) ──► sm ──► (receive `%}`) ──► idle - ``` - - ### submachine: `expression_parser` - handles the Pratt-style operator precedence and function calls. - - ```text - expecting_operand ──► (receive Identifier) ──► expecting_operator - ├──► (receive Numeric) ──► expecting_operator - └──► (receive `(`) ──► sm (nested) - - expecting_operator ──► (receive `+`) ──► expecting_operand - ├──► (receive `(`) ──► sm - └──► (receive `]`) ──► (pop stack) - ``` - - ## responsibilities - - 1. **one-time heap allocation**: - - AST nodes (like `binary_expression` or `identifier`) are dynamically allocated via `std::make_unique`. - - `ast_list` uses `std::vector>`. - - memory is released naturally when the `program` struct is destroyed. - - 2. **explicit bounds**: - - the SML context must track `stack_depth`. if an expression nests too deeply (e.g. `(((((...))))`), the `push_stack` action must trigger a transition to `errored` to prevent malicious nesting attacks. - - 3. **deterministic error recovery**: - - if a syntax error occurs (e.g. an unexpected token type for the current state), the machine immediately transitions to `errored` and sets `error_pos` in the context. subsequent `token_received` events are ignored or result in `sml::unexpected_event` routing. - - ## error codes - - this actor can produce the following error codes: - - - `EMEL_ERR_TEMPLATE_SYNTAX` — unexpected token or malformed grammar construct. - - `EMEL_ERR_TEMPLATE_UNSUPPORTED` — valid syntax for a construct that is not supported by this implementation. - - `EMEL_ERR_CAPACITY` — node arena or parsing stack capacity exceeded. - - `EMEL_ERR_INVALID_ARGUMENT` — invalid token stream, buffer pointers, or limit values. -*/ - - -/* -design doc: docs/designs/text/jinja/lexer.design.md - --- - title: jinja/lexer architecture design - status: draft - --- - - # jinja/lexer architecture design - - this document defines the jinja lexer actor. it replaces the legacy procedural `while`-loop tokenizer with a pure `boost::sml` state machine, acting as a character-pump automaton to cleanly identify boundaries, handle escapes, and emit tokens without complex nested loops or procedural lookahead spaghetti. - - ## role - - act as a pure SML actor that consumes a stream of `char` (or `event::char_received`). - - emit a sequence of `jinja::token` objects via synchronous callbacks or by accumulating them in a context array. - - gracefully handle malformed templates (e.g., unterminated strings) using strict state transitions rather than complex procedural bounds checking. - - ## architecture shift: the character-pump automaton - in the legacy design, the lexer was a procedural class that ran a massive `while` loop, manually checking indices (`pos + 1`, `pos + 2`) and advancing pointers. this is fragile and violates the actor model paradigm. - - the new architecture turns the lexer into a true **Finite State Machine**: - 1. **character events**: the orchestrator iterates over the `std::string_view` of the template and dispatches an `event::char_received` for each character (plus an `event::eof` at the end). - 2. **state-driven tokenization**: the lexer starts in a `text` state. if it receives `{`, it transitions to a `potential_block_open` state. if the next char is `%`, it emits a `text` token (if any was accumulated) and transitions to the `in_statement` state. - 3. **zero-allocation tracking**: the context holds a fixed-size `scratch_buffer` for accumulating the current token's characters (or simply tracks the `start_pos` and `current_pos` indices to yield `std::string_view` tokens). - - ## events - - `event::start_lex` - - inputs: optional synchronous callbacks for emitting tokens. - - outputs: initializes context pointers and transitions to `idle`. - - `event::char_received` - - inputs: `char c`, `size_t pos`. - - outputs: drives the state machine. may accumulate characters, transition states, or emit completed tokens. - - `event::eof` - - inputs: none. - - outputs: forces the emission of the final token (if valid). if the machine is in an unterminated state (like `in_string`), it transitions to `errored`. - - ## state model - - ```text - uninitialized ──► idle (accumulating text) - │ - ├──► (receive `{`) ──► potential_open - │ ├──► (receive `%`) ──► in_statement - │ ├──► (receive `{`) ──► in_expression - │ └──► (receive `#`) ──► in_comment - │ - └──► (receive EOF) ──► done - ``` - - ### sub-states (e.g., `in_expression`) - when inside an expression or statement, the machine parses numbers, identifiers, strings, and operators: - - ```text - in_expression ──► (receive `"`) ──► in_string ──► (receive `"`) ──► in_expression - ├──► (receive digit) ──► in_number ──► (receive non-digit) ──► in_expression - ├──► (receive letter) ──► in_identifier ──► (receive non-word) ──► in_expression - └──► (receive `}`) ──► potential_close ──► (receive `}`) ──► idle - ``` - - ## responsibilities & constraints - - 1. **index-based string views**: - - to avoid heap-allocating `std::string` for every token, the lexer context should merely track the `start_pos` of the current token. - - when a token is complete, it emits a `jinja::token` containing the `token_type` and the `std::string_view` derived from the original template string. - - 2. **escape sequence handling**: - - when inside `in_string`, receiving `\\` transitions to an `in_escape` state. the next character is decoded and the machine returns to `in_string`. if `EOF` is received during `in_escape` or `in_string`, the lexer transitions to `errored`. - - 3. **deterministic error recovery**: - - invalid syntax (e.g., an illegal character in a variable name or an unterminated string) causes an immediate transition to `errored`. - - the error position is recorded in the context, and subsequent `char_received` events are ignored via `unexpected_event` routing. -*/ - - -/* -design doc: docs/designs/jinja.design.md - --- - title: jinja architecture design - status: draft - --- - - # jinja architecture design - - this document defines the jinja templating subsystem. it provides a high-performance, allocation-bounded jinja2 template engine used for chat formatting and prompt construction within the `text/formatter` and `text/conditioner` pipelines. - - ## role - - provide a lightweight, dependency-free jinja2 implementation tailored for LLM chat templates. - - execute entirely within the run-to-completion (RTC) actor model. - - enforce strict memory and computation bounds during both parsing and formatting to prevent denial-of-service from malicious templates. - - ## architecture: explicit decoupling - the jinja subsystem is divided into three distinct phases to ensure strict separation of concerns and allow offline compilation of templates in the future: - 1. **`lexer`**: a pure, synchronous utility that converts a raw template string into a bounded array of tokens. - 2. **`parser::sm`**: an SML actor that consumes a token stream and builds an abstract syntax tree (`program`). - 3. **`formatter::sm`**: an SML actor that evaluates a `program` against a set of `globals` and emits formatted UTF-8 bytes. - - ## lexer - the lexer is a pure utility (not an SML actor) that takes a `std::string_view` and returns a `lexer_result` containing a sequence of tokens. - - **responsibilities**: handle whitespace trimming (`{%-`, `-%}`), detect blocks, comments, and expressions, and reject invalid syntax (like unterminated strings). - - **invariants**: operates sequentially without heap allocating individual tokens, typically returning a flat array/vector of bounded tokens. - - ## parser (parser::sm) - the parser is an SML actor that converts the token stream into a structured `program` (AST). - - ### events - - `event::parse` - - inputs: `template_text`, `program_out` (AST destination), `error_out`, and optional synchronous callbacks (`dispatch_done`, `dispatch_error`). - - outputs: tokenizes and parses the template text, populates the provided `program`, and invokes the appropriate callback before returning. - - ### state model - ```text - uninitialized ──► initialized - │ - initialized ──► parse_decision ──► (done | errored) - ▲ │ - └────────────────────────────────────┘ - ``` - - `initialized` — awaiting parse intent. - - `parse_decision` — synchronously runs the internal parsing logic (via recursive descent) and branches based on success/failure. - - `done` — parsing complete and AST populated. - - `errored` — syntax error or invalid argument. - - unexpected events route to `unexpected`. - - ## formatter (formatter::sm) - the formatter is an SML actor that evaluates a parsed `program` and emits the final formatted text. it supports variables, control flow (if/for/macro), filters, and tests. - - ### events - - `event::format` - - inputs: `program` (parsed AST), `globals` (object containing user variables and model metadata), caller-provided `output` buffer + capacity, `output_length`, `error_out`, and optional synchronous callbacks (`dispatch_done`, `dispatch_error`). - - outputs: evaluates the AST, writes UTF-8 bytes directly to the `output` buffer, updates `output_length`, and invokes the callback. - - ### state model - ```text - uninitialized ──► initialized - │ - initialized ──► setup ──► eval_stmt ──► format_decision ──► (done | errored) - ▲ │ ▲ │ - │ ▼ │ │ - │ eval_expr ──► write_output │ - │ │ │ - └─────────────────────────────────────┴───────────────────────┘ - ``` - - `initialized` — idle state awaiting format intent. - - `setup` — initializes the evaluation context and seeds the initial statement work. - - `eval_stmt` — steps through statements in the current scope. - - `eval_expr` — evaluates pending expressions (variable resolution, math, filters). - - `write_output` — emits the evaluated values to the output buffer. - - `format_decision` — checks if the evaluation completed successfully or hit an error/limit. - - `done` — formatting complete. - - `errored` — hit an execution limit (e.g., max scope depth, out of buffer capacity, division by zero) or invalid arguments. - - unexpected events route to `unexpected`. - - ### responsibilities & constraints - 1. **bounded evaluation**: the formatter strictly limits evaluation depth, scope counts, call stack limits, and array/object sizes to prevent infinite loops or memory exhaustion (enforced by constants like `k_max_scopes`, `k_max_array_items`, `k_max_callables`). - 2. **zero-allocation formatting**: outputs are written directly to caller-provided buffers (`format_io`). intermediate strings are not dynamically heap-allocated during template evaluation; they are either string views of the original template or written into a fixed-size scratch buffer in the SML context. - 3. **synchronous callbacks**: results are communicated via `dispatch_done` and `dispatch_error` callbacks before the SML dispatch returns, eliminating the need for the caller to inspect the state machine's internal context. -*/ - - #include +#include "emel/sm.hpp" #include "emel/text/jinja/parser/actions.hpp" +#include "emel/text/jinja/parser/classifier_parser/sm.hpp" +#include "emel/text/jinja/parser/context.hpp" #include "emel/text/jinja/parser/events.hpp" #include "emel/text/jinja/parser/guards.hpp" -#include "emel/sm.hpp" +#include "emel/text/jinja/parser/lexer/sm.hpp" +#include "emel/text/jinja/parser/program_parser/sm.hpp" namespace emel::text::jinja::parser { struct initialized {}; -struct parse_decision {}; +struct request_decision {}; +struct tokenize_begin {}; +struct tokenize_next {}; +struct tokenize_result_decision {}; +struct tokenize_append {}; +struct classify_result_decision {}; +struct parse_result_decision {}; struct done {}; struct errored {}; struct unexpected {}; -/** - * jinja parser orchestration model. - * - * state purposes: - * - `initialized`: idle state awaiting parse intent. - * - `parse_decision`: run parsing step and branch based on phase results. - * - `done`/`errored`: terminal outcomes. - * - `unexpected`: sequencing contract violation. - * - * guard semantics: - * - `valid_parse`/`invalid_parse` validate request pointers and parameters. - * - `phase_*` guards observe errors set by actions. - * - * action side effects: - * - `run_parse` returns format unsupported until parser is implemented. - * - `on_unexpected` reports any event sequencing violations. - */ struct model { auto operator()() const { namespace sml = boost::sml; + // clang-format off return sml::make_transition_table( - *sml::state + - sml::event[guard::valid_parse{}] / - action::run_parse = sml::state, - sml::state + - sml::event[guard::invalid_parse{}] / - action::reject_invalid_parse = sml::state, - - sml::state + sml::event[guard::valid_parse{}] / - action::run_parse = sml::state, - sml::state + sml::event[guard::invalid_parse{}] / - action::reject_invalid_parse = - sml::state, - - sml::state + sml::event[guard::valid_parse{}] / - action::run_parse = sml::state, - sml::state + sml::event[guard::invalid_parse{}] / - action::reject_invalid_parse = - sml::state, - - sml::state + - sml::event[guard::valid_parse{}] / - action::run_parse = sml::state, - sml::state + - sml::event[guard::invalid_parse{}] / - action::reject_invalid_parse = sml::state, - - sml::state[guard::phase_ok{}] = - sml::state, - sml::state[guard::phase_failed{}] = - sml::state, - - sml::state + - sml::unexpected_event / - action::on_unexpected = sml::state, - sml::state + - sml::unexpected_event / - action::on_unexpected = sml::state, - sml::state + sml::unexpected_event / - action::on_unexpected = sml::state, - sml::state + sml::unexpected_event / - action::on_unexpected = - sml::state, - sml::state + - sml::unexpected_event / - action::on_unexpected = sml::state); + //------------------------------------------------------------------------------// + // Request validation. + sml::state <= *sml::state + + sml::event[ guard::valid_parse{} ] + / action::begin_parse + , sml::state <= sml::state + + sml::event[ guard::invalid_parse{} ] + / action::reject_invalid_parse + + , sml::state <= sml::state + + sml::event[ guard::valid_parse{} ] + / action::begin_parse + , sml::state <= sml::state + + sml::event[ guard::invalid_parse{} ] + / action::reject_invalid_parse + + , sml::state <= sml::state + + sml::event[ guard::valid_parse{} ] + / action::begin_parse + , sml::state <= sml::state + + sml::event[ guard::invalid_parse{} ] + / action::reject_invalid_parse + + , sml::state <= sml::state + + sml::event[ guard::valid_parse{} ] + / action::begin_parse + , sml::state <= sml::state + + sml::event[ guard::invalid_parse{} ] + / action::reject_invalid_parse + + //------------------------------------------------------------------------------// + // Pipeline phases. + , sml::state <= sml::state + + sml::completion + / action::begin_tokenization + + , sml::state <= sml::state + + sml::completion + / action::request_next_lex_token + + , sml::state <= sml::state + + sml::completion + + , sml::state <= sml::state + + sml::completion[ guard::lexer_at_eof{} ] + + , sml::state <= sml::state + + sml::completion[ guard::lexer_has_token{} ] + / action::append_lex_token + + , sml::state <= sml::state + + sml::completion[ guard::phase_failed{} ] + / action::commit_lex_error + + , sml::state <= sml::state + + sml::completion + / action::request_next_lex_token + + , sml::state <= sml::state + + sml::completion + + , sml::state <= sml::state + + sml::completion[ guard::phase_ok{} ] + , sml::state <= sml::state + + sml::completion[ guard::phase_failed{} ] + + , sml::state <= sml::state + + sml::completion + + , sml::state <= sml::state + + sml::completion[ guard::phase_ok{} ] + / action::dispatch_done + , sml::state <= sml::state + + sml::completion[ guard::phase_failed{} ] + / action::dispatch_error + + //------------------------------------------------------------------------------// + // Unexpected events. + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + , sml::state <= sml::state + sml::unexpected_event + / action::on_unexpected + ); + // clang-format on } }; -struct sm : public emel::sm { - using base_type = emel::sm; +struct sm : public emel::sm { + using base_type = emel::sm; using base_type::base_type; + using base_type::is; using base_type::process_event; + using base_type::visit_current_states; + + bool process_event(const event::parse &ev) { + event::parse_ctx runtime_ctx{ + ev.template_text, + ev.error_out, + ev.error_pos_out, + }; + event::parse_runtime runtime_ev{ev, runtime_ctx}; + const bool accepted = base_type::process_event(runtime_ev); + return accepted && runtime_ctx.err == error::none; + } }; +using Parser = sm; + } // namespace emel::text::jinja::parser diff --git a/src/emel/text/jinja/types.hpp b/src/emel/text/jinja/types.hpp index d2b083e4..8275d93a 100644 --- a/src/emel/text/jinja/types.hpp +++ b/src/emel/text/jinja/types.hpp @@ -1,3 +1,3 @@ #pragma once -#include "emel/text/jinja/ast.hpp" +#include "emel/text/jinja/parser/detail.hpp" diff --git a/src/emel/text/jinja/value.hpp b/src/emel/text/jinja/value.hpp deleted file mode 100644 index c7fbf7f7..00000000 --- a/src/emel/text/jinja/value.hpp +++ /dev/null @@ -1,70 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace emel::text::jinja { - -enum class value_type : uint8_t { - undefined = 0, - none = 1, - boolean = 2, - integer = 3, - floating = 4, - string = 5, - array = 6, - object = 7, - function = 8 -}; - -struct value; -struct object_entry; - -struct string_value { - std::string_view view = {}; - bool is_input = false; -}; - -struct array_value { - value * items = nullptr; - size_t count = 0; - size_t capacity = 0; -}; - -struct object_value { - object_entry * entries = nullptr; - size_t count = 0; - size_t capacity = 0; - bool has_builtins = true; -}; - -enum class function_kind : uint8_t { - builtin = 0, - macro = 1, - caller = 2 -}; - -struct function_ref { - function_kind kind = function_kind::builtin; - const void * data = nullptr; -}; - -struct value { - value_type type = value_type::undefined; - bool bool_v = false; - int64_t int_v = 0; - double float_v = 0.0; - string_value string_v = {}; - array_value array_v = {}; - object_value object_v = {}; - function_ref func_v = {}; - std::string_view hint = {}; -}; - -struct object_entry { - value key = {}; - value val = {}; -}; - -} // namespace emel::text::jinja diff --git a/tests/text/jinja/lexer_tests.cpp b/tests/text/jinja/lexer_tests.cpp index 95a4bb04..4cd39746 100644 --- a/tests/text/jinja/lexer_tests.cpp +++ b/tests/text/jinja/lexer_tests.cpp @@ -1,199 +1,284 @@ #include -#include "emel/emel.h" -#include "emel/text/jinja/lexer.hpp" +#include +#include + +#include "emel/text/jinja/parser/detail.hpp" +#include "emel/text/jinja/parser/errors.hpp" +#include "emel/text/jinja/parser/lexer/actions.hpp" +#include "emel/text/jinja/parser/lexer/sm.hpp" + +namespace { + +using emel::text::jinja::lexer_result; +using emel::text::jinja::lexer::cursor; +using emel::text::jinja::lexer::event::next; +using emel::text::jinja::lexer::events::next_done; +using emel::text::jinja::lexer::events::next_error; +using emel::text::jinja::parser::error; + +constexpr int32_t k_ok = emel::text::jinja::parser::to_error_code(error::none); +constexpr int32_t k_parse_failed = + emel::text::jinja::parser::to_error_code(error::parse_failed); + +struct token_step_result { + bool done_called = false; + bool error_called = false; + emel::text::jinja::token token = {}; + bool has_token = false; + cursor next_cursor = {}; + int32_t err = k_ok; + size_t error_pos = 0; + + bool on_done(const next_done &ev) { + done_called = true; + token = ev.token; + has_token = ev.has_token; + next_cursor = ev.next_cursor; + return true; + } + + bool on_error(const next_error &ev) { + error_called = true; + err = ev.err; + error_pos = ev.error_pos; + return true; + } +}; + +lexer_result tokenize_with_machine(std::string_view source) { + lexer_result result{}; + result.source = std::string(source); + emel::text::jinja::parser::lexer::detail::normalize_source(result.source); + + emel::text::jinja::parser::lexer::action::context ctx{}; + emel::text::jinja::parser::lexer::sm machine{ctx}; + cursor cur{ + result.source, + 0, + 0, + 0, + emel::text::jinja::token_type::close_statement, + false, + false, + }; + + for (;;) { + token_step_result step{}; + const next ev{ + cur, + next::done_callback::from(&step), + next::error_callback::from(&step), + }; + const auto scan = emel::text::jinja::lexer::detail::scan_next_token_safe(cur); + const emel::text::jinja::parser::lexer::event::next_runtime runtime_ev{ + ev, + scan, + }; + const bool accepted = machine.process_event(runtime_ev); + if (!accepted) { + result.error = step.error_called ? step.err : k_parse_failed; + result.error_pos = step.error_called ? step.error_pos : cur.offset; + break; + } + if (step.error_called) { + result.error = step.err; + result.error_pos = step.error_pos; + break; + } + if (!step.done_called || !step.has_token) { + break; + } + result.tokens.push_back(step.token); + cur = step.next_cursor; + } + + return result; +} + +} // namespace TEST_CASE("jinja_lexer_tokenizes_expression") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("hello {{ name }}"); + lexer_result result = tokenize_with_machine("hello {{ name }}"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(result.tokens.size() >= 4); CHECK(result.tokens[0].type == emel::text::jinja::token_type::text); CHECK(result.tokens[0].value == "hello "); - CHECK(result.tokens[1].type == emel::text::jinja::token_type::open_expression); + CHECK(result.tokens[1].type == + emel::text::jinja::token_type::open_expression); CHECK(result.tokens[2].type == emel::text::jinja::token_type::identifier); CHECK(result.tokens[2].value == "name"); - CHECK(result.tokens[3].type == emel::text::jinja::token_type::close_expression); + CHECK(result.tokens[3].type == + emel::text::jinja::token_type::close_expression); } TEST_CASE("jinja_lexer_handles_empty_input") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize(""); + lexer_result result = tokenize_with_machine(""); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); CHECK(result.tokens.empty()); } TEST_CASE("jinja_lexer_tokenizes_comment") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("text{# note #}{{ x }}"); + lexer_result result = tokenize_with_machine("text{# note #}{{ x }}"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(result.tokens.size() >= 4); CHECK(result.tokens[0].type == emel::text::jinja::token_type::text); CHECK(result.tokens[1].type == emel::text::jinja::token_type::comment); - CHECK(result.tokens[2].type == emel::text::jinja::token_type::open_expression); + CHECK(result.tokens[2].type == + emel::text::jinja::token_type::open_expression); CHECK(result.tokens[3].type == emel::text::jinja::token_type::identifier); } TEST_CASE("jinja_lexer_rejects_invalid_escape") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{{ \"\\x\" }}"); + lexer_result result = tokenize_with_machine("{{ \"\\x\" }}"); - CHECK(result.error == EMEL_ERR_PARSE_FAILED); + CHECK(result.error == k_parse_failed); CHECK(result.error_pos > 0); } TEST_CASE("jinja_lexer_rejects_unterminated_escape") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{{ \"foo\\"); + lexer_result result = tokenize_with_machine("{{ \"foo\\"); - CHECK(result.error == EMEL_ERR_PARSE_FAILED); + CHECK(result.error == k_parse_failed); } TEST_CASE("jinja_lexer_rejects_unterminated_comment") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{# comment"); + lexer_result result = tokenize_with_machine("{# comment"); - CHECK(result.error == EMEL_ERR_PARSE_FAILED); + CHECK(result.error == k_parse_failed); } TEST_CASE("jinja_lexer_rejects_unexpected_character") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{{ @ }}"); + lexer_result result = tokenize_with_machine("{{ @ }}"); - CHECK(result.error == EMEL_ERR_PARSE_FAILED); + CHECK(result.error == k_parse_failed); } TEST_CASE("jinja_lexer_handles_lonely_brace") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{"); + lexer_result result = tokenize_with_machine("{"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(result.tokens.size() == 1); CHECK(result.tokens[0].type == emel::text::jinja::token_type::text); } TEST_CASE("jinja_lexer_handles_double_dash_expression") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{{--}}"); + lexer_result result = tokenize_with_machine("{{--}}"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(result.tokens.size() >= 2); - CHECK(result.tokens[0].type == emel::text::jinja::token_type::open_expression); - CHECK(result.tokens.back().type == emel::text::jinja::token_type::close_expression); + CHECK(result.tokens[0].type == + emel::text::jinja::token_type::open_expression); + CHECK(result.tokens.back().type == + emel::text::jinja::token_type::close_expression); } TEST_CASE("jinja_lexer_rejects_unterminated_string") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{{ \"foo }}"); + lexer_result result = tokenize_with_machine("{{ \"foo }}"); - CHECK(result.error == EMEL_ERR_PARSE_FAILED); + CHECK(result.error == k_parse_failed); } TEST_CASE("jinja_lexer_handles_unary_numeric") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{{ -1 + 2 }}"); + lexer_result result = tokenize_with_machine("{{ -1 + 2 }}"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(result.tokens.size() >= 4); - CHECK(result.tokens[0].type == emel::text::jinja::token_type::open_expression); - CHECK(result.tokens[1].type == emel::text::jinja::token_type::numeric_literal); + CHECK(result.tokens[0].type == + emel::text::jinja::token_type::open_expression); + CHECK(result.tokens[1].type == + emel::text::jinja::token_type::numeric_literal); CHECK(result.tokens[1].value == "-1"); } TEST_CASE("jinja_lexer_handles_nested_objects") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{{ {'a': {'b': 1}} }}"); + lexer_result result = tokenize_with_machine("{{ {'a': {'b': 1}} }}"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(result.tokens.size() >= 6); - CHECK(result.tokens[0].type == emel::text::jinja::token_type::open_expression); - CHECK(result.tokens[1].type == emel::text::jinja::token_type::open_curly_bracket); + CHECK(result.tokens[0].type == + emel::text::jinja::token_type::open_expression); + CHECK(result.tokens[1].type == + emel::text::jinja::token_type::open_curly_bracket); } TEST_CASE("jinja_lexer_handles_decimal_numbers") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{{ 1.25 }}"); + lexer_result result = tokenize_with_machine("{{ 1.25 }}"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(result.tokens.size() >= 3); - CHECK(result.tokens[1].type == emel::text::jinja::token_type::numeric_literal); + CHECK(result.tokens[1].type == + emel::text::jinja::token_type::numeric_literal); CHECK(result.tokens[1].value == "1.25"); } TEST_CASE("jinja_lexer_handles_escaped_strings") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = - lex.tokenize("{{ \"a\\n\\t\\r\\b\\f\\v\\\\\\\"\" }}"); + lexer_result result = + tokenize_with_machine("{{ \"a\\n\\t\\r\\b\\f\\v\\\\\\\"\" }}"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(result.tokens.size() >= 3); CHECK(result.tokens[1].type == emel::text::jinja::token_type::string_literal); } TEST_CASE("jinja_lexer_normalizes_crlf") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("a\r\nb\r{{ x }}\n"); + lexer_result result = tokenize_with_machine("a\r\nb\r{{ x }}\n"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); CHECK(result.source.find('\r') == std::string::npos); } TEST_CASE("jinja_lexer_trims_blocks") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = - lex.tokenize(" \n {%- if x -%}\ntext"); + lexer_result result = tokenize_with_machine(" \n {%- if x -%}\ntext"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(!result.tokens.empty()); CHECK(result.tokens[0].type == emel::text::jinja::token_type::open_statement); } TEST_CASE("jinja_lexer_trims_whitespace_before_block") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize(" {%- if x %}"); + lexer_result result = tokenize_with_machine(" {%- if x %}"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(!result.tokens.empty()); CHECK(result.tokens[0].type == emel::text::jinja::token_type::open_statement); } TEST_CASE("jinja_lexer_trims_whitespace_after_block") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{%- if x -%} "); + lexer_result result = tokenize_with_machine("{%- if x -%} "); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(!result.tokens.empty()); CHECK(result.tokens[0].type == emel::text::jinja::token_type::open_statement); } TEST_CASE("jinja_lexer_trims_newline_after_block") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{% if x %}\ntext"); + lexer_result result = tokenize_with_machine("{% if x %}\ntext"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(result.tokens.size() >= 5); CHECK(result.tokens.back().type == emel::text::jinja::token_type::text); CHECK(result.tokens.back().value == "text"); } TEST_CASE("jinja_lexer_handles_single_quote_escape") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{{ 'it\\'s' }}"); + lexer_result result = tokenize_with_machine("{{ 'it\\'s' }}"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(result.tokens.size() >= 3); CHECK(result.tokens[1].type == emel::text::jinja::token_type::string_literal); CHECK(result.tokens[1].value == "it's"); } TEST_CASE("jinja_lexer_lstrip_block_trims_trailing_whitespace") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("hello {%- if x %}"); + lexer_result result = tokenize_with_machine("hello {%- if x %}"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(result.tokens.size() >= 2); CHECK(result.tokens[0].type == emel::text::jinja::token_type::text); CHECK(result.tokens[0].value == "hello"); @@ -201,10 +286,10 @@ TEST_CASE("jinja_lexer_lstrip_block_trims_trailing_whitespace") { } TEST_CASE("jinja_lexer_handles_open_expression_trailing_dash") { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result result = lex.tokenize("{{-"); + lexer_result result = tokenize_with_machine("{{-"); - CHECK(result.error == EMEL_OK); + CHECK(result.error == k_ok); REQUIRE(result.tokens.size() == 1); - CHECK(result.tokens[0].type == emel::text::jinja::token_type::open_expression); + CHECK(result.tokens[0].type == + emel::text::jinja::token_type::open_expression); } diff --git a/tests/text/jinja/parser_tests.cpp b/tests/text/jinja/parser_tests.cpp index 7cd3015a..467480f6 100644 --- a/tests/text/jinja/parser_tests.cpp +++ b/tests/text/jinja/parser_tests.cpp @@ -1,29 +1,52 @@ #include #include +#include #include -#include "emel/emel.h" -#include "emel/text/jinja/ast.hpp" -#include "emel/text/jinja/parser/actions.hpp" +#include "emel/text/jinja/parser/detail.hpp" +#include "emel/text/jinja/parser/errors.hpp" #include "emel/text/jinja/parser/events.hpp" #include "emel/text/jinja/parser/sm.hpp" -#include "emel/text/jinja/types.hpp" namespace { -bool dispatch_done_test(void * owner, - const emel::text::jinja::events::parsing_done &) { - *static_cast(owner) = true; +using emel::text::jinja::event::parse; +using emel::text::jinja::events::parsing_done; +using emel::text::jinja::events::parsing_error; +using done_cb = parse::done_callback; +using error_cb = parse::error_callback; + +bool ignore_done_callback(const parsing_done &) { return true; } -bool dispatch_error_test(void * owner, - const emel::text::jinja::events::parsing_error &) { - *static_cast(owner) = true; +bool ignore_error_callback(const parsing_error &) { return true; } -} // namespace +constexpr done_cb k_ignore_done_callback = done_cb::from<&ignore_done_callback>(); +constexpr error_cb k_ignore_error_callback = error_cb::from<&ignore_error_callback>(); + +struct callback_tracker { + bool done_called = false; + bool error_called = false; + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 0; + + bool on_done(const parsing_done &) { + done_called = true; + return true; + } + + bool on_error(const parsing_error & ev) { + error_called = true; + err = ev.err; + error_pos = ev.error_pos; + return true; + } +}; + +} // namespace TEST_CASE("jinja_parser_starts_initialized") { emel::text::jinja::parser::action::context ctx{}; @@ -34,107 +57,114 @@ TEST_CASE("jinja_parser_starts_initialized") { TEST_CASE("jinja_parser_valid_parse_reaches_done") { emel::text::jinja::parser::action::context ctx{}; emel::text::jinja::parser::sm machine{ctx}; - int32_t error = -1; emel::text::jinja::program program{}; - bool done_called = false; - bool error_called = false; - - ::emel::text::jinja::event::parse ev{ - .template_text = "{{ foo }}", - .program_out = &program, - .error_out = &error, - .owner_sm = &error_called, - .dispatch_done = - ::emel::callback( - &done_called, dispatch_done_test), - .dispatch_error = - ::emel::callback( - &error_called, dispatch_error_test)}; + int32_t err = -1; + size_t error_pos = 999; + callback_tracker tracker{}; + + parse ev{ + "{{ foo }}", + program, + done_cb::from(&tracker), + error_cb::from(&tracker), + err, + error_pos, + }; CHECK(machine.process_event(ev)); CHECK(machine.is(boost::sml::state)); - CHECK(done_called); - CHECK_FALSE(error_called); - CHECK(error == EMEL_OK); + CHECK(tracker.done_called); + CHECK_FALSE(tracker.error_called); + CHECK(err == static_cast(emel::text::jinja::parser::error::none)); + CHECK(error_pos == 0); CHECK(program.body.size() == 1); CHECK(dynamic_cast(program.body[0].get()) != nullptr); } -TEST_CASE("jinja_parser_invalid_parse_reaches_errored") { +TEST_CASE("jinja_parser_invalid_request_with_callbacks_dispatches_error") { emel::text::jinja::parser::action::context ctx{}; emel::text::jinja::parser::sm machine{ctx}; - int32_t error = EMEL_OK; emel::text::jinja::program program{}; - bool error_called = false; - - ::emel::text::jinja::event::parse ev{ - .template_text = "", - .program_out = &program, - .error_out = &error, - .owner_sm = &error_called, - .dispatch_error = - ::emel::callback( - &error_called, dispatch_error_test)}; + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 999; + callback_tracker tracker{}; + + parse ev{ + "", + program, + done_cb::from(&tracker), + error_cb::from(&tracker), + err, + error_pos, + }; CHECK_FALSE(machine.process_event(ev)); CHECK(machine.is(boost::sml::state)); - CHECK(error_called); - CHECK(error == EMEL_ERR_INVALID_ARGUMENT); + CHECK_FALSE(tracker.done_called); + CHECK(tracker.error_called); + CHECK(err == static_cast(emel::text::jinja::parser::error::invalid_request)); + CHECK(error_pos == 0); CHECK(program.body.empty()); } -TEST_CASE("jinja_parser_parse_failure_reports_parse_error") { +TEST_CASE("jinja_parser_parse_failure_reports_error") { emel::text::jinja::parser::action::context ctx{}; emel::text::jinja::parser::sm machine{ctx}; - int32_t error = EMEL_OK; emel::text::jinja::program program{}; - bool error_called = false; - - ::emel::text::jinja::event::parse ev{ - .template_text = "{{ }}", - .program_out = &program, - .error_out = &error, - .owner_sm = &error_called, - .dispatch_error = - ::emel::callback( - &error_called, dispatch_error_test)}; + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 0; + callback_tracker tracker{}; + + parse ev{ + "{{ }}", + program, + done_cb::from(&tracker), + error_cb::from(&tracker), + err, + error_pos, + }; CHECK_FALSE(machine.process_event(ev)); CHECK(machine.is(boost::sml::state)); - CHECK(error_called); - CHECK(error == EMEL_ERR_PARSE_FAILED); + CHECK_FALSE(tracker.done_called); + CHECK(tracker.error_called); + CHECK(err == static_cast(emel::text::jinja::parser::error::parse_failed)); + CHECK(error_pos > 0); CHECK(program.body.empty()); } -TEST_CASE("jinja_parser_lex_failure_reports_parse_error") { +TEST_CASE("jinja_parser_lex_failure_reports_error") { emel::text::jinja::parser::action::context ctx{}; emel::text::jinja::parser::sm machine{ctx}; - int32_t error = EMEL_OK; emel::text::jinja::program program{}; - bool error_called = false; - - ::emel::text::jinja::event::parse ev{ - .template_text = "{{ \"\\x\" }}", - .program_out = &program, - .error_out = &error, - .owner_sm = &error_called, - .dispatch_error = - ::emel::callback( - &error_called, dispatch_error_test)}; + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 0; + callback_tracker tracker{}; + + parse ev{ + "{{ \"\\x\" }}", + program, + done_cb::from(&tracker), + error_cb::from(&tracker), + err, + error_pos, + }; CHECK_FALSE(machine.process_event(ev)); CHECK(machine.is(boost::sml::state)); - CHECK(error_called); - CHECK(error == EMEL_ERR_PARSE_FAILED); + CHECK_FALSE(tracker.done_called); + CHECK(tracker.error_called); + CHECK(err == static_cast(emel::text::jinja::parser::error::parse_failed)); + CHECK(error_pos > 0); CHECK(program.body.empty()); } TEST_CASE("jinja_parser_parses_control_statements") { emel::text::jinja::parser::action::context ctx{}; emel::text::jinja::parser::sm machine{ctx}; - int32_t error = EMEL_OK; emel::text::jinja::program program{}; - bool done_called = false; + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 0; const std::string_view tmpl = "{% if cond %}{{ value }}{% elif other %}x{% else %}y{% endif %}" @@ -147,27 +177,27 @@ TEST_CASE("jinja_parser_parses_control_statements") { "{% filter upper(value) %}x{% endfilter %}" "{% generation %}{% endgeneration %}"; - ::emel::text::jinja::event::parse ev{ - .template_text = tmpl, - .program_out = &program, - .error_out = &error, - .owner_sm = &done_called, - .dispatch_done = - ::emel::callback( - &done_called, dispatch_done_test)}; + parse ev{ + tmpl, + program, + k_ignore_done_callback, + k_ignore_error_callback, + err, + error_pos, + }; CHECK(machine.process_event(ev)); CHECK(machine.is(boost::sml::state)); - CHECK(done_called); - CHECK(error == EMEL_OK); + CHECK(err == static_cast(emel::text::jinja::parser::error::none)); CHECK(program.body.size() >= 7); } TEST_CASE("jinja_parser_parses_expressions") { emel::text::jinja::parser::action::context ctx{}; emel::text::jinja::parser::sm machine{ctx}; - int32_t error = EMEL_OK; emel::text::jinja::program program{}; + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 0; const std::string_view tmpl = "{{ foo.bar[1:2:3](a=1, *args)|filter }}" @@ -180,23 +210,27 @@ TEST_CASE("jinja_parser_parses_expressions") { "{{ 'a' 'b' }}" "{{ -1 + 2 * 3 }}"; - ::emel::text::jinja::event::parse ev{ - .template_text = tmpl, - .program_out = &program, - .error_out = &error, + parse ev{ + tmpl, + program, + k_ignore_done_callback, + k_ignore_error_callback, + err, + error_pos, }; CHECK(machine.process_event(ev)); CHECK(machine.is(boost::sml::state)); - CHECK(error == EMEL_OK); + CHECK(err == static_cast(emel::text::jinja::parser::error::none)); CHECK(program.body.size() >= 9); } TEST_CASE("jinja_parser_parses_additional_expressions") { emel::text::jinja::parser::action::context ctx{}; emel::text::jinja::parser::sm machine{ctx}; - int32_t error = EMEL_OK; emel::text::jinja::program program{}; + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 0; const std::string_view tmpl = "{# note #}" @@ -209,92 +243,78 @@ TEST_CASE("jinja_parser_parses_additional_expressions") { "{{ foo()() }}" "{{ 1.5 }}"; - ::emel::text::jinja::event::parse ev{ - .template_text = tmpl, - .program_out = &program, - .error_out = &error, + parse ev{ + tmpl, + program, + k_ignore_done_callback, + k_ignore_error_callback, + err, + error_pos, }; CHECK(machine.process_event(ev)); CHECK(machine.is(boost::sml::state)); - CHECK(error == EMEL_OK); + CHECK(err == static_cast(emel::text::jinja::parser::error::none)); CHECK(program.body.size() >= 8); } TEST_CASE("jinja_parser_parses_slices_and_loops") { emel::text::jinja::parser::action::context ctx{}; emel::text::jinja::parser::sm machine{ctx}; - int32_t error = EMEL_OK; emel::text::jinja::program program{}; + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 0; const std::string_view tmpl = "{% set name = value %}" "{% for item in items %}{% break %}{% continue %}{% endfor %}" "{{ arr[:] }}{{ arr[1:] }}{{ arr[:2] }}{{ arr[1:2] }}"; - ::emel::text::jinja::event::parse ev{ - .template_text = tmpl, - .program_out = &program, - .error_out = &error, + parse ev{ + tmpl, + program, + k_ignore_done_callback, + k_ignore_error_callback, + err, + error_pos, }; CHECK(machine.process_event(ev)); CHECK(machine.is(boost::sml::state)); - CHECK(error == EMEL_OK); + CHECK(err == static_cast(emel::text::jinja::parser::error::none)); CHECK(program.body.size() >= 5); } TEST_CASE("jinja_parser_rejects_unknown_statement") { emel::text::jinja::parser::action::context ctx{}; emel::text::jinja::parser::sm machine{ctx}; - int32_t error = EMEL_OK; emel::text::jinja::program program{}; - - ::emel::text::jinja::event::parse ev{ - .template_text = "{% unknown %}", - .program_out = &program, - .error_out = &error, + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 0; + + parse ev{ + "{% unknown %}", + program, + k_ignore_done_callback, + k_ignore_error_callback, + err, + error_pos, }; CHECK_FALSE(machine.process_event(ev)); CHECK(machine.is(boost::sml::state)); - CHECK(error == EMEL_ERR_PARSE_FAILED); + CHECK(err == static_cast(emel::text::jinja::parser::error::parse_failed)); CHECK(program.body.empty()); } -TEST_CASE("jinja_parser_action_rejects_missing_program") { - emel::text::jinja::parser::action::context ctx{}; - int32_t error = EMEL_OK; - bool error_called = false; - - ::emel::text::jinja::event::parse ev{ - .template_text = "{{ foo }}", - .program_out = nullptr, - .error_out = &error, - .owner_sm = &error_called, - .dispatch_error = - ::emel::callback( - &error_called, dispatch_error_test)}; - - emel::text::jinja::parser::action::run_parse(ev, ctx); - CHECK(ctx.last_error == EMEL_ERR_INVALID_ARGUMENT); - CHECK(ctx.phase_error == EMEL_ERR_INVALID_ARGUMENT); - CHECK(error == EMEL_ERR_INVALID_ARGUMENT); - CHECK(error_called); -} +TEST_CASE("jinja_parser_unexpected_event_transitions_state") { + struct unknown_event { + int value = 0; + }; -TEST_CASE("jinja_parser_on_unexpected_sets_backend_error") { emel::text::jinja::parser::action::context ctx{}; - emel::text::jinja::program program{}; - int32_t error = EMEL_OK; - - ::emel::text::jinja::event::parse ev{ - .template_text = "{{ foo }}", - .program_out = &program, - .error_out = &error, - }; + emel::text::jinja::parser::sm machine{ctx}; + machine.process_event(unknown_event{}); - emel::text::jinja::parser::action::on_unexpected(ev, ctx); - CHECK(ctx.last_error == EMEL_ERR_BACKEND); - CHECK(ctx.phase_error == EMEL_ERR_BACKEND); + CHECK(machine.is(boost::sml::state)); } diff --git a/tools/bench/text/jinja/formatter_bench.cpp b/tools/bench/text/jinja/formatter_bench.cpp index 14ebbb90..14c5d285 100644 --- a/tools/bench/text/jinja/formatter_bench.cpp +++ b/tools/bench/text/jinja/formatter_bench.cpp @@ -3,9 +3,9 @@ #include #include -#include "emel/text/jinja/lexer.hpp" -#include "emel/text/jinja/parser/detail.hpp" #include "emel/text/jinja/formatter/sm.hpp" +#include "emel/text/jinja/parser/errors.hpp" +#include "emel/text/jinja/parser/sm.hpp" #include "jinja/lexer.h" #include "jinja/parser.h" @@ -13,6 +13,14 @@ namespace { +bool parser_done_sink(const emel::text::jinja::events::parsing_done &) { + return true; +} + +bool parser_error_sink(const emel::text::jinja::events::parsing_error &) { + return true; +} + std::string make_long_template() { std::string out; out.reserve(2048); @@ -24,16 +32,22 @@ std::string make_long_template() { } emel::text::jinja::program parse_emel(const std::string & templ) { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result lex_res = lex.tokenize(templ); - if (lex_res.error != EMEL_OK) { - std::fprintf(stderr, "error: emel jinja lexer failed at %zu\n", lex_res.error_pos); - std::abort(); - } emel::text::jinja::program program{}; - emel::text::jinja::parser::detail::recursive_descent_parser parser{program}; - if (!parser.parse(lex_res)) { - std::fprintf(stderr, "error: emel jinja parser failed at %zu\n", parser.error_pos()); + int32_t parse_err = static_cast(emel::text::jinja::parser::error::none); + size_t parse_error_pos = 0; + emel::text::jinja::parser::action::context parse_ctx{}; + emel::text::jinja::parser::sm parser{parse_ctx}; + const emel::text::jinja::event::parse parse_ev{ + templ, + program, + emel::text::jinja::event::parse::done_callback::from<&parser_done_sink>(), + emel::text::jinja::event::parse::error_callback::from<&parser_error_sink>(), + parse_err, + parse_error_pos, + }; + if (!parser.process_event(parse_ev) || + parse_err != static_cast(emel::text::jinja::parser::error::none)) { + std::fprintf(stderr, "error: emel jinja parser failed at %zu\n", parse_error_pos); std::abort(); } return program; diff --git a/tools/bench/text/jinja/parser_bench.cpp b/tools/bench/text/jinja/parser_bench.cpp index e5433e2a..87df6210 100644 --- a/tools/bench/text/jinja/parser_bench.cpp +++ b/tools/bench/text/jinja/parser_bench.cpp @@ -3,8 +3,8 @@ #include #include -#include "emel/text/jinja/lexer.hpp" -#include "emel/text/jinja/parser/detail.hpp" +#include "emel/text/jinja/parser/errors.hpp" +#include "emel/text/jinja/parser/sm.hpp" #include "jinja/lexer.h" #include "jinja/parser.h" @@ -21,17 +21,32 @@ std::string make_long_template() { return out; } +bool parser_done_sink(const emel::text::jinja::events::parsing_done &) { + return true; +} + +bool parser_error_sink(const emel::text::jinja::events::parsing_error &) { + return true; +} + void ensure_emel_parses(const std::string & templ) { - emel::text::jinja::lexer lex; - emel::text::jinja::lexer_result lex_res = lex.tokenize(templ); - if (lex_res.error != EMEL_OK) { - std::fprintf(stderr, "error: emel jinja lexer failed at %zu\n", lex_res.error_pos); - std::abort(); - } emel::text::jinja::program program{}; - emel::text::jinja::parser::detail::recursive_descent_parser parser{program}; - if (!parser.parse(lex_res)) { - std::fprintf(stderr, "error: emel jinja parser failed at %zu\n", parser.error_pos()); + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 0; + emel::text::jinja::parser::action::context ctx{}; + emel::text::jinja::parser::sm machine{ctx}; + + const emel::text::jinja::event::parse ev{ + templ, + program, + emel::text::jinja::event::parse::done_callback::from<&parser_done_sink>(), + emel::text::jinja::event::parse::error_callback::from<&parser_error_sink>(), + err, + error_pos, + }; + const bool ok = machine.process_event(ev); + if (!ok || err != static_cast(emel::text::jinja::parser::error::none)) { + std::fprintf(stderr, "error: emel jinja parser failed at %zu\n", error_pos); std::abort(); } } @@ -58,22 +73,60 @@ void append_emel_jinja_parser_cases(std::vector & results, const config ensure_emel_parses(short_template); ensure_emel_parses(long_template); - emel::text::jinja::lexer lex; + emel::text::jinja::parser::action::context parser_ctx{}; + emel::text::jinja::parser::sm machine{parser_ctx}; + const emel::text::jinja::event::parse::done_callback done_cb = + emel::text::jinja::event::parse::done_callback::from<&parser_done_sink>(); + const emel::text::jinja::event::parse::error_callback error_cb = + emel::text::jinja::event::parse::error_callback::from<&parser_error_sink>(); + static volatile uint64_t sink = 0; + auto short_fn = [&]() { - emel::text::jinja::lexer_result lex_res = lex.tokenize(short_template); emel::text::jinja::program program{}; - emel::text::jinja::parser::detail::recursive_descent_parser parser{program}; - (void)parser.parse(lex_res); + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 0; + const emel::text::jinja::event::parse ev{ + short_template, + program, + done_cb, + error_cb, + err, + error_pos, + }; + const bool ok = machine.process_event(ev); + if (!ok || err != static_cast(emel::text::jinja::parser::error::none)) { + std::abort(); + } + sink += program.body.size(); + if (!program.body.empty()) { + sink += static_cast(program.body.back()->pos); + } }; results.push_back(measure_case("text/jinja/parser_short", cfg, short_fn)); auto long_fn = [&]() { - emel::text::jinja::lexer_result lex_res = lex.tokenize(long_template); emel::text::jinja::program program{}; - emel::text::jinja::parser::detail::recursive_descent_parser parser{program}; - (void)parser.parse(lex_res); + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 0; + const emel::text::jinja::event::parse ev{ + long_template, + program, + done_cb, + error_cb, + err, + error_pos, + }; + const bool ok = machine.process_event(ev); + if (!ok || err != static_cast(emel::text::jinja::parser::error::none)) { + std::abort(); + } + sink += program.body.size(); + if (!program.body.empty()) { + sink += static_cast(program.body.back()->pos); + } }; results.push_back(measure_case("text/jinja/parser_long", cfg, long_fn)); + (void)sink; } void append_reference_jinja_parser_cases(std::vector & results, const config & cfg) { diff --git a/tools/docsgen/docsgen.cpp b/tools/docsgen/docsgen.cpp index 5b0dbccb..f527f7d1 100644 --- a/tools/docsgen/docsgen.cpp +++ b/tools/docsgen/docsgen.cpp @@ -21,7 +21,7 @@ #include "emel/docs/detail.hpp" #include "emel/text/jinja/parser/sm.hpp" #include "emel/text/jinja/formatter/sm.hpp" -#include "emel/text/jinja/value.hpp" +#include "emel/text/jinja/parser/detail.hpp" namespace fs = std::filesystem; @@ -295,6 +295,14 @@ bool formatter_render_error_sink(const emel::text::jinja::events::rendering_erro return true; } +bool parser_parse_done_sink(const emel::text::jinja::events::parsing_done &) { + return true; +} + +bool parser_parse_error_sink(const emel::text::jinja::events::parsing_error &) { + return true; +} + std::optional render_template(const fs::path & template_path, const std::vector & vars) { const std::string template_text = read_file(template_path); @@ -305,17 +313,21 @@ std::optional render_template(const fs::path & template_path, } emel::text::jinja::program program; - int32_t parse_err = EMEL_OK; + int32_t parse_err = static_cast(emel::text::jinja::parser::error::none); + size_t parse_error_pos = 0; emel::text::jinja::parser::action::context parse_ctx; emel::text::jinja::parser::sm parser{parse_ctx}; emel::text::jinja::event::parse parse_ev{ - .template_text = template_text, - .program_out = &program, - .error_out = &parse_err, + template_text, + program, + emel::text::jinja::event::parse::done_callback::from<&parser_parse_done_sink>(), + emel::text::jinja::event::parse::error_callback::from<&parser_parse_error_sink>(), + parse_err, + parse_error_pos, }; parser.process_event(parse_ev); - if (parse_err != EMEL_OK || + if (parse_err != static_cast(emel::text::jinja::parser::error::none) || !parser.is(boost::sml::state)) { std::fprintf(stderr, "error: jinja parse failed\n"); return std::nullopt; From 27d4199b9874c590f0fe8d060e38a7dbd3bd8fc2 Mon Sep 17 00:00:00 2001 From: gabewillen Date: Mon, 2 Mar 2026 15:35:11 -0600 Subject: [PATCH 3/3] text/jinja: harden callback and unexpected-event handling - require parser done/error callbacks in valid request guards and split invalid request routing for missing callbacks - avoid dispatching parser error callbacks when callbacks are absent - make formatter unexpected render_runtime path mark invalid_request and emit rendering_error - add parser coverage for missing-callback request behavior --- src/emel/text/jinja/formatter/actions.hpp | 10 +++++++--- src/emel/text/jinja/parser/guards.hpp | 23 ++++++++++++++++++++--- src/emel/text/jinja/parser/sm.hpp | 20 ++++++++++++++++---- tests/text/jinja/parser_tests.cpp | 23 +++++++++++++++++++++++ 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/src/emel/text/jinja/formatter/actions.hpp b/src/emel/text/jinja/formatter/actions.hpp index 1f6f702b..c939e40d 100644 --- a/src/emel/text/jinja/formatter/actions.hpp +++ b/src/emel/text/jinja/formatter/actions.hpp @@ -80,9 +80,13 @@ struct dispatch_error { }; struct on_unexpected { - template - void operator()(const runtime_event_type & ev) const noexcept { - (void)ev; + void operator()(const event::render_runtime & ev) const noexcept { + detail::mark_error(ev.ctx, error::invalid_request, true, 0); + detail::emit_error(ev.request, ev.ctx); + } + + template + void operator()(const event_type &) const noexcept { } }; diff --git a/src/emel/text/jinja/parser/guards.hpp b/src/emel/text/jinja/parser/guards.hpp index a3e476bd..26473624 100644 --- a/src/emel/text/jinja/parser/guards.hpp +++ b/src/emel/text/jinja/parser/guards.hpp @@ -24,6 +24,12 @@ valid_parse_request(const emel::text::jinja::event::parse &ev) noexcept { return ev.template_text.data() != nullptr && !ev.template_text.empty(); } +inline bool +callbacks_present(const emel::text::jinja::event::parse &ev) noexcept { + return static_cast(ev.dispatch_done) && + static_cast(ev.dispatch_error); +} + } // namespace helper struct valid_parse { @@ -31,16 +37,27 @@ struct valid_parse { bool operator()(const runtime_event_type &ev, const action::context &) const noexcept { const auto &runtime_ev = helper::unwrap_runtime_event(ev); - return helper::valid_parse_request(runtime_ev.request); + return helper::valid_parse_request(runtime_ev.request) && + helper::callbacks_present(runtime_ev.request); + } +}; + +struct invalid_parse_with_callbacks { + template + bool operator()(const runtime_event_type &ev, + const action::context &) const noexcept { + const auto &runtime_ev = helper::unwrap_runtime_event(ev); + return !helper::valid_parse_request(runtime_ev.request) && + helper::callbacks_present(runtime_ev.request); } }; -struct invalid_parse { +struct invalid_parse_without_callbacks { template bool operator()(const runtime_event_type &ev, const action::context &) const noexcept { const auto &runtime_ev = helper::unwrap_runtime_event(ev); - return !helper::valid_parse_request(runtime_ev.request); + return !helper::callbacks_present(runtime_ev.request); } }; diff --git a/src/emel/text/jinja/parser/sm.hpp b/src/emel/text/jinja/parser/sm.hpp index 0d71cd54..066c5fe2 100644 --- a/src/emel/text/jinja/parser/sm.hpp +++ b/src/emel/text/jinja/parser/sm.hpp @@ -37,28 +37,40 @@ struct model { + sml::event[ guard::valid_parse{} ] / action::begin_parse , sml::state <= sml::state - + sml::event[ guard::invalid_parse{} ] + + sml::event[ guard::invalid_parse_with_callbacks{} ] + / action::reject_invalid_parse + , sml::state <= sml::state + + sml::event[ guard::invalid_parse_without_callbacks{} ] / action::reject_invalid_parse , sml::state <= sml::state + sml::event[ guard::valid_parse{} ] / action::begin_parse , sml::state <= sml::state - + sml::event[ guard::invalid_parse{} ] + + sml::event[ guard::invalid_parse_with_callbacks{} ] + / action::reject_invalid_parse + , sml::state <= sml::state + + sml::event[ guard::invalid_parse_without_callbacks{} ] / action::reject_invalid_parse , sml::state <= sml::state + sml::event[ guard::valid_parse{} ] / action::begin_parse , sml::state <= sml::state - + sml::event[ guard::invalid_parse{} ] + + sml::event[ guard::invalid_parse_with_callbacks{} ] + / action::reject_invalid_parse + , sml::state <= sml::state + + sml::event[ guard::invalid_parse_without_callbacks{} ] / action::reject_invalid_parse , sml::state <= sml::state + sml::event[ guard::valid_parse{} ] / action::begin_parse , sml::state <= sml::state - + sml::event[ guard::invalid_parse{} ] + + sml::event[ guard::invalid_parse_with_callbacks{} ] + / action::reject_invalid_parse + , sml::state <= sml::state + + sml::event[ guard::invalid_parse_without_callbacks{} ] / action::reject_invalid_parse //------------------------------------------------------------------------------// diff --git a/tests/text/jinja/parser_tests.cpp b/tests/text/jinja/parser_tests.cpp index 467480f6..410c3f0f 100644 --- a/tests/text/jinja/parser_tests.cpp +++ b/tests/text/jinja/parser_tests.cpp @@ -107,6 +107,29 @@ TEST_CASE("jinja_parser_invalid_request_with_callbacks_dispatches_error") { CHECK(program.body.empty()); } +TEST_CASE("jinja_parser_missing_callbacks_returns_error_without_dispatch") { + emel::text::jinja::parser::action::context ctx{}; + emel::text::jinja::parser::sm machine{ctx}; + emel::text::jinja::program program{}; + int32_t err = static_cast(emel::text::jinja::parser::error::none); + size_t error_pos = 999; + + parse ev{ + "{{ foo }}", + program, + done_cb{}, + error_cb{}, + err, + error_pos, + }; + + CHECK_FALSE(machine.process_event(ev)); + CHECK(machine.is(boost::sml::state)); + CHECK(err == static_cast(emel::text::jinja::parser::error::invalid_request)); + CHECK(error_pos == 0); + CHECK(program.body.empty()); +} + TEST_CASE("jinja_parser_parse_failure_reports_error") { emel::text::jinja::parser::action::context ctx{}; emel::text::jinja::parser::sm machine{ctx};