From ba5be9906ff41c3acb89ca0ca5cc6b55600df733 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Tue, 3 Feb 2026 00:19:23 +0800 Subject: [PATCH 1/9] chore: Show less error message --- src/sphinxnotes/data/render/datanodes.py | 65 ++++++++++++++---------- src/sphinxnotes/data/utils/__init__.py | 15 +++--- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/src/sphinxnotes/data/render/datanodes.py b/src/sphinxnotes/data/render/datanodes.py index 3dae467..8d4616e 100644 --- a/src/sphinxnotes/data/render/datanodes.py +++ b/src/sphinxnotes/data/render/datanodes.py @@ -71,25 +71,34 @@ def render(self, host: Host) -> None: assert not self.rendered self.rendered = True - report = Report( - 'Render Debug Report', 'DEBUG', source=self.source, line=self.line - ) + # Clear empty reports. + Reporter(self).clear_empty() + + dbg = Report('Render Report', 'DEBUG', source=self.source, line=self.line) + + def get_error_report() -> Report: + if self.template.debug: + # Reuse the debug report as possible. + dbg['type'] = 'ERROR' + return dbg + return Report('Render Report', 'ERROR', source=self.source, line=self.line) # 1. Prepare context for Jinja template. if isinstance(self.data, PendingData): - report.text('Raw data:') - report.code(pformat(self.data.raw), lang='python') - report.text('Schema:') - report.code(pformat(self.data.schema), lang='python') + dbg.text('Raw data:') + dbg.code(pformat(self.data.raw), lang='python') + dbg.text('Schema:') + dbg.code(pformat(self.data.schema), lang='python') for hook in self._raw_data_hooks: hook(self, self.data.raw) try: data = self.data = self.data.parse() - except ValueError: + except ValueError as e: + report = get_error_report() report.text('Failed to parse raw data:') - report.excption() + report.exception(e) self += report return else: @@ -98,45 +107,47 @@ def render(self, host: Host) -> None: for hook in self._parsed_data_hooks: hook(self, data) - report.text(f'Parsed data (type: {type(data)}):') - report.code(pformat(data), lang='python') - report.text('Extra context (only keys):') - report.code(pformat(list(self.extra.keys())), lang='python') - report.text(f'Template (phase: {self.template.phase}):') - report.code(self.template.text, lang='jinja') + dbg.text(f'Parsed data (type: {type(data)}):') + dbg.code(pformat(data), lang='python') + dbg.text('Extra context (only keys):') + dbg.code(pformat(list(self.extra.keys())), lang='python') + dbg.text(f'Template (phase: {self.template.phase}):') + dbg.code(self.template.text, lang='jinja') # 2. Render the template and data to markup text. try: markup = TemplateRenderer(self.template.text).render(data, extra=self.extra) - except Exception: + except Exception as e: + report = get_error_report() report.text('Failed to render Jinja template:') - report.excption() + report.exception(e) self += report return for hook in self._markup_text_hooks: markup = hook(self, markup) - report.text('Rendered markup text:') - report.code(markup, lang='rst') + dbg.text('Rendered markup text:') + dbg.code(markup, lang='rst') # 3. Render the markup text to doctree nodes. try: ns, msgs = MarkupRenderer(host).render(markup, inline=self.inline) - except Exception: + except Exception as e: + report = get_error_report() report.text( 'Failed to render markup text ' f'to {"inline " if self.inline else ""}nodes:' ) - report.excption() + report.exception(e) self += report return - report.text(f'Rendered nodes (inline: {self.inline}):') - report.code('\n\n'.join([n.pformat() for n in ns]), lang='xml') + dbg.text(f'Rendered nodes (inline: {self.inline}):') + dbg.code('\n\n'.join([n.pformat() for n in ns]), lang='xml') if msgs: - report.text('Systemd messages:') - [report.node(msg) for msg in msgs] + dbg.text('Systemd messages:') + [dbg.node(msg) for msg in msgs] # 4. Add rendered nodes to container. for hook in self._rendered_nodes_hooks: @@ -145,9 +156,7 @@ def render(self, host: Host) -> None: self += ns if self.template.debug: - self += report - - Reporter(self).clear_empty() + self += dbg return diff --git a/src/sphinxnotes/data/utils/__init__.py b/src/sphinxnotes/data/utils/__init__.py index 58f1530..62b4f35 100644 --- a/src/sphinxnotes/data/utils/__init__.py +++ b/src/sphinxnotes/data/utils/__init__.py @@ -97,15 +97,14 @@ def find_nearest_block_element(node: nodes.Node | None) -> nodes.Element | None: class Report(nodes.system_message): - type Level = Literal['DEBUG', 'INFO', 'WARNING', 'ERROR'] + type Type = Literal['DEBUG', 'INFO', 'WARNING', 'ERROR'] - level: Level title: str def __init__( - self, title: str, level: Level = 'DEBUG', *children, **attributes + self, title: str, typ: Type = 'DEBUG', *children, **attributes ) -> None: - super().__init__(title + ':', type=level, level=2, *children, **attributes) + super().__init__(title + ':', type=typ, level=2, *children, **attributes) self.title = title def empty(self) -> bool: @@ -143,12 +142,16 @@ def list(self, lines: Iterable[str]) -> None: self.node(bullet_list) - def excption(self) -> None: + def traceback(self) -> None: # https://pygments.org/docs/lexers/#pygments.lexers.python.PythonTracebackLexer self.code(traceback.format_exc(), lang='pytb') + def exception(self, e: Exception) -> None: + # https://pygments.org/docs/lexers/#pygments.lexers.python.PythonTracebackLexer + self.code(str(e), lang='pytb') + def is_error(self) -> bool: - return self.level == 'ERROR' + return self['type'] == 'ERROR' type Inliner = RstInliner | tuple[nodes.document, nodes.Element] From a51d3a0beaef9c775c1ee7395e6838def385d3aa Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Tue, 3 Feb 2026 00:37:08 +0800 Subject: [PATCH 2/9] refactor: Impl _ParsedHook with Transform --- src/sphinxnotes/data/render/extractx.py | 8 ++-- src/sphinxnotes/data/render/markup.py | 8 +++- src/sphinxnotes/data/render/pipeline.py | 53 ++++++------------------- src/sphinxnotes/data/render/render.py | 3 +- tests/test_data.py | 2 +- 5 files changed, 25 insertions(+), 49 deletions(-) diff --git a/src/sphinxnotes/data/render/extractx.py b/src/sphinxnotes/data/render/extractx.py index e05277b..9a2db65 100644 --- a/src/sphinxnotes/data/render/extractx.py +++ b/src/sphinxnotes/data/render/extractx.py @@ -40,7 +40,7 @@ def generate(self, host: TransformHost) -> Any: ... class ExtraContextRegistry: names: set[str] parsing: dict[str, ParsePhaseExtraContext] - parsed: dict[str, ParsePhaseExtraContext] + parsed: dict[str, TransformPhaseExtraContext] post_transform: dict[str, TransformPhaseExtraContext] global_: dict[str, GlobalExtraContxt] @@ -70,7 +70,7 @@ def add_parsing_phase_context( self.parsing['_' + name] = ctxgen def add_parsed_phase_context( - self, name: str, ctxgen: ParsePhaseExtraContext + self, name: str, ctxgen: TransformPhaseExtraContext ) -> None: self._name_dedup(name) self.parsed['_' + name] = ctxgen @@ -163,7 +163,7 @@ def on_parsing(self, host: ParseHost) -> None: for name, ctxgen in self.registry.parsing.items(): self._safegen(name, lambda: ctxgen.generate(host)) - def on_parsed(self, host: ParseHost) -> None: + def on_parsed(self, host: TransformHost) -> None: for name, ctxgen in self.registry.parsed.items(): self._safegen(name, lambda: ctxgen.generate(host)) @@ -177,7 +177,7 @@ def _safegen(self, name: str, gen: Callable[[], Any]): self.node.extra[name] = gen() except Exception: self.report.text(f'Failed to generate extra context "{name}":') - self.report.excption() + self.report.traceback() def setup(app: Sphinx): diff --git a/src/sphinxnotes/data/render/markup.py b/src/sphinxnotes/data/render/markup.py index 198ab16..06c5dec 100644 --- a/src/sphinxnotes/data/render/markup.py +++ b/src/sphinxnotes/data/render/markup.py @@ -43,9 +43,13 @@ def _render(self, text: str) -> list[Node]: elif isinstance(self.host, SphinxTransform): # TODO: dont create parser for every time if version_info[0] >= 9: - parser = self.host.app.registry.create_source_parser('rst', env=self.host.env, config=self.host.config) + parser = self.host.app.registry.create_source_parser( + 'rst', env=self.host.env, config=self.host.config + ) else: - parser = self.host.app.registry.create_source_parser(self.host.app, 'rst') + parser = self.host.app.registry.create_source_parser( + self.host.app, 'rst' + ) settings = self.host.document.settings doc = new_document('', settings=settings) parser.parse(text, doc) diff --git a/src/sphinxnotes/data/render/pipeline.py b/src/sphinxnotes/data/render/pipeline.py index 7d9ba50..10bc3f2 100644 --- a/src/sphinxnotes/data/render/pipeline.py +++ b/src/sphinxnotes/data/render/pipeline.py @@ -48,6 +48,7 @@ from docutils import nodes from sphinx.util import logging from sphinx.util.docutils import SphinxDirective, SphinxRole +from sphinx.transforms import SphinxTransform from sphinx.transforms.post_transforms import SphinxPostTransform, ReferencesResolver from .render import HostWrapper, Phase, Template, Host, ParseHost, TransformHost @@ -232,70 +233,42 @@ def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]: return ns, msgs -class _ParsedHook(SphinxDirective, Pipeline): +class _ParsedHook(SphinxTransform, Pipeline): + # Before almost all others. + default_priority = 100 + @override def process_pending_node(self, n: pending_node) -> bool: - self.state.document.note_source(n.source, n.line) # type: ignore[arg-type] - - # Generate and save parsed extra context for later use. - ExtraContextGenerator(n).on_parsed(cast(ParseHost, self)) - + ExtraContextGenerator(n).on_parsed(cast(TransformHost, self)) return n.template.phase == Phase.Parsed @override - def run(self) -> list[nodes.Node]: - for pending in self.state.document.findall(pending_node): + def apply(self, **kwargs): + for pending in self.document.findall(pending_node): self.queue_pending_node(pending) - # Hook system_message method to let it report the - # correct line number. - # TODO: self.state.document.note_source(source, line) # type: ignore[arg-type] - # def fix_lineno(level, message, *children, **kwargs): - # kwargs['line'] = pending.line - # return orig_sysmsg(level, message, *children, **kwargs) - - # self.state_machine.reporter.system_message = fix_lineno - - ns = self.render_queue() - assert len(ns) == 0 - - return [] # nothing to return - - -def _insert_parsed_hook(app, docname, content): - # NOTE: content is a single element list, representing the content of the - # source file. - # - # .. seealso:: https://www.sphinx-doc.org/en/master/extdev/event_callbacks.html#event-source-read - # - # TODO: markdown? - # TODO: rst_prelog? - content[-1] = content[-1] + '\n\n.. data.parsed-hook::' + self.render_queue() class _ResolvingHook(SphinxPostTransform, Pipeline): - # After resolving pending_xref. - default_priority = (ReferencesResolver.default_priority or 10) + 5 + # After resolving pending_xref + default_priority = (ReferencesResolver.default_priority or 10) + 5 @override def process_pending_node(self, n: pending_node) -> bool: - # Generate and save post transform extra context for later use. ExtraContextGenerator(n).on_post_transform(cast(TransformHost, self)) - - return n.template.phase == Phase.PostTranform + return n.template.phase == Phase.Resolving @override def apply(self, **kwargs): for pending in self.document.findall(pending_node): self.queue_pending_node(pending) - ns = self.render_queue() assert len(ns) == 0 def setup(app: Sphinx) -> None: # Hook for Phase.Parsed. - app.add_directive('data.parsed-hook', _ParsedHook) - app.connect('source-read', _insert_parsed_hook) + app.add_transform(_ParsedHook) # Hook for Phase.Resolving. app.add_post_transform(_ResolvingHook) diff --git a/src/sphinxnotes/data/render/render.py b/src/sphinxnotes/data/render/render.py index e54254e..02472a5 100644 --- a/src/sphinxnotes/data/render/render.py +++ b/src/sphinxnotes/data/render/render.py @@ -10,8 +10,7 @@ class Phase(Enum): Parsing = 'parsing' Parsed = 'parsed' - PostTranform = 'post-transform' - # TODO: transform? + Resolving = 'resolving' @classmethod def default(cls) -> Phase: diff --git a/tests/test_data.py b/tests/test_data.py index f91b34e..477cd0f 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -6,8 +6,8 @@ from data.data import Field, REGISTRY -class TestFieldParser(unittest.TestCase): +class TestFieldParser(unittest.TestCase): # ========================== # Basic Types # ========================== From 75f45442a19210be991d665fb9c7625160e0fd4e Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Tue, 3 Feb 2026 01:38:55 +0800 Subject: [PATCH 3/9] feat: Phase.Resolving works now --- src/sphinxnotes/data/data.py | 2 +- src/sphinxnotes/data/render/datanodes.py | 111 +++++++++++++---------- src/sphinxnotes/data/render/pipeline.py | 17 +++- src/sphinxnotes/data/utils/ctxproxy.py | 4 +- 4 files changed, 78 insertions(+), 56 deletions(-) diff --git a/src/sphinxnotes/data/data.py b/src/sphinxnotes/data/data.py index b1c93e3..1eae6b3 100644 --- a/src/sphinxnotes/data/data.py +++ b/src/sphinxnotes/data/data.py @@ -451,7 +451,7 @@ def by_option_store_value_error(opt: ByOption) -> ValueError: @dataclass(frozen=True) -class Schema(object): +class Schema(Unpicklable): name: Field | None attrs: dict[str, Field] | Field content: Field | None diff --git a/src/sphinxnotes/data/render/datanodes.py b/src/sphinxnotes/data/render/datanodes.py index 8d4616e..d212fcd 100644 --- a/src/sphinxnotes/data/render/datanodes.py +++ b/src/sphinxnotes/data/render/datanodes.py @@ -10,7 +10,6 @@ from .template import TemplateRenderer from ..data import RawData, PendingData, ParsedData from ..utils import ( - Unpicklable, Report, Reporter, find_nearest_block_element, @@ -24,7 +23,7 @@ class Base(nodes.Element): ... -class pending_node(Base, Unpicklable): +class pending_node(Base): # The data to be rendered by Jinja template. data: PendingData | ParsedData | dict[str, Any] # The extra context for Jina template. @@ -35,6 +34,8 @@ class pending_node(Base, Unpicklable): inline: bool #: Whether the rendering pipeline is finished (failed is also finished). rendered: bool + #: The report of render pipepine. + report: Report def __init__( self, @@ -51,6 +52,9 @@ def __init__( self.template = tmpl self.inline = inline self.rendered = False + self.report = Report( + 'Render Report', 'DEBUG', source=self.source, line=self.line + ) # Init hook lists. self._raw_data_hooks = [] @@ -58,11 +62,48 @@ def __init__( self._markup_text_hooks = [] self._rendered_nodes_hooks = [] + def get_error_report(self) -> Report: + if self.template.debug: + # Reuse the render report as possible. + self.report['type'] = 'ERROR' + return self.report + return Report('Render Report', 'ERROR', source=self.source, line=self.line) + + def ensure_data_parsed(self) -> ParsedData | dict[str, Any] | None: + """ + Ensure self.data is parsed (instance of ParsedData | dict[str, Any]). + if no, parse it. + """ + if not isinstance(self.data, PendingData): + return self.data + + self.report.text('Raw data:') + self.report.code(pformat(self.data.raw), lang='python') + self.report.text('Schema:') + self.report.code(pformat(self.data.schema), lang='python') + + for hook in self._raw_data_hooks: + hook(self, self.data.raw) + + try: + data = self.data = self.data.parse() + except ValueError as e: + report = self.get_error_report() + report.text('Failed to parse raw data:') + report.exception(e) + self += report + return None + + for hook in self._parsed_data_hooks: + hook(self, data) + + return data + def render(self, host: Host) -> None: """ The core function for rendering data to docutils nodes. - 1. Schema.parse(RawData) -> ParsedData + 1. Schema.parse(RawData) -> ParsedData (self.parse_data) 2. TemplateRenderer.render(ParsedData) -> Markup Text (``str``) 3. MarkupRenderer.render(Markup Text) -> doctree Nodes (list[nodes.Node]) """ @@ -74,51 +115,22 @@ def render(self, host: Host) -> None: # Clear empty reports. Reporter(self).clear_empty() - dbg = Report('Render Report', 'DEBUG', source=self.source, line=self.line) - - def get_error_report() -> Report: - if self.template.debug: - # Reuse the debug report as possible. - dbg['type'] = 'ERROR' - return dbg - return Report('Render Report', 'ERROR', source=self.source, line=self.line) - # 1. Prepare context for Jinja template. - if isinstance(self.data, PendingData): - dbg.text('Raw data:') - dbg.code(pformat(self.data.raw), lang='python') - dbg.text('Schema:') - dbg.code(pformat(self.data.schema), lang='python') - - for hook in self._raw_data_hooks: - hook(self, self.data.raw) - - try: - data = self.data = self.data.parse() - except ValueError as e: - report = get_error_report() - report.text('Failed to parse raw data:') - report.exception(e) - self += report - return - else: - data = self.data + if (data := self.ensure_data_parsed()) is None: + return # parse failure - for hook in self._parsed_data_hooks: - hook(self, data) - - dbg.text(f'Parsed data (type: {type(data)}):') - dbg.code(pformat(data), lang='python') - dbg.text('Extra context (only keys):') - dbg.code(pformat(list(self.extra.keys())), lang='python') - dbg.text(f'Template (phase: {self.template.phase}):') - dbg.code(self.template.text, lang='jinja') + self.report.text(f'Parsed data (type: {type(data)}):') + self.report.code(pformat(data), lang='python') + self.report.text('Extra context (only keys):') + self.report.code(pformat(list(self.extra.keys())), lang='python') + self.report.text(f'Template (phase: {self.template.phase}):') + self.report.code(self.template.text, lang='jinja') # 2. Render the template and data to markup text. try: markup = TemplateRenderer(self.template.text).render(data, extra=self.extra) except Exception as e: - report = get_error_report() + report = self.get_error_report() report.text('Failed to render Jinja template:') report.exception(e) self += report @@ -127,14 +139,14 @@ def get_error_report() -> Report: for hook in self._markup_text_hooks: markup = hook(self, markup) - dbg.text('Rendered markup text:') - dbg.code(markup, lang='rst') + self.report.text('Rendered markup text:') + self.report.code(markup, lang='rst') # 3. Render the markup text to doctree nodes. try: ns, msgs = MarkupRenderer(host).render(markup, inline=self.inline) except Exception as e: - report = get_error_report() + report = self.get_error_report() report.text( 'Failed to render markup text ' f'to {"inline " if self.inline else ""}nodes:' @@ -143,20 +155,21 @@ def get_error_report() -> Report: self += report return - dbg.text(f'Rendered nodes (inline: {self.inline}):') - dbg.code('\n\n'.join([n.pformat() for n in ns]), lang='xml') + self.report.text(f'Rendered nodes (inline: {self.inline}):') + self.report.code('\n\n'.join([n.pformat() for n in ns]), lang='xml') if msgs: - dbg.text('Systemd messages:') - [dbg.node(msg) for msg in msgs] + self.report.text('Systemd messages:') + [self.report.node(msg) for msg in msgs] # 4. Add rendered nodes to container. for hook in self._rendered_nodes_hooks: hook(self, ns) + # TODO: set_source_info? self += ns if self.template.debug: - self += dbg + self += self.report return diff --git a/src/sphinxnotes/data/render/pipeline.py b/src/sphinxnotes/data/render/pipeline.py index 10bc3f2..c5b580f 100644 --- a/src/sphinxnotes/data/render/pipeline.py +++ b/src/sphinxnotes/data/render/pipeline.py @@ -73,7 +73,7 @@ def process_pending_node(self, n: pending_node) -> bool: """ You can add hooks to pending node here. - Return ``true`` if you want to render the pending node *immediately*, + Return ``true`` if you want to render the pending node *now*, otherwise it will be inserted to doctree directly andwaiting to later rendering """ @@ -123,7 +123,8 @@ def render_queue(self) -> list[pending_node]: while self._q: pending = self._q.pop() - if not self.process_pending_node(pending): + render_now = self.process_pending_node(pending) + if not render_now: ns.append(pending) continue @@ -246,12 +247,18 @@ def process_pending_node(self, n: pending_node) -> bool: def apply(self, **kwargs): for pending in self.document.findall(pending_node): self.queue_pending_node(pending) - self.render_queue() + + for n in self.render_queue(): + # NOTE: In the next Phase, doctrees will be pickled to disk. + # As :cls:`data.Schema` is **Unpicklable**, we should ensure + # ``pending_node.data`` is parsed, which means pending_node dropped + # the reference to Schema. + n.ensure_data_parsed() class _ResolvingHook(SphinxPostTransform, Pipeline): # After resolving pending_xref - default_priority = (ReferencesResolver.default_priority or 10) + 5 + default_priority = (ReferencesResolver.default_priority or 10) + 5 @override def process_pending_node(self, n: pending_node) -> bool: @@ -263,6 +270,8 @@ def apply(self, **kwargs): for pending in self.document.findall(pending_node): self.queue_pending_node(pending) ns = self.render_queue() + + # NOTE: Should no node left. assert len(ns) == 0 diff --git a/src/sphinxnotes/data/utils/ctxproxy.py b/src/sphinxnotes/data/utils/ctxproxy.py index f48c092..cd9816c 100644 --- a/src/sphinxnotes/data/utils/ctxproxy.py +++ b/src/sphinxnotes/data/utils/ctxproxy.py @@ -8,7 +8,6 @@ from sphinx.config import Config as SphinxConfig from ..utils import find_first_child -from ..utils import Unpicklable logger = logging.getLogger(__name__) @@ -22,8 +21,9 @@ def wrapped(self: Proxy) -> Any: return property(wrapped) +# FIXME: Unpicklable? @dataclass(frozen=True) -class Proxy(Unpicklable): +class Proxy: """ Proxy complex objects into context for convenient and secure access within Jinja templates. From 4fa2d3f7f0443796e3e0061399149af88d424e2a Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Tue, 3 Feb 2026 22:43:22 +0800 Subject: [PATCH 4/9] refactor: Distingust DataSource and RawDataSource --- src/sphinxnotes/data/__init__.py | 10 +- src/sphinxnotes/data/examples/datadomain.py | 7 +- src/sphinxnotes/data/render/__init__.py | 16 ++- src/sphinxnotes/data/render/pipeline.py | 132 ++++++++++-------- .../{examples/strict.py => render/sources.py} | 57 +++++++- 5 files changed, 151 insertions(+), 71 deletions(-) rename src/sphinxnotes/data/{examples/strict.py => render/sources.py} (50%) diff --git a/src/sphinxnotes/data/__init__.py b/src/sphinxnotes/data/__init__.py index 07affe4..31f5010 100644 --- a/src/sphinxnotes/data/__init__.py +++ b/src/sphinxnotes/data/__init__.py @@ -28,12 +28,14 @@ Template, Host, pending_node, - BaseDataDefineRole, - BaseDataDefineDirective, + BaseDataRole, + BaseDataDirective, ExtraContextRegistry, ExtraContextGenerator, + BaseDataDefineRole, + BaseDataDefineDirective, + StrictDataDefineDirective, ) -from .examples.strict import StrictDataDefineDirective if TYPE_CHECKING: from sphinx.application import Sphinx @@ -53,6 +55,8 @@ 'Template', 'Host', 'pending_node', + 'BaseDataRole', + 'BaseDataDirective', 'BaseDataDefineRole', 'BaseDataDefineDirective', 'StrictDataDefineDirective', diff --git a/src/sphinxnotes/data/examples/datadomain.py b/src/sphinxnotes/data/examples/datadomain.py index caeadc2..3358b76 100644 --- a/src/sphinxnotes/data/examples/datadomain.py +++ b/src/sphinxnotes/data/examples/datadomain.py @@ -128,9 +128,10 @@ def __init__(self, orig_name: str) -> None: self.orig_name = orig_name @override - def current_data(self) -> RawData: - _, _, name = self.orig_name.partition('+') - return RawData(name, {}, self.text) + def current_raw_data(self) -> RawData: + data = super().current_raw_data() + _, _, data.name = self.orig_name.partition('+') + return data @override def current_schema(self) -> Schema: diff --git a/src/sphinxnotes/data/render/__init__.py b/src/sphinxnotes/data/render/__init__.py index 2716368..6ff7259 100644 --- a/src/sphinxnotes/data/render/__init__.py +++ b/src/sphinxnotes/data/render/__init__.py @@ -4,10 +4,15 @@ from .render import Phase, Template, Host from .datanodes import pending_node from .pipeline import ( - BaseDataDefineRole, - BaseDataDefineDirective, + BaseDataRole, + BaseDataDirective, ) from .extractx import ExtraContextRegistry, ExtraContextGenerator +from .sources import ( + BaseDataDefineDirective, + StrictDataDefineDirective, + BaseDataDefineRole, +) if TYPE_CHECKING: from sphinx.application import Sphinx @@ -18,10 +23,13 @@ 'Template', 'Host', 'pending_node', - 'BaseDataDefineRole', - 'BaseDataDefineDirective', + 'BaseDataRole', + 'BaseDataDirective', 'ExtraContextRegistry', 'ExtraContextGenerator', + 'BaseDataDefineDirective', + 'StrictDataDefineDirective', + 'BaseDataDefineRole', ] diff --git a/src/sphinxnotes/data/render/pipeline.py b/src/sphinxnotes/data/render/pipeline.py index c5b580f..fd7e7cc 100644 --- a/src/sphinxnotes/data/render/pipeline.py +++ b/src/sphinxnotes/data/render/pipeline.py @@ -1,6 +1,6 @@ """ -sphinxnotes.data.pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~ +sphinxnotes.data.render.pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :copyright: Copyright 2026 by the Shengyu Zhang. :license: BSD, see LICENSE for details. @@ -10,32 +10,36 @@ The Pipline =========== -1. Define data: BaseDataDefiner generates a :cls:`pending_node`, which contains: +1. Define data: BaseDataSource generates a :cls:`pending_node`, which contains: - - Data and possible extra contexts - - Schema for validating Data + - Data: :cls:`PendingData` (data to be validated), :cls:`ParsedData`, or + simple ``dict[str, Any]``. - Template for rendering data to markup text + - Possible extra contexts + + See also :cls:`BaseDataSource`. 2. Render data: the ``pending_node`` nodes will be rendered - (by calling :meth:`pending_node.render`) at some point, depending on :cls:`Phase`. + (by calling :meth:`pending_node.render`) at some point, depending on + :attr:`pending_node.template.phase`. The one who calls ``pending_node.render`` is called ``Host``. - The ``Host`` host is responsible for rendering the markup text into doctree + The ``Host`` host is responsible for rendering the markup text into docutils nodes (See :cls:`MarkupRenderer`). Phases: :``Phase.Parsing``: - Called by BaseDataDefiner ('s subclasses) + Called by BaseDataSource ('s subclasses) :``Phase.Parsed``: - Called by :cls:`_ParsedHook`. + Called by :cls:`ParsedHookTransform`. :``Phase.Resolving``: - Called by :cls:`_ResolvingHook`. + Called by :cls:`ResolvingHookTransform`. -How :cls:`RawData` be rendered ``list[nodes.Node]`` -=================================================== +How data be rendered ``list[nodes.Node]`` +========================================= .. seealso:: :meth:`.datanodes.pending_node.render`. @@ -64,6 +68,25 @@ class Pipeline(ABC): + """ + The core class defines the pipleing of rendering :cls:`pending_node`s. + + Subclass is responsible to: + + - call ``queue_xxx`` to add pendin nodes into queue. + - override :meth:`process_pending_node` to control when a pending node gets + rendered. In this method subclass can also call ``queue_xxx`` to add more + pending nodes. + - call :meth:`render_queue` to process all queued nodes and + returns any that couldn't be rendered in the current phase. + + See Also: + + - :class:`BaseDataSource`: Data source implementation and hook for Phase.Parsing + - :class:`ParsedHookTransform`: Built-in hook for Phase.Parsed + - :class:`ResolvingHookTransform`: Built-in hook for Phase.Resolving + """ + #: Queue of pending node to be rendered. _q: list[pending_node] | None = None @@ -87,25 +110,14 @@ def queue_pending_node(self, n: pending_node) -> None: self._q = [] self._q.append(n) - @final - def queue_raw_data( - self, data: RawData, schema: Schema, tmpl: Template - ) -> pending_node: - pending = pending_node(PendingData(data, schema), tmpl) - self.queue_pending_node(pending) - return pending @final - def queue_parsed_data(self, data: ParsedData, tmpl: Template) -> pending_node: + def queue_data(self, data: PendingData | ParsedData | dict[str, Any], tmpl: Template + ) -> pending_node: pending = pending_node(data, tmpl) self.queue_pending_node(pending) return pending - @final - def queue_any_data(self, data: Any, tmpl: Template) -> pending_node: - pending = pending_node(data, tmpl) - self.queue_pending_node(pending) - return pending @final def render_queue(self) -> list[pending_node]: @@ -147,25 +159,44 @@ def render_queue(self) -> list[pending_node]: return ns -class BaseDataDefiner(Pipeline): +class BaseDataSource(Pipeline): """ - A abstract class that owns :cls:`RawData` and support - validating and rendering the data at the appropriate time. + Abstract base class for generateing data, as the source of the rendering + pipeline. - The subclasses *MUST* be subclass of :cls:`SphinxDirective` or - :cls:`SphinxRole`. + This class also responsible to render data in Phase.Parsing. So the final + implementations MUST be subclass of :class:`SphinxDirective` or + :class:`SphinxRole`, which provide the execution context and interface for + processing reStructuredText markup. """ """Methods to be implemented.""" @abstractmethod - def current_data(self) -> RawData: ... + def current_data(self) -> PendingData | ParsedData | dict[str, Any]: + """ + Return the data to be rendered. - @abstractmethod - def current_schema(self) -> Schema: ... + This method should be implemented to provide the actual data content + that will be rendered. The returned data can be a PendingData object, + a ParsedData object, or a plain dictionary. + + Returns: + The data content to render into nodes. + """ @abstractmethod - def current_template(self) -> Template: ... + def current_template(self) -> Template: + """ + Return the template for rendering the data. + + This method should be implemented to provide the Jinja2 template + that will render the data into markup text. The template determines + the phase at which rendering occurs. + + Returns: + The template to use for rendering. + """ """Methods override from parent.""" @@ -181,20 +212,10 @@ def process_pending_node(self, n: pending_node) -> bool: return n.template.phase == Phase.Parsing -class BaseDataDefineDirective(BaseDataDefiner, SphinxDirective): - @override - def current_data(self) -> RawData: - return RawData( - ' '.join(self.arguments) if self.arguments else None, - self.options.copy(), - '\n'.join(self.content) if self.has_content else None, - ) - +class BaseDataDirective(BaseDataSource, SphinxDirective): @override def run(self) -> list[nodes.Node]: - self.queue_raw_data( - self.current_data(), self.current_schema(), self.current_template() - ) + self.queue_data(self.current_data(), self.current_template()) ns = [] for x in self.render_queue(): @@ -206,11 +227,7 @@ def run(self) -> list[nodes.Node]: return ns -class BaseDataDefineRole(BaseDataDefiner, SphinxRole): - @override - def current_data(self) -> RawData: - return RawData(None, {}, self.text) - +class BaseDataRole(BaseDataSource, SphinxRole): @override def process_pending_node(self, n: pending_node) -> bool: n.inline = True @@ -218,9 +235,8 @@ def process_pending_node(self, n: pending_node) -> bool: @override def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]: - self.queue_raw_data( - self.current_data(), self.current_schema(), self.current_template() - ) + pending = self.queue_data(self.current_data(), self.current_template()) + pending.inline = True ns, msgs = [], [] for n in self.render_queue(): @@ -234,7 +250,7 @@ def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]: return ns, msgs -class _ParsedHook(SphinxTransform, Pipeline): +class ParsedHookTransform(SphinxTransform, Pipeline): # Before almost all others. default_priority = 100 @@ -256,7 +272,7 @@ def apply(self, **kwargs): n.ensure_data_parsed() -class _ResolvingHook(SphinxPostTransform, Pipeline): +class ResolvingHookTransform(SphinxPostTransform, Pipeline): # After resolving pending_xref default_priority = (ReferencesResolver.default_priority or 10) + 5 @@ -277,7 +293,7 @@ def apply(self, **kwargs): def setup(app: Sphinx) -> None: # Hook for Phase.Parsed. - app.add_transform(_ParsedHook) + app.add_transform(ParsedHookTransform) # Hook for Phase.Resolving. - app.add_post_transform(_ResolvingHook) + app.add_post_transform(ResolvingHookTransform) diff --git a/src/sphinxnotes/data/examples/strict.py b/src/sphinxnotes/data/render/sources.py similarity index 50% rename from src/sphinxnotes/data/examples/strict.py rename to src/sphinxnotes/data/render/sources.py index f0ed2fc..6cf8c1d 100644 --- a/src/sphinxnotes/data/examples/strict.py +++ b/src/sphinxnotes/data/render/sources.py @@ -1,10 +1,61 @@ +""" +sphinxnotes.data.render.sources +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:copyright: Copyright 2026 by the Shengyu Zhang. +:license: BSD, see LICENSE for details. + +This module provides helpful BaseDataSource subclasses. +""" + from __future__ import annotations -from typing import override +from typing import TYPE_CHECKING, override +from abc import abstractmethod from docutils.parsers.rst import directives +from docutils.parsers.rst.directives import directive + +from ..data import Field, RawData, PendingData, ParsedData, Schema +from .render import Template +from .pipeline import BaseDataSource, BaseDataDirective, BaseDataRole + +if TYPE_CHECKING: + from typing import Any + +class BaseRawDataSource(BaseDataSource): + """ + A BaseDataRenderer subclass, which itself is a definition of raw data + """ + + """Methods to be implemented.""" + + @abstractmethod + def current_raw_data(self) -> RawData: ... + + @abstractmethod + def current_schema(self) -> Schema: ... -from ..data import Field, Schema -from ..render import Template, BaseDataDefineDirective + """Methods to be overrided.""" + + @override + def current_data(self) -> PendingData | ParsedData | dict[str, Any]: + return PendingData(self.current_raw_data(), self.current_schema()) + + +class BaseDataDefineDirective(BaseRawDataSource, BaseDataDirective): + @override + def current_raw_data(self) -> RawData: + return RawData( + ' '.join(self.arguments) if self.arguments else None, + self.options.copy(), + '\n'.join(self.content) if self.has_content else None, + ) + + +class BaseDataDefineRole(BaseRawDataSource, BaseDataRole): + @override + def current_raw_data(self) -> RawData: + return RawData(self.name, self.options.copy(), self.text) class StrictDataDefineDirective(BaseDataDefineDirective): From 5e127f68277c5fe2f3389a15de80f94b2f34fc34 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Thu, 5 Feb 2026 10:42:17 +0800 Subject: [PATCH 5/9] feat: Introduce PendingContextRef --- src/sphinxnotes/data/__init__.py | 18 +- src/sphinxnotes/data/data.py | 23 ++- src/sphinxnotes/data/render/__init__.py | 22 ++- src/sphinxnotes/data/render/ctx.py | 96 ++++++++++ .../data/render/{datanodes.py => ctxnodes.py} | 168 +++++++++--------- src/sphinxnotes/data/render/extractx.py | 2 +- src/sphinxnotes/data/render/pipeline.py | 64 +++---- src/sphinxnotes/data/render/sources.py | 27 +-- 8 files changed, 266 insertions(+), 154 deletions(-) create mode 100644 src/sphinxnotes/data/render/ctx.py rename src/sphinxnotes/data/render/{datanodes.py => ctxnodes.py} (55%) diff --git a/src/sphinxnotes/data/__init__.py b/src/sphinxnotes/data/__init__.py index 31f5010..67129e5 100644 --- a/src/sphinxnotes/data/__init__.py +++ b/src/sphinxnotes/data/__init__.py @@ -28,8 +28,13 @@ Template, Host, pending_node, - BaseDataRole, - BaseDataDirective, + Context, + PendingContext, + PendingContextRef, + PendingContextStorage, + ResolvedContext, + BaseContextRole, + BaseContextDirective, ExtraContextRegistry, ExtraContextGenerator, BaseDataDefineRole, @@ -54,9 +59,14 @@ 'Phase', 'Template', 'Host', + 'Context', + 'PendingContext', + 'PendingContextRef', + 'PendingContextStorage', + 'ResolvedContext', 'pending_node', - 'BaseDataRole', - 'BaseDataDirective', + 'BaseContextRole', + 'BaseContextDirective', 'BaseDataDefineRole', 'BaseDataDefineDirective', 'StrictDataDefineDirective', diff --git a/src/sphinxnotes/data/data.py b/src/sphinxnotes/data/data.py index 1eae6b3..3365c85 100644 --- a/src/sphinxnotes/data/data.py +++ b/src/sphinxnotes/data/data.py @@ -217,14 +217,8 @@ class RawData: attrs: dict[str, str] content: str | None - -@dataclass -class PendingData(Unpicklable): - raw: RawData - schema: Schema - - def parse(self) -> ParsedData: - return self.schema.parse(self.raw) + def __hash__(self) -> int: + return hash((self.name, frozenset(self.attrs.items()), self.content)) @dataclass @@ -233,6 +227,9 @@ class ParsedData: attrs: dict[str, Value] content: Value + def __hash__(self) -> int: + return hash((self.name, frozenset(self.attrs.items()), self.content)) + def asdict(self) -> dict[str, Any]: """ Convert Data to a dict for usage of Jinja2 context. @@ -268,6 +265,9 @@ class Field(Unpicklable): required: bool = False sep: str | None = None + def __hash__(self) -> int: + return hash((self.etype, self.ctype, frozenset(self.flags.items()), self.dsl)) + @classmethod def from_dsl(cls, dsl: str) -> Self: self = cls() @@ -456,6 +456,13 @@ class Schema(Unpicklable): attrs: dict[str, Field] | Field content: Field | None + def __hash__(self) -> int: + if isinstance(self.attrs, Field): + attrs_hash = hash(self.attrs) + else: + attrs_hash = hash(frozenset(self.attrs.items())) + return hash((self.name, attrs_hash, self.content)) + @classmethod def from_dsl( cls, diff --git a/src/sphinxnotes/data/render/__init__.py b/src/sphinxnotes/data/render/__init__.py index 6ff7259..3e8a89b 100644 --- a/src/sphinxnotes/data/render/__init__.py +++ b/src/sphinxnotes/data/render/__init__.py @@ -1,11 +1,16 @@ from __future__ import annotations from typing import TYPE_CHECKING -from .render import Phase, Template, Host -from .datanodes import pending_node +from .render import ( + Phase, + Template, + Host, +) +from .ctx import Context, PendingContext, PendingContextRef, PendingContextStorage, ResolvedContext +from .ctxnodes import pending_node from .pipeline import ( - BaseDataRole, - BaseDataDirective, + BaseContextRole, + BaseContextDirective, ) from .extractx import ExtraContextRegistry, ExtraContextGenerator from .sources import ( @@ -22,9 +27,14 @@ 'Phase', 'Template', 'Host', + 'Context', + 'PendingContext', + 'PendingContextRef', + 'PendingContextStorage', + 'ResolvedContext', 'pending_node', - 'BaseDataRole', - 'BaseDataDirective', + 'BaseContextRole', + 'BaseContextDirective', 'ExtraContextRegistry', 'ExtraContextGenerator', 'BaseDataDefineDirective', diff --git a/src/sphinxnotes/data/render/ctx.py b/src/sphinxnotes/data/render/ctx.py new file mode 100644 index 0000000..08f2a91 --- /dev/null +++ b/src/sphinxnotes/data/render/ctx.py @@ -0,0 +1,96 @@ +""" +sphinxnotes.data.render.data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:copyright: Copyright 2026 by the Shengyu Zhang. +:license: BSD, see LICENSE for details. + +This module wraps the :mod:`..data` module to make it work well with the render +pipeline (:mod:`pipeline`). +""" + +from typing import TYPE_CHECKING, override +from abc import ABC, abstractmethod +from collections.abc import Hashable +from dataclasses import dataclass + +from sphinxnotes.data.utils import Unpicklable + +from ..data import ParsedData, RawData, Schema + +if TYPE_CHECKING: + from typing import Any + +type Context = PendingContextRef | ResolvedContext + +type ResolvedContext = ParsedData | dict[str, Any] + + +@dataclass +class PendingContextRef: + """A abstract class that references to :cls:`PendingCtx`.""" + + ref: int + chksum: int + + def __hash__(self) -> int: + return hash((self.ref, self.chksum)) + + +class PendingContext(ABC, Unpicklable, Hashable): + """A abstract represent of context that is not currently available. + + Call :meth:`resolve` to get data. + """ + + @abstractmethod + def resolve(self) -> ResolvedContext: ... + + +class PendingContextStorage: + """Area for temporarily storing PendingData. + + This class is indented to resolve the problem that some datas are Unpicklable + and can not be hold by :cls:`pending_node` (as ``pending_node`` will be + pickled along with the docutils doctree) + + This class maintains a mapping from :cls:`PendingDataRef` -> :cls:`PendingData`. + ``pending_node`` owns the ``PendingDataRef``, and the PendingData is Unpicklable, + pending_node can get it by calling :meth:`retrieve`. + """ + + _next_id: int + _data: dict[PendingContextRef, PendingContext] = {} + + def __init__(self) -> None: + self._next_id = 0 + self._data = {} + + def stash(self, pending: PendingContext) -> PendingContextRef: + ref = PendingContextRef(self._next_id, hash(pending)) + self._next_id += 1 + self._data[ref] = pending + return ref + + def retrieve(self, ref: PendingContextRef) -> PendingContext | None: + data = self._data.pop(ref, None) + return data + + +PENDING_CONTEXT_STORAGE = PendingContextStorage() + + +@dataclass +class UnparsedData(PendingContext): + """A implementation of PendingData, contains raw data and its schema.""" + + raw: RawData + schema: Schema + + @override + def resolve(self) -> ResolvedContext: + return self.schema.parse(self.raw) + + @override + def __hash__(self) -> int: + return hash((self.raw, self.schema)) diff --git a/src/sphinxnotes/data/render/datanodes.py b/src/sphinxnotes/data/render/ctxnodes.py similarity index 55% rename from src/sphinxnotes/data/render/datanodes.py rename to src/sphinxnotes/data/render/ctxnodes.py index d212fcd..00b50f1 100644 --- a/src/sphinxnotes/data/render/datanodes.py +++ b/src/sphinxnotes/data/render/ctxnodes.py @@ -6,9 +6,9 @@ from docutils.parsers.rst.states import Inliner from .render import Template +from .ctx import PENDING_CONTEXT_STORAGE, PendingContextRef from .markup import MarkupRenderer from .template import TemplateRenderer -from ..data import RawData, PendingData, ParsedData from ..utils import ( Report, Reporter, @@ -18,15 +18,13 @@ if TYPE_CHECKING: from typing import Any, Callable from .markup import Host + from .ctx import Context, PendingContext, ResolvedContext -class Base(nodes.Element): ... - - -class pending_node(Base): - # The data to be rendered by Jinja template. - data: PendingData | ParsedData | dict[str, Any] - # The extra context for Jina template. +class pending_node(nodes.Element): + # The context to be rendered by Jinja template. + ctx: Context + # The extra context as supplement to ctx. extra: dict[str, Any] #: Jinja template for rendering the context. template: Template @@ -34,12 +32,10 @@ class pending_node(Base): inline: bool #: Whether the rendering pipeline is finished (failed is also finished). rendered: bool - #: The report of render pipepine. - report: Report def __init__( self, - data: PendingData | ParsedData | dict[str, Any], + ctx: Context, tmpl: Template, inline: bool = False, rawsource='', @@ -47,64 +43,24 @@ def __init__( **attributes, ) -> None: super().__init__(rawsource, *children, **attributes) - self.data = data + self.ctx = ctx self.extra = {} self.template = tmpl self.inline = inline self.rendered = False - self.report = Report( - 'Render Report', 'DEBUG', source=self.source, line=self.line - ) # Init hook lists. - self._raw_data_hooks = [] - self._parsed_data_hooks = [] + self._pending_data_hooks = [] + self._resoloved_data_hooks = [] self._markup_text_hooks = [] self._rendered_nodes_hooks = [] - def get_error_report(self) -> Report: - if self.template.debug: - # Reuse the render report as possible. - self.report['type'] = 'ERROR' - return self.report - return Report('Render Report', 'ERROR', source=self.source, line=self.line) - - def ensure_data_parsed(self) -> ParsedData | dict[str, Any] | None: - """ - Ensure self.data is parsed (instance of ParsedData | dict[str, Any]). - if no, parse it. - """ - if not isinstance(self.data, PendingData): - return self.data - - self.report.text('Raw data:') - self.report.code(pformat(self.data.raw), lang='python') - self.report.text('Schema:') - self.report.code(pformat(self.data.schema), lang='python') - - for hook in self._raw_data_hooks: - hook(self, self.data.raw) - - try: - data = self.data = self.data.parse() - except ValueError as e: - report = self.get_error_report() - report.text('Failed to parse raw data:') - report.exception(e) - self += report - return None - - for hook in self._parsed_data_hooks: - hook(self, data) - - return data - def render(self, host: Host) -> None: """ - The core function for rendering data to docutils nodes. + The core function for rendering context to docutils nodes. - 1. Schema.parse(RawData) -> ParsedData (self.parse_data) - 2. TemplateRenderer.render(ParsedData) -> Markup Text (``str``) + 1. PendingDataRef + PENDING_DATA_RESOLVER -> PendingData -> ResolvedContext + 2. TemplateRenderer.render(ResolvedData) -> Markup Text (``str``) 3. MarkupRenderer.render(Markup Text) -> doctree Nodes (list[nodes.Node]) """ @@ -112,25 +68,63 @@ def render(self, host: Host) -> None: assert not self.rendered self.rendered = True - # Clear empty reports. + # Clear previous empty reports. Reporter(self).clear_empty() + # Create debug report. + report = Report('Render Report', 'DEBUG', source=self.source, line=self.line) - # 1. Prepare context for Jinja template. - if (data := self.ensure_data_parsed()) is None: - return # parse failure - - self.report.text(f'Parsed data (type: {type(data)}):') - self.report.code(pformat(data), lang='python') - self.report.text('Extra context (only keys):') - self.report.code(pformat(list(self.extra.keys())), lang='python') - self.report.text(f'Template (phase: {self.template.phase}):') - self.report.code(self.template.text, lang='jinja') + # Constructor for error report. + def err_report() -> Report: + if self.template.debug: + # Reuse the render report as possible. + report['type'] = 'ERROR' + return report + return Report('Render Report', 'ERROR', source=self.source, line=self.line) - # 2. Render the template and data to markup text. + # 1. Prepare context for Jinja template. + if isinstance(self.ctx, PendingContextRef): + report.text('Pending context ref:') + report.code(pformat(self.ctx), lang='python') + + pdata = PENDING_CONTEXT_STORAGE.retrieve(self.ctx) + if pdata is None: + report = err_report() + report.text(f'Failed to retrieve pending context from ref {self.ctx}') + self += report + return None + + report.text('Pending context:') + report.code(pformat(pdata), lang='python') + + for hook in self._pending_data_hooks: + hook(self, pdata) + + try: + ctx = self.ctx = pdata.resolve() + except ValueError as e: + report = err_report() + report.text('Failed to resolve pending context:') + report.exception(e) + self += report + return None + else: + ctx = self.ctx + + for hook in self._resoloved_data_hooks: + hook(self, ctx) + + report.text(f'Resolved context (type: {type(ctx)}):') + report.code(pformat(ctx), lang='python') + report.text('Extra context (only keys):') + report.code(pformat(list(self.extra.keys())), lang='python') + report.text(f'Template (phase: {self.template.phase}):') + report.code(self.template.text, lang='jinja') + + # 2. Render the template and context to markup text. try: - markup = TemplateRenderer(self.template.text).render(data, extra=self.extra) + markup = TemplateRenderer(self.template.text).render(ctx, extra=self.extra) except Exception as e: - report = self.get_error_report() + report = err_report() report.text('Failed to render Jinja template:') report.exception(e) self += report @@ -139,14 +133,14 @@ def render(self, host: Host) -> None: for hook in self._markup_text_hooks: markup = hook(self, markup) - self.report.text('Rendered markup text:') - self.report.code(markup, lang='rst') + report.text('Rendered markup text:') + report.code(markup, lang='rst') # 3. Render the markup text to doctree nodes. try: ns, msgs = MarkupRenderer(host).render(markup, inline=self.inline) except Exception as e: - report = self.get_error_report() + report = err_report() report.text( 'Failed to render markup text ' f'to {"inline " if self.inline else ""}nodes:' @@ -155,11 +149,11 @@ def render(self, host: Host) -> None: self += report return - self.report.text(f'Rendered nodes (inline: {self.inline}):') - self.report.code('\n\n'.join([n.pformat() for n in ns]), lang='xml') + report.text(f'Rendered nodes (inline: {self.inline}):') + report.code('\n\n'.join([n.pformat() for n in ns]), lang='xml') if msgs: - self.report.text('Systemd messages:') - [self.report.node(msg) for msg in msgs] + report.text('Systemd messages:') + [report.node(msg) for msg in msgs] # 4. Add rendered nodes to container. for hook in self._rendered_nodes_hooks: @@ -169,7 +163,7 @@ def render(self, host: Host) -> None: self += ns if self.template.debug: - self += self.report + self += report return @@ -211,21 +205,21 @@ def unwrap_and_replace_self_inline(self, inliner: Report.Inliner) -> None: """Hooks for procssing render intermediate products. """ - type RawDataHook = Callable[[pending_node, RawData], None] - type ParsedDataHook = Callable[[pending_node, ParsedData | dict[str, Any]], None] + type PendingDataHook = Callable[[pending_node, PendingContext], None] + type ResolvedDataHook = Callable[[pending_node, ResolvedContext], None] type MarkupTextHook = Callable[[pending_node, str], str] type RenderedNodesHook = Callable[[pending_node, list[nodes.Node]], None] - _raw_data_hooks: list[RawDataHook] - _parsed_data_hooks: list[ParsedDataHook] + _pending_data_hooks: list[PendingDataHook] + _resoloved_data_hooks: list[ResolvedDataHook] _markup_text_hooks: list[MarkupTextHook] _rendered_nodes_hooks: list[RenderedNodesHook] - def hook_raw_data(self, hook: RawDataHook) -> None: - self._raw_data_hooks.append(hook) + def hook_pending_data(self, hook: PendingDataHook) -> None: + self._pending_data_hooks.append(hook) - def hook_parsed_data(self, hook: ParsedDataHook) -> None: - self._parsed_data_hooks.append(hook) + def hook_resolved_data(self, hook: ResolvedDataHook) -> None: + self._resoloved_data_hooks.append(hook) def hook_markup_text(self, hook: MarkupTextHook) -> None: self._markup_text_hooks.append(hook) diff --git a/src/sphinxnotes/data/render/extractx.py b/src/sphinxnotes/data/render/extractx.py index 9a2db65..ce42174 100644 --- a/src/sphinxnotes/data/render/extractx.py +++ b/src/sphinxnotes/data/render/extractx.py @@ -7,7 +7,7 @@ from docutils.parsers.rst.roles import _roles from .render import HostWrapper -from .datanodes import pending_node +from .ctxnodes import pending_node from ..utils import find_current_section, Report, Reporter from ..utils.ctxproxy import proxy diff --git a/src/sphinxnotes/data/render/pipeline.py b/src/sphinxnotes/data/render/pipeline.py index fd7e7cc..7ba700b 100644 --- a/src/sphinxnotes/data/render/pipeline.py +++ b/src/sphinxnotes/data/render/pipeline.py @@ -5,12 +5,12 @@ :copyright: Copyright 2026 by the Shengyu Zhang. :license: BSD, see LICENSE for details. -This modeule defines pipeline for rendering data to nodes. +This module defines pipeline for rendering data to nodes. The Pipline =========== -1. Define data: BaseDataSource generates a :cls:`pending_node`, which contains: +1. Define context: BaseDataSource generates a :cls:`pending_node`, which contains: - Data: :cls:`PendingData` (data to be validated), :cls:`ParsedData`, or simple ``dict[str, Any]``. @@ -41,7 +41,7 @@ How data be rendered ``list[nodes.Node]`` ========================================= -.. seealso:: :meth:`.datanodes.pending_node.render`. +.. seealso:: :meth:`.ctxnodes.pending_node.render`. """ @@ -56,12 +56,12 @@ from sphinx.transforms.post_transforms import SphinxPostTransform, ReferencesResolver from .render import HostWrapper, Phase, Template, Host, ParseHost, TransformHost -from .datanodes import pending_node +from .ctx import Context +from .ctxnodes import pending_node from .extractx import ExtraContextGenerator -from ..data import RawData, PendingData, ParsedData, Schema + if TYPE_CHECKING: - from typing import Any from sphinx.application import Sphinx logger = logging.getLogger(__name__) @@ -82,7 +82,7 @@ class Pipeline(ABC): See Also: - - :class:`BaseDataSource`: Data source implementation and hook for Phase.Parsing + - :class:`BaseDataSource`: Context source implementation and hook for Phase.Parsing - :class:`ParsedHookTransform`: Built-in hook for Phase.Parsed - :class:`ResolvingHookTransform`: Built-in hook for Phase.Resolving """ @@ -110,14 +110,17 @@ def queue_pending_node(self, n: pending_node) -> None: self._q = [] self._q.append(n) - @final - def queue_data(self, data: PendingData | ParsedData | dict[str, Any], tmpl: Template - ) -> pending_node: - pending = pending_node(data, tmpl) + def queue_context(self, ctx: Context, tmpl: Template) -> pending_node: + pending = pending_node(ctx, tmpl) self.queue_pending_node(pending) return pending + @final + def queue_resolved_data(self, ctx: Context, tmpl: Template) -> pending_node: + pending = pending_node(ctx, tmpl) + self.queue_pending_node(pending) + return pending @final def render_queue(self) -> list[pending_node]: @@ -159,12 +162,12 @@ def render_queue(self) -> list[pending_node]: return ns -class BaseDataSource(Pipeline): +class BaseContextSource(Pipeline): """ - Abstract base class for generateing data, as the source of the rendering + Abstract base class for generateing context, as the source of the rendering pipeline. - This class also responsible to render data in Phase.Parsing. So the final + This class also responsible to render context in Phase.Parsing. So the final implementations MUST be subclass of :class:`SphinxDirective` or :class:`SphinxRole`, which provide the execution context and interface for processing reStructuredText markup. @@ -173,30 +176,23 @@ class BaseDataSource(Pipeline): """Methods to be implemented.""" @abstractmethod - def current_data(self) -> PendingData | ParsedData | dict[str, Any]: - """ - Return the data to be rendered. - - This method should be implemented to provide the actual data content - that will be rendered. The returned data can be a PendingData object, - a ParsedData object, or a plain dictionary. - - Returns: - The data content to render into nodes. - """ + def current_context(self) -> Context: + """Return the context to be rendered.""" + ... @abstractmethod def current_template(self) -> Template: """ - Return the template for rendering the data. + Return the template for rendering the context. This method should be implemented to provide the Jinja2 template - that will render the data into markup text. The template determines + that will render the context into markup text. The template determines the phase at which rendering occurs. Returns: The template to use for rendering. """ + ... """Methods override from parent.""" @@ -212,10 +208,10 @@ def process_pending_node(self, n: pending_node) -> bool: return n.template.phase == Phase.Parsing -class BaseDataDirective(BaseDataSource, SphinxDirective): +class BaseContextDirective(BaseContextSource, SphinxDirective): @override def run(self) -> list[nodes.Node]: - self.queue_data(self.current_data(), self.current_template()) + self.queue_context(self.current_context(), self.current_template()) ns = [] for x in self.render_queue(): @@ -227,7 +223,7 @@ def run(self) -> list[nodes.Node]: return ns -class BaseDataRole(BaseDataSource, SphinxRole): +class BaseContextRole(BaseContextSource, SphinxRole): @override def process_pending_node(self, n: pending_node) -> bool: n.inline = True @@ -235,7 +231,7 @@ def process_pending_node(self, n: pending_node) -> bool: @override def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]: - pending = self.queue_data(self.current_data(), self.current_template()) + pending = self.queue_context(self.current_context(), self.current_template()) pending.inline = True ns, msgs = [], [] @@ -265,11 +261,7 @@ def apply(self, **kwargs): self.queue_pending_node(pending) for n in self.render_queue(): - # NOTE: In the next Phase, doctrees will be pickled to disk. - # As :cls:`data.Schema` is **Unpicklable**, we should ensure - # ``pending_node.data`` is parsed, which means pending_node dropped - # the reference to Schema. - n.ensure_data_parsed() + ... class ResolvingHookTransform(SphinxPostTransform, Pipeline): diff --git a/src/sphinxnotes/data/render/sources.py b/src/sphinxnotes/data/render/sources.py index 6cf8c1d..a7dde93 100644 --- a/src/sphinxnotes/data/render/sources.py +++ b/src/sphinxnotes/data/render/sources.py @@ -5,7 +5,7 @@ :copyright: Copyright 2026 by the Shengyu Zhang. :license: BSD, see LICENSE for details. -This module provides helpful BaseDataSource subclasses. +This module provides helpful BaseContextSource subclasses. """ from __future__ import annotations @@ -13,18 +13,19 @@ from abc import abstractmethod from docutils.parsers.rst import directives -from docutils.parsers.rst.directives import directive -from ..data import Field, RawData, PendingData, ParsedData, Schema +from ..data import Field, RawData, Schema +from .ctx import Context, UnparsedData, PENDING_CONTEXT_STORAGE from .render import Template -from .pipeline import BaseDataSource, BaseDataDirective, BaseDataRole +from .pipeline import BaseContextSource, BaseContextDirective, BaseContextRole if TYPE_CHECKING: - from typing import Any + pass -class BaseRawDataSource(BaseDataSource): + +class BaseRawDataSource(BaseContextSource): """ - A BaseDataRenderer subclass, which itself is a definition of raw data + A BaseContextRenderer subclass, which itself is a definition of raw data. """ """Methods to be implemented.""" @@ -38,11 +39,13 @@ def current_schema(self) -> Schema: ... """Methods to be overrided.""" @override - def current_data(self) -> PendingData | ParsedData | dict[str, Any]: - return PendingData(self.current_raw_data(), self.current_schema()) + def current_context(self) -> Context: + data = UnparsedData(self.current_raw_data(), self.current_schema()) + ref = PENDING_CONTEXT_STORAGE.stash(data) + return ref -class BaseDataDefineDirective(BaseRawDataSource, BaseDataDirective): +class BaseDataDefineDirective(BaseRawDataSource, BaseContextDirective): @override def current_raw_data(self) -> RawData: return RawData( @@ -52,7 +55,7 @@ def current_raw_data(self) -> RawData: ) -class BaseDataDefineRole(BaseRawDataSource, BaseDataRole): +class BaseDataDefineRole(BaseRawDataSource, BaseContextRole): @override def current_raw_data(self) -> RawData: return RawData(self.name, self.options.copy(), self.text) @@ -98,7 +101,7 @@ def derive( # Generate directive class return type( - '%sStrictDataDefineDirective' % name.title(), + 'Strict%sDataDefineDirective' % name.title(), (cls,), { 'schema': schema, From 566988cc125c8c07e5c0803115440e8b361f7aac Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Thu, 5 Feb 2026 11:18:47 +0800 Subject: [PATCH 6/9] refactor: No need to export Context, PendingContextRef and PendingContextStorage --- src/sphinxnotes/data/__init__.py | 8 +--- src/sphinxnotes/data/data.py | 5 ++- src/sphinxnotes/data/render/__init__.py | 7 ++-- src/sphinxnotes/data/render/ctx.py | 32 +++------------- src/sphinxnotes/data/render/ctxnodes.py | 51 +++++++++++++++---------- src/sphinxnotes/data/render/pipeline.py | 21 ++++------ src/sphinxnotes/data/render/sources.py | 25 +++++++++--- 7 files changed, 72 insertions(+), 77 deletions(-) diff --git a/src/sphinxnotes/data/__init__.py b/src/sphinxnotes/data/__init__.py index 67129e5..a2bed29 100644 --- a/src/sphinxnotes/data/__init__.py +++ b/src/sphinxnotes/data/__init__.py @@ -28,15 +28,13 @@ Template, Host, pending_node, - Context, PendingContext, - PendingContextRef, - PendingContextStorage, ResolvedContext, BaseContextRole, BaseContextDirective, ExtraContextRegistry, ExtraContextGenerator, + UnparsedData, BaseDataDefineRole, BaseDataDefineDirective, StrictDataDefineDirective, @@ -59,14 +57,12 @@ 'Phase', 'Template', 'Host', - 'Context', 'PendingContext', - 'PendingContextRef', - 'PendingContextStorage', 'ResolvedContext', 'pending_node', 'BaseContextRole', 'BaseContextDirective', + 'UnparsedData', 'BaseDataDefineRole', 'BaseDataDefineDirective', 'StrictDataDefineDirective', diff --git a/src/sphinxnotes/data/data.py b/src/sphinxnotes/data/data.py index 3365c85..577f870 100644 --- a/src/sphinxnotes/data/data.py +++ b/src/sphinxnotes/data/data.py @@ -266,7 +266,10 @@ class Field(Unpicklable): sep: str | None = None def __hash__(self) -> int: - return hash((self.etype, self.ctype, frozenset(self.flags.items()), self.dsl)) + flags = { + k: v if not isinstance(v, list) else tuple(v) for k, v in self.flags.items() + } + return hash((self.etype, self.ctype, frozenset(flags.items()), self.dsl)) @classmethod def from_dsl(cls, dsl: str) -> Self: diff --git a/src/sphinxnotes/data/render/__init__.py b/src/sphinxnotes/data/render/__init__.py index 3e8a89b..b6856fc 100644 --- a/src/sphinxnotes/data/render/__init__.py +++ b/src/sphinxnotes/data/render/__init__.py @@ -6,7 +6,7 @@ Template, Host, ) -from .ctx import Context, PendingContext, PendingContextRef, PendingContextStorage, ResolvedContext +from .ctx import PendingContext, ResolvedContext from .ctxnodes import pending_node from .pipeline import ( BaseContextRole, @@ -14,6 +14,7 @@ ) from .extractx import ExtraContextRegistry, ExtraContextGenerator from .sources import ( + UnparsedData, BaseDataDefineDirective, StrictDataDefineDirective, BaseDataDefineRole, @@ -27,16 +28,14 @@ 'Phase', 'Template', 'Host', - 'Context', 'PendingContext', - 'PendingContextRef', - 'PendingContextStorage', 'ResolvedContext', 'pending_node', 'BaseContextRole', 'BaseContextDirective', 'ExtraContextRegistry', 'ExtraContextGenerator', + 'UnparsedData', 'BaseDataDefineDirective', 'StrictDataDefineDirective', 'BaseDataDefineRole', diff --git a/src/sphinxnotes/data/render/ctx.py b/src/sphinxnotes/data/render/ctx.py index 08f2a91..ed2e0f1 100644 --- a/src/sphinxnotes/data/render/ctx.py +++ b/src/sphinxnotes/data/render/ctx.py @@ -9,19 +9,16 @@ pipeline (:mod:`pipeline`). """ -from typing import TYPE_CHECKING, override +from typing import TYPE_CHECKING from abc import ABC, abstractmethod from collections.abc import Hashable from dataclasses import dataclass from sphinxnotes.data.utils import Unpicklable -from ..data import ParsedData, RawData, Schema - if TYPE_CHECKING: from typing import Any - -type Context = PendingContextRef | ResolvedContext + from ..data import ParsedData type ResolvedContext = ParsedData | dict[str, Any] @@ -48,14 +45,14 @@ def resolve(self) -> ResolvedContext: ... class PendingContextStorage: - """Area for temporarily storing PendingData. + """Area for temporarily storing PendingContext. This class is indented to resolve the problem that some datas are Unpicklable and can not be hold by :cls:`pending_node` (as ``pending_node`` will be pickled along with the docutils doctree) - This class maintains a mapping from :cls:`PendingDataRef` -> :cls:`PendingData`. - ``pending_node`` owns the ``PendingDataRef``, and the PendingData is Unpicklable, + This class maintains a mapping from :cls:`PendingContextRef` -> :cls:`PendingContext`. + ``pending_node`` owns the ``PendingContextRef``, and the PendingContext is Unpicklable, pending_node can get it by calling :meth:`retrieve`. """ @@ -75,22 +72,3 @@ def stash(self, pending: PendingContext) -> PendingContextRef: def retrieve(self, ref: PendingContextRef) -> PendingContext | None: data = self._data.pop(ref, None) return data - - -PENDING_CONTEXT_STORAGE = PendingContextStorage() - - -@dataclass -class UnparsedData(PendingContext): - """A implementation of PendingData, contains raw data and its schema.""" - - raw: RawData - schema: Schema - - @override - def resolve(self) -> ResolvedContext: - return self.schema.parse(self.raw) - - @override - def __hash__(self) -> int: - return hash((self.raw, self.schema)) diff --git a/src/sphinxnotes/data/render/ctxnodes.py b/src/sphinxnotes/data/render/ctxnodes.py index 00b50f1..d029826 100644 --- a/src/sphinxnotes/data/render/ctxnodes.py +++ b/src/sphinxnotes/data/render/ctxnodes.py @@ -6,7 +6,7 @@ from docutils.parsers.rst.states import Inliner from .render import Template -from .ctx import PENDING_CONTEXT_STORAGE, PendingContextRef +from .ctx import PendingContextRef, PendingContext, PendingContextStorage from .markup import MarkupRenderer from .template import TemplateRenderer from ..utils import ( @@ -16,14 +16,14 @@ ) if TYPE_CHECKING: - from typing import Any, Callable + from typing import Any, Callable, ClassVar from .markup import Host - from .ctx import Context, PendingContext, ResolvedContext + from .ctx import ResolvedContext class pending_node(nodes.Element): # The context to be rendered by Jinja template. - ctx: Context + ctx: PendingContextRef | ResolvedContext # The extra context as supplement to ctx. extra: dict[str, Any] #: Jinja template for rendering the context. @@ -33,9 +33,15 @@ class pending_node(nodes.Element): #: Whether the rendering pipeline is finished (failed is also finished). rendered: bool + #: Mapping of PendingContextRef -> PendingContext. + #: + #: NOTE: ``PendingContextStorage`` holds Unpicklable data (``PendingContext``) + #: but it is doesn't matters :-), cause pickle doesn't deal with ClassVar. + _PENDING_CONTEXTS: ClassVar[PendingContextStorage] = PendingContextStorage() + def __init__( self, - ctx: Context, + ctx: PendingContext | PendingContextRef | ResolvedContext, tmpl: Template, inline: bool = False, rawsource='', @@ -43,15 +49,18 @@ def __init__( **attributes, ) -> None: super().__init__(rawsource, *children, **attributes) - self.ctx = ctx + if not isinstance(ctx, PendingContext): + self.ctx = ctx + else: + self.ctx = self._PENDING_CONTEXTS.stash(ctx) self.extra = {} self.template = tmpl self.inline = inline self.rendered = False # Init hook lists. - self._pending_data_hooks = [] - self._resoloved_data_hooks = [] + self._pending_context_hooks = [] + self._resolved_data_hooks = [] self._markup_text_hooks = [] self._rendered_nodes_hooks = [] @@ -59,8 +68,8 @@ def render(self, host: Host) -> None: """ The core function for rendering context to docutils nodes. - 1. PendingDataRef + PENDING_DATA_RESOLVER -> PendingData -> ResolvedContext - 2. TemplateRenderer.render(ResolvedData) -> Markup Text (``str``) + 1. PendingContextRef -> PendingContext -> ResolvedContext + 2. TemplateRenderer.render(ResolvedContext) -> Markup Text (``str``) 3. MarkupRenderer.render(Markup Text) -> doctree Nodes (list[nodes.Node]) """ @@ -86,7 +95,7 @@ def err_report() -> Report: report.text('Pending context ref:') report.code(pformat(self.ctx), lang='python') - pdata = PENDING_CONTEXT_STORAGE.retrieve(self.ctx) + pdata = self._PENDING_CONTEXTS.retrieve(self.ctx) if pdata is None: report = err_report() report.text(f'Failed to retrieve pending context from ref {self.ctx}') @@ -96,7 +105,7 @@ def err_report() -> Report: report.text('Pending context:') report.code(pformat(pdata), lang='python') - for hook in self._pending_data_hooks: + for hook in self._pending_context_hooks: hook(self, pdata) try: @@ -110,7 +119,7 @@ def err_report() -> Report: else: ctx = self.ctx - for hook in self._resoloved_data_hooks: + for hook in self._resolved_data_hooks: hook(self, ctx) report.text(f'Resolved context (type: {type(ctx)}):') @@ -205,21 +214,21 @@ def unwrap_and_replace_self_inline(self, inliner: Report.Inliner) -> None: """Hooks for procssing render intermediate products. """ - type PendingDataHook = Callable[[pending_node, PendingContext], None] - type ResolvedDataHook = Callable[[pending_node, ResolvedContext], None] + type PendingContextHook = Callable[[pending_node, PendingContext], None] + type ResolvedContextHook = Callable[[pending_node, ResolvedContext], None] type MarkupTextHook = Callable[[pending_node, str], str] type RenderedNodesHook = Callable[[pending_node, list[nodes.Node]], None] - _pending_data_hooks: list[PendingDataHook] - _resoloved_data_hooks: list[ResolvedDataHook] + _pending_context_hooks: list[PendingContextHook] + _resolved_data_hooks: list[ResolvedContextHook] _markup_text_hooks: list[MarkupTextHook] _rendered_nodes_hooks: list[RenderedNodesHook] - def hook_pending_data(self, hook: PendingDataHook) -> None: - self._pending_data_hooks.append(hook) + def hook_pending_context(self, hook: PendingContextHook) -> None: + self._pending_context_hooks.append(hook) - def hook_resolved_data(self, hook: ResolvedDataHook) -> None: - self._resoloved_data_hooks.append(hook) + def hook_resolved_context(self, hook: ResolvedContextHook) -> None: + self._resolved_data_hooks.append(hook) def hook_markup_text(self, hook: MarkupTextHook) -> None: self._markup_text_hooks.append(hook) diff --git a/src/sphinxnotes/data/render/pipeline.py b/src/sphinxnotes/data/render/pipeline.py index 7ba700b..4b750a8 100644 --- a/src/sphinxnotes/data/render/pipeline.py +++ b/src/sphinxnotes/data/render/pipeline.py @@ -12,8 +12,7 @@ 1. Define context: BaseDataSource generates a :cls:`pending_node`, which contains: - - Data: :cls:`PendingData` (data to be validated), :cls:`ParsedData`, or - simple ``dict[str, Any]``. + - Context - Template for rendering data to markup text - Possible extra contexts @@ -38,8 +37,8 @@ :``Phase.Resolving``: Called by :cls:`ResolvingHookTransform`. -How data be rendered ``list[nodes.Node]`` -========================================= +How context be rendered ``list[nodes.Node]`` +============================================ .. seealso:: :meth:`.ctxnodes.pending_node.render`. @@ -56,7 +55,7 @@ from sphinx.transforms.post_transforms import SphinxPostTransform, ReferencesResolver from .render import HostWrapper, Phase, Template, Host, ParseHost, TransformHost -from .ctx import Context +from .ctx import PendingContext, ResolvedContext from .ctxnodes import pending_node from .extractx import ExtraContextGenerator @@ -111,13 +110,9 @@ def queue_pending_node(self, n: pending_node) -> None: self._q.append(n) @final - def queue_context(self, ctx: Context, tmpl: Template) -> pending_node: - pending = pending_node(ctx, tmpl) - self.queue_pending_node(pending) - return pending - - @final - def queue_resolved_data(self, ctx: Context, tmpl: Template) -> pending_node: + def queue_context( + self, ctx: PendingContext | ResolvedContext, tmpl: Template + ) -> pending_node: pending = pending_node(ctx, tmpl) self.queue_pending_node(pending) return pending @@ -176,7 +171,7 @@ class BaseContextSource(Pipeline): """Methods to be implemented.""" @abstractmethod - def current_context(self) -> Context: + def current_context(self) -> PendingContext | ResolvedContext: """Return the context to be rendered.""" ... diff --git a/src/sphinxnotes/data/render/sources.py b/src/sphinxnotes/data/render/sources.py index a7dde93..66991fa 100644 --- a/src/sphinxnotes/data/render/sources.py +++ b/src/sphinxnotes/data/render/sources.py @@ -11,11 +11,12 @@ from __future__ import annotations from typing import TYPE_CHECKING, override from abc import abstractmethod +from dataclasses import dataclass from docutils.parsers.rst import directives from ..data import Field, RawData, Schema -from .ctx import Context, UnparsedData, PENDING_CONTEXT_STORAGE +from .ctx import PendingContext, ResolvedContext from .render import Template from .pipeline import BaseContextSource, BaseContextDirective, BaseContextRole @@ -23,6 +24,22 @@ pass +@dataclass +class UnparsedData(PendingContext): + """A implementation of PendingContext, contains raw data and its schema.""" + + raw: RawData + schema: Schema + + @override + def resolve(self) -> ResolvedContext: + return self.schema.parse(self.raw) + + @override + def __hash__(self) -> int: + return hash((self.raw, self.schema)) + + class BaseRawDataSource(BaseContextSource): """ A BaseContextRenderer subclass, which itself is a definition of raw data. @@ -39,10 +56,8 @@ def current_schema(self) -> Schema: ... """Methods to be overrided.""" @override - def current_context(self) -> Context: - data = UnparsedData(self.current_raw_data(), self.current_schema()) - ref = PENDING_CONTEXT_STORAGE.stash(data) - return ref + def current_context(self) -> PendingContext | ResolvedContext: + return UnparsedData(self.current_raw_data(), self.current_schema()) class BaseDataDefineDirective(BaseRawDataSource, BaseContextDirective): From 9ae39f9414f44eeb5dd8f3e53354ed91ffa79f25 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Thu, 5 Feb 2026 12:23:17 +0800 Subject: [PATCH 7/9] fix: Catch broader exception in pending context resolution --- src/sphinxnotes/data/render/ctxnodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sphinxnotes/data/render/ctxnodes.py b/src/sphinxnotes/data/render/ctxnodes.py index d029826..811104d 100644 --- a/src/sphinxnotes/data/render/ctxnodes.py +++ b/src/sphinxnotes/data/render/ctxnodes.py @@ -110,7 +110,7 @@ def err_report() -> Report: try: ctx = self.ctx = pdata.resolve() - except ValueError as e: + except Exception as e: report = err_report() report.text('Failed to resolve pending context:') report.exception(e) From 00b286415ef8436ae309b226881387fb4cee5db9 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Thu, 5 Feb 2026 12:56:32 +0800 Subject: [PATCH 8/9] fix: Fix image node generated from separate source parser --- src/sphinxnotes/data/render/markup.py | 43 ++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/src/sphinxnotes/data/render/markup.py b/src/sphinxnotes/data/render/markup.py index 06c5dec..54a29f0 100644 --- a/src/sphinxnotes/data/render/markup.py +++ b/src/sphinxnotes/data/render/markup.py @@ -15,9 +15,10 @@ from docutils import nodes from docutils.parsers.rst.states import Struct from docutils.utils import new_document +from sphinx import version_info from sphinx.util.docutils import SphinxDirective, SphinxRole from sphinx.transforms import SphinxTransform -from sphinx import version_info +from sphinx.environment.collectors.asset import ImageCollector from .render import Host @@ -53,6 +54,12 @@ def _render(self, text: str) -> list[Node]: settings = self.host.document.settings doc = new_document('', settings=settings) parser.parse(text, doc) + + # NOTE: Nodes produced by standalone source parser should be fixed + # before returning, cause they missed the processing by certain + # Sphinx transforms. + self._fix_document(doc) + return doc.children else: assert False @@ -79,3 +86,37 @@ def _render_inline(self, text: str) -> tuple[list[Node], list[system_message]]: return ns, [] else: assert False + + def _fix_document(self, document: nodes.document) -> None: + assert isinstance(self.host, SphinxTransform) + + """For documents generated by a separate source parser, some preprocessing + may be missing. For example: + + - the lack of an ImageCollector result in an incorrect "node['uri']" and + a missing "node['candidates']" :cls:`nodes.images + + .. note:: + + Since this depends on Sphinx's internal implementation, there may be + many cases that have not been considered. We can only do our best + and fix each error we encounter as possible. + """ + + ImageCollector().process_doc(self.host.app, document) + + def _fix_image_candidates(self, node: nodes.image) -> None: + """This is another way for fixing images node instead of calling + ImageCollector().process_doc. Keep it for now. + """ + assert isinstance(self.host, SphinxTransform) + + # NOTE: + # :meth:`sphinx.environment.collectors.ImageCollector.process_doc` add + # a 'candidates' key to nodes.image, and subclasses of + # :cls:`sphinx.transforms.post_transforms.BaseImageConverter` require + # the key. + node['candidates'] = {} + + # Update `node['uri']` to a relative path from srcdir. + node['uri'], _ = self.host.env.relfn2path(node['uri']) From cc8f503df6c67017c4ec60636e55329d823cf2a3 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Fri, 6 Feb 2026 02:00:18 +0800 Subject: [PATCH 9/9] chore: Minor docstring fixes --- src/sphinxnotes/data/render/ctx.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/sphinxnotes/data/render/ctx.py b/src/sphinxnotes/data/render/ctx.py index ed2e0f1..5391c4a 100644 --- a/src/sphinxnotes/data/render/ctx.py +++ b/src/sphinxnotes/data/render/ctx.py @@ -1,12 +1,11 @@ """ -sphinxnotes.data.render.data -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +sphinxnotes.data.render.ctx +~~~~~~~~~~~~~~~~~~~~~~~~~~~ :copyright: Copyright 2026 by the Shengyu Zhang. :license: BSD, see LICENSE for details. -This module wraps the :mod:`..data` module to make it work well with the render -pipeline (:mod:`pipeline`). +This module wraps the :mod:`..data` into context for rendering the template. """ from typing import TYPE_CHECKING @@ -35,9 +34,10 @@ def __hash__(self) -> int: class PendingContext(ABC, Unpicklable, Hashable): - """A abstract represent of context that is not currently available. + """A abstract representation of context that is not currently available. - Call :meth:`resolve` to get data. + Call :meth:`resolve` at the right time (depends on the implment) to get + context available. """ @abstractmethod @@ -47,13 +47,15 @@ def resolve(self) -> ResolvedContext: ... class PendingContextStorage: """Area for temporarily storing PendingContext. - This class is indented to resolve the problem that some datas are Unpicklable - and can not be hold by :cls:`pending_node` (as ``pending_node`` will be - pickled along with the docutils doctree) + This class is indented to resolve the problem that: + + Some of the PendingContext are :cls:Unpicklable` and they can not be hold + by :cls:`pending_node` (as ``pending_node`` will be pickled along with + the docutils doctree) This class maintains a mapping from :cls:`PendingContextRef` -> :cls:`PendingContext`. - ``pending_node`` owns the ``PendingContextRef``, and the PendingContext is Unpicklable, - pending_node can get it by calling :meth:`retrieve`. + ``pending_node`` owns the ``PendingContextRef``, and can retrieve the context + by calling :meth:`retrieve`. """ _next_id: int