diff --git a/src/sphinxnotes/data/__init__.py b/src/sphinxnotes/data/__init__.py index 07affe4..a2bed29 100644 --- a/src/sphinxnotes/data/__init__.py +++ b/src/sphinxnotes/data/__init__.py @@ -28,12 +28,17 @@ Template, Host, pending_node, - BaseDataDefineRole, - BaseDataDefineDirective, + PendingContext, + ResolvedContext, + BaseContextRole, + BaseContextDirective, ExtraContextRegistry, ExtraContextGenerator, + UnparsedData, + BaseDataDefineRole, + BaseDataDefineDirective, + StrictDataDefineDirective, ) -from .examples.strict import StrictDataDefineDirective if TYPE_CHECKING: from sphinx.application import Sphinx @@ -52,7 +57,12 @@ 'Phase', 'Template', 'Host', + 'PendingContext', + 'ResolvedContext', 'pending_node', + 'BaseContextRole', + 'BaseContextDirective', + 'UnparsedData', 'BaseDataDefineRole', 'BaseDataDefineDirective', 'StrictDataDefineDirective', diff --git a/src/sphinxnotes/data/data.py b/src/sphinxnotes/data/data.py index b1c93e3..577f870 100644 --- a/src/sphinxnotes/data/data.py +++ b/src/sphinxnotes/data/data.py @@ -217,14 +217,8 @@ class RawData: attrs: dict[str, str] content: str | None - -@dataclass -class PendingData(Unpicklable): - raw: RawData - schema: Schema - - def parse(self) -> ParsedData: - return self.schema.parse(self.raw) + def __hash__(self) -> int: + return hash((self.name, frozenset(self.attrs.items()), self.content)) @dataclass @@ -233,6 +227,9 @@ class ParsedData: attrs: dict[str, Value] content: Value + def __hash__(self) -> int: + return hash((self.name, frozenset(self.attrs.items()), self.content)) + def asdict(self) -> dict[str, Any]: """ Convert Data to a dict for usage of Jinja2 context. @@ -268,6 +265,12 @@ class Field(Unpicklable): required: bool = False sep: str | None = None + def __hash__(self) -> int: + flags = { + k: v if not isinstance(v, list) else tuple(v) for k, v in self.flags.items() + } + return hash((self.etype, self.ctype, frozenset(flags.items()), self.dsl)) + @classmethod def from_dsl(cls, dsl: str) -> Self: self = cls() @@ -451,11 +454,18 @@ def by_option_store_value_error(opt: ByOption) -> ValueError: @dataclass(frozen=True) -class Schema(object): +class Schema(Unpicklable): name: Field | None attrs: dict[str, Field] | Field content: Field | None + def __hash__(self) -> int: + if isinstance(self.attrs, Field): + attrs_hash = hash(self.attrs) + else: + attrs_hash = hash(frozenset(self.attrs.items())) + return hash((self.name, attrs_hash, self.content)) + @classmethod def from_dsl( cls, diff --git a/src/sphinxnotes/data/examples/datadomain.py b/src/sphinxnotes/data/examples/datadomain.py index caeadc2..3358b76 100644 --- a/src/sphinxnotes/data/examples/datadomain.py +++ b/src/sphinxnotes/data/examples/datadomain.py @@ -128,9 +128,10 @@ def __init__(self, orig_name: str) -> None: self.orig_name = orig_name @override - def current_data(self) -> RawData: - _, _, name = self.orig_name.partition('+') - return RawData(name, {}, self.text) + def current_raw_data(self) -> RawData: + data = super().current_raw_data() + _, _, data.name = self.orig_name.partition('+') + return data @override def current_schema(self) -> Schema: diff --git a/src/sphinxnotes/data/examples/strict.py b/src/sphinxnotes/data/examples/strict.py deleted file mode 100644 index f0ed2fc..0000000 --- a/src/sphinxnotes/data/examples/strict.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations -from typing import override - -from docutils.parsers.rst import directives - -from ..data import Field, Schema -from ..render import Template, BaseDataDefineDirective - - -class StrictDataDefineDirective(BaseDataDefineDirective): - final_argument_whitespace = True - - schema: Schema - template: Template - - @override - def current_template(self) -> Template: - return self.template - - @override - def current_schema(self) -> Schema: - return self.schema - - @classmethod - def derive( - cls, name: str, schema: Schema, tmpl: Template - ) -> type[StrictDataDefineDirective]: - if not schema.name: - required_arguments = 0 - optional_arguments = 0 - elif schema.name.required: - required_arguments = 1 - optional_arguments = 0 - else: - required_arguments = 0 - optional_arguments = 1 - - assert not isinstance(schema.attrs, Field) - option_spec = {} - for name, field in schema.attrs.items(): - if field.required: - option_spec[name] = directives.unchanged_required - else: - option_spec[name] = directives.unchanged - - has_content = schema.content is not None - - # Generate directive class - return type( - '%sStrictDataDefineDirective' % name.title(), - (cls,), - { - 'schema': schema, - 'template': tmpl, - 'has_content': has_content, - 'required_arguments': required_arguments, - 'optional_arguments': optional_arguments, - 'option_spec': option_spec, - }, - ) diff --git a/src/sphinxnotes/data/render/__init__.py b/src/sphinxnotes/data/render/__init__.py index 2716368..b6856fc 100644 --- a/src/sphinxnotes/data/render/__init__.py +++ b/src/sphinxnotes/data/render/__init__.py @@ -1,13 +1,24 @@ from __future__ import annotations from typing import TYPE_CHECKING -from .render import Phase, Template, Host -from .datanodes import pending_node +from .render import ( + Phase, + Template, + Host, +) +from .ctx import PendingContext, ResolvedContext +from .ctxnodes import pending_node from .pipeline import ( - BaseDataDefineRole, - BaseDataDefineDirective, + BaseContextRole, + BaseContextDirective, ) from .extractx import ExtraContextRegistry, ExtraContextGenerator +from .sources import ( + UnparsedData, + BaseDataDefineDirective, + StrictDataDefineDirective, + BaseDataDefineRole, +) if TYPE_CHECKING: from sphinx.application import Sphinx @@ -17,11 +28,17 @@ 'Phase', 'Template', 'Host', + 'PendingContext', + 'ResolvedContext', 'pending_node', - 'BaseDataDefineRole', - 'BaseDataDefineDirective', + 'BaseContextRole', + 'BaseContextDirective', 'ExtraContextRegistry', 'ExtraContextGenerator', + 'UnparsedData', + 'BaseDataDefineDirective', + 'StrictDataDefineDirective', + 'BaseDataDefineRole', ] diff --git a/src/sphinxnotes/data/render/ctx.py b/src/sphinxnotes/data/render/ctx.py new file mode 100644 index 0000000..5391c4a --- /dev/null +++ b/src/sphinxnotes/data/render/ctx.py @@ -0,0 +1,76 @@ +""" +sphinxnotes.data.render.ctx +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:copyright: Copyright 2026 by the Shengyu Zhang. +:license: BSD, see LICENSE for details. + +This module wraps the :mod:`..data` into context for rendering the template. +""" + +from typing import TYPE_CHECKING +from abc import ABC, abstractmethod +from collections.abc import Hashable +from dataclasses import dataclass + +from sphinxnotes.data.utils import Unpicklable + +if TYPE_CHECKING: + from typing import Any + from ..data import ParsedData + +type ResolvedContext = ParsedData | dict[str, Any] + + +@dataclass +class PendingContextRef: + """A abstract class that references to :cls:`PendingCtx`.""" + + ref: int + chksum: int + + def __hash__(self) -> int: + return hash((self.ref, self.chksum)) + + +class PendingContext(ABC, Unpicklable, Hashable): + """A abstract representation of context that is not currently available. + + Call :meth:`resolve` at the right time (depends on the implment) to get + context available. + """ + + @abstractmethod + def resolve(self) -> ResolvedContext: ... + + +class PendingContextStorage: + """Area for temporarily storing PendingContext. + + This class is indented to resolve the problem that: + + Some of the PendingContext are :cls:Unpicklable` and they can not be hold + by :cls:`pending_node` (as ``pending_node`` will be pickled along with + the docutils doctree) + + This class maintains a mapping from :cls:`PendingContextRef` -> :cls:`PendingContext`. + ``pending_node`` owns the ``PendingContextRef``, and can retrieve the context + by calling :meth:`retrieve`. + """ + + _next_id: int + _data: dict[PendingContextRef, PendingContext] = {} + + def __init__(self) -> None: + self._next_id = 0 + self._data = {} + + def stash(self, pending: PendingContext) -> PendingContextRef: + ref = PendingContextRef(self._next_id, hash(pending)) + self._next_id += 1 + self._data[ref] = pending + return ref + + def retrieve(self, ref: PendingContextRef) -> PendingContext | None: + data = self._data.pop(ref, None) + return data diff --git a/src/sphinxnotes/data/render/datanodes.py b/src/sphinxnotes/data/render/ctxnodes.py similarity index 57% rename from src/sphinxnotes/data/render/datanodes.py rename to src/sphinxnotes/data/render/ctxnodes.py index 3dae467..811104d 100644 --- a/src/sphinxnotes/data/render/datanodes.py +++ b/src/sphinxnotes/data/render/ctxnodes.py @@ -6,28 +6,25 @@ from docutils.parsers.rst.states import Inliner from .render import Template +from .ctx import PendingContextRef, PendingContext, PendingContextStorage from .markup import MarkupRenderer from .template import TemplateRenderer -from ..data import RawData, PendingData, ParsedData from ..utils import ( - Unpicklable, Report, Reporter, find_nearest_block_element, ) if TYPE_CHECKING: - from typing import Any, Callable + from typing import Any, Callable, ClassVar from .markup import Host + from .ctx import ResolvedContext -class Base(nodes.Element): ... - - -class pending_node(Base, Unpicklable): - # The data to be rendered by Jinja template. - data: PendingData | ParsedData | dict[str, Any] - # The extra context for Jina template. +class pending_node(nodes.Element): + # The context to be rendered by Jinja template. + ctx: PendingContextRef | ResolvedContext + # The extra context as supplement to ctx. extra: dict[str, Any] #: Jinja template for rendering the context. template: Template @@ -36,9 +33,15 @@ class pending_node(Base, Unpicklable): #: Whether the rendering pipeline is finished (failed is also finished). rendered: bool + #: Mapping of PendingContextRef -> PendingContext. + #: + #: NOTE: ``PendingContextStorage`` holds Unpicklable data (``PendingContext``) + #: but it is doesn't matters :-), cause pickle doesn't deal with ClassVar. + _PENDING_CONTEXTS: ClassVar[PendingContextStorage] = PendingContextStorage() + def __init__( self, - data: PendingData | ParsedData | dict[str, Any], + ctx: PendingContext | PendingContextRef | ResolvedContext, tmpl: Template, inline: bool = False, rawsource='', @@ -46,24 +49,27 @@ def __init__( **attributes, ) -> None: super().__init__(rawsource, *children, **attributes) - self.data = data + if not isinstance(ctx, PendingContext): + self.ctx = ctx + else: + self.ctx = self._PENDING_CONTEXTS.stash(ctx) self.extra = {} self.template = tmpl self.inline = inline self.rendered = False # Init hook lists. - self._raw_data_hooks = [] - self._parsed_data_hooks = [] + self._pending_context_hooks = [] + self._resolved_data_hooks = [] self._markup_text_hooks = [] self._rendered_nodes_hooks = [] def render(self, host: Host) -> None: """ - The core function for rendering data to docutils nodes. + The core function for rendering context to docutils nodes. - 1. Schema.parse(RawData) -> ParsedData - 2. TemplateRenderer.render(ParsedData) -> Markup Text (``str``) + 1. PendingContextRef -> PendingContext -> ResolvedContext + 2. TemplateRenderer.render(ResolvedContext) -> Markup Text (``str``) 3. MarkupRenderer.render(Markup Text) -> doctree Nodes (list[nodes.Node]) """ @@ -71,46 +77,65 @@ def render(self, host: Host) -> None: assert not self.rendered self.rendered = True - report = Report( - 'Render Debug Report', 'DEBUG', source=self.source, line=self.line - ) + # Clear previous empty reports. + Reporter(self).clear_empty() + # Create debug report. + report = Report('Render Report', 'DEBUG', source=self.source, line=self.line) + + # Constructor for error report. + def err_report() -> Report: + if self.template.debug: + # Reuse the render report as possible. + report['type'] = 'ERROR' + return report + return Report('Render Report', 'ERROR', source=self.source, line=self.line) # 1. Prepare context for Jinja template. - if isinstance(self.data, PendingData): - report.text('Raw data:') - report.code(pformat(self.data.raw), lang='python') - report.text('Schema:') - report.code(pformat(self.data.schema), lang='python') + if isinstance(self.ctx, PendingContextRef): + report.text('Pending context ref:') + report.code(pformat(self.ctx), lang='python') + + pdata = self._PENDING_CONTEXTS.retrieve(self.ctx) + if pdata is None: + report = err_report() + report.text(f'Failed to retrieve pending context from ref {self.ctx}') + self += report + return None + + report.text('Pending context:') + report.code(pformat(pdata), lang='python') - for hook in self._raw_data_hooks: - hook(self, self.data.raw) + for hook in self._pending_context_hooks: + hook(self, pdata) try: - data = self.data = self.data.parse() - except ValueError: - report.text('Failed to parse raw data:') - report.excption() + ctx = self.ctx = pdata.resolve() + except Exception as e: + report = err_report() + report.text('Failed to resolve pending context:') + report.exception(e) self += report - return + return None else: - data = self.data + ctx = self.ctx - for hook in self._parsed_data_hooks: - hook(self, data) + for hook in self._resolved_data_hooks: + hook(self, ctx) - report.text(f'Parsed data (type: {type(data)}):') - report.code(pformat(data), lang='python') + report.text(f'Resolved context (type: {type(ctx)}):') + report.code(pformat(ctx), lang='python') report.text('Extra context (only keys):') report.code(pformat(list(self.extra.keys())), lang='python') report.text(f'Template (phase: {self.template.phase}):') report.code(self.template.text, lang='jinja') - # 2. Render the template and data to markup text. + # 2. Render the template and context to markup text. try: - markup = TemplateRenderer(self.template.text).render(data, extra=self.extra) - except Exception: + markup = TemplateRenderer(self.template.text).render(ctx, extra=self.extra) + except Exception as e: + report = err_report() report.text('Failed to render Jinja template:') - report.excption() + report.exception(e) self += report return @@ -123,12 +148,13 @@ def render(self, host: Host) -> None: # 3. Render the markup text to doctree nodes. try: ns, msgs = MarkupRenderer(host).render(markup, inline=self.inline) - except Exception: + except Exception as e: + report = err_report() report.text( 'Failed to render markup text ' f'to {"inline " if self.inline else ""}nodes:' ) - report.excption() + report.exception(e) self += report return @@ -141,14 +167,13 @@ def render(self, host: Host) -> None: # 4. Add rendered nodes to container. for hook in self._rendered_nodes_hooks: hook(self, ns) + # TODO: set_source_info? self += ns if self.template.debug: self += report - Reporter(self).clear_empty() - return def unwrap(self) -> list[nodes.Node]: @@ -189,21 +214,21 @@ def unwrap_and_replace_self_inline(self, inliner: Report.Inliner) -> None: """Hooks for procssing render intermediate products. """ - type RawDataHook = Callable[[pending_node, RawData], None] - type ParsedDataHook = Callable[[pending_node, ParsedData | dict[str, Any]], None] + type PendingContextHook = Callable[[pending_node, PendingContext], None] + type ResolvedContextHook = Callable[[pending_node, ResolvedContext], None] type MarkupTextHook = Callable[[pending_node, str], str] type RenderedNodesHook = Callable[[pending_node, list[nodes.Node]], None] - _raw_data_hooks: list[RawDataHook] - _parsed_data_hooks: list[ParsedDataHook] + _pending_context_hooks: list[PendingContextHook] + _resolved_data_hooks: list[ResolvedContextHook] _markup_text_hooks: list[MarkupTextHook] _rendered_nodes_hooks: list[RenderedNodesHook] - def hook_raw_data(self, hook: RawDataHook) -> None: - self._raw_data_hooks.append(hook) + def hook_pending_context(self, hook: PendingContextHook) -> None: + self._pending_context_hooks.append(hook) - def hook_parsed_data(self, hook: ParsedDataHook) -> None: - self._parsed_data_hooks.append(hook) + def hook_resolved_context(self, hook: ResolvedContextHook) -> None: + self._resolved_data_hooks.append(hook) def hook_markup_text(self, hook: MarkupTextHook) -> None: self._markup_text_hooks.append(hook) diff --git a/src/sphinxnotes/data/render/extractx.py b/src/sphinxnotes/data/render/extractx.py index e05277b..ce42174 100644 --- a/src/sphinxnotes/data/render/extractx.py +++ b/src/sphinxnotes/data/render/extractx.py @@ -7,7 +7,7 @@ from docutils.parsers.rst.roles import _roles from .render import HostWrapper -from .datanodes import pending_node +from .ctxnodes import pending_node from ..utils import find_current_section, Report, Reporter from ..utils.ctxproxy import proxy @@ -40,7 +40,7 @@ def generate(self, host: TransformHost) -> Any: ... class ExtraContextRegistry: names: set[str] parsing: dict[str, ParsePhaseExtraContext] - parsed: dict[str, ParsePhaseExtraContext] + parsed: dict[str, TransformPhaseExtraContext] post_transform: dict[str, TransformPhaseExtraContext] global_: dict[str, GlobalExtraContxt] @@ -70,7 +70,7 @@ def add_parsing_phase_context( self.parsing['_' + name] = ctxgen def add_parsed_phase_context( - self, name: str, ctxgen: ParsePhaseExtraContext + self, name: str, ctxgen: TransformPhaseExtraContext ) -> None: self._name_dedup(name) self.parsed['_' + name] = ctxgen @@ -163,7 +163,7 @@ def on_parsing(self, host: ParseHost) -> None: for name, ctxgen in self.registry.parsing.items(): self._safegen(name, lambda: ctxgen.generate(host)) - def on_parsed(self, host: ParseHost) -> None: + def on_parsed(self, host: TransformHost) -> None: for name, ctxgen in self.registry.parsed.items(): self._safegen(name, lambda: ctxgen.generate(host)) @@ -177,7 +177,7 @@ def _safegen(self, name: str, gen: Callable[[], Any]): self.node.extra[name] = gen() except Exception: self.report.text(f'Failed to generate extra context "{name}":') - self.report.excption() + self.report.traceback() def setup(app: Sphinx): diff --git a/src/sphinxnotes/data/render/markup.py b/src/sphinxnotes/data/render/markup.py index 198ab16..54a29f0 100644 --- a/src/sphinxnotes/data/render/markup.py +++ b/src/sphinxnotes/data/render/markup.py @@ -15,9 +15,10 @@ from docutils import nodes from docutils.parsers.rst.states import Struct from docutils.utils import new_document +from sphinx import version_info from sphinx.util.docutils import SphinxDirective, SphinxRole from sphinx.transforms import SphinxTransform -from sphinx import version_info +from sphinx.environment.collectors.asset import ImageCollector from .render import Host @@ -43,12 +44,22 @@ def _render(self, text: str) -> list[Node]: elif isinstance(self.host, SphinxTransform): # TODO: dont create parser for every time if version_info[0] >= 9: - parser = self.host.app.registry.create_source_parser('rst', env=self.host.env, config=self.host.config) + parser = self.host.app.registry.create_source_parser( + 'rst', env=self.host.env, config=self.host.config + ) else: - parser = self.host.app.registry.create_source_parser(self.host.app, 'rst') + parser = self.host.app.registry.create_source_parser( + self.host.app, 'rst' + ) settings = self.host.document.settings doc = new_document('', settings=settings) parser.parse(text, doc) + + # NOTE: Nodes produced by standalone source parser should be fixed + # before returning, cause they missed the processing by certain + # Sphinx transforms. + self._fix_document(doc) + return doc.children else: assert False @@ -75,3 +86,37 @@ def _render_inline(self, text: str) -> tuple[list[Node], list[system_message]]: return ns, [] else: assert False + + def _fix_document(self, document: nodes.document) -> None: + assert isinstance(self.host, SphinxTransform) + + """For documents generated by a separate source parser, some preprocessing + may be missing. For example: + + - the lack of an ImageCollector result in an incorrect "node['uri']" and + a missing "node['candidates']" :cls:`nodes.images + + .. note:: + + Since this depends on Sphinx's internal implementation, there may be + many cases that have not been considered. We can only do our best + and fix each error we encounter as possible. + """ + + ImageCollector().process_doc(self.host.app, document) + + def _fix_image_candidates(self, node: nodes.image) -> None: + """This is another way for fixing images node instead of calling + ImageCollector().process_doc. Keep it for now. + """ + assert isinstance(self.host, SphinxTransform) + + # NOTE: + # :meth:`sphinx.environment.collectors.ImageCollector.process_doc` add + # a 'candidates' key to nodes.image, and subclasses of + # :cls:`sphinx.transforms.post_transforms.BaseImageConverter` require + # the key. + node['candidates'] = {} + + # Update `node['uri']` to a relative path from srcdir. + node['uri'], _ = self.host.env.relfn2path(node['uri']) diff --git a/src/sphinxnotes/data/render/pipeline.py b/src/sphinxnotes/data/render/pipeline.py index 7d9ba50..4b750a8 100644 --- a/src/sphinxnotes/data/render/pipeline.py +++ b/src/sphinxnotes/data/render/pipeline.py @@ -1,43 +1,46 @@ """ -sphinxnotes.data.pipeline -~~~~~~~~~~~~~~~~~~~~~~~~~ +sphinxnotes.data.render.pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :copyright: Copyright 2026 by the Shengyu Zhang. :license: BSD, see LICENSE for details. -This modeule defines pipeline for rendering data to nodes. +This module defines pipeline for rendering data to nodes. The Pipline =========== -1. Define data: BaseDataDefiner generates a :cls:`pending_node`, which contains: +1. Define context: BaseDataSource generates a :cls:`pending_node`, which contains: - - Data and possible extra contexts - - Schema for validating Data + - Context - Template for rendering data to markup text + - Possible extra contexts + + See also :cls:`BaseDataSource`. 2. Render data: the ``pending_node`` nodes will be rendered - (by calling :meth:`pending_node.render`) at some point, depending on :cls:`Phase`. + (by calling :meth:`pending_node.render`) at some point, depending on + :attr:`pending_node.template.phase`. The one who calls ``pending_node.render`` is called ``Host``. - The ``Host`` host is responsible for rendering the markup text into doctree + The ``Host`` host is responsible for rendering the markup text into docutils nodes (See :cls:`MarkupRenderer`). Phases: :``Phase.Parsing``: - Called by BaseDataDefiner ('s subclasses) + Called by BaseDataSource ('s subclasses) :``Phase.Parsed``: - Called by :cls:`_ParsedHook`. + Called by :cls:`ParsedHookTransform`. :``Phase.Resolving``: - Called by :cls:`_ResolvingHook`. + Called by :cls:`ResolvingHookTransform`. -How :cls:`RawData` be rendered ``list[nodes.Node]`` -=================================================== +How context be rendered ``list[nodes.Node]`` +============================================ -.. seealso:: :meth:`.datanodes.pending_node.render`. +.. seealso:: :meth:`.ctxnodes.pending_node.render`. """ @@ -48,21 +51,41 @@ from docutils import nodes from sphinx.util import logging from sphinx.util.docutils import SphinxDirective, SphinxRole +from sphinx.transforms import SphinxTransform from sphinx.transforms.post_transforms import SphinxPostTransform, ReferencesResolver from .render import HostWrapper, Phase, Template, Host, ParseHost, TransformHost -from .datanodes import pending_node +from .ctx import PendingContext, ResolvedContext +from .ctxnodes import pending_node from .extractx import ExtraContextGenerator -from ..data import RawData, PendingData, ParsedData, Schema + if TYPE_CHECKING: - from typing import Any from sphinx.application import Sphinx logger = logging.getLogger(__name__) class Pipeline(ABC): + """ + The core class defines the pipleing of rendering :cls:`pending_node`s. + + Subclass is responsible to: + + - call ``queue_xxx`` to add pendin nodes into queue. + - override :meth:`process_pending_node` to control when a pending node gets + rendered. In this method subclass can also call ``queue_xxx`` to add more + pending nodes. + - call :meth:`render_queue` to process all queued nodes and + returns any that couldn't be rendered in the current phase. + + See Also: + + - :class:`BaseDataSource`: Context source implementation and hook for Phase.Parsing + - :class:`ParsedHookTransform`: Built-in hook for Phase.Parsed + - :class:`ResolvingHookTransform`: Built-in hook for Phase.Resolving + """ + #: Queue of pending node to be rendered. _q: list[pending_node] | None = None @@ -72,7 +95,7 @@ def process_pending_node(self, n: pending_node) -> bool: """ You can add hooks to pending node here. - Return ``true`` if you want to render the pending node *immediately*, + Return ``true`` if you want to render the pending node *now*, otherwise it will be inserted to doctree directly andwaiting to later rendering """ @@ -87,22 +110,10 @@ def queue_pending_node(self, n: pending_node) -> None: self._q.append(n) @final - def queue_raw_data( - self, data: RawData, schema: Schema, tmpl: Template + def queue_context( + self, ctx: PendingContext | ResolvedContext, tmpl: Template ) -> pending_node: - pending = pending_node(PendingData(data, schema), tmpl) - self.queue_pending_node(pending) - return pending - - @final - def queue_parsed_data(self, data: ParsedData, tmpl: Template) -> pending_node: - pending = pending_node(data, tmpl) - self.queue_pending_node(pending) - return pending - - @final - def queue_any_data(self, data: Any, tmpl: Template) -> pending_node: - pending = pending_node(data, tmpl) + pending = pending_node(ctx, tmpl) self.queue_pending_node(pending) return pending @@ -122,7 +133,8 @@ def render_queue(self) -> list[pending_node]: while self._q: pending = self._q.pop() - if not self.process_pending_node(pending): + render_now = self.process_pending_node(pending) + if not render_now: ns.append(pending) continue @@ -145,25 +157,37 @@ def render_queue(self) -> list[pending_node]: return ns -class BaseDataDefiner(Pipeline): +class BaseContextSource(Pipeline): """ - A abstract class that owns :cls:`RawData` and support - validating and rendering the data at the appropriate time. + Abstract base class for generateing context, as the source of the rendering + pipeline. - The subclasses *MUST* be subclass of :cls:`SphinxDirective` or - :cls:`SphinxRole`. + This class also responsible to render context in Phase.Parsing. So the final + implementations MUST be subclass of :class:`SphinxDirective` or + :class:`SphinxRole`, which provide the execution context and interface for + processing reStructuredText markup. """ """Methods to be implemented.""" @abstractmethod - def current_data(self) -> RawData: ... + def current_context(self) -> PendingContext | ResolvedContext: + """Return the context to be rendered.""" + ... @abstractmethod - def current_schema(self) -> Schema: ... + def current_template(self) -> Template: + """ + Return the template for rendering the context. - @abstractmethod - def current_template(self) -> Template: ... + This method should be implemented to provide the Jinja2 template + that will render the context into markup text. The template determines + the phase at which rendering occurs. + + Returns: + The template to use for rendering. + """ + ... """Methods override from parent.""" @@ -179,20 +203,10 @@ def process_pending_node(self, n: pending_node) -> bool: return n.template.phase == Phase.Parsing -class BaseDataDefineDirective(BaseDataDefiner, SphinxDirective): - @override - def current_data(self) -> RawData: - return RawData( - ' '.join(self.arguments) if self.arguments else None, - self.options.copy(), - '\n'.join(self.content) if self.has_content else None, - ) - +class BaseContextDirective(BaseContextSource, SphinxDirective): @override def run(self) -> list[nodes.Node]: - self.queue_raw_data( - self.current_data(), self.current_schema(), self.current_template() - ) + self.queue_context(self.current_context(), self.current_template()) ns = [] for x in self.render_queue(): @@ -204,11 +218,7 @@ def run(self) -> list[nodes.Node]: return ns -class BaseDataDefineRole(BaseDataDefiner, SphinxRole): - @override - def current_data(self) -> RawData: - return RawData(None, {}, self.text) - +class BaseContextRole(BaseContextSource, SphinxRole): @override def process_pending_node(self, n: pending_node) -> bool: n.inline = True @@ -216,9 +226,8 @@ def process_pending_node(self, n: pending_node) -> bool: @override def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]: - self.queue_raw_data( - self.current_data(), self.current_schema(), self.current_template() - ) + pending = self.queue_context(self.current_context(), self.current_template()) + pending.inline = True ns, msgs = [], [] for n in self.render_queue(): @@ -232,70 +241,46 @@ def run(self) -> tuple[list[nodes.Node], list[nodes.system_message]]: return ns, msgs -class _ParsedHook(SphinxDirective, Pipeline): +class ParsedHookTransform(SphinxTransform, Pipeline): + # Before almost all others. + default_priority = 100 + @override def process_pending_node(self, n: pending_node) -> bool: - self.state.document.note_source(n.source, n.line) # type: ignore[arg-type] - - # Generate and save parsed extra context for later use. - ExtraContextGenerator(n).on_parsed(cast(ParseHost, self)) - + ExtraContextGenerator(n).on_parsed(cast(TransformHost, self)) return n.template.phase == Phase.Parsed @override - def run(self) -> list[nodes.Node]: - for pending in self.state.document.findall(pending_node): + def apply(self, **kwargs): + for pending in self.document.findall(pending_node): self.queue_pending_node(pending) - # Hook system_message method to let it report the - # correct line number. - # TODO: self.state.document.note_source(source, line) # type: ignore[arg-type] - # def fix_lineno(level, message, *children, **kwargs): - # kwargs['line'] = pending.line - # return orig_sysmsg(level, message, *children, **kwargs) - # self.state_machine.reporter.system_message = fix_lineno - - ns = self.render_queue() - assert len(ns) == 0 - - return [] # nothing to return - - -def _insert_parsed_hook(app, docname, content): - # NOTE: content is a single element list, representing the content of the - # source file. - # - # .. seealso:: https://www.sphinx-doc.org/en/master/extdev/event_callbacks.html#event-source-read - # - # TODO: markdown? - # TODO: rst_prelog? - content[-1] = content[-1] + '\n\n.. data.parsed-hook::' + for n in self.render_queue(): + ... -class _ResolvingHook(SphinxPostTransform, Pipeline): - # After resolving pending_xref. +class ResolvingHookTransform(SphinxPostTransform, Pipeline): + # After resolving pending_xref default_priority = (ReferencesResolver.default_priority or 10) + 5 @override def process_pending_node(self, n: pending_node) -> bool: - # Generate and save post transform extra context for later use. ExtraContextGenerator(n).on_post_transform(cast(TransformHost, self)) - - return n.template.phase == Phase.PostTranform + return n.template.phase == Phase.Resolving @override def apply(self, **kwargs): for pending in self.document.findall(pending_node): self.queue_pending_node(pending) - ns = self.render_queue() + + # NOTE: Should no node left. assert len(ns) == 0 def setup(app: Sphinx) -> None: # Hook for Phase.Parsed. - app.add_directive('data.parsed-hook', _ParsedHook) - app.connect('source-read', _insert_parsed_hook) + app.add_transform(ParsedHookTransform) # Hook for Phase.Resolving. - app.add_post_transform(_ResolvingHook) + app.add_post_transform(ResolvingHookTransform) diff --git a/src/sphinxnotes/data/render/render.py b/src/sphinxnotes/data/render/render.py index e54254e..02472a5 100644 --- a/src/sphinxnotes/data/render/render.py +++ b/src/sphinxnotes/data/render/render.py @@ -10,8 +10,7 @@ class Phase(Enum): Parsing = 'parsing' Parsed = 'parsed' - PostTranform = 'post-transform' - # TODO: transform? + Resolving = 'resolving' @classmethod def default(cls) -> Phase: diff --git a/src/sphinxnotes/data/render/sources.py b/src/sphinxnotes/data/render/sources.py new file mode 100644 index 0000000..66991fa --- /dev/null +++ b/src/sphinxnotes/data/render/sources.py @@ -0,0 +1,129 @@ +""" +sphinxnotes.data.render.sources +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:copyright: Copyright 2026 by the Shengyu Zhang. +:license: BSD, see LICENSE for details. + +This module provides helpful BaseContextSource subclasses. +""" + +from __future__ import annotations +from typing import TYPE_CHECKING, override +from abc import abstractmethod +from dataclasses import dataclass + +from docutils.parsers.rst import directives + +from ..data import Field, RawData, Schema +from .ctx import PendingContext, ResolvedContext +from .render import Template +from .pipeline import BaseContextSource, BaseContextDirective, BaseContextRole + +if TYPE_CHECKING: + pass + + +@dataclass +class UnparsedData(PendingContext): + """A implementation of PendingContext, contains raw data and its schema.""" + + raw: RawData + schema: Schema + + @override + def resolve(self) -> ResolvedContext: + return self.schema.parse(self.raw) + + @override + def __hash__(self) -> int: + return hash((self.raw, self.schema)) + + +class BaseRawDataSource(BaseContextSource): + """ + A BaseContextRenderer subclass, which itself is a definition of raw data. + """ + + """Methods to be implemented.""" + + @abstractmethod + def current_raw_data(self) -> RawData: ... + + @abstractmethod + def current_schema(self) -> Schema: ... + + """Methods to be overrided.""" + + @override + def current_context(self) -> PendingContext | ResolvedContext: + return UnparsedData(self.current_raw_data(), self.current_schema()) + + +class BaseDataDefineDirective(BaseRawDataSource, BaseContextDirective): + @override + def current_raw_data(self) -> RawData: + return RawData( + ' '.join(self.arguments) if self.arguments else None, + self.options.copy(), + '\n'.join(self.content) if self.has_content else None, + ) + + +class BaseDataDefineRole(BaseRawDataSource, BaseContextRole): + @override + def current_raw_data(self) -> RawData: + return RawData(self.name, self.options.copy(), self.text) + + +class StrictDataDefineDirective(BaseDataDefineDirective): + final_argument_whitespace = True + + schema: Schema + template: Template + + @override + def current_template(self) -> Template: + return self.template + + @override + def current_schema(self) -> Schema: + return self.schema + + @classmethod + def derive( + cls, name: str, schema: Schema, tmpl: Template + ) -> type[StrictDataDefineDirective]: + if not schema.name: + required_arguments = 0 + optional_arguments = 0 + elif schema.name.required: + required_arguments = 1 + optional_arguments = 0 + else: + required_arguments = 0 + optional_arguments = 1 + + assert not isinstance(schema.attrs, Field) + option_spec = {} + for name, field in schema.attrs.items(): + if field.required: + option_spec[name] = directives.unchanged_required + else: + option_spec[name] = directives.unchanged + + has_content = schema.content is not None + + # Generate directive class + return type( + 'Strict%sDataDefineDirective' % name.title(), + (cls,), + { + 'schema': schema, + 'template': tmpl, + 'has_content': has_content, + 'required_arguments': required_arguments, + 'optional_arguments': optional_arguments, + 'option_spec': option_spec, + }, + ) diff --git a/src/sphinxnotes/data/utils/__init__.py b/src/sphinxnotes/data/utils/__init__.py index 58f1530..62b4f35 100644 --- a/src/sphinxnotes/data/utils/__init__.py +++ b/src/sphinxnotes/data/utils/__init__.py @@ -97,15 +97,14 @@ def find_nearest_block_element(node: nodes.Node | None) -> nodes.Element | None: class Report(nodes.system_message): - type Level = Literal['DEBUG', 'INFO', 'WARNING', 'ERROR'] + type Type = Literal['DEBUG', 'INFO', 'WARNING', 'ERROR'] - level: Level title: str def __init__( - self, title: str, level: Level = 'DEBUG', *children, **attributes + self, title: str, typ: Type = 'DEBUG', *children, **attributes ) -> None: - super().__init__(title + ':', type=level, level=2, *children, **attributes) + super().__init__(title + ':', type=typ, level=2, *children, **attributes) self.title = title def empty(self) -> bool: @@ -143,12 +142,16 @@ def list(self, lines: Iterable[str]) -> None: self.node(bullet_list) - def excption(self) -> None: + def traceback(self) -> None: # https://pygments.org/docs/lexers/#pygments.lexers.python.PythonTracebackLexer self.code(traceback.format_exc(), lang='pytb') + def exception(self, e: Exception) -> None: + # https://pygments.org/docs/lexers/#pygments.lexers.python.PythonTracebackLexer + self.code(str(e), lang='pytb') + def is_error(self) -> bool: - return self.level == 'ERROR' + return self['type'] == 'ERROR' type Inliner = RstInliner | tuple[nodes.document, nodes.Element] diff --git a/src/sphinxnotes/data/utils/ctxproxy.py b/src/sphinxnotes/data/utils/ctxproxy.py index f48c092..cd9816c 100644 --- a/src/sphinxnotes/data/utils/ctxproxy.py +++ b/src/sphinxnotes/data/utils/ctxproxy.py @@ -8,7 +8,6 @@ from sphinx.config import Config as SphinxConfig from ..utils import find_first_child -from ..utils import Unpicklable logger = logging.getLogger(__name__) @@ -22,8 +21,9 @@ def wrapped(self: Proxy) -> Any: return property(wrapped) +# FIXME: Unpicklable? @dataclass(frozen=True) -class Proxy(Unpicklable): +class Proxy: """ Proxy complex objects into context for convenient and secure access within Jinja templates. diff --git a/tests/test_data.py b/tests/test_data.py index f91b34e..477cd0f 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -6,8 +6,8 @@ from data.data import Field, REGISTRY -class TestFieldParser(unittest.TestCase): +class TestFieldParser(unittest.TestCase): # ========================== # Basic Types # ==========================