diff --git a/src/extensions/score_metamodel/docs/generate_metamodel_rst.py b/src/extensions/score_metamodel/docs/generate_metamodel_rst.py index 76b7b7c91..0a0e959d5 100644 --- a/src/extensions/score_metamodel/docs/generate_metamodel_rst.py +++ b/src/extensions/score_metamodel/docs/generate_metamodel_rst.py @@ -46,6 +46,7 @@ import sys from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, replace +from hashlib import sha256 from pathlib import Path from typing import Any, cast @@ -78,6 +79,22 @@ def _split_targets(targets: str) -> list[str]: return [target.strip() for target in targets.split(",") if target.strip()] +def _mermaid_identifier(name: str) -> str: + """Return a Mermaid-safe identifier for a metamodel type name. + + Metamodel type names are also directive names and may contain hyphens. + Mermaid class declarations accept those names, but Mermaid's ``style`` + statements do not accept a hyphenated target. A short stable digest + distinguishes names that normalize to the same identifier, while the + original name remains the displayed class label. + """ + normalized = name.replace("-", "_") + if normalized == name: + return name + suffix = sha256(name.encode("utf-8")).hexdigest()[:8] + return f"{normalized}_{suffix}" + + @dataclass(frozen=True) class NeedLink: """A link definition belonging to one need type.""" @@ -209,6 +226,7 @@ class MermaidNode: """A generic class-diagram node.""" name: str + identifier: str attributes: tuple[str, ...] = () optional_attributes: tuple[str, ...] = () color: str | None = None @@ -284,8 +302,13 @@ def render(self, diagram: MermaidDiagram) -> str: def _class_declarations(self, diagram: MermaidDiagram) -> list[str]: lines: list[str] = [] for node in sorted(diagram.nodes, key=lambda item: item.name): + label = ( + f'["{self._escape_label(node.name)}"]' + if node.identifier != node.name + else "" + ) if node.attributes or node.optional_attributes: - lines.append(f"class {node.name} {{") + lines.append(f"class {node.identifier}{label} {{") lines.extend(f" +{attribute}" for attribute in sorted(node.attributes)) # Mermaid's trailing '*' member classifier renders the complete # attribute line in italics. Keep the textual marker as part @@ -298,29 +321,32 @@ def _class_declarations(self, diagram: MermaidDiagram) -> list[str]: ) lines.append("}") else: - lines.append(f"class {node.name}") + lines.append(f"class {node.identifier}{label}") return lines def _edge_declarations(self, diagram: MermaidDiagram) -> list[str]: lines: list[str] = [] seen: set[tuple[str, str, str]] = set() + identifiers = {node.name: node.identifier for node in diagram.nodes} for edge in diagram.edges: key = (edge.source, edge.target, edge.label) if key in seen: continue seen.add(key) arrow = "..>" if edge.optional else "-->" - lines.append(f"{edge.source} {arrow} {edge.target} : {edge.label}") + source = identifiers.get(edge.source, edge.source) + target = identifiers.get(edge.target, edge.target) + lines.append(f"{source} {arrow} {target} : {edge.label}") return lines def _style_declarations(self, diagram: MermaidDiagram) -> list[str]: lines = [ - f"style {node.name} fill:{node.color},stroke:#666,color:#000" + f"style {node.identifier} fill:{node.color},stroke:#666,color:#000" for node in sorted(diagram.nodes, key=lambda item: item.name) if node.color and node.style_override is None ] lines.extend( - f"style {node.name} {node.style_override}" + f"style {node.identifier} {node.style_override}" for node in sorted(diagram.nodes, key=lambda item: item.name) if node.style_override ) @@ -331,12 +357,17 @@ def _link_declarations(self, diagram: MermaidDiagram) -> list[str]: for node in sorted(diagram.nodes, key=lambda item: item.name): if node.href is None: continue - line = f'click {node.name} href "{node.href}"' + line = f'click {node.identifier} href "{node.href}"' if node.tooltip: line += f' "{node.tooltip}"' lines.append(line) return lines + @staticmethod + def _escape_label(value: str) -> str: + """Escape a type name used inside a Mermaid quoted class label.""" + return value.replace("\\", "\\\\").replace('"', '\\"') + @dataclass(frozen=True) class NeedDiagramBuilder: @@ -375,6 +406,7 @@ def build(self, focal_name: str | None = None) -> MermaidDiagram: def _node(need_type: NeedType) -> MermaidNode: return MermaidNode( name=need_type.name, + identifier=_mermaid_identifier(need_type.name), attributes=tuple(need_type.mandatory_options), optional_attributes=tuple(need_type.optional_options), color=need_type.color, diff --git a/src/extensions/score_metamodel/tests/test_rules_file_based.py b/src/extensions/score_metamodel/tests/test_rules_file_based.py index 4d47fe90d..00998d472 100644 --- a/src/extensions/score_metamodel/tests/test_rules_file_based.py +++ b/src/extensions/score_metamodel/tests/test_rules_file_based.py @@ -13,7 +13,8 @@ import re import shutil -from collections.abc import Callable +import sys +from collections.abc import Callable, Collection from dataclasses import dataclass, field from pathlib import Path from typing import Any, cast @@ -32,6 +33,7 @@ # Relative paths of all rst files in RST_DIR RST_FILES = [str(f.relative_to(RST_DIR)) for f in Path(RST_DIR).rglob("*.rst")] +_NEED_DIRECTIVE_PATTERN = re.compile(r"^\s*\.\.\s+([A-Za-z][\w-]*)::") @pytest.fixture @@ -94,17 +96,47 @@ class RstData: metadata: dict[str, list[str] | str] = field(default_factory=dict) -def count_need_objects(rst_file: Path) -> RstData: +def count_need_objects(rst_file: Path, need_directives: Collection[str]) -> RstData: rst_data = RstData(filename=str(rst_file.relative_to(RST_DIR))) + need_directive_names = set(need_directives) | {"needextend"} with open(rst_file) as f: for no, line in enumerate(f, start=1): - # Beginning of new need - # We filter for '::' as well so we ONLY get directives not comments - if line.startswith(".. ") and "::" in line: + # Nested Needs are indented below their containing Need. Parse the + # directive name so nested notes, code blocks, and other ordinary + # reStructuredText directives are not mistaken for Needs. + match = _NEED_DIRECTIVE_PATTERN.match(line) + if match and match.group(1) in need_directive_names: rst_data.found_objects.append(no) return rst_data +def test_count_need_objects_ignores_nested_non_need_directives( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Only configured Need directives count, including an explicit needextend.""" + monkeypatch.setattr(sys.modules[__name__], "RST_DIR", tmp_path) + rst_file = tmp_path / "nested.rst" + rst_file.write_text( + """.. stkh_req:: Parent + :id: stkh_req__parent + + .. note:: + This ordinary directive must not be counted. + + .. code-block:: text + + This ordinary directive must not be counted either. + +.. needextend:: stkh_req__parent +""", + encoding="utf-8", + ) + + rst_data = count_need_objects(rst_file, {"stkh_req"}) + + assert rst_data.found_objects == [1, 11] + + def filter_warnings_by_position( rst_data: RstData, line_nr: int, @@ -284,13 +316,16 @@ def test_rst_files( request: pytest.FixtureRequest, ) -> None: ### Build the given rst file with Sphinx and check expected/unexpected warnings. - rst_data = count_need_objects(RST_DIR / rst_file) - # Build the documentation app = sphinx_app_setup(RST_DIR / rst_file) monkeypatch.chdir(app.srcdir) # Sphinx resolves paths relative to the source dir app.build() + need_directives = { + str(need_type["directive"]) for need_type in app.config.needs_types + } + rst_data = count_need_objects(RST_DIR / rst_file, need_directives) + # Get & parse metadata needs needs_data = SphinxNeedsData(app.env) needs_view = needs_data.get_needs_view() diff --git a/src/extensions/score_sphinx_bundle/__init__.py b/src/extensions/score_sphinx_bundle/__init__.py index 523649be0..1628647fc 100644 --- a/src/extensions/score_sphinx_bundle/__init__.py +++ b/src/extensions/score_sphinx_bundle/__init__.py @@ -53,8 +53,8 @@ def setup(app: Sphinx) -> dict[str, object]: # Global settings # Note: the "sub-extensions" also set their own config values - # Same as current VS Code extension - config_setdefault(app.config, "mermaid_version", "11.6.0") + # Match the current GitHub and VS Code Mermaid renderers. + config_setdefault(app.config, "mermaid_version", "11.17.2") config_setdefault(app.config, "mermaid_d3_zoom", True) config_setdefault(app.config, "mermaid_include_elk", True) diff --git a/src/extensions/score_sphinx_needs_templates/BUILD b/src/extensions/score_sphinx_needs_templates/BUILD index a57da451c..21ebcfa31 100644 --- a/src/extensions/score_sphinx_needs_templates/BUILD +++ b/src/extensions/score_sphinx_needs_templates/BUILD @@ -12,6 +12,7 @@ # ******************************************************************************* load("@aspect_rules_py//py:defs.bzl", "py_library") load("@docs_as_code_hub_env//:requirements.bzl", "all_requirements") +load("//:score_pytest.bzl", "score_pytest") filegroup( name = "all_sources", @@ -28,3 +29,10 @@ py_library( "@score_docs_as_code//src/helper_lib", ], ) + +score_pytest( + name = "unit_tests", + srcs = ["tests/test_needs_templates.py"], + deps = [":score_sphinx_needs_templates"], + pytest_config = "//:pyproject.toml", +) diff --git a/src/extensions/score_sphinx_needs_templates/README.md b/src/extensions/score_sphinx_needs_templates/README.md index 56e540156..fa76b5a7a 100644 --- a/src/extensions/score_sphinx_needs_templates/README.md +++ b/src/extensions/score_sphinx_needs_templates/README.md @@ -24,6 +24,7 @@ The extension provides: * the shared `src/needs_templates` directory as the Sphinx-Needs template directory; * the `linked_needs(need_id, link_name)` helper for traversing Need links; +* the `needs_of_type(need_type)` helper for selecting Needs by type; * support for graph-driven `post_template`s that are rendered after parallel Need collection has been merged; * ordinary Sphinx page navigation for sections generated by those @@ -51,7 +52,9 @@ with Sphinx-Needs' `:post_template:` option. The extension then purges and rereads the affected report page once after parallel Need collection has been merged, so `linked_needs` can see the complete model. -The generated content should use normal reStructuredText sections instead of +Both outgoing links and backlink fields ending in ``_back`` can be traversed, +which is useful for generated testcase evidence and nested Need children. The +generated content should use normal reStructuredText sections instead of rubrics. Sphinx-Needs parses `post_template` output after the Need and with section matching enabled, so section IDs and the local page ToC are collected by Sphinx itself. Ordinary `:template:` use keeps its normal Sphinx-Needs diff --git a/src/extensions/score_sphinx_needs_templates/__init__.py b/src/extensions/score_sphinx_needs_templates/__init__.py index 3863046af..a7b33e859 100644 --- a/src/extensions/score_sphinx_needs_templates/__init__.py +++ b/src/extensions/score_sphinx_needs_templates/__init__.py @@ -19,11 +19,18 @@ from src.helper_lib import config_setdefault -_template_environment: BuildEnvironment | None = None +_build_environment: BuildEnvironment | None = None + # Post-templates containing this marker need a second read after parallel Need # collection has been merged. _RENDER_AFTER_NEEDS_COLLECTION_MARKER = "score: render-after-needs-collection" +# During that second read, ``env.clear_doc()`` temporarily removes all Needs +# belonging to the document from Sphinx's collection. Keep those temporarily +# removed Needs available so templates can resolve links while the document is +# reread. +_temporarily_removed_needs: dict[str, NeedItem] = {} + def _base_need_id(need_id: str) -> str: """Strip link conditions from an ID used to look up a merged Need.""" @@ -46,6 +53,24 @@ def _find_need(needs: dict[str, NeedItem], need_id: str) -> NeedItem | None: return None +def _get_available_needs() -> dict[str, NeedItem]: + """Return the current Needs, including temporarily removed Needs.""" + if _build_environment is None: + # The render helpers are registered before ``builder-inited`` captures + # the environment. There is no Need collection to query before then. + return {} + + needs = SphinxNeedsData(_build_environment).get_needs_mutable() + if _temporarily_removed_needs: + # ``env.clear_doc()`` removes the Needs belonging to the page being + # reread. Preserve the live collection and overlay the saved entries + # so templates can resolve links during that reread without mutating + # Sphinx's collection while it is being rebuilt. + needs = dict(needs) + needs.update(_temporarily_removed_needs) + return needs + + def _needs_template_folder() -> Path: """Locate the shared ``.need`` template directory for Sphinx-Needs.""" template_folder = Path(__file__).parents[2] / "needs_templates" @@ -60,8 +85,9 @@ class _LinkedNeeds: """Provide link traversal to Need templates as a pickleable callable. Calling the object with a Need ID and a link field returns the target - ``NeedItem`` objects in the order declared by the source Need. This lets a - template derive sections from the Need graph instead of embedding IDs. + ``NeedItem`` objects in the order declared by the source Need. Backlink + fields ending in ``_back`` are also supported. This lets a template derive + sections from the Need graph instead of embedding IDs. The object is deliberately a top-level class instance because Sphinx puts the render context into its parallel-reader configuration. A plain @@ -69,17 +95,66 @@ class _LinkedNeeds: is kept process-local and captured once Sphinx has created ``app.env``. """ + @staticmethod + def _resolve_backlinks( + needs: dict[str, NeedItem], + source: NeedItem | None, + need_id: str, + link_type: str, + ) -> list[NeedItem]: + """Merge indexed backlinks with links found in the live Need fields.""" + linked: list[NeedItem] = [] + linked_ids: set[str] = set() + if source is not None: + # Prefer Sphinx-Needs' backlink index when the source Need is + # present. Keep these results first, but do not assume that a + # non-empty index is complete: links injected later in the build + # may only be visible on the outgoing Need fields. + for link in source.get_backlinks(link_type, as_str=False): + target = _find_need(needs, link.to_link_string()) + if target is not None and target["id"] not in linked_ids: + linked.append(target) + linked_ids.add(target["id"]) + + # During a post-template reread, Sphinx-Needs may not have rebuilt + # backlink caches yet. The current Need may also be temporarily absent + # from the live environment while its document is reread. Derive the + # reverse relation from outgoing links in all cases and merge it with + # the indexed results above. This catches new links while preserving + # the index order and avoids duplicate Needs. + source_id = _base_need_id(need_id) + for candidate in needs.values(): + points_to_source = any( + _base_need_id(link.to_link_string()) == source_id + for link in candidate.get_links(link_type, as_str=False) + ) + if points_to_source and candidate["id"] not in linked_ids: + linked.append(candidate) + linked_ids.add(candidate["id"]) + return linked + def __call__(self, need_id: str, link_name: str) -> list[NeedItem]: - if _template_environment is None: + needs = _get_available_needs() + if not needs: return [] - needs = SphinxNeedsData(_template_environment).get_needs_mutable() source = _find_need(needs, need_id) - if source is None: - return [] + if link_name.endswith("_back"): + # A ``*_back`` name asks for the reverse of an ordinary outgoing + # link. Strip the suffix because Sphinx-Needs stores backlinks + # under the original link type. + link_type = link_name.removesuffix("_back") + return self._resolve_backlinks(needs, source, need_id, link_type) + else: + if source is None: + return [] + # For an ordinary link name, Sphinx-Needs already stores the + # outgoing links on the source Need. Resolve those links against + # the combined live-and-snapshot collection below. + links = source.get_links(link_name, as_str=False) linked: list[NeedItem] = [] - for link in source.get_links(link_name, as_str=False): + for link in links: target = _find_need(needs, link.to_link_string()) if target is not None: linked.append(target) @@ -89,7 +164,21 @@ def __call__(self, need_id: str, link_name: str) -> list[NeedItem]: _linked_needs_callable = _LinkedNeeds() -def _complex_post_template_names(app: Sphinx) -> set[str]: +class _NeedsOfType: + """Provide all Needs of a given type to graph-driven templates.""" + + def __call__(self, need_type: str) -> list[NeedItem]: + return [ + need + for need in _get_available_needs().values() + if need["type"] == need_type + ] + + +_needs_of_type_callable = _NeedsOfType() + + +def _post_templates_requiring_reread(app: Sphinx) -> set[str]: """Return post-template names opting into the post-merge rendering pass.""" template_folder = _needs_template_folder() return { @@ -102,9 +191,7 @@ def _complex_post_template_names(app: Sphinx) -> set[str]: } -def _rerender_pages_with_complex_post_templates( - app: Sphinx, env: BuildEnvironment -) -> list[str]: +def _reread_post_template_pages(app: Sphinx, env: BuildEnvironment) -> list[str]: """Re-read marked post-template pages after Need environments are merged. Post-templates are expanded while source documents are read. A parallel @@ -115,44 +202,55 @@ def _rerender_pages_with_complex_post_templates( if app.builder.name != "html": return [] - complex_post_templates = _complex_post_template_names(app) - if not complex_post_templates: + post_templates_requiring_reread = _post_templates_requiring_reread(app) + if not post_templates_requiring_reread: return [] needs_data = SphinxNeedsData(env) if needs_data.needs_is_post_processed: return [] - complex_post_template_docs: set[str] = set() + post_template_docs: set[str] = set() for need in needs_data.get_needs_mutable().values(): post_template = need.get("post_template") if ( not isinstance(post_template, str) - or post_template not in complex_post_templates + or post_template not in post_templates_requiring_reread ): continue docname = need["docname"] if isinstance(docname, str) and docname: - complex_post_template_docs.add(docname) - - pages_to_rerender = sorted(complex_post_template_docs) - for docname in pages_to_rerender: - app.emit("env-purge-doc", env, docname) - env.clear_doc(docname) - app.builder.read_doc(docname) + post_template_docs.add(docname) + + pages_to_reread = sorted(post_template_docs) + global _temporarily_removed_needs + _temporarily_removed_needs = { + need["id"]: need + for need in needs_data.get_needs_mutable().values() + if need.get("docname") in pages_to_reread + } + try: + for docname in pages_to_reread: + app.emit("env-purge-doc", env, docname) + env.clear_doc(docname) + app.builder.read_doc(docname) + finally: + # Always clear the temporary snapshot, whether rereading succeeds or + # fails. Any reread exception still propagates after this cleanup. + _temporarily_removed_needs = {} - return pages_to_rerender + return pages_to_reread -def _capture_template_environment(app: Sphinx) -> None: +def _capture_build_environment(app: Sphinx) -> None: """Give the link helper the environment in which it should resolve Needs. The helper is registered during ``setup()``, but Sphinx creates ``app.env`` only after extension setup has completed. ``builder-inited`` is the first lifecycle event at which the final build environment is available. """ - global _template_environment - _template_environment = app.env + global _build_environment + _build_environment = app.env def setup(app: Sphinx) -> dict[str, object]: @@ -163,8 +261,12 @@ def setup(app: Sphinx) -> dict[str, object]: app.config, "needs_template_folder", str(_needs_template_folder()) ) app.config.needs_render_context.setdefault("linked_needs", _linked_needs_callable) - app.connect("builder-inited", _capture_template_environment) - app.connect("env-updated", _rerender_pages_with_complex_post_templates) + app.config.needs_render_context.setdefault("needs_of_type", _needs_of_type_callable) + app.connect("builder-inited", _capture_build_environment) + # Run after the source-code linker has injected generated testcase Needs and + # their verification backlinks (priority 525), so report templates can + # include those testcases in their traceability tables. + app.connect("env-updated", _reread_post_template_pages, priority=600) return { "version": "1.0.0", diff --git a/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py b/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py new file mode 100644 index 000000000..0cc018d98 --- /dev/null +++ b/src/extensions/score_sphinx_needs_templates/tests/test_needs_templates.py @@ -0,0 +1,70 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Tests for graph traversal helpers used by Sphinx-Needs templates.""" + +from collections.abc import Iterable + +import pytest +import score_sphinx_needs_templates as templates + + +class FakeLink: + """Small test double for the Sphinx-Needs link objects.""" + + def __init__(self, target: str): + self.target = target + + def to_link_string(self) -> str: + return self.target + + +class FakeNeed(dict[str, str]): + """Need-shaped test double with independently controlled link indexes.""" + + def __init__( + self, + need_id: str, + *, + backlinks: Iterable[FakeLink] = (), + links: Iterable[FakeLink] = (), + ): + super().__init__(id=need_id, type="test") + self._backlinks = list(backlinks) + self._links = list(links) + + def get_backlinks(self, link_name: str, *, as_str: bool) -> list[FakeLink]: + del link_name, as_str + return self._backlinks + + def get_links(self, link_name: str, *, as_str: bool) -> list[FakeLink]: + del link_name, as_str + return self._links + + +def test_backlinks_merge_indexed_and_new_outgoing_links( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A stale non-empty backlink index must not hide links added later.""" + requirement = FakeNeed( + "REQ", + backlinks=[FakeLink("TC-old")], + ) + old_testcase = FakeNeed("TC-old", links=[FakeLink("REQ")]) + new_testcase = FakeNeed("TC-new", links=[FakeLink("REQ")]) + needs = {need["id"]: need for need in (requirement, old_testcase, new_testcase)} + monkeypatch.setattr(templates, "_get_available_needs", lambda: needs) + + linked_needs_class = vars(templates)["_LinkedNeeds"] + linked = linked_needs_class()("REQ", "fully_verifies_back") + + assert [need["id"] for need in linked] == ["TC-old", "TC-new"] diff --git a/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/docs/data_test/index.html b/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/docs/data_test/index.html index b551a3006..17090d47f 100644 --- a/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/docs/data_test/index.html +++ b/src/tests/docs_bzl/scenarios/data_files_runfiles/_expected/docs/data_test/index.html @@ -63,7 +63,7 @@ -