Include all subtests in HTML report with outcome styling

contrib/tests/README.md

@@ -152,16 +152,18 @@ rendering.  The relationship is a property of the environment definition
 
 On failure, the assertion reports mean and max FLIP error, and saves a
 magma heatmap (`*_diff.png`) next to the rendered image.  When
-generating an HTML report (`--html`), failed subtests include
-side-by-side thumbnails of the reference image, rendered image, and
-FLIP heatmap.
+generating an HTML report (`--html`), all subtests (passed, failed,
+and skipped) include side-by-side panels showing the reference image,
+rendered image, and FLIP heatmap, styled by outcome.  Heatmaps are
+generated whenever `--html` is active or when FLIP exceeds the
+threshold.
 
 The `--flip-threshold` CLI option (default `0.05`) controls the mean
 FLIP error threshold.
 
 **Important:** the reference environment must render first.  If
-reference images are missing, the comparison is skipped with a message.
-In practice this means running `test_render.py` (stdlib) before
+reference images are missing, the comparison fails.  In practice this
+means running `test_render.py` (stdlib) before
 `test_render_metashade.py`.
 
 ## CLI Options

contrib/tests/conftest.py

@@ -84,10 +84,16 @@ def cli_options(request, repo_root):
     output_opt = request.config.getoption("--output-dir")
     output_root = Path(output_opt) if output_opt else repo_root / "contrib"
 
+    try:
+        html_report = bool(request.config.getoption("htmlpath"))
+    except ValueError:
+        html_report = False
+
     return CliOptions(
         output_root=output_root,
         no_render=request.config.getoption("--no-render"),
         flip_threshold=request.config.getoption("--flip-threshold"),
+        html_report=html_report,
     )
 
 
@@ -268,158 +274,9 @@ def pytest_runtest_makereport(item, call):
 
 @pytest.hookimpl(tryfirst=True)
 def pytest_runtest_logreport(report):
-    """Append visual comparisons for failed subtests to the HTML report."""
-    if type(report).__name__ == "SubtestReport" and report.failed:
-        try:
-            from pytest_html import extras
-        except ImportError:
-            return
-
-        funcargs = _node_funcargs.get(report.nodeid)
-        if not funcargs:
-            return
-
-        from test_render import RenderEnvironment, RenderTestCase
-
-        case = funcargs.get("case")
-        if not isinstance(case, RenderTestCase):
-            return
-
-        env = None
-        for arg_val in funcargs.values():
-            if isinstance(arg_val, RenderEnvironment):
-                env = arg_val
-                break
-        if not env:
-            return
-
-        context = getattr(report, "context", None)
-        subtest_name = context.msg if context else None
-        if not subtest_name:
-            return
-
-        output_path = env.get_output_path(case)
-
-        if not output_path or not output_path.exists():
-            return
-
-        # Find the rendered file
-        import MaterialX as mx
-        valid_elem_name = mx.createValidName(subtest_name)
-        rendered_files = list(output_path.glob(f"{valid_elem_name}_*.png"))
-        rendered_files = [
-            f for f in rendered_files if not f.name.endswith("_diff.png")
-        ]
-
-        if not rendered_files:
-            return
-
-        rendered_file = rendered_files[0]
-
-        # Derive ref image from the environment's image_ref_env_subpath
-        image_ref_dir = env.get_image_ref_dir(output_path)
-        if image_ref_dir:
-            baseline_file = image_ref_dir / rendered_file.name
-            if not baseline_file.exists():
-                baseline_file = None
-        else:
-            baseline_file = None
-        heatmap_file = rendered_file.parent / f"{rendered_file.stem}_diff.png"
-
-        # Determine HTML report directory to compute relative paths for images
-        import os
-        try:
-            htmlpath_str = (
-                _pytest_config.getoption("htmlpath")
-                if _pytest_config
-                else None
-            )
-        except ValueError:
-            htmlpath_str = None
-        html_dir = (
-            Path(htmlpath_str).parent.resolve() if htmlpath_str else None
-        )
-
-        try:
-            is_self_contained = (
-                _pytest_config.getoption("self_contained_html")
-                if _pytest_config
-                else False
-            )
-        except ValueError:
-            is_self_contained = False
-
-        def get_image_src(path: Path) -> str:
-            if not path or not path.exists():
-                return ""
-            if is_self_contained:
-                import base64
-                try:
-                    with open(path, "rb") as f:
-                        encoded = base64.b64encode(f.read()).decode("utf-8")
-                        return f"data:image/png;base64,{encoded}"
-                except Exception:
-                    pass
-            elif html_dir:
-                try:
-                    return os.path.relpath(
-                        path.resolve(), html_dir,
-                    ).replace("\\", "/")
-                except ValueError:
-                    return path.resolve().as_uri()
-            return path.resolve().as_uri()
-
-        rendered_src = get_image_src(rendered_file)
-        baseline_src = get_image_src(baseline_file)
-        heatmap_src = get_image_src(heatmap_file)
-
-        if not rendered_src:
-            return
-
-        if baseline_src:
-            baseline_img_tag = (
-                f'<img src="{baseline_src}" style="max-width: 100%; '
-                f'height: auto; border: 1px solid #ccc; border-radius: 4px;" />'
-            )
-        else:
-            baseline_img_tag = (
-                '<div style="padding: 50px 10px; background: #eee; '
-                'border: 1px dashed #ccc; border-radius: 4px; color: #666; '
-                'font-size: 12px;">Baseline image missing</div>'
-            )
-
-        if heatmap_src:
-            heatmap_img_tag = (
-                f'<img src="{heatmap_src}" style="max-width: 100%; '
-                f'height: auto; border: 1px solid #ccc; border-radius: 4px;" />'
-            )
-        else:
-            heatmap_img_tag = (
-                '<div style="padding: 50px 10px; background: #eee; '
-                'border: 1px dashed #ccc; border-radius: 4px; color: #666; '
-                'font-size: 12px;">No heatmap (comparison passed or skipped)</div>'
-            )
-
-        html_content = f"""
-        <div style="margin-top: 15px; padding: 15px; border: 1px solid #e74c3c; border-radius: 6px; background: #fdf2f2; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;">
-            <h4 style="margin: 0 0 12px 0; color: #c0392b; font-size: 14px;">Visual Comparison for {subtest_name}</h4>
-            <div style="display: flex; gap: 15px; flex-wrap: wrap;">
-                <div style="flex: 1; min-width: 220px; text-align: center;">
-                    <div style="font-weight: bold; margin-bottom: 6px; font-size: 12px; color: #555;">Baseline (Reference)</div>
-                    {baseline_img_tag}
-                </div>
-                <div style="flex: 1; min-width: 220px; text-align: center;">
-                    <div style="font-weight: bold; margin-bottom: 6px; font-size: 12px; color: #555;">Rendered (Current)</div>
-                    <img src="{rendered_src}" style="max-width: 100%; height: auto; border: 1px solid #ccc; border-radius: 4px;" />
-                </div>
-                <div style="flex: 1; min-width: 220px; text-align: center;">
-                    <div style="font-weight: bold; margin-bottom: 6px; font-size: 12px; color: #555;">FLIP Heatmap</div>
-                    {heatmap_img_tag}
-                </div>
-            </div>
-        </div>
-        """
-        _subtest_html_extras[report.nodeid].append(extras.html(html_content))
+    """Append visual comparisons for every subtest to the HTML report."""
+    if type(report).__name__ == "SubtestReport":
+        _append_visual_extras(report)
 
     elif type(report).__name__ == "TestReport" and report.when == "teardown":
         if report.nodeid in _subtest_html_extras:
@@ -431,3 +288,196 @@ def get_image_src(path: Path) -> str:
             extra.extend(_subtest_html_extras[report.nodeid])
             report.extras = extra
             del _subtest_html_extras[report.nodeid]
+
+
+_OUTCOME_STYLES = {
+    "failed": {
+        "border_color": "#e74c3c",
+        "bg_color": "#fdf2f2",
+        "heading_color": "#c0392b",
+        "label": "FAILED",
+    },
+    "passed": {
+        "border_color": "#27ae60",
+        "bg_color": "#f2fdf6",
+        "heading_color": "#1e8449",
+        "label": "PASSED",
+    },
+    "skipped": {
+        "border_color": "#95a5a6",
+        "bg_color": "#f9f9f9",
+        "heading_color": "#7f8c8d",
+        "label": "SKIPPED",
+    },
+}
+
+
+def _append_visual_extras(report):
+    """Build an HTML panel with rendered images for a single subtest."""
+    try:
+        from pytest_html import extras
+    except ImportError:
+        return
+
+    funcargs = _node_funcargs.get(report.nodeid)
+    if not funcargs:
+        return
+
+    from test_render import RenderEnvironment, RenderTestCase
+
+    case = funcargs.get("case")
+    if not isinstance(case, RenderTestCase):
+        return
+
+    env = None
+    for arg_val in funcargs.values():
+        if isinstance(arg_val, RenderEnvironment):
+            env = arg_val
+            break
+    if not env:
+        return
+
+    context = getattr(report, "context", None)
+    subtest_name = context.msg if context else None
+    if not subtest_name:
+        return
+
+    output_path = env.get_output_path(case)
+    if not output_path or not output_path.exists():
+        return
+
+    import MaterialX as mx
+    valid_elem_name = mx.createValidName(subtest_name)
+    rendered_files = list(output_path.glob(f"{valid_elem_name}_*.png"))
+    rendered_files = [
+        f for f in rendered_files if not f.name.endswith("_diff.png")
+    ]
+    if not rendered_files:
+        return
+
+    rendered_file = rendered_files[0]
+
+    image_ref_dir = env.get_image_ref_dir(output_path)
+    baseline_file = None
+    if image_ref_dir:
+        candidate = image_ref_dir / rendered_file.name
+        if candidate.exists():
+            baseline_file = candidate
+    heatmap_file = rendered_file.parent / f"{rendered_file.stem}_diff.png"
+
+    import os
+    try:
+        htmlpath_str = (
+            _pytest_config.getoption("htmlpath")
+            if _pytest_config
+            else None
+        )
+    except ValueError:
+        htmlpath_str = None
+    html_dir = (
+        Path(htmlpath_str).parent.resolve() if htmlpath_str else None
+    )
+
+    try:
+        is_self_contained = (
+            _pytest_config.getoption("self_contained_html")
+            if _pytest_config
+            else False
+        )
+    except ValueError:
+        is_self_contained = False
+
+    def get_image_src(path: Path) -> str:
+        if not path or not path.exists():
+            return ""
+        if is_self_contained:
+            import base64
+            try:
+                with open(path, "rb") as f:
+                    encoded = base64.b64encode(f.read()).decode("utf-8")
+                    return f"data:image/png;base64,{encoded}"
+            except Exception:
+                pass
+        elif html_dir:
+            try:
+                return os.path.relpath(
+                    path.resolve(), html_dir,
+                ).replace("\\", "/")
+            except ValueError:
+                return path.resolve().as_uri()
+        return path.resolve().as_uri()
+
+    rendered_src = get_image_src(rendered_file)
+    if not rendered_src:
+        return
+
+    baseline_src = get_image_src(baseline_file)
+    heatmap_src = get_image_src(heatmap_file)
+
+    if report.failed:
+        outcome = "failed"
+    elif report.skipped:
+        outcome = "skipped"
+    else:
+        outcome = "passed"
+    style = _OUTCOME_STYLES[outcome]
+
+    _IMG_STYLE = (
+        "max-width:100%; height:auto; border:1px solid #ccc; "
+        "border-radius:4px; background:black;"
+    )
+    _PLACEHOLDER_STYLE = (
+        "padding:50px 10px; background:#eee; border:1px dashed #ccc; "
+        "border-radius:4px; color:#666; font-size:12px;"
+    )
+
+    panels = []
+
+    if baseline_src:
+        panels.append(
+            f'<div style="flex:1; min-width:200px; text-align:center;">'
+            f'<div style="font-weight:bold; margin-bottom:6px; font-size:12px; color:#555;">Baseline</div>'
+            f'<img src="{baseline_src}" style="{_IMG_STYLE}" />'
+            f'</div>'
+        )
+    elif image_ref_dir:
+        panels.append(
+            f'<div style="flex:1; min-width:200px; text-align:center;">'
+            f'<div style="font-weight:bold; margin-bottom:6px; font-size:12px; color:#555;">Baseline</div>'
+            f'<div style="{_PLACEHOLDER_STYLE}">Baseline image missing</div>'
+            f'</div>'
+        )
+
+    panels.append(
+        f'<div style="flex:1; min-width:200px; text-align:center;">'
+        f'<div style="font-weight:bold; margin-bottom:6px; font-size:12px; color:#555;">Rendered</div>'
+        f'<img src="{rendered_src}" style="{_IMG_STYLE}" />'
+        f'</div>'
+    )
+
+    if heatmap_src:
+        panels.append(
+            f'<div style="flex:1; min-width:200px; text-align:center;">'
+            f'<div style="font-weight:bold; margin-bottom:6px; font-size:12px; color:#555;">FLIP Heatmap</div>'
+            f'<img src="{heatmap_src}" style="{_IMG_STYLE}" />'
+            f'</div>'
+        )
+
+    html_content = (
+        f'<div style="margin-top:15px; padding:15px; '
+        f'border:2px solid {style["border_color"]}; border-radius:6px; '
+        f'background:{style["bg_color"]}; '
+        f'font-family:-apple-system,BlinkMacSystemFont,\'Segoe UI\',Roboto,sans-serif; '
+        f'print-color-adjust:exact; -webkit-print-color-adjust:exact;">'
+        f'<h4 style="margin:0 0 12px 0; color:{style["heading_color"]}; font-size:14px;">'
+        f'{subtest_name}'
+        f'<span style="margin-left:10px; padding:2px 8px; border-radius:3px; '
+        f'font-size:11px; font-weight:bold; color:white; '
+        f'background:{style["border_color"]};">{style["label"]}</span>'
+        f'</h4>'
+        f'<div style="display:flex; gap:15px; flex-wrap:wrap;">'
+        f'{"".join(panels)}'
+        f'</div>'
+        f'</div>'
+    )
+    _subtest_html_extras[report.nodeid].append(extras.html(html_content))