From 0adf9850b8a912154b322893884a66000edabb44 Mon Sep 17 00:00:00 2001 From: Pavlo Penenko Date: Mon, 24 Aug 2026 15:09:40 -0400 Subject: [PATCH 1/4] Include all subtests in HTML report with outcome styling Show visual comparison panels (baseline, rendered, FLIP heatmap) for every subtest outcome, not just failures. Panels use green borders for passes, red for failures, and grey for skips, with a colored badge label. Image cells use black backgrounds for transparent renders and print-color-adjust for PDF fidelity. Generate FLIP heatmaps conditionally: only when --html is active (for the report) or when FLIP exceeds the threshold (for debugging). Expand all report rows by default via render_collapsed. --- contrib/tests/conftest.py | 354 ++++++++++++++++++++--------------- contrib/tests/pyproject.toml | 1 + contrib/tests/test_render.py | 64 ++++--- 3 files changed, 240 insertions(+), 179 deletions(-) diff --git a/contrib/tests/conftest.py b/contrib/tests/conftest.py index 21d60806ce..242f0c6ab7 100644 --- a/contrib/tests/conftest.py +++ b/contrib/tests/conftest.py @@ -84,10 +84,16 @@ def cli_options(request, repo_root): output_opt = request.config.getoption("--output-dir") output_root = Path(output_opt) if output_opt else repo_root / "contrib" + try: + html_report = bool(request.config.getoption("htmlpath")) + except ValueError: + html_report = False + return CliOptions( output_root=output_root, no_render=request.config.getoption("--no-render"), flip_threshold=request.config.getoption("--flip-threshold"), + html_report=html_report, ) @@ -268,158 +274,9 @@ def pytest_runtest_makereport(item, call): @pytest.hookimpl(tryfirst=True) def pytest_runtest_logreport(report): - """Append visual comparisons for failed subtests to the HTML report.""" - if type(report).__name__ == "SubtestReport" and report.failed: - try: - from pytest_html import extras - except ImportError: - return - - funcargs = _node_funcargs.get(report.nodeid) - if not funcargs: - return - - from test_render import RenderEnvironment, RenderTestCase - - case = funcargs.get("case") - if not isinstance(case, RenderTestCase): - return - - env = None - for arg_val in funcargs.values(): - if isinstance(arg_val, RenderEnvironment): - env = arg_val - break - if not env: - return - - context = getattr(report, "context", None) - subtest_name = context.msg if context else None - if not subtest_name: - return - - output_path = env.get_output_path(case) - - if not output_path or not output_path.exists(): - return - - # Find the rendered file - import MaterialX as mx - valid_elem_name = mx.createValidName(subtest_name) - rendered_files = list(output_path.glob(f"{valid_elem_name}_*.png")) - rendered_files = [ - f for f in rendered_files if not f.name.endswith("_diff.png") - ] - - if not rendered_files: - return - - rendered_file = rendered_files[0] - - # Derive ref image from the environment's image_ref_env_subpath - image_ref_dir = env.get_image_ref_dir(output_path) - if image_ref_dir: - baseline_file = image_ref_dir / rendered_file.name - if not baseline_file.exists(): - baseline_file = None - else: - baseline_file = None - heatmap_file = rendered_file.parent / f"{rendered_file.stem}_diff.png" - - # Determine HTML report directory to compute relative paths for images - import os - try: - htmlpath_str = ( - _pytest_config.getoption("htmlpath") - if _pytest_config - else None - ) - except ValueError: - htmlpath_str = None - html_dir = ( - Path(htmlpath_str).parent.resolve() if htmlpath_str else None - ) - - try: - is_self_contained = ( - _pytest_config.getoption("self_contained_html") - if _pytest_config - else False - ) - except ValueError: - is_self_contained = False - - def get_image_src(path: Path) -> str: - if not path or not path.exists(): - return "" - if is_self_contained: - import base64 - try: - with open(path, "rb") as f: - encoded = base64.b64encode(f.read()).decode("utf-8") - return f"data:image/png;base64,{encoded}" - except Exception: - pass - elif html_dir: - try: - return os.path.relpath( - path.resolve(), html_dir, - ).replace("\\", "/") - except ValueError: - return path.resolve().as_uri() - return path.resolve().as_uri() - - rendered_src = get_image_src(rendered_file) - baseline_src = get_image_src(baseline_file) - heatmap_src = get_image_src(heatmap_file) - - if not rendered_src: - return - - if baseline_src: - baseline_img_tag = ( - f'' - ) - else: - baseline_img_tag = ( - '
Baseline image missing
' - ) - - if heatmap_src: - heatmap_img_tag = ( - f'' - ) - else: - heatmap_img_tag = ( - '
No heatmap (comparison passed or skipped)
' - ) - - html_content = f""" -
-

Visual Comparison for {subtest_name}

-
-
-
Baseline (Reference)
- {baseline_img_tag} -
-
-
Rendered (Current)
- -
-
-
FLIP Heatmap
- {heatmap_img_tag} -
-
-
- """ - _subtest_html_extras[report.nodeid].append(extras.html(html_content)) + """Append visual comparisons for every subtest to the HTML report.""" + if type(report).__name__ == "SubtestReport": + _append_visual_extras(report) elif type(report).__name__ == "TestReport" and report.when == "teardown": if report.nodeid in _subtest_html_extras: @@ -431,3 +288,196 @@ def get_image_src(path: Path) -> str: extra.extend(_subtest_html_extras[report.nodeid]) report.extras = extra del _subtest_html_extras[report.nodeid] + + +_OUTCOME_STYLES = { + "failed": { + "border_color": "#e74c3c", + "bg_color": "#fdf2f2", + "heading_color": "#c0392b", + "label": "FAILED", + }, + "passed": { + "border_color": "#27ae60", + "bg_color": "#f2fdf6", + "heading_color": "#1e8449", + "label": "PASSED", + }, + "skipped": { + "border_color": "#95a5a6", + "bg_color": "#f9f9f9", + "heading_color": "#7f8c8d", + "label": "SKIPPED", + }, +} + + +def _append_visual_extras(report): + """Build an HTML panel with rendered images for a single subtest.""" + try: + from pytest_html import extras + except ImportError: + return + + funcargs = _node_funcargs.get(report.nodeid) + if not funcargs: + return + + from test_render import RenderEnvironment, RenderTestCase + + case = funcargs.get("case") + if not isinstance(case, RenderTestCase): + return + + env = None + for arg_val in funcargs.values(): + if isinstance(arg_val, RenderEnvironment): + env = arg_val + break + if not env: + return + + context = getattr(report, "context", None) + subtest_name = context.msg if context else None + if not subtest_name: + return + + output_path = env.get_output_path(case) + if not output_path or not output_path.exists(): + return + + import MaterialX as mx + valid_elem_name = mx.createValidName(subtest_name) + rendered_files = list(output_path.glob(f"{valid_elem_name}_*.png")) + rendered_files = [ + f for f in rendered_files if not f.name.endswith("_diff.png") + ] + if not rendered_files: + return + + rendered_file = rendered_files[0] + + image_ref_dir = env.get_image_ref_dir(output_path) + baseline_file = None + if image_ref_dir: + candidate = image_ref_dir / rendered_file.name + if candidate.exists(): + baseline_file = candidate + heatmap_file = rendered_file.parent / f"{rendered_file.stem}_diff.png" + + import os + try: + htmlpath_str = ( + _pytest_config.getoption("htmlpath") + if _pytest_config + else None + ) + except ValueError: + htmlpath_str = None + html_dir = ( + Path(htmlpath_str).parent.resolve() if htmlpath_str else None + ) + + try: + is_self_contained = ( + _pytest_config.getoption("self_contained_html") + if _pytest_config + else False + ) + except ValueError: + is_self_contained = False + + def get_image_src(path: Path) -> str: + if not path or not path.exists(): + return "" + if is_self_contained: + import base64 + try: + with open(path, "rb") as f: + encoded = base64.b64encode(f.read()).decode("utf-8") + return f"data:image/png;base64,{encoded}" + except Exception: + pass + elif html_dir: + try: + return os.path.relpath( + path.resolve(), html_dir, + ).replace("\\", "/") + except ValueError: + return path.resolve().as_uri() + return path.resolve().as_uri() + + rendered_src = get_image_src(rendered_file) + if not rendered_src: + return + + baseline_src = get_image_src(baseline_file) + heatmap_src = get_image_src(heatmap_file) + + if report.failed: + outcome = "failed" + elif report.skipped: + outcome = "skipped" + else: + outcome = "passed" + style = _OUTCOME_STYLES[outcome] + + _IMG_STYLE = ( + "max-width:100%; height:auto; border:1px solid #ccc; " + "border-radius:4px; background:black;" + ) + _PLACEHOLDER_STYLE = ( + "padding:50px 10px; background:#eee; border:1px dashed #ccc; " + "border-radius:4px; color:#666; font-size:12px;" + ) + + panels = [] + + if baseline_src: + panels.append( + f'
' + f'
Baseline
' + f'' + f'
' + ) + elif image_ref_dir: + panels.append( + f'
' + f'
Baseline
' + f'
Baseline image missing
' + f'
' + ) + + panels.append( + f'
' + f'
Rendered
' + f'' + f'
' + ) + + if heatmap_src: + panels.append( + f'
' + f'
FLIP Heatmap
' + f'' + f'
' + ) + + html_content = ( + f'
' + f'

' + f'{subtest_name}' + f'{style["label"]}' + f'

' + f'
' + f'{"".join(panels)}' + f'
' + f'
' + ) + _subtest_html_extras[report.nodeid].append(extras.html(html_content)) diff --git a/contrib/tests/pyproject.toml b/contrib/tests/pyproject.toml index 502046ce9a..323671f188 100644 --- a/contrib/tests/pyproject.toml +++ b/contrib/tests/pyproject.toml @@ -16,6 +16,7 @@ test = [ ] [tool.pytest.ini_options] +render_collapsed = "" testpaths = ["."] python_files = ["test_*.py"] python_classes = ["Test*"] diff --git a/contrib/tests/test_render.py b/contrib/tests/test_render.py index 707a061e0c..eb7220cf1a 100644 --- a/contrib/tests/test_render.py +++ b/contrib/tests/test_render.py @@ -64,6 +64,7 @@ class CliOptions: output_root: Path no_render: bool = False flip_threshold: float = 0.05 + html_report: bool = False # --------------------------------------------------------------------------- @@ -483,13 +484,17 @@ def _check_shader_baselines(result, baseline_dir: Path): # Image comparison # --------------------------------------------------------------------------- -def _compare_render(result, image_ref_dir: Path, threshold: float): +def _compare_render( + result, image_ref_dir: Path, threshold: float, + *, save_heatmap: bool = False, +): """Compare a rendered image against the reference environment's render. - Uses NVIDIA FLIP to compute a perceptual difference metric and - saves a magma heatmap (``*_diff.png``) next to the rendered image. - Asserts that the mean FLIP error is within *threshold*. Skips - gracefully when reference images are missing. + Uses NVIDIA FLIP to compute a perceptual difference metric. + Saves a magma heatmap (``*_diff.png``) next to the rendered image + when *save_heatmap* is ``True`` (for the HTML report) or when the + mean FLIP error exceeds *threshold*. Skips gracefully when + reference images are missing. Requires ``flip_evaluator`` (``pip install flip-evaluator``). """ @@ -513,30 +518,34 @@ def _compare_render(result, image_ref_dir: Path, threshold: float): ) mean_flip = float(mean_flip) - if mean_flip <= threshold: - return + failed = mean_flip > threshold - max_flip = float(np.array(flip_map).max()) + if save_heatmap or failed: + from PIL import Image - heatmap_path = result.output_path.parent / f"{result.output_path.stem}_diff.png" - heatmap_img, _, _ = flip.evaluate( - str(ref_image), str(result.output_path), - "LDR", inputsRGB=True, applyMagma=True, - computeMeanError=False, parameters={"ppd": 70.0}, - ) - from PIL import Image - heatmap_arr = np.array(heatmap_img) - if heatmap_arr.max() <= 1.0: - heatmap_arr = (heatmap_arr * 255).astype(np.uint8) - Image.fromarray(heatmap_arr).save(heatmap_path) - - assert False, ( - f"FLIP mean {mean_flip:.6f} exceeds threshold {threshold:.6f} " - f"(max {max_flip:.6f}) for {result.output_path.name}\n" - f" ref: {ref_image}\n" - f" rendered: {result.output_path}\n" - f" heatmap: {heatmap_path}" - ) + heatmap_path = ( + result.output_path.parent + / f"{result.output_path.stem}_diff.png" + ) + heatmap_img, _, _ = flip.evaluate( + str(ref_image), str(result.output_path), + "LDR", inputsRGB=True, applyMagma=True, + computeMeanError=False, parameters={"ppd": 70.0}, + ) + heatmap_arr = np.array(heatmap_img) + if heatmap_arr.max() <= 1.0: + heatmap_arr = (heatmap_arr * 255).astype(np.uint8) + Image.fromarray(heatmap_arr).save(heatmap_path) + + if failed: + max_flip = float(np.array(flip_map).max()) + assert False, ( + f"FLIP mean {mean_flip:.6f} exceeds threshold {threshold:.6f} " + f"(max {max_flip:.6f}) for {result.output_path.name}\n" + f" ref: {ref_image}\n" + f" rendered: {result.output_path}\n" + f" heatmap: {heatmap_path}" + ) # --------------------------------------------------------------------------- @@ -612,6 +621,7 @@ def _render_elements( _compare_render( result, image_ref_dir, env.cli_options.flip_threshold, + save_heatmap=env.cli_options.html_report, ) From 449c3189598e888d8aa87f7a0576a7b365a47dfe Mon Sep 17 00:00:00 2001 From: Pavlo Penenko Date: Mon, 24 Aug 2026 17:09:02 -0400 Subject: [PATCH 2/4] Fix NaN handling in FLIP comparison and update README Preserve the original semantics where NaN FLIP values fail the comparison instead of silently passing. Update README to document all-outcome visual panels and conditional heatmap generation. --- contrib/tests/README.md | 8 +++++--- contrib/tests/test_render.py | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/contrib/tests/README.md b/contrib/tests/README.md index 9fff0d4078..fc37cf038f 100644 --- a/contrib/tests/README.md +++ b/contrib/tests/README.md @@ -152,9 +152,11 @@ rendering. The relationship is a property of the environment definition On failure, the assertion reports mean and max FLIP error, and saves a magma heatmap (`*_diff.png`) next to the rendered image. When -generating an HTML report (`--html`), failed subtests include -side-by-side thumbnails of the reference image, rendered image, and -FLIP heatmap. +generating an HTML report (`--html`), all subtests (passed, failed, +and skipped) include side-by-side panels showing the reference image, +rendered image, and FLIP heatmap, styled by outcome. Heatmaps are +generated whenever `--html` is active or when FLIP exceeds the +threshold. The `--flip-threshold` CLI option (default `0.05`) controls the mean FLIP error threshold. diff --git a/contrib/tests/test_render.py b/contrib/tests/test_render.py index eb7220cf1a..a43cbdd453 100644 --- a/contrib/tests/test_render.py +++ b/contrib/tests/test_render.py @@ -518,7 +518,9 @@ def _compare_render( ) mean_flip = float(mean_flip) - failed = mean_flip > threshold + # `not <=` instead of `>` so that NaN comparisons fail rather than + # silently pass (NaN > x is False, but NaN <= x is also False). + failed = not (mean_flip <= threshold) if save_heatmap or failed: from PIL import Image From 7c62627d912dd9c9333d16f4a7fe15abb26ebb6c Mon Sep 17 00:00:00 2001 From: Pavlo Penenko Date: Tue, 25 Aug 2026 17:50:00 -0400 Subject: [PATCH 3/4] Fail instead of skip when FLIP reference render is missing Missing baselines should surface as test failures, not silent skips that hide the problem in CI and reports. Signed-off-by: Pavlo Penenko --- contrib/tests/test_render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/tests/test_render.py b/contrib/tests/test_render.py index a43cbdd453..0064ea69fe 100644 --- a/contrib/tests/test_render.py +++ b/contrib/tests/test_render.py @@ -503,7 +503,7 @@ def _compare_render( ref_image = image_ref_dir / result.output_path.name if not ref_image.exists(): - pytest.skip( + pytest.fail( f"Reference render not found: {ref_image}\n" f"Run the reference environment first." ) From 3701656fbf64f58e5bf9be83f32220a023273922 Mon Sep 17 00:00:00 2001 From: Pavlo Penenko Date: Wed, 26 Aug 2026 13:34:39 -0400 Subject: [PATCH 4/4] Update docs to reflect fail-on-missing-baseline behavior The docstring for _compare_render and the README still said missing references are skipped; they now fail. Signed-off-by: Pavlo Penenko --- contrib/tests/README.md | 4 ++-- contrib/tests/test_render.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/tests/README.md b/contrib/tests/README.md index fc37cf038f..69738e5f49 100644 --- a/contrib/tests/README.md +++ b/contrib/tests/README.md @@ -162,8 +162,8 @@ The `--flip-threshold` CLI option (default `0.05`) controls the mean FLIP error threshold. **Important:** the reference environment must render first. If -reference images are missing, the comparison is skipped with a message. -In practice this means running `test_render.py` (stdlib) before +reference images are missing, the comparison fails. In practice this +means running `test_render.py` (stdlib) before `test_render_metashade.py`. ## CLI Options diff --git a/contrib/tests/test_render.py b/contrib/tests/test_render.py index 0064ea69fe..56f51dcd34 100644 --- a/contrib/tests/test_render.py +++ b/contrib/tests/test_render.py @@ -493,8 +493,8 @@ def _compare_render( Uses NVIDIA FLIP to compute a perceptual difference metric. Saves a magma heatmap (``*_diff.png``) next to the rendered image when *save_heatmap* is ``True`` (for the HTML report) or when the - mean FLIP error exceeds *threshold*. Skips gracefully when - reference images are missing. + mean FLIP error exceeds *threshold*. Fails when reference images + are missing. Requires ``flip_evaluator`` (``pip install flip-evaluator``). """