diff --git a/contrib/tests/README.md b/contrib/tests/README.md index 9fff0d4078..69738e5f49 100644 --- a/contrib/tests/README.md +++ b/contrib/tests/README.md @@ -152,16 +152,18 @@ rendering. The relationship is a property of the environment definition On failure, the assertion reports mean and max FLIP error, and saves a magma heatmap (`*_diff.png`) next to the rendered image. When -generating an HTML report (`--html`), failed subtests include -side-by-side thumbnails of the reference image, rendered image, and -FLIP heatmap. +generating an HTML report (`--html`), all subtests (passed, failed, +and skipped) include side-by-side panels showing the reference image, +rendered image, and FLIP heatmap, styled by outcome. Heatmaps are +generated whenever `--html` is active or when FLIP exceeds the +threshold. The `--flip-threshold` CLI option (default `0.05`) controls the mean FLIP error threshold. **Important:** the reference environment must render first. If -reference images are missing, the comparison is skipped with a message. -In practice this means running `test_render.py` (stdlib) before +reference images are missing, the comparison fails. In practice this +means running `test_render.py` (stdlib) before `test_render_metashade.py`. ## CLI Options diff --git a/contrib/tests/conftest.py b/contrib/tests/conftest.py index 21d60806ce..242f0c6ab7 100644 --- a/contrib/tests/conftest.py +++ b/contrib/tests/conftest.py @@ -84,10 +84,16 @@ def cli_options(request, repo_root): output_opt = request.config.getoption("--output-dir") output_root = Path(output_opt) if output_opt else repo_root / "contrib" + try: + html_report = bool(request.config.getoption("htmlpath")) + except ValueError: + html_report = False + return CliOptions( output_root=output_root, no_render=request.config.getoption("--no-render"), flip_threshold=request.config.getoption("--flip-threshold"), + html_report=html_report, ) @@ -268,158 +274,9 @@ def pytest_runtest_makereport(item, call): @pytest.hookimpl(tryfirst=True) def pytest_runtest_logreport(report): - """Append visual comparisons for failed subtests to the HTML report.""" - if type(report).__name__ == "SubtestReport" and report.failed: - try: - from pytest_html import extras - except ImportError: - return - - funcargs = _node_funcargs.get(report.nodeid) - if not funcargs: - return - - from test_render import RenderEnvironment, RenderTestCase - - case = funcargs.get("case") - if not isinstance(case, RenderTestCase): - return - - env = None - for arg_val in funcargs.values(): - if isinstance(arg_val, RenderEnvironment): - env = arg_val - break - if not env: - return - - context = getattr(report, "context", None) - subtest_name = context.msg if context else None - if not subtest_name: - return - - output_path = env.get_output_path(case) - - if not output_path or not output_path.exists(): - return - - # Find the rendered file - import MaterialX as mx - valid_elem_name = mx.createValidName(subtest_name) - rendered_files = list(output_path.glob(f"{valid_elem_name}_*.png")) - rendered_files = [ - f for f in rendered_files if not f.name.endswith("_diff.png") - ] - - if not rendered_files: - return - - rendered_file = rendered_files[0] - - # Derive ref image from the environment's image_ref_env_subpath - image_ref_dir = env.get_image_ref_dir(output_path) - if image_ref_dir: - baseline_file = image_ref_dir / rendered_file.name - if not baseline_file.exists(): - baseline_file = None - else: - baseline_file = None - heatmap_file = rendered_file.parent / f"{rendered_file.stem}_diff.png" - - # Determine HTML report directory to compute relative paths for images - import os - try: - htmlpath_str = ( - _pytest_config.getoption("htmlpath") - if _pytest_config - else None - ) - except ValueError: - htmlpath_str = None - html_dir = ( - Path(htmlpath_str).parent.resolve() if htmlpath_str else None - ) - - try: - is_self_contained = ( - _pytest_config.getoption("self_contained_html") - if _pytest_config - else False - ) - except ValueError: - is_self_contained = False - - def get_image_src(path: Path) -> str: - if not path or not path.exists(): - return "" - if is_self_contained: - import base64 - try: - with open(path, "rb") as f: - encoded = base64.b64encode(f.read()).decode("utf-8") - return f"data:image/png;base64,{encoded}" - except Exception: - pass - elif html_dir: - try: - return os.path.relpath( - path.resolve(), html_dir, - ).replace("\\", "/") - except ValueError: - return path.resolve().as_uri() - return path.resolve().as_uri() - - rendered_src = get_image_src(rendered_file) - baseline_src = get_image_src(baseline_file) - heatmap_src = get_image_src(heatmap_file) - - if not rendered_src: - return - - if baseline_src: - baseline_img_tag = ( - f'' - ) - else: - baseline_img_tag = ( - '
Baseline image missing
' - ) - - if heatmap_src: - heatmap_img_tag = ( - f'' - ) - else: - heatmap_img_tag = ( - '
No heatmap (comparison passed or skipped)
' - ) - - html_content = f""" -
-

Visual Comparison for {subtest_name}

-
-
-
Baseline (Reference)
- {baseline_img_tag} -
-
-
Rendered (Current)
- -
-
-
FLIP Heatmap
- {heatmap_img_tag} -
-
-
- """ - _subtest_html_extras[report.nodeid].append(extras.html(html_content)) + """Append visual comparisons for every subtest to the HTML report.""" + if type(report).__name__ == "SubtestReport": + _append_visual_extras(report) elif type(report).__name__ == "TestReport" and report.when == "teardown": if report.nodeid in _subtest_html_extras: @@ -431,3 +288,196 @@ def get_image_src(path: Path) -> str: extra.extend(_subtest_html_extras[report.nodeid]) report.extras = extra del _subtest_html_extras[report.nodeid] + + +_OUTCOME_STYLES = { + "failed": { + "border_color": "#e74c3c", + "bg_color": "#fdf2f2", + "heading_color": "#c0392b", + "label": "FAILED", + }, + "passed": { + "border_color": "#27ae60", + "bg_color": "#f2fdf6", + "heading_color": "#1e8449", + "label": "PASSED", + }, + "skipped": { + "border_color": "#95a5a6", + "bg_color": "#f9f9f9", + "heading_color": "#7f8c8d", + "label": "SKIPPED", + }, +} + + +def _append_visual_extras(report): + """Build an HTML panel with rendered images for a single subtest.""" + try: + from pytest_html import extras + except ImportError: + return + + funcargs = _node_funcargs.get(report.nodeid) + if not funcargs: + return + + from test_render import RenderEnvironment, RenderTestCase + + case = funcargs.get("case") + if not isinstance(case, RenderTestCase): + return + + env = None + for arg_val in funcargs.values(): + if isinstance(arg_val, RenderEnvironment): + env = arg_val + break + if not env: + return + + context = getattr(report, "context", None) + subtest_name = context.msg if context else None + if not subtest_name: + return + + output_path = env.get_output_path(case) + if not output_path or not output_path.exists(): + return + + import MaterialX as mx + valid_elem_name = mx.createValidName(subtest_name) + rendered_files = list(output_path.glob(f"{valid_elem_name}_*.png")) + rendered_files = [ + f for f in rendered_files if not f.name.endswith("_diff.png") + ] + if not rendered_files: + return + + rendered_file = rendered_files[0] + + image_ref_dir = env.get_image_ref_dir(output_path) + baseline_file = None + if image_ref_dir: + candidate = image_ref_dir / rendered_file.name + if candidate.exists(): + baseline_file = candidate + heatmap_file = rendered_file.parent / f"{rendered_file.stem}_diff.png" + + import os + try: + htmlpath_str = ( + _pytest_config.getoption("htmlpath") + if _pytest_config + else None + ) + except ValueError: + htmlpath_str = None + html_dir = ( + Path(htmlpath_str).parent.resolve() if htmlpath_str else None + ) + + try: + is_self_contained = ( + _pytest_config.getoption("self_contained_html") + if _pytest_config + else False + ) + except ValueError: + is_self_contained = False + + def get_image_src(path: Path) -> str: + if not path or not path.exists(): + return "" + if is_self_contained: + import base64 + try: + with open(path, "rb") as f: + encoded = base64.b64encode(f.read()).decode("utf-8") + return f"data:image/png;base64,{encoded}" + except Exception: + pass + elif html_dir: + try: + return os.path.relpath( + path.resolve(), html_dir, + ).replace("\\", "/") + except ValueError: + return path.resolve().as_uri() + return path.resolve().as_uri() + + rendered_src = get_image_src(rendered_file) + if not rendered_src: + return + + baseline_src = get_image_src(baseline_file) + heatmap_src = get_image_src(heatmap_file) + + if report.failed: + outcome = "failed" + elif report.skipped: + outcome = "skipped" + else: + outcome = "passed" + style = _OUTCOME_STYLES[outcome] + + _IMG_STYLE = ( + "max-width:100%; height:auto; border:1px solid #ccc; " + "border-radius:4px; background:black;" + ) + _PLACEHOLDER_STYLE = ( + "padding:50px 10px; background:#eee; border:1px dashed #ccc; " + "border-radius:4px; color:#666; font-size:12px;" + ) + + panels = [] + + if baseline_src: + panels.append( + f'
' + f'
Baseline
' + f'' + f'
' + ) + elif image_ref_dir: + panels.append( + f'
' + f'
Baseline
' + f'
Baseline image missing
' + f'
' + ) + + panels.append( + f'
' + f'
Rendered
' + f'' + f'
' + ) + + if heatmap_src: + panels.append( + f'
' + f'
FLIP Heatmap
' + f'' + f'
' + ) + + html_content = ( + f'
' + f'

' + f'{subtest_name}' + f'{style["label"]}' + f'

' + f'
' + f'{"".join(panels)}' + f'
' + f'
' + ) + _subtest_html_extras[report.nodeid].append(extras.html(html_content)) diff --git a/contrib/tests/pyproject.toml b/contrib/tests/pyproject.toml index 502046ce9a..323671f188 100644 --- a/contrib/tests/pyproject.toml +++ b/contrib/tests/pyproject.toml @@ -16,6 +16,7 @@ test = [ ] [tool.pytest.ini_options] +render_collapsed = "" testpaths = ["."] python_files = ["test_*.py"] python_classes = ["Test*"] diff --git a/contrib/tests/test_render.py b/contrib/tests/test_render.py index 707a061e0c..56f51dcd34 100644 --- a/contrib/tests/test_render.py +++ b/contrib/tests/test_render.py @@ -64,6 +64,7 @@ class CliOptions: output_root: Path no_render: bool = False flip_threshold: float = 0.05 + html_report: bool = False # --------------------------------------------------------------------------- @@ -483,13 +484,17 @@ def _check_shader_baselines(result, baseline_dir: Path): # Image comparison # --------------------------------------------------------------------------- -def _compare_render(result, image_ref_dir: Path, threshold: float): +def _compare_render( + result, image_ref_dir: Path, threshold: float, + *, save_heatmap: bool = False, +): """Compare a rendered image against the reference environment's render. - Uses NVIDIA FLIP to compute a perceptual difference metric and - saves a magma heatmap (``*_diff.png``) next to the rendered image. - Asserts that the mean FLIP error is within *threshold*. Skips - gracefully when reference images are missing. + Uses NVIDIA FLIP to compute a perceptual difference metric. + Saves a magma heatmap (``*_diff.png``) next to the rendered image + when *save_heatmap* is ``True`` (for the HTML report) or when the + mean FLIP error exceeds *threshold*. Fails when reference images + are missing. Requires ``flip_evaluator`` (``pip install flip-evaluator``). """ @@ -498,7 +503,7 @@ def _compare_render(result, image_ref_dir: Path, threshold: float): ref_image = image_ref_dir / result.output_path.name if not ref_image.exists(): - pytest.skip( + pytest.fail( f"Reference render not found: {ref_image}\n" f"Run the reference environment first." ) @@ -513,30 +518,36 @@ def _compare_render(result, image_ref_dir: Path, threshold: float): ) mean_flip = float(mean_flip) - if mean_flip <= threshold: - return + # `not <=` instead of `>` so that NaN comparisons fail rather than + # silently pass (NaN > x is False, but NaN <= x is also False). + failed = not (mean_flip <= threshold) - max_flip = float(np.array(flip_map).max()) + if save_heatmap or failed: + from PIL import Image - heatmap_path = result.output_path.parent / f"{result.output_path.stem}_diff.png" - heatmap_img, _, _ = flip.evaluate( - str(ref_image), str(result.output_path), - "LDR", inputsRGB=True, applyMagma=True, - computeMeanError=False, parameters={"ppd": 70.0}, - ) - from PIL import Image - heatmap_arr = np.array(heatmap_img) - if heatmap_arr.max() <= 1.0: - heatmap_arr = (heatmap_arr * 255).astype(np.uint8) - Image.fromarray(heatmap_arr).save(heatmap_path) - - assert False, ( - f"FLIP mean {mean_flip:.6f} exceeds threshold {threshold:.6f} " - f"(max {max_flip:.6f}) for {result.output_path.name}\n" - f" ref: {ref_image}\n" - f" rendered: {result.output_path}\n" - f" heatmap: {heatmap_path}" - ) + heatmap_path = ( + result.output_path.parent + / f"{result.output_path.stem}_diff.png" + ) + heatmap_img, _, _ = flip.evaluate( + str(ref_image), str(result.output_path), + "LDR", inputsRGB=True, applyMagma=True, + computeMeanError=False, parameters={"ppd": 70.0}, + ) + heatmap_arr = np.array(heatmap_img) + if heatmap_arr.max() <= 1.0: + heatmap_arr = (heatmap_arr * 255).astype(np.uint8) + Image.fromarray(heatmap_arr).save(heatmap_path) + + if failed: + max_flip = float(np.array(flip_map).max()) + assert False, ( + f"FLIP mean {mean_flip:.6f} exceeds threshold {threshold:.6f} " + f"(max {max_flip:.6f}) for {result.output_path.name}\n" + f" ref: {ref_image}\n" + f" rendered: {result.output_path}\n" + f" heatmap: {heatmap_path}" + ) # --------------------------------------------------------------------------- @@ -612,6 +623,7 @@ def _render_elements( _compare_render( result, image_ref_dir, env.cli_options.flip_threshold, + save_heatmap=env.cli_options.html_report, )