Test/ref differ test 241221

.github/workflows/python-package.yml

@@ -59,7 +59,7 @@ jobs:
       run: |
         Add-Content $env:GITHUB_PATH $env:GITHUB_WORKSPACE\.dxc\bin\x64
         Add-Content $env:GITHUB_PATH $env:GITHUB_WORKSPACE\.glslang\bin
-        Add-Content $env:GITHUB_ENV METASHADE_PYTEST_OUT_DIR=$env:GITHUB_WORKSPACE/tests/out
+        Add-Content $env:GITHUB_ENV METASHADE_GLTFSAMPLE_PYTEST_OUT_DIR=$env:GITHUB_WORKSPACE/tests/out
     - name: Test with pytest
       run: |
         pytest -rP tests

.gitmodules

@@ -5,3 +5,6 @@
 	path = glTFSample
 	url = https://github.com/metashade/glTFSample.git
 	branch = metashade_demo
+[submodule "tests/ref"]
+	path = tests/ref
+	url = https://github.com/metashade/metashade-glTFSample-testrefs.git

src/_impl.py

@@ -537,6 +537,8 @@ def _sample_material_texture(texture_name : str):
 
     # Finally, the pixel shader entry point
     with sh.entry_point(entry_point_name, sh.PsOut)(psIn = sh.VsOut):
+        sh // "Test diffing in the test"
+
         sh.Vw = (sh.g_cameraPw - sh.psIn.Pw).normalize()
         sh.Nw = sh.getNormal(psIn = sh.psIn)
 

src/generate.py

@@ -21,6 +21,7 @@
 from metashade.util import perf, spirv_cross
 from metashade.hlsl.util import dxc
 from metashade.glsl.util import glslang, glslc
+from metashade.util.tests import RefDiffer
 
 import _impl
 
@@ -55,25 +56,38 @@ class CompileResult(NamedTuple):
         success : bool
 
     @abc.abstractmethod
-    def compile(self, to_glsl : bool) -> CompileResult:
+    def _compile(self, to_glsl : bool) -> bool:
         pass
 
-def _compile_shader(shader, to_glsl : bool) -> _Shader.CompileResult:
+    def compile(self, to_glsl : bool, ref_differ : RefDiffer) -> CompileResult:
+        log = io.StringIO()
+        log, sys.stdout = sys.stdout, log
+
+        if ref_differ is not None:
+            ref_differ(self._file_path)
+
+        success = self._compile(to_glsl)
+
+        log, sys.stdout = sys.stdout, log
+        return _Shader.CompileResult(log.getvalue(), success)
+
+def _compile_shader(
+    shader,
+    to_glsl : bool,
+    ref_differ : RefDiffer
+) -> _Shader.CompileResult:
     '''
     Helper function to compile a shader in a process pool.
     Without it, the pool would not be able to pickle the method.
     '''
-    return shader.compile(to_glsl)
+    return shader.compile(to_glsl, ref_differ)
 
 class _HlslShader(_Shader):
     @abc.abstractmethod
     def _get_hlsl_profile():
         pass
 
-    def compile(self, to_glsl : bool) -> _Shader.CompileResult:
-        log = io.StringIO()
-        log, sys.stdout = sys.stdout, log
-
+    def _compile(self, to_glsl : bool) -> bool:
         try:
             dxc_output_path = Path(self._file_path).with_suffix(
                 '.hlsl.spv' if to_glsl else '.cso'
@@ -102,12 +116,9 @@ def compile(self, to_glsl : bool) -> _Shader.CompileResult:
                     entry_point_name = _impl.entry_point_name,
                     output_path = spv_path
                 )
-            success = True
+            return True
         except subprocess.CalledProcessError as err:
-            success = False
-
-        log, sys.stdout = sys.stdout, log
-        return _Shader.CompileResult(log.getvalue(), success)
+            return False
 
 class _HlslVertexShader(_HlslShader):
     def __init__(
@@ -154,10 +165,7 @@ def _generate(self, shader_file, material, primitive):
         )
 
 class _GlslShader(_Shader):
-    def compile(self, to_glsl : bool) -> _Shader.CompileResult:
-        log = io.StringIO()
-        log, sys.stdout = sys.stdout, log
-
+    def _compile(self, to_glsl : bool) -> bool:
         try:
             glsl_output_path = Path(self._file_path).with_suffix('.spv')
             glslang.compile(
@@ -166,12 +174,9 @@ def compile(self, to_glsl : bool) -> _Shader.CompileResult:
                 shader_stage = 'frag',
                 output_path = glsl_output_path
             )
-            success = True
+            return True
         except subprocess.CalledProcessError as err:
-            success = False
-
-        log, sys.stdout = sys.stdout, log
-        return _Shader.CompileResult(log.getvalue(), success)
+            return False
 
 class _GlslFragmentShader(_GlslShader):
     def __init__(
@@ -237,7 +242,8 @@ def generate(
     compile : bool,
     to_glsl : bool,
     skip_codegen : bool,
-    serial : bool
+    serial : bool,
+    ref_differ : RefDiffer
 ):
     if not gltf_dir_path.is_dir():
         raise NotADirectoryError(gltf_dir_path)
@@ -280,7 +286,10 @@ def generate(
 
         if serial:
             for shader in shaders:
-                result = shader.compile(to_glsl = to_glsl)
+                result = shader.compile(
+                    to_glsl = to_glsl,
+                    ref_differ = ref_differ
+        )
                 if not result.success:
                     num_failed += 1
                 print(result.log, end = '')
@@ -289,7 +298,8 @@ def generate(
                 for result in pool.imap_unordered(
                     functools.partial(
                         _compile_shader,
-                        to_glsl = to_glsl
+                        to_glsl = to_glsl,
+                        ref_differ = ref_differ
                     ),
                     shaders
                 ):
@@ -311,6 +321,8 @@ def generate(
     )
     parser.add_argument("--gltf-dir", help = "Path to the source glTF assets")
     parser.add_argument("--out-dir", help = "Path to the output directory")
+    parser.add_argument("--ref-dir", help = "Path to the test references")
+
     parser.add_argument(
         "--compile",
         action = 'store_true',
@@ -340,5 +352,6 @@ def generate(
         compile = args.compile,
         to_glsl = args.to_glsl,
         skip_codegen = args.skip_codegen,
-        serial = args.serial
+        serial = args.serial,
+        ref_differ = RefDiffer(Path(args.ref_dir)) if args.ref_dir else None
     )

tests/test_generate.py

@@ -17,24 +17,46 @@
 
 tests_dir_path = Path(__file__).parent
 repo_root_dir_path = tests_dir_path.parent
+
+# Add these directories to PYTHONPATH
 src_dir_path = (repo_root_dir_path / 'src').resolve()
 metashade_dir_path = (repo_root_dir_path / 'metashade').resolve()
-
 sys.path += [str(src_dir_path), str(metashade_dir_path)]
 
+from metashade.util.tests import RefDiffer
 import generate
 
 class TestGenerate:
+    @classmethod
+    def setup_class(cls):
+        out_dir = os.getenv('METASHADE_GLTFSAMPLE_PYTEST_OUT_DIR', None)
+        ref_dir = repo_root_dir_path / 'tests' / 'ref' / 'content'
+
+        if out_dir is None:
+            # Don't compare against references explicitly in the script.
+            # Instead, overwrite the references with the generated files.
+            # This is useful for diffing or updating the references manually with
+            # git.
+            cls._out_dir = ref_dir
+            cls._ref_differ = None
+        else:
+            cls._out_dir = Path(out_dir).resolve()
+            print(f'Reference directory: {ref_dir}')
+            cls._ref_differ = RefDiffer(ref_dir)
+
+        print(f'Test output directory: {cls._out_dir}')
+        os.makedirs(cls._out_dir, exist_ok = True)
+
     def test_generate(self):
         gltf_sample_dir_path = repo_root_dir_path / 'glTFSample'
         gltf_dir_path = gltf_sample_dir_path / 'media' / 'Cauldron-Media'
-        out_dir_path = gltf_sample_dir_path / 'build' / 'DX12' / 'metashade-out'
 
         generate.generate(
             gltf_dir_path = gltf_dir_path,
-            out_dir_path = out_dir_path,
+            out_dir_path = self._out_dir,
             compile = True,
             to_glsl = False,
             skip_codegen = False,
-            serial = False
+            serial = False,
+            ref_differ = self._ref_differ
         )