Merge OBJECT/SS screen video with DV audio for Panopto split-stream recordings

Some Panopto sessions store the screen-recording (OBJECT/SS) without an audio
track and put the lecturer's microphone in a separate DV stream. Previously
we'd reject the audio-less screen stream and download the DV camera video,
which destroyed slide content for the frame extractor. We now classify each
candidate as video-bearing or audio-bearing, prefer the screen stream as the
video source, and merge the two streams via ffmpeg when they differ.

Includes the test fixes for the dir_key-based section/image-cache API and a
new regression suite covering the split-stream selector + the merge helper.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: nodeeeeee

.gitignore

@@ -15,6 +15,8 @@ anthropic_key.txt
 85397/
 85427/
 manifest.json
+# Stray notes generated when AUTONOTE_DATA_DIR isn't set land at the repo root
+*_notes.md
 
 # ── Python ────────────────────────────────────────────────────────────────────
 __pycache__/

downloader.py

@@ -1022,6 +1022,46 @@ def _hls_has_audio(stream_url: str, ff_bin: str | None = None) -> bool:
         return True  # don't block download on probe failure
 
 
+def _run_ffmpeg_hls_merge(video_url: str, audio_url: str,
+                          out_path: Path, progress_cb) -> None:
+    """Merge a video-only HLS stream with an audio-only HLS stream.
+
+    Used when the screen-recording stream (OBJECT/SS) has no audio track and
+    the audio lives in a separate DV/AUDIO stream. Produces one MP4 with the
+    screen video and the lecturer's audio.
+    """
+    ff_bin = _resolve_ffmpeg()
+    if not ff_bin:
+        raise RuntimeError(
+            "ffmpeg not found. Install ffmpeg or `pip install imageio-ffmpeg`."
+        )
+    cmd = [
+        ff_bin, "-y", "-loglevel", "error",
+        "-i", video_url,
+        "-i", audio_url,
+        "-map", "0:v:0", "-map", "1:a:0",
+        "-c", "copy",
+        "-shortest",
+        str(out_path),
+    ]
+    try:
+        from ffmpeg_progress_yield import FfmpegProgress
+        ff = FfmpegProgress(cmd)
+        for pct in ff.run_command_with_progress():
+            progress_cb(pct)
+        if not out_path.exists() or out_path.stat().st_size < 1024:
+            raise RuntimeError(
+                f"ffmpeg produced no output for {out_path.name} — "
+                f"video/audio merge failed."
+            )
+    except ImportError:
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=3600)
+        if result.returncode != 0:
+            tail = (result.stderr or "").strip().splitlines()[-10:]
+            raise RuntimeError(f"ffmpeg merge failed:\n" + "\n".join(tail))
+        progress_cb(100)
+
+
 def _run_ffmpeg_hls(stream_url: str, out_path: Path, progress_cb) -> None:
     """Download an HLS master.m3u8 stream to *out_path* via ffmpeg.
 
@@ -1158,36 +1198,69 @@ def download_video(video: dict, manifest: dict, base_dir: Path) -> bool | None:
         return False
 
     # Record which tags existed so the frame extractor can tell a true camera
-    # recording from a DV-with-audio fallback whose OBJECT/SS screen stream
-    # was only rejected for lacking an audio track.
+    # recording from a split-stream case where the audio is in DV but the
+    # screen video is in OBJECT/SS.
     available_tags = [ct for (_, _, ct) in candidates]
     has_screen_stream = any(t in ("SS", "OBJECT") for t in available_tags)
 
-    # Probe each candidate for an audio track. Panopto screen-recording
-    # streams (SS/OBJECT) sometimes pack audio only in the DV variant, so
-    # we fall back through all available streams rather than trusting the
-    # first by tag.
-    stream_url = dl_headers = stream_tag = None
+    # Classify each candidate as "video-with-audio" or "video-only" up front.
+    # Panopto screen recordings often store SS/OBJECT (screen content, no
+    # audio) alongside DV (camera + audio). When both exist, we want the
+    # screen video AND the DV audio — merged by ffmpeg — so notes see slides
+    # as frames AND the transcript carries lecture audio.
+    video_cand = audio_cand = None   # (url, headers, tag, is_m3u8)
     tried = []
     for (cu, ch, ct) in candidates:
         tried.append(ct)
-        # Only HLS master playlists are audio-probable from here. Direct
-        # authenticated URLs (non-m3u8) go straight through.
-        if "master.m3u8" not in cu or _hls_has_audio(cu):
-            stream_url, dl_headers, stream_tag = cu, ch, ct
-            break
-        tqdm.write(f"  Stream '{ct}' has no audio — trying next candidate")
-    if stream_url is None:
+        is_m3u8 = "master.m3u8" in cu
+        has_audio = (not is_m3u8) or _hls_has_audio(cu)
+        # Preferred screen stream (first one we see that is SS/OBJECT) becomes
+        # the chosen VIDEO source — regardless of whether it has audio.
+        if video_cand is None and ct in ("SS", "OBJECT"):
+            video_cand = (cu, ch, ct, is_m3u8, has_audio)
+        # First stream with audio becomes the audio source.
+        if audio_cand is None and has_audio:
+            audio_cand = (cu, ch, ct, is_m3u8, has_audio)
+
+    # If we never saw a screen stream, fall back to the audio-bearing stream
+    # (treat its video as the video source too).
+    if video_cand is None:
+        if audio_cand is None:
+            tqdm.write(
+                f"  [error] No stream with audio found for {title} "
+                f"(tried: {', '.join(tried)}) — skipping; the recording "
+                f"itself appears to be video-only.")
+            manifest[key] = {
+                "status": "error", "title": title,
+                "error": "no-audio-track",
+            }
+            raise RuntimeError(f"No audio track in any Panopto stream for '{title}'")
+        video_cand = audio_cand
+    if audio_cand is None:
+        # Screen stream exists but nothing has audio — still skip.
         tqdm.write(
-            f"  [error] No stream with audio found for {title} "
-            f"(tried: {', '.join(tried)}) — skipping; the recording itself "
-            f"appears to be video-only.")
+            f"  [error] Video streams exist but none has audio for {title} "
+            f"(tried: {', '.join(tried)}) — skipping.")
         manifest[key] = {
             "status": "error", "title": title,
             "error": "no-audio-track",
         }
         raise RuntimeError(f"No audio track in any Panopto stream for '{title}'")
-    tqdm.write(f"  Stream type: {stream_tag}")
+
+    merge_needed = (
+        video_cand[0] != audio_cand[0]        # different stream URLs
+        and video_cand[3] and audio_cand[3]   # both HLS master playlists
+    )
+    stream_url    = video_cand[0]
+    dl_headers    = video_cand[1]
+    stream_tag    = video_cand[2]
+    audio_url     = audio_cand[0] if merge_needed else None
+    audio_tag     = audio_cand[2] if merge_needed else None
+    if merge_needed:
+        tqdm.write(
+            f"  Stream type: {stream_tag} video + {audio_tag} audio (merging)")
+    else:
+        tqdm.write(f"  Stream type: {stream_tag}")
 
     bar = tqdm(total=100, desc=f"  {title[:50]}", unit="%",
                bar_format="{desc} |{bar}| {n:3d}/{total}%", leave=True)
@@ -1200,6 +1273,9 @@ def progress_cb(pct: float) -> None:
         if dl_headers:
             # Direct authenticated download (e.g. REST API DownloadUrl)
             _download_authenticated(stream_url, out_path, dl_headers, progress_cb)
+        elif audio_url:
+            # Split-stream case: screen video (no audio) + separate audio stream.
+            _run_ffmpeg_hls_merge(stream_url, audio_url, out_path, progress_cb)
         elif "master.m3u8" in stream_url:
             # HLS stream — always use ffmpeg. PanoptoDownloader's HLS path
             # is broken for URLs with query strings (its endswith('master.m3u8')

test/test_language_and_skip.py

@@ -365,18 +365,22 @@ class TestNoteGenerationSkipLogic:
 
     def test_section_cache_respected(self, tmp_path):
         """generate_section should return cached content without --force."""
-        from note_generation import _section_path
+        from note_generation import _section_path, LectureData
 
         sections_dir = tmp_path / "sections"
         sections_dir.mkdir()
 
-        # Create a cached section file (must be > 500 bytes to pass cache check)
-        sec_file = _section_path(sections_dir, 1, 1, 1)
+        slide_path = tmp_path / "lec1.pdf"
+        slide_path.write_bytes(b"%PDF-1.4 dummy")
+        ld = LectureData(num=1, slide_path=slide_path, alignment_path=None)
+
+        sec_file = _section_path(sections_dir, ld, 1)
         content = "### 1.1 Cached Content\n\n" + ("This was previously generated. " * 30)
         sec_file.write_text(content)
 
         assert sec_file.exists()
         assert sec_file.stat().st_size > 50
+        assert ld.dir_key in sec_file.name
 
     def test_help_shows_force_flag(self):
         r = _run("note_generation.py", "--help")

test/test_note_generation.py

@@ -176,6 +176,8 @@ def _make_lecture_data(self, slides_with_text: list[tuple[int, str, str]]):
         ld = MagicMock()
         ld.num = 1
         ld.file_idx = 1
+        ld.dir_key = "L01"
+        ld.source = "slides"
         ld.img_cache = {}
         ld.slides = [
             ng.SlideInfo(idx, label, text)

test/test_v0_13_split_stream.py

@@ -0,0 +1,135 @@
+"""
+Regression tests for the v0.13.0 Panopto split-stream download path.
+
+When a Panopto recording stores screen video in OBJECT/SS (no audio) and the
+microphone audio in a separate DV stream, downloader.py must:
+  • Prefer the screen-recording stream as the VIDEO source
+  • Pick the first audio-bearing stream as the AUDIO source
+  • Use ffmpeg to merge them into one MP4 (so the frame extractor sees slides
+    AND the transcript sees lecture audio)
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+PROJECT_DIR = Path(__file__).parent.parent
+sys.path.insert(0, str(PROJECT_DIR))
+
+
+def _select_split(candidates, hls_audio_lookup):
+    """Reimplement downloader.py's video/audio classification.
+
+    Mirrors the logic in download_video so tests don't need to monkeypatch
+    Panopto auth, ffprobe, or the progress bar. ``candidates`` is a list of
+    (url, headers, tag) and ``hls_audio_lookup`` maps url → bool (has audio).
+    Returns (video_cand, audio_cand, merge_needed).
+    """
+    video_cand = audio_cand = None
+    for cu, ch, ct in candidates:
+        is_m3u8 = "master.m3u8" in cu
+        has_audio = (not is_m3u8) or hls_audio_lookup.get(cu, True)
+        if video_cand is None and ct in ("SS", "OBJECT"):
+            video_cand = (cu, ch, ct, is_m3u8, has_audio)
+        if audio_cand is None and has_audio:
+            audio_cand = (cu, ch, ct, is_m3u8, has_audio)
+    if video_cand is None and audio_cand is not None:
+        video_cand = audio_cand
+    merge = (
+        video_cand is not None and audio_cand is not None
+        and video_cand[0] != audio_cand[0]
+        and video_cand[3] and audio_cand[3]
+    )
+    return video_cand, audio_cand, merge
+
+
+class TestSplitStreamClassification:
+    """Verify the OBJECT-video + DV-audio split-stream selector."""
+
+    def test_object_video_dv_audio_merges(self):
+        cands = [
+            ("https://cdn/obj.master.m3u8", None, "OBJECT"),
+            ("https://cdn/dv.master.m3u8",  None, "DV"),
+        ]
+        audio = {
+            "https://cdn/obj.master.m3u8": False,   # screen has no audio
+            "https://cdn/dv.master.m3u8":  True,    # camera carries audio
+        }
+        v, a, merge = _select_split(cands, audio)
+        assert v[2] == "OBJECT"
+        assert a[2] == "DV"
+        assert merge is True
+
+    def test_ss_with_audio_no_merge(self):
+        """If the screen stream already has audio, no merge is needed."""
+        cands = [
+            ("https://cdn/ss.master.m3u8", None, "SS"),
+            ("https://cdn/dv.master.m3u8", None, "DV"),
+        ]
+        audio = {
+            "https://cdn/ss.master.m3u8": True,
+            "https://cdn/dv.master.m3u8": True,
+        }
+        v, a, merge = _select_split(cands, audio)
+        assert v[2] == "SS"
+        assert a[2] == "SS"   # same stream → audio source is video source
+        assert merge is False
+
+    def test_dv_only_camera_recording(self):
+        """No screen stream — fall back to DV as both video and audio."""
+        cands = [("https://cdn/dv.master.m3u8", None, "DV")]
+        audio = {"https://cdn/dv.master.m3u8": True}
+        v, a, merge = _select_split(cands, audio)
+        assert v[2] == "DV"
+        assert a[2] == "DV"
+        assert merge is False
+
+    def test_screen_without_audio_anywhere_returns_no_audio(self):
+        """OBJECT exists but no candidate has audio — caller should error."""
+        cands = [
+            ("https://cdn/obj.master.m3u8", None, "OBJECT"),
+            ("https://cdn/dv.master.m3u8",  None, "DV"),
+        ]
+        audio = {
+            "https://cdn/obj.master.m3u8": False,
+            "https://cdn/dv.master.m3u8":  False,
+        }
+        v, a, merge = _select_split(cands, audio)
+        assert v is not None
+        assert a is None
+        assert merge is False
+
+    def test_direct_url_treated_as_audio_bearing(self):
+        """Non-HLS authenticated URLs skip the audio probe (always assumed audio)."""
+        cands = [("https://cdn/dv.mp4?token=abc", {"Auth": "x"}, "DV")]
+        v, a, merge = _select_split(cands, {})
+        assert v[2] == "DV"
+        assert merge is False
+        assert v[4] is True   # has_audio
+
+
+class TestMergeFunctionExists:
+    """The new merge helper must be importable with the right signature."""
+
+    def test_run_ffmpeg_hls_merge_signature(self):
+        from downloader import _run_ffmpeg_hls_merge
+        import inspect
+
+        sig = inspect.signature(_run_ffmpeg_hls_merge)
+        params = list(sig.parameters.keys())
+        assert params == ["video_url", "audio_url", "out_path", "progress_cb"]
+
+    def test_merge_raises_when_ffmpeg_missing(self, tmp_path):
+        from downloader import _run_ffmpeg_hls_merge
+
+        with patch("downloader._resolve_ffmpeg", return_value=None):
+            with pytest.raises(RuntimeError, match="ffmpeg not found"):
+                _run_ffmpeg_hls_merge(
+                    "https://cdn/v.m3u8",
+                    "https://cdn/a.m3u8",
+                    tmp_path / "out.mp4",
+                    lambda pct: None,
+                )