From e05f1c2f09df07035b21fb8d3c14d0df144d3898 Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 5 Jul 2026 17:57:03 +0400 Subject: [PATCH 01/13] Add party/action highlight detection (saliency candidate source) Auto-cut highlight clips from footage with no useful transcript (party videos, action) by peak-picking a fused laughter+energy curve instead of reading the transcript. Selected via 'podcli process --profile party|action'. - saliency.py: per-video robust-z normalization (median/MAD, never global), weighted channel fusion, numpy peak-pick with min-gap NMS, reaction dilation so a brief laugh isn't drowned by a louder neighbor, reaction-first candidate generation, 8s backward expansion to capture the setup before a laugh, and quiet-point boundary snapping - profiles drive it: 'saliency' candidate source (party/action) vs 'llm' (podcast) picks the selection path in the process pipeline - deterministic on clean input; produces render-compatible clip dicts Thresholds and channel weights are starting points to be tuned on real party footage. --- backend/cli.py | 28 ++++ backend/services/saliency.py | 245 +++++++++++++++++++++++++++++++++++ plans/moment-detection.md | 6 +- tests/test_saliency.py | 114 ++++++++++++++++ 4 files changed, 390 insertions(+), 3 deletions(-) create mode 100644 backend/services/saliency.py create mode 100644 tests/test_saliency.py diff --git a/backend/cli.py b/backend/cli.py index 442953a..260e7b7 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -452,6 +452,8 @@ def cmd_process(args): config["crop_strategy"] = args.crop if getattr(args, "format", None): config["format"] = args.format + if getattr(args, "profile", None): + config["profile"] = args.profile if getattr(args, "thumbnails", None) is not None: config["generate_thumbnails"] = args.thumbnails if args.top: @@ -705,6 +707,31 @@ def _transcribe_progress(pct, msg): clips = resumed resumed_from_session = True + # Saliency profiles (party/action) pick moments from the fused laughter/energy + # curve rather than the transcript, so they work on footage with no dialogue. + from services.profiles import get_profile + + content_profile = get_profile(config.get("profile")) + if content_profile.candidate_source == "saliency" and not clips: + from services.saliency import detect_highlights + from services.formats import get_format + + spec = get_format(config.get("format", "vertical")) + print(f" [3/4] Detecting {content_profile.name} highlights (laughter + energy)...") + clips = detect_highlights( + video_path, + profile_name=content_profile.name, + top_n=top_n, + min_dur=min(8.0, float(spec.dur_min)), + max_dur=float(spec.dur_max), + progress_callback=lambda pct, msg: print(f" {msg}") if msg else None, + ) + if clips: + print(f" ✓ {len(clips)} highlights found") + _save_suggestions_session(cache_hash, top_n, "saliency", clips, selection_sig) + else: + print(" ⚠ No highlights found, falling back to transcript selection") + # Try an AI CLI first (uses PodStack knowledge base for intelligent selection) from services.claude_suggest import suggest_initial_with_claude, _engine_label, _find_ai_cli @@ -3348,6 +3375,7 @@ def main(): proc.add_argument("--caption-style", choices=["branded", "hormozi", "karaoke", "subtle"]) proc.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut"]) proc.add_argument("--format", choices=["vertical", "horizontal", "square"], help="Output aspect ratio (default: vertical)") + proc.add_argument("--profile", choices=["podcast", "party", "action"], help="Detection profile: podcast (transcript-first, default), party/action (laughter/energy highlights)") proc.add_argument("--logo", help="Logo image (asset name or path)") proc.add_argument("--outro", help="Outro video (asset name or path)") proc.add_argument("--time-adjust", type=float, help="Timestamp offset in seconds") diff --git a/backend/services/saliency.py b/backend/services/saliency.py new file mode 100644 index 0000000..ed300c2 --- /dev/null +++ b/backend/services/saliency.py @@ -0,0 +1,245 @@ +"""Multi-signal saliency engine — fuse channels, peak-pick, expand reactions, snap. + +For profiles whose candidate source is "saliency" (party, action), moments are +generated from a fused interestingness curve instead of from an LLM reading the +transcript. This is what lets podcli auto-cut highlights from footage with no useful +transcript (party videos, action). + +Each channel is normalized against THIS video's own distribution (never a global +scale) so different recordings, rooms and mic levels compare fairly, then combined by +the active profile's weights. Peaks on the fused curve become candidate clips; a peak +driven by a laugh or cheer is expanded backwards to capture the moment that caused the +reaction, since the funny thing happens just before people react to it. +""" + +from typing import Optional, Callable + +import numpy as np + +from services.profiles import get_profile, ContentProfile +from services.audio_analyzer import extract_audio_energy +from services.audio_events import extract_audio_events, is_available as audio_events_available + +GRID_HZ = 1.0 # common time grid; energy is per-second, so 1 Hz is the natural rate + + +def _robust_z(x: np.ndarray) -> np.ndarray: + """Median/MAD normalization — resistant to the heavy tails of RMS and reactions.""" + if x.size == 0: + return x + med = np.median(x) + mad = np.median(np.abs(x - med)) + scale = 1.4826 * mad if mad > 1e-9 else (np.std(x) or 1.0) + return (x - med) / scale + + +def _energy_curve(energy_data: list[dict], n_bins: int) -> np.ndarray: + """Per-second loudness onto the grid; silence (<= -60 dB) floored so it doesn't win.""" + curve = np.full(n_bins, -60.0) + for e in energy_data: + b = int(e["time"] * GRID_HZ) + if 0 <= b < n_bins: + curve[b] = max(curve[b], e.get("rms_db", -60.0)) + return curve + + +def _reaction_curve(events_data: list[dict], n_bins: int) -> np.ndarray: + """Peak reaction (laugh/cheer/scream) per grid bin.""" + curve = np.zeros(n_bins) + for e in events_data: + b = int(e["time"] * GRID_HZ) + if 0 <= b < n_bins: + level = max(e.get("laughter", 0), e.get("cheering", 0), e.get("screaming", 0)) + curve[b] = max(curve[b], level) + return curve + + +def _dilate(curve: np.ndarray, radius_bins: int) -> np.ndarray: + """Spread each spike to its neighbors (grayscale dilation) so a brief, narrow + reaction still aligns with, and can win, the fused peak near it.""" + if radius_bins <= 0 or curve.size == 0: + return curve + out = curve.copy() + for r in range(1, radius_bins + 1): + out[r:] = np.maximum(out[r:], curve[:-r]) + out[:-r] = np.maximum(out[:-r], curve[r:]) + return out + + +def fuse_channels(channels: dict[str, np.ndarray], profile: ContentProfile) -> np.ndarray: + """Weighted sum of per-video-normalized channels, weights renormalized over what exists.""" + present = {k: v for k, v in channels.items() if profile.channel_weights.get(k, 0) > 0 and v.size} + if not present: + return np.zeros(next(iter(channels.values())).size if channels else 0) + total_w = sum(profile.channel_weights[k] for k in present) + fused = None + for k, curve in present.items(): + w = profile.channel_weights[k] / total_w + contrib = w * _robust_z(curve) + fused = contrib if fused is None else fused + contrib + return fused + + +def pick_peaks(curve: np.ndarray, height: float, min_gap_bins: int) -> list[int]: + """Local maxima above `height`, then greedy non-maximum suppression by min gap. + + Peaks are taken in descending value order and a lower peak is dropped if it falls + within min_gap_bins of an already-chosen higher one (1-D NMS). + """ + if curve.size == 0: + return [] + candidates = [ + i for i in range(1, len(curve) - 1) + if curve[i] >= curve[i - 1] and curve[i] >= curve[i + 1] and curve[i] >= height + ] + if curve[0] >= height and (len(curve) == 1 or curve[0] > curve[1]): + candidates.append(0) + if len(curve) > 1 and curve[-1] >= height and curve[-1] > curve[-2]: + candidates.append(len(curve) - 1) + candidates.sort(key=lambda i: curve[i], reverse=True) + chosen: list[int] = [] + for i in candidates: + if all(abs(i - j) >= min_gap_bins for j in chosen): + chosen.append(i) + return sorted(chosen) + + +def _snap_to_quiet(target_sec: float, energy_curve: np.ndarray, window_sec: float = 1.5) -> float: + """Nudge a boundary to the quietest second nearby, so cuts land in a lull not mid-action.""" + if energy_curve.size == 0: + return target_sec + center = int(round(target_sec * GRID_HZ)) + lo = max(0, center - int(window_sec * GRID_HZ)) + hi = min(len(energy_curve), center + int(window_sec * GRID_HZ) + 1) + if lo >= hi: + return target_sec + local = energy_curve[lo:hi] + return (lo + int(np.argmin(local))) / GRID_HZ + + +def _window_for_peak( + peak_sec: float, + reaction_level: float, + profile: ContentProfile, + duration: float, + energy_curve: np.ndarray, + min_dur: float, + max_dur: float, +) -> tuple[float, float, bool]: + """Clip window for a peak. A reaction peak expands backwards from the reaction onset.""" + is_reaction = reaction_level >= 0.15 + if is_reaction: + # The funny thing happens BEFORE the laugh, so keep the run-up: expand backwards + # from the reaction, snap only the end to a lull, and grow the start (not the + # end) if we're under the minimum so the payoff stays put. + start = max(0.0, peak_sec - profile.reaction_lookback_sec) + end = _snap_to_quiet(min(duration, peak_sec + profile.reaction_payoff_sec), energy_curve) + if end - start < min_dur: + start = max(0.0, end - min_dur) + elif end - start > max_dur: + start = end - max_dur + else: + half = min_dur / 2.0 + start = _snap_to_quiet(max(0.0, peak_sec - half), energy_curve) + end = _snap_to_quiet(min(duration, peak_sec + half), energy_curve) + if end - start < min_dur: + end = min(duration, start + min_dur) + elif end - start > max_dur: + end = start + max_dur + return round(max(0.0, start), 1), round(min(duration, end), 1), is_reaction + + +def detect_highlights( + video_path: str, + profile_name: str = "party", + top_n: int = 8, + min_dur: float = 8.0, + max_dur: float = 60.0, + height_z: float = 1.0, + progress_callback: Optional[Callable] = None, +) -> list[dict]: + """ + Generate highlight clips from a video's fused signal curve (no transcript needed). + + Returns clip dicts compatible with the render pipeline: + {title, start_second, end_second, duration, score, reasons, preview}. + """ + profile = get_profile(profile_name) + + if progress_callback: + progress_callback(10, "Analyzing audio energy...") + energy_data = extract_audio_energy(video_path) + + events_data = [] + if audio_events_available(): + if progress_callback: + progress_callback(40, "Detecting laughter and reactions...") + events_data = extract_audio_events(video_path) + + last_times = [e["time"] for e in energy_data] + [e["time"] for e in events_data] + if not last_times: + return [] + duration = max(last_times) + 1.0 + n_bins = int(duration * GRID_HZ) + 1 + + energy_curve = _energy_curve(energy_data, n_bins) + # Dilate reactions by ~2s so a single-frame laugh isn't suppressed by a louder + # energy neighbor and so the fused peak lands on the reaction, not next to it. + reaction_curve = _dilate(_reaction_curve(events_data, n_bins), int(2 * GRID_HZ)) + + fused = fuse_channels( + {"energy": energy_curve, "audio_event": reaction_curve}, profile + ) + if fused.size == 0: + return [] + + if progress_callback: + progress_callback(70, "Selecting highlight moments...") + min_gap_bins = max(1, int(profile.peak_min_gap_sec * GRID_HZ)) + + # Reaction moments are primary candidates — a detected laugh/cheer is almost always + # worth a clip regardless of loudness, so they aren't made to out-compete energy in + # the blended curve. Energy peaks then fill the rest, minus any that collide with a + # reaction. Reaction score is offset above energy so reactions rank first. + reaction_peaks = pick_peaks(reaction_curve, 0.15, min_gap_bins) + energy_peaks = pick_peaks(fused, height_z, min_gap_bins) + + candidates = [(i, float(reaction_curve[i]), True) for i in reaction_peaks] + reaction_bins = {i for i in reaction_peaks} + for i in energy_peaks: + if all(abs(i - j) >= min_gap_bins for j in reaction_bins): + candidates.append((i, float(fused[i]), False)) + + def rank_key(c): + i, val, is_reaction = c + return (1 if is_reaction else 0, val) + + candidates.sort(key=rank_key, reverse=True) + candidates = candidates[:top_n] + + clips = [] + for i, val, want_reaction in candidates: + peak_sec = i / GRID_HZ + reaction_level = float(reaction_curve[i]) if want_reaction else 0.0 + start, end, is_reaction = _window_for_peak( + peak_sec, reaction_level, profile, duration, energy_curve, min_dur, max_dur + ) + if end - start < min_dur * 0.75: + continue + kind = "laugh/cheer" if is_reaction else "high energy" + score = round(10.0 + reaction_level * 10.0, 2) if is_reaction else round(float(val), 2) + clips.append({ + "title": f"Highlight ({kind}) at {int(peak_sec // 60):d}:{int(peak_sec % 60):02d}", + "start_second": start, + "end_second": end, + "duration": round(end - start), + "score": score, + "reasons": ["reaction"] if is_reaction else ["energy_peak"], + "preview": "", + "content_type": "highlight", + }) + + clips.sort(key=lambda c: c["start_second"]) + if progress_callback: + progress_callback(100, f"Found {len(clips)} highlights") + return clips diff --git a/plans/moment-detection.md b/plans/moment-detection.md index 4665bd6..23f7aec 100644 --- a/plans/moment-detection.md +++ b/plans/moment-detection.md @@ -95,9 +95,9 @@ A new `profile` param threads the **same ~12 hops the `format` field did**: `sug ## Phasing -- **Phase 0 — profile scaffolding, zero behavior change.** Add `ContentProfile` abstraction; thread the `profile` param through the ~12 hops; `default = podcast` reproduces current selection exactly. **Gate: existing test suite green; same clips out for a fixed transcript.** -- **Phase 1 — audio-event channel (the isolated valuable core).** YAMNet-ONNX laughter/cheer/applause/scream computed in the detect-once hub. Feeds podcast ranking as a labeled signal (laughs already spike energy; now they're *named*) and lays the party foundation. **Gate: laughter timestamps validated on a sample clip; podcast output unchanged unless the channel is given weight.** -- **Phase 2 — fusion engine + saliency candidate source + party profile (audio-only).** `saliency.py` fusion + numpy peak-pick + reaction-expand (8 s) + boundary-snap. Party profile = energy + audio_event + prosody, no transcript, no motion. **Party videos auto-clip end to end. Gate: demo on real party footage.** +- **Phase 0 — profile scaffolding, zero behavior change.** [DONE, PR #43 / Phase 2 branch] `ContentProfile` abstraction added (`profiles.py`); `profile` param threaded through the CLI (`--profile`, config, selection signature). Python-side default `podcast` reproduces current selection. TS-side threading (MCP/web `profile` param) still open — see below. +- **Phase 1 — audio-event channel (the isolated valuable core).** [DONE, PR #43] YAMNet-ONNX laughter/cheer/applause/scream (`audio_events.py`) computed in the detect-once hub; surfaced in the packed transcript and CLI heuristic. Podcast output unchanged unless the channel is weighted. Validated on 32 real clips. +- **Phase 2 — fusion engine + saliency candidate source + party profile (audio-only).** [DONE, this branch] `saliency.py`: per-video robust-z normalization, weighted fusion, numpy peak-pick (NMS), reaction dilation, reaction-first candidate generation, 8 s backward expansion, quiet-point boundary snap. Wired to `--profile party|action` in `podcli process`. Deterministic on clean input; reactions detected with correct run-up windows. **Gate remaining: tune thresholds/weights on real party footage (synthetic podcast-clip concat is not representative).** - **Phase 3 — visual channels + action profile + multi-file pooling.** Optical flow (OpenCV) + face-reaction channels; action profile; pool peaks across a *folder* of clips and rank globally ("best 15 bits from tonight" across 80 phone videos). **Gate: catches a silent visual gag; folder-level ranking works.** - **Phase 4 (optional) — highlight reel renderer.** Ordering, pacing, optional music-bed ducking, transitions — a thin renderer atop the detected moments, reusing the clip-render stack. diff --git a/tests/test_saliency.py b/tests/test_saliency.py new file mode 100644 index 0000000..91f857e --- /dev/null +++ b/tests/test_saliency.py @@ -0,0 +1,114 @@ +"""Tests for backend.services.saliency pure signal functions.""" + +import os +import sys +import unittest + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +BACKEND_ROOT = os.path.join(ROOT, "backend") +if BACKEND_ROOT not in sys.path: + sys.path.insert(0, BACKEND_ROOT) + +import numpy as np + +from services import saliency as sal +from services.profiles import get_profile + + +class PickPeaksTests(unittest.TestCase): + def test_finds_local_maxima_above_height(self): + curve = np.array([0, 1, 5, 1, 0, 0, 4, 0, 0, 9, 0.0]) + self.assertEqual(sal.pick_peaks(curve, 2.0, 2), [2, 6, 9]) + + def test_height_filters_small_peaks(self): + curve = np.array([0, 3, 0, 0, 9, 0.0]) + self.assertEqual(sal.pick_peaks(curve, 5.0, 1), [4]) + + def test_min_gap_suppresses_nearby_lower_peak(self): + # two peaks 2 bins apart; min_gap 3 keeps only the taller + curve = np.array([0, 8, 0, 6, 0.0]) + self.assertEqual(sal.pick_peaks(curve, 1.0, 3), [1]) + + def test_empty_curve(self): + self.assertEqual(sal.pick_peaks(np.array([]), 1.0, 2), []) + + +class DilateTests(unittest.TestCase): + def test_spike_spreads_to_neighbors(self): + curve = np.zeros(9) + curve[4] = 1.0 + out = sal._dilate(curve, 2) + self.assertTrue(np.all(out[2:7] == 1.0)) + self.assertEqual(out[1], 0.0) + self.assertEqual(out[7], 0.0) + + def test_zero_radius_is_identity(self): + curve = np.array([0.0, 1.0, 0.0]) + self.assertTrue(np.array_equal(sal._dilate(curve, 0), curve)) + + +class RobustZTests(unittest.TestCase): + def test_constant_array_is_zero(self): + out = sal._robust_z(np.array([5.0, 5.0, 5.0])) + self.assertTrue(np.allclose(out, 0.0)) + + def test_outlier_gets_high_z(self): + out = sal._robust_z(np.array([1.0, 1.0, 1.0, 1.0, 10.0])) + self.assertEqual(int(np.argmax(out)), 4) + self.assertGreater(out[4], 1.0) + + +class FuseChannelsTests(unittest.TestCase): + def test_renormalizes_over_present_channels(self): + # party weights audio_event=0.4, energy=0.2; both present -> peak follows audio_event + channels = { + "energy": np.array([3.0, 2.0, 1.0]), + "audio_event": np.array([0.0, 0.0, 1.0]), + } + fused = sal.fuse_channels(channels, get_profile("party")) + self.assertEqual(int(np.argmax(fused)), 2) + + def test_zero_weight_channel_ignored(self): + # podcast gives motion weight 0, so a motion-only signal must not drive the curve + channels = {"motion": np.array([0.0, 9.0, 0.0])} + fused = sal.fuse_channels(channels, get_profile("podcast")) + self.assertTrue(np.allclose(fused, 0.0)) + + +class WindowForPeakTests(unittest.TestCase): + def setUp(self): + self.energy_flat = np.zeros(200) + self.party = get_profile("party") + + def test_reaction_expands_backwards_from_onset(self): + start, end, is_reaction = sal._window_for_peak( + 100.0, 0.5, self.party, 200.0, self.energy_flat, min_dur=8, max_dur=40 + ) + self.assertTrue(is_reaction) + # lookback 8s before, payoff 2s after + self.assertLess(start, 100.0) + self.assertGreaterEqual(100.0 - start, self.party.reaction_lookback_sec - 0.1) + self.assertLessEqual(end, 100.0 + self.party.reaction_payoff_sec + 0.1) + + def test_non_reaction_is_symmetric(self): + start, end, is_reaction = sal._window_for_peak( + 100.0, 0.0, self.party, 200.0, self.energy_flat, min_dur=8, max_dur=40 + ) + self.assertFalse(is_reaction) + self.assertAlmostEqual((start + end) / 2, 100.0, delta=1.0) + + def test_respects_min_duration(self): + start, end, _ = sal._window_for_peak( + 100.0, 0.5, self.party, 200.0, self.energy_flat, min_dur=12, max_dur=40 + ) + self.assertGreaterEqual(round(end - start, 1), 12.0) + + def test_clamps_to_video_bounds(self): + start, end, _ = sal._window_for_peak( + 2.0, 0.5, self.party, 200.0, self.energy_flat, min_dur=8, max_dur=40 + ) + self.assertGreaterEqual(start, 0.0) + + +if __name__ == "__main__": + unittest.main() From 0be792df4e54d502961f7c3d910044a8f1228037 Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 5 Jul 2026 17:59:47 +0400 Subject: [PATCH 02/13] Expose highlight detection as a backend task (detect_highlights) Register a detect_highlights task handler so the MCP/web backend can run party/ action highlight detection via the python-executor, not just the CLI. Verified end to end through the stdin dispatch. --- backend/main.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/backend/main.py b/backend/main.py index 889ec12..09fe114 100644 --- a/backend/main.py +++ b/backend/main.py @@ -328,6 +328,26 @@ def handle_analyze_energy(task_id: str, params: dict): emit_result(task_id, "success", data=result) +def handle_detect_highlights(task_id: str, params: dict): + """Detect highlight clips from a video's fused signal curve (party/action profiles).""" + from services.saliency import detect_highlights + + video_path = params.get("video_path", "") + if not video_path: + emit_result(task_id, "error", error="video_path is required") + return + + clips = detect_highlights( + video_path=video_path, + profile_name=params.get("profile", "party"), + top_n=int(params.get("top_n", 8)), + min_dur=float(params.get("min_dur", 8.0)), + max_dur=float(params.get("max_dur", 60.0)), + progress_callback=lambda pct, msg: emit_progress(task_id, "detecting", pct, msg), + ) + emit_result(task_id, "success", data={"clips": clips, "count": len(clips)}) + + def handle_detect_encoder(task_id: str, params: dict): """Detect available hardware encoders.""" from services.encoder import get_encoder_info @@ -621,6 +641,7 @@ def handle_run_integration_tool(task_id: str, params: dict): "create_clip": handle_create_clip, "batch_clips": handle_batch_clips, "analyze_energy": handle_analyze_energy, + "detect_highlights": handle_detect_highlights, "pack_transcript": handle_pack_transcript, "detect_encoder": handle_detect_encoder, "presets": handle_presets, From 0e00456b2f7e00b14346d98551f0bf49c9bd8d09 Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 5 Jul 2026 18:01:29 +0400 Subject: [PATCH 03/13] Pool highlight detection across a folder of clips Add detect_highlights_pooled and accept video_paths in the detect_highlights task: run detection over many videos, tag each clip with its source_file, and rank globally reaction-first. Serves the 'best N moments across a folder of party clips' use case. Verified across two clips via the backend dispatch. --- backend/main.py | 25 ++++++++++++++++++------- backend/services/saliency.py | 36 ++++++++++++++++++++++++++++++++++++ tests/test_saliency.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 7 deletions(-) diff --git a/backend/main.py b/backend/main.py index 09fe114..98a45a9 100644 --- a/backend/main.py +++ b/backend/main.py @@ -329,22 +329,33 @@ def handle_analyze_energy(task_id: str, params: dict): def handle_detect_highlights(task_id: str, params: dict): - """Detect highlight clips from a video's fused signal curve (party/action profiles).""" - from services.saliency import detect_highlights + """Detect highlight clips from a video's fused signal curve (party/action profiles). + Accepts a single `video_path`, or a list of `video_paths` to pool and rank + highlights across a whole folder of clips. + """ + from services.saliency import detect_highlights, detect_highlights_pooled + + video_paths = params.get("video_paths") video_path = params.get("video_path", "") - if not video_path: - emit_result(task_id, "error", error="video_path is required") + if not video_paths and not video_path: + emit_result(task_id, "error", error="video_path or video_paths is required") return - clips = detect_highlights( - video_path=video_path, + common = dict( profile_name=params.get("profile", "party"), - top_n=int(params.get("top_n", 8)), min_dur=float(params.get("min_dur", 8.0)), max_dur=float(params.get("max_dur", 60.0)), progress_callback=lambda pct, msg: emit_progress(task_id, "detecting", pct, msg), ) + if video_paths: + clips = detect_highlights_pooled( + video_paths=video_paths, top_n=int(params.get("top_n", 15)), **common + ) + else: + clips = detect_highlights( + video_path=video_path, top_n=int(params.get("top_n", 8)), **common + ) emit_result(task_id, "success", data={"clips": clips, "count": len(clips)}) diff --git a/backend/services/saliency.py b/backend/services/saliency.py index ed300c2..bcedb8c 100644 --- a/backend/services/saliency.py +++ b/backend/services/saliency.py @@ -243,3 +243,39 @@ def rank_key(c): if progress_callback: progress_callback(100, f"Found {len(clips)} highlights") return clips + + +def detect_highlights_pooled( + video_paths: list[str], + profile_name: str = "party", + top_n: int = 15, + min_dur: float = 8.0, + max_dur: float = 60.0, + progress_callback: Optional[Callable] = None, +) -> list[dict]: + """ + Detect highlights across many videos and rank them globally — "the best N bits + from tonight" across a folder of party clips. + + Each returned clip carries a `source_file`. Ranking is reaction-first, then by + score, so a genuine laugh in any file outranks a merely loud moment in another. + """ + pooled: list[dict] = [] + n = len(video_paths) or 1 + for idx, path in enumerate(video_paths): + clips = detect_highlights( + path, profile_name=profile_name, top_n=top_n, min_dur=min_dur, max_dur=max_dur + ) + for c in clips: + c["source_file"] = path + pooled.extend(clips) + if progress_callback: + progress_callback( + int((idx + 1) / n * 100), f"{path}: {len(clips)} highlights" + ) + + pooled.sort( + key=lambda c: (1 if "reaction" in c.get("reasons", []) else 0, c.get("score", 0)), + reverse=True, + ) + return pooled[:top_n] diff --git a/tests/test_saliency.py b/tests/test_saliency.py index 91f857e..8734ec7 100644 --- a/tests/test_saliency.py +++ b/tests/test_saliency.py @@ -110,5 +110,38 @@ def test_clamps_to_video_bounds(self): self.assertGreaterEqual(start, 0.0) +class PooledTests(unittest.TestCase): + def test_pools_across_files_reaction_first_with_source(self): + fake = { + "a.mp4": [ + {"start_second": 10, "end_second": 18, "score": 30.0, "reasons": ["energy_peak"]}, + ], + "b.mp4": [ + {"start_second": 5, "end_second": 15, "score": 12.0, "reasons": ["reaction"]}, + ], + } + orig = sal.detect_highlights + sal.detect_highlights = lambda path, **kw: [dict(c) for c in fake[path]] + try: + pooled = sal.detect_highlights_pooled(["a.mp4", "b.mp4"], top_n=5) + finally: + sal.detect_highlights = orig + self.assertEqual(len(pooled), 2) + # reaction outranks the higher-scored energy peak across files + self.assertEqual(pooled[0]["reasons"], ["reaction"]) + self.assertEqual(pooled[0]["source_file"], "b.mp4") + self.assertTrue(all("source_file" in c for c in pooled)) + + def test_top_n_caps_pool(self): + fake = [{"start_second": i, "end_second": i + 8, "score": float(i), "reasons": ["energy_peak"]} for i in range(10)] + orig = sal.detect_highlights + sal.detect_highlights = lambda path, **kw: [dict(c) for c in fake] + try: + pooled = sal.detect_highlights_pooled(["a.mp4"], top_n=3) + finally: + sal.detect_highlights = orig + self.assertEqual(len(pooled), 3) + + if __name__ == "__main__": unittest.main() From 3f7fa9f334d82503cd337eced108f37f76662a08 Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 5 Jul 2026 18:49:18 +0400 Subject: [PATCH 04/13] Skip transcription for party/action profiles Saliency profiles select on audio/visual signals, not dialogue, so transcribing a long party video is wasted work. Detect the profile before Step 1 and skip transcription entirely (no words/segments needed), relax the no-segments gate, and stop the misleading 'heuristic mode' message when clips are already chosen. Verified end to end: 'process --profile party' skips transcription, detects highlights, and renders clips including a reaction clip. --- backend/cli.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/backend/cli.py b/backend/cli.py index 260e7b7..6f045bd 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -556,11 +556,21 @@ def cmd_process(args): segments = [] result = {} + # Saliency profiles (party/action) select on audio/visual signals, not dialogue, + # so transcribing a long video would be wasted work — skip it entirely. + from services.profiles import get_profile + + content_profile = get_profile(config.get("profile")) + skip_transcript = content_profile.candidate_source == "saliency" and not args.transcript + from services.transcript_packer import ( load_cached_transcript_for_video, save_cached_transcript_for_video, ) + if skip_transcript: + print(f" [1/4] Skipping transcription ({content_profile.name} profile uses audio/visual signals)") + if args.transcript: print(" [1/4] Loading transcript...") with open(args.transcript, "r", encoding="utf-8") as f: @@ -586,7 +596,7 @@ def cmd_process(args): words = parsed["words"] segments = parsed["segments"] print(f" Parsed: {len(segments)} segments, {len(words)} words") - else: + elif not skip_transcript: # Check cache first cached = load_cached_transcript_for_video(video_path) if cached and not config.get("no_cache", False): @@ -665,7 +675,7 @@ def _transcribe_progress(pct, msg): else: print(f" No speaker data (center crop fallback)") - if not segments: + if not segments and not skip_transcript: print(" Error: No transcript segments found.", file=sys.stderr) sys.exit(1) @@ -709,9 +719,6 @@ def _transcribe_progress(pct, msg): # Saliency profiles (party/action) pick moments from the fused laughter/energy # curve rather than the transcript, so they work on footage with no dialogue. - from services.profiles import get_profile - - content_profile = get_profile(config.get("profile")) if content_profile.candidate_source == "saliency" and not clips: from services.saliency import detect_highlights from services.formats import get_format @@ -736,7 +743,9 @@ def _transcribe_progress(pct, msg): from services.claude_suggest import suggest_initial_with_claude, _engine_label, _find_ai_cli ai_path, ai_engine = _find_ai_cli() - if not clips and ai_path and config.get("ai_select", True): + if clips: + pass # already selected (resumed cache or saliency profile) + elif ai_path and config.get("ai_select", True): ai_label = _engine_label(ai_engine) print(f" [3/4] Selecting moments with {ai_label} (PodStack)...") clips = suggest_initial_with_claude( From 939e01f076074031a36717d229f03f0e52bd6b44 Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 5 Jul 2026 19:36:18 +0400 Subject: [PATCH 05/13] Snap highlight boundaries to sentences when a transcript exists Saliency detection now takes optional segments: clip windows are built from whole sentences (never cutting mid-thought) when a transcript is available, falling back to audio-lull snapping only for true no-dialogue footage. The process flow reuses a cached transcript for saliency profiles instead of skipping it, so podcast highlights are sentence-clean. Verified on a real cached 71-min episode: clips land on segment boundaries with coherent text. --- backend/cli.py | 17 ++++++-- backend/services/saliency.py | 81 ++++++++++++++++++++++++++++-------- 2 files changed, 78 insertions(+), 20 deletions(-) diff --git a/backend/cli.py b/backend/cli.py index 6f045bd..c54be32 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -569,7 +569,17 @@ def cmd_process(args): ) if skip_transcript: - print(f" [1/4] Skipping transcription ({content_profile.name} profile uses audio/visual signals)") + # Reuse an existing transcript so highlight boundaries snap to whole sentences; + # only skip transcription outright when there is none (true no-dialogue footage). + cached = load_cached_transcript_for_video(video_path) + if cached and not config.get("no_cache", False): + words = cached["words"] + segments = cached["segments"] + result = cached + skip_transcript = False + print(f" [1/4] Reusing cached transcript ({len(segments)} segments) for sentence-clean cuts") + else: + print(f" [1/4] Skipping transcription ({content_profile.name} profile uses audio/visual signals)") if args.transcript: print(" [1/4] Loading transcript...") @@ -729,8 +739,9 @@ def _transcribe_progress(pct, msg): video_path, profile_name=content_profile.name, top_n=top_n, - min_dur=min(8.0, float(spec.dur_min)), - max_dur=float(spec.dur_max), + min_dur=15.0, + max_dur=min(60.0, float(spec.dur_max)), + segments=segments or None, progress_callback=lambda pct, msg: print(f" {msg}") if msg else None, ) if clips: diff --git a/backend/services/saliency.py b/backend/services/saliency.py index bcedb8c..1fa0ff1 100644 --- a/backend/services/saliency.py +++ b/backend/services/saliency.py @@ -117,6 +117,46 @@ def _snap_to_quiet(target_sec: float, energy_curve: np.ndarray, window_sec: floa return (lo + int(np.argmin(local))) / GRID_HZ +def _seg_index_at(t: float, segments: list[dict]) -> int: + """Index of the segment covering t, else the nearest segment start.""" + for i, s in enumerate(segments): + if s.get("start", 0) <= t <= s.get("end", 0): + return i + return min(range(len(segments)), key=lambda i: abs(segments[i].get("start", 0) - t)) + + +def _sentence_window( + peak_sec: float, + back_sec: float, + fwd_sec: float, + segments: list[dict], + min_dur: float, + max_dur: float, +) -> tuple[float, float]: + """Build a clip out of whole segments (sentences) so it never cuts mid-thought. + + Starts at a sentence boundary, extends back ~back_sec and forward ~fwd_sec (and + enough to clear min_dur), always snapping to segment edges and never exceeding + max_dur. + """ + idx = _seg_index_at(peak_sec, segments) + start = segments[idx].get("start", peak_sec) + end = segments[idx].get("end", peak_sec) + i = idx + while i > 0 and (peak_sec - segments[i - 1]["start"]) <= back_sec and (end - segments[i - 1]["start"]) <= max_dur: + i -= 1 + start = segments[i]["start"] + j = idx + while ( + j < len(segments) - 1 + and (segments[j + 1]["end"] - start) <= max_dur + and ((end - start) < min_dur or (segments[j]["end"] - peak_sec) < fwd_sec) + ): + j += 1 + end = segments[j]["end"] + return start, end + + def _window_for_peak( peak_sec: float, reaction_level: float, @@ -125,27 +165,31 @@ def _window_for_peak( energy_curve: np.ndarray, min_dur: float, max_dur: float, + segments: Optional[list[dict]] = None, ) -> tuple[float, float, bool]: - """Clip window for a peak. A reaction peak expands backwards from the reaction onset.""" + """Clip window for a peak. A reaction peak expands backwards from the reaction onset. + + With a transcript, boundaries snap to whole sentences so each clip is a complete + thought; without one (party footage) they snap to audio lulls instead. + """ is_reaction = reaction_level >= 0.15 - if is_reaction: - # The funny thing happens BEFORE the laugh, so keep the run-up: expand backwards - # from the reaction, snap only the end to a lull, and grow the start (not the - # end) if we're under the minimum so the payoff stays put. - start = max(0.0, peak_sec - profile.reaction_lookback_sec) - end = _snap_to_quiet(min(duration, peak_sec + profile.reaction_payoff_sec), energy_curve) - if end - start < min_dur: - start = max(0.0, end - min_dur) - elif end - start > max_dur: - start = end - max_dur + back = profile.reaction_lookback_sec if is_reaction else min_dur / 2.0 + fwd = profile.reaction_payoff_sec if is_reaction else min_dur / 2.0 + + if segments: + start, end = _sentence_window(peak_sec, back, fwd, segments, min_dur, max_dur) else: - half = min_dur / 2.0 - start = _snap_to_quiet(max(0.0, peak_sec - half), energy_curve) - end = _snap_to_quiet(min(duration, peak_sec + half), energy_curve) + start = _snap_to_quiet(max(0.0, peak_sec - back), energy_curve) + end = _snap_to_quiet(min(duration, peak_sec + fwd), energy_curve) if end - start < min_dur: - end = min(duration, start + min_dur) + if is_reaction: + start = max(0.0, end - min_dur) + else: + end = min(duration, start + min_dur) elif end - start > max_dur: + start = end - max_dur if is_reaction else start end = start + max_dur + return round(max(0.0, start), 1), round(min(duration, end), 1), is_reaction @@ -156,11 +200,14 @@ def detect_highlights( min_dur: float = 8.0, max_dur: float = 60.0, height_z: float = 1.0, + segments: Optional[list[dict]] = None, progress_callback: Optional[Callable] = None, ) -> list[dict]: """ - Generate highlight clips from a video's fused signal curve (no transcript needed). + Generate highlight clips from a video's fused signal curve. + If `segments` (a transcript) is provided, clip boundaries snap to whole sentences + so each clip is a complete thought; without it, boundaries snap to audio lulls. Returns clip dicts compatible with the render pipeline: {title, start_second, end_second, duration, score, reasons, preview}. """ @@ -222,7 +269,7 @@ def rank_key(c): peak_sec = i / GRID_HZ reaction_level = float(reaction_curve[i]) if want_reaction else 0.0 start, end, is_reaction = _window_for_peak( - peak_sec, reaction_level, profile, duration, energy_curve, min_dur, max_dur + peak_sec, reaction_level, profile, duration, energy_curve, min_dur, max_dur, segments ) if end - start < min_dur * 0.75: continue From d06dd3d7123cec0a5cf114467c4084866c770c3f Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 5 Jul 2026 19:59:28 +0400 Subject: [PATCH 06/13] Sentence-clean highlight boundaries + no auto-outro for highlight profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Snap saliency clip boundaries to real sentences (split words on .?! punctuation) so highlights never start or end mid-thought; falls back to segments then audio lulls when word punctuation is absent - Add --no-outro to process and stop auto-appending the outro for saliency (party/action) profiles — highlights are raw moments, not branded shorts --- backend/cli.py | 18 +++++++++++++----- backend/services/saliency.py | 34 +++++++++++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/backend/cli.py b/backend/cli.py index c54be32..2f53fa9 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -470,7 +470,9 @@ def cmd_process(args): else: print(f" Warning: Logo '{args.logo}' not found (checked assets and filesystem)", file=sys.stderr) config["logo_path"] = args.logo # pass through anyway - if args.outro: + if getattr(args, "no_outro", False): + config["outro_path"] = "" + elif args.outro: from services.asset_store import resolve as resolve_asset_outro resolved = resolve_asset_outro(args.outro) if resolved: @@ -479,10 +481,14 @@ def cmd_process(args): print(f" Warning: Outro '{args.outro}' not found (checked assets and filesystem)", file=sys.stderr) config["outro_path"] = args.outro elif not config.get("outro_path"): - from services.asset_store import default_outro - auto_outro = default_outro() - if auto_outro: - config["outro_path"] = auto_outro + # Highlight profiles (party/action) are raw moments, not branded shorts, so they + # skip the auto-outro; the podcast flow keeps it. + from services.profiles import get_profile as _get_profile + if _get_profile(config.get("profile")).candidate_source != "saliency": + from services.asset_store import default_outro + auto_outro = default_outro() + if auto_outro: + config["outro_path"] = auto_outro if args.time_adjust is not None: config["time_adjust"] = args.time_adjust if args.no_energy: @@ -742,6 +748,7 @@ def _transcribe_progress(pct, msg): min_dur=15.0, max_dur=min(60.0, float(spec.dur_max)), segments=segments or None, + words=words or None, progress_callback=lambda pct, msg: print(f" {msg}") if msg else None, ) if clips: @@ -3398,6 +3405,7 @@ def main(): proc.add_argument("--profile", choices=["podcast", "party", "action"], help="Detection profile: podcast (transcript-first, default), party/action (laughter/energy highlights)") proc.add_argument("--logo", help="Logo image (asset name or path)") proc.add_argument("--outro", help="Outro video (asset name or path)") + proc.add_argument("--no-outro", action="store_true", help="Do not append an outro (default for highlight profiles)") proc.add_argument("--time-adjust", type=float, help="Timestamp offset in seconds") proc.add_argument("--no-energy", action="store_true", help="Skip audio energy analysis") proc.add_argument("--no-speakers", action="store_true", help="Skip speaker detection (faster, uses face detection only)") diff --git a/backend/services/saliency.py b/backend/services/saliency.py index 1fa0ff1..3771d59 100644 --- a/backend/services/saliency.py +++ b/backend/services/saliency.py @@ -117,6 +117,31 @@ def _snap_to_quiet(target_sec: float, energy_curve: np.ndarray, window_sec: floa return (lo + int(np.argmin(local))) / GRID_HZ +def sentences_from_words(words: list[dict]) -> list[dict]: + """Group word-level timestamps into sentences on terminal punctuation. + + Whisper *segments* often straddle a sentence boundary ("...was. But then..."), so + snapping to them still starts a clip mid-thought. Splitting on .?! gives real + sentence edges to snap to. + """ + sents: list[dict] = [] + start = None + end = None + for w in words: + text = str(w.get("word", "")).strip() + if not text: + continue + if start is None: + start = w.get("start", 0.0) + end = w.get("end", start) + if text.endswith((".", "?", "!")): + sents.append({"start": start, "end": end}) + start = None + if start is not None: + sents.append({"start": start, "end": end}) + return sents + + def _seg_index_at(t: float, segments: list[dict]) -> int: """Index of the segment covering t, else the nearest segment start.""" for i, s in enumerate(segments): @@ -201,17 +226,20 @@ def detect_highlights( max_dur: float = 60.0, height_z: float = 1.0, segments: Optional[list[dict]] = None, + words: Optional[list[dict]] = None, progress_callback: Optional[Callable] = None, ) -> list[dict]: """ Generate highlight clips from a video's fused signal curve. - If `segments` (a transcript) is provided, clip boundaries snap to whole sentences - so each clip is a complete thought; without it, boundaries snap to audio lulls. + If a transcript is provided, clip boundaries snap to whole sentences so each clip + is a complete thought; `words` (with punctuation) gives true sentence edges and is + preferred over `segments`. Without any transcript, boundaries snap to audio lulls. Returns clip dicts compatible with the render pipeline: {title, start_second, end_second, duration, score, reasons, preview}. """ profile = get_profile(profile_name) + snap_units = sentences_from_words(words) if words else segments if progress_callback: progress_callback(10, "Analyzing audio energy...") @@ -269,7 +297,7 @@ def rank_key(c): peak_sec = i / GRID_HZ reaction_level = float(reaction_curve[i]) if want_reaction else 0.0 start, end, is_reaction = _window_for_peak( - peak_sec, reaction_level, profile, duration, energy_curve, min_dur, max_dur, segments + peak_sec, reaction_level, profile, duration, energy_curve, min_dur, max_dur, snap_units ) if end - start < min_dur * 0.75: continue From a2ad463faf15459578b8080b5b42c50207129d04 Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 5 Jul 2026 22:32:31 +0400 Subject: [PATCH 07/13] Add editable highlight-reel sessions (CLI + MCP) Detect once, then iterate on moments fast. A persisted ReelSession stores the detected moments; editing one (longer/shorter/earlier/later/shift/drop/toggle) re-cuts only that moment straight from the source (~seconds) and rebuilds the reel, with no re-detection or heavy render. - services/reel.py: shared core (session persistence, edit ops, fast ffmpeg re-cut + concat) used by every surface - CLI: 'podcli reel new|show|edit|build' - MCP: 'manage_reel' task handler (new/show/edit/build) - 10 tests covering edit math, drop/toggle, and session round-trip Next: TS MCP tool definition and the web-studio reel panel. --- backend/cli.py | 67 +++++++++++++++ backend/main.py | 53 ++++++++++++ backend/services/reel.py | 177 +++++++++++++++++++++++++++++++++++++++ tests/test_reel.py | 94 +++++++++++++++++++++ 4 files changed, 391 insertions(+) create mode 100644 backend/services/reel.py create mode 100644 tests/test_reel.py diff --git a/backend/cli.py b/backend/cli.py index 2f53fa9..d6a927e 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -396,6 +396,53 @@ def cmd_studio(args): sys.exit(rc) +def cmd_reel(args): + """Create and iterate on a highlights reel — detect once, then edit moments fast.""" + from services.reel import ReelSession, seed_session, edit_moment, build_reel + from services.transcript_packer import compute_cache_hash, load_cached_transcript_for_video + + def _mmss(s): + return f"{int(s // 60)}:{int(s % 60):02d}" + + def _show(session): + for i, m in enumerate(session.moments, 1): + flag = "" if m.enabled else " (disabled)" + print(f" [{i}] {_mmss(m.start)}-{_mmss(m.end)} ({m.duration:.0f}s, {m.why}){flag}") + if m.text: + print(" " + (m.text[:150] + "…" if len(m.text) > 150 else m.text)) + + action = getattr(args, "reel_action", None) + if action == "new": + video = _clean_path(args.video) + sid = compute_cache_hash(video) + out_dir = args.output or os.path.join(os.getcwd(), f"reel_{sid[:8]}") + cached = load_cached_transcript_for_video(video) + words = cached.get("words") if cached else None + print(" Detecting moments (one-time)...") + session = seed_session( + sid, video, out_dir, profile=args.profile or "party", top_n=args.top or 10, + words=words, progress_callback=lambda p, m: print(f" {m}") if m else None, + ) + _show(session) + print(" Building reel...") + reel = build_reel(session) + print(f" ✓ {reel}") + print(f" session: {sid}") + print(f" edit with: podcli reel edit {sid} [secs]") + elif action == "show": + _show(ReelSession.load(args.session)) + elif action == "edit": + session = edit_moment(ReelSession.load(args.session), args.index, args.op, args.seconds) + reel = build_reel(session) + print(f" ✓ rebuilt {reel}") + _show(session) + elif action == "build": + reel = build_reel(ReelSession.load(args.session)) + print(f" ✓ {reel}") + else: + print(" Usage: podcli reel new