diff --git a/backend/cli.py b/backend/cli.py index 5d21292..1524668 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -444,6 +444,8 @@ def cmd_process(args): config["crop_strategy"] = args.crop if getattr(args, "format", None): config["format"] = args.format + if getattr(args, "profile", None): + config["profile"] = args.profile if getattr(args, "thumbnails", None) is not None: config["generate_thumbnails"] = args.thumbnails if args.top: @@ -694,6 +696,31 @@ def _transcribe_progress(pct, msg): clips = resumed resumed_from_session = True + # Saliency profiles (party/action) pick moments from the fused laughter/energy + # curve rather than the transcript, so they work on footage with no dialogue. + from services.profiles import get_profile + + content_profile = get_profile(config.get("profile")) + if content_profile.candidate_source == "saliency" and not clips: + from services.saliency import detect_highlights + from services.formats import get_format + + spec = get_format(config.get("format", "vertical")) + print(f" [3/4] Detecting {content_profile.name} highlights (laughter + energy)...") + clips = detect_highlights( + video_path, + profile_name=content_profile.name, + top_n=top_n, + min_dur=min(8.0, float(spec.dur_min)), + max_dur=float(spec.dur_max), + progress_callback=lambda pct, msg: print(f" {msg}") if msg else None, + ) + if clips: + print(f" ✓ {len(clips)} highlights found") + _save_suggestions_session(cache_hash, top_n, "saliency", clips, selection_sig) + else: + print(" ⚠ No highlights found, falling back to transcript selection") + # Try an AI CLI first (uses PodStack knowledge base for intelligent selection) from services.claude_suggest import suggest_initial_with_claude, _engine_label, _find_ai_cli @@ -3338,6 +3365,7 @@ def main(): proc.add_argument("--caption-style", choices=["branded", "hormozi", "karaoke", "subtle"]) proc.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut"]) proc.add_argument("--format", choices=["vertical", "horizontal", "square"], help="Output aspect ratio (default: vertical)") + proc.add_argument("--profile", choices=["podcast", "party", "action"], help="Detection profile: podcast (transcript-first, default), party/action (laughter/energy highlights)") proc.add_argument("--logo", help="Logo image (asset name or path)") proc.add_argument("--outro", help="Outro video (asset name or path)") proc.add_argument("--time-adjust", type=float, help="Timestamp offset in seconds") diff --git a/backend/main.py b/backend/main.py index 9355357..03a4d93 100644 --- a/backend/main.py +++ b/backend/main.py @@ -308,6 +308,37 @@ def handle_analyze_energy(task_id: str, params: dict): emit_result(task_id, "success", data=result) +def handle_detect_highlights(task_id: str, params: dict): + """Detect highlight clips from a video's fused signal curve (party/action profiles). + + Accepts a single `video_path`, or a list of `video_paths` to pool and rank + highlights across a whole folder of clips. + """ + from services.saliency import detect_highlights, detect_highlights_pooled + + video_paths = params.get("video_paths") + video_path = params.get("video_path", "") + if not video_paths and not video_path: + emit_result(task_id, "error", error="video_path or video_paths is required") + return + + common = dict( + profile_name=params.get("profile", "party"), + min_dur=float(params.get("min_dur", 8.0)), + max_dur=float(params.get("max_dur", 60.0)), + progress_callback=lambda pct, msg: emit_progress(task_id, "detecting", pct, msg), + ) + if video_paths: + clips = detect_highlights_pooled( + video_paths=video_paths, top_n=int(params.get("top_n", 15)), **common + ) + else: + clips = detect_highlights( + video_path=video_path, top_n=int(params.get("top_n", 8)), **common + ) + emit_result(task_id, "success", data={"clips": clips, "count": len(clips)}) + + def handle_detect_encoder(task_id: str, params: dict): """Detect available hardware encoders.""" from services.encoder import get_encoder_info @@ -601,6 +632,7 @@ def handle_run_integration_tool(task_id: str, params: dict): "create_clip": handle_create_clip, "batch_clips": handle_batch_clips, "analyze_energy": handle_analyze_energy, + "detect_highlights": handle_detect_highlights, "pack_transcript": handle_pack_transcript, "detect_encoder": handle_detect_encoder, "presets": handle_presets, diff --git a/backend/services/saliency.py b/backend/services/saliency.py new file mode 100644 index 0000000..bcedb8c --- /dev/null +++ b/backend/services/saliency.py @@ -0,0 +1,281 @@ +"""Multi-signal saliency engine — fuse channels, peak-pick, expand reactions, snap. + +For profiles whose candidate source is "saliency" (party, action), moments are +generated from a fused interestingness curve instead of from an LLM reading the +transcript. This is what lets podcli auto-cut highlights from footage with no useful +transcript (party videos, action). + +Each channel is normalized against THIS video's own distribution (never a global +scale) so different recordings, rooms and mic levels compare fairly, then combined by +the active profile's weights. Peaks on the fused curve become candidate clips; a peak +driven by a laugh or cheer is expanded backwards to capture the moment that caused the +reaction, since the funny thing happens just before people react to it. +""" + +from typing import Optional, Callable + +import numpy as np + +from services.profiles import get_profile, ContentProfile +from services.audio_analyzer import extract_audio_energy +from services.audio_events import extract_audio_events, is_available as audio_events_available + +GRID_HZ = 1.0 # common time grid; energy is per-second, so 1 Hz is the natural rate + + +def _robust_z(x: np.ndarray) -> np.ndarray: + """Median/MAD normalization — resistant to the heavy tails of RMS and reactions.""" + if x.size == 0: + return x + med = np.median(x) + mad = np.median(np.abs(x - med)) + scale = 1.4826 * mad if mad > 1e-9 else (np.std(x) or 1.0) + return (x - med) / scale + + +def _energy_curve(energy_data: list[dict], n_bins: int) -> np.ndarray: + """Per-second loudness onto the grid; silence (<= -60 dB) floored so it doesn't win.""" + curve = np.full(n_bins, -60.0) + for e in energy_data: + b = int(e["time"] * GRID_HZ) + if 0 <= b < n_bins: + curve[b] = max(curve[b], e.get("rms_db", -60.0)) + return curve + + +def _reaction_curve(events_data: list[dict], n_bins: int) -> np.ndarray: + """Peak reaction (laugh/cheer/scream) per grid bin.""" + curve = np.zeros(n_bins) + for e in events_data: + b = int(e["time"] * GRID_HZ) + if 0 <= b < n_bins: + level = max(e.get("laughter", 0), e.get("cheering", 0), e.get("screaming", 0)) + curve[b] = max(curve[b], level) + return curve + + +def _dilate(curve: np.ndarray, radius_bins: int) -> np.ndarray: + """Spread each spike to its neighbors (grayscale dilation) so a brief, narrow + reaction still aligns with, and can win, the fused peak near it.""" + if radius_bins <= 0 or curve.size == 0: + return curve + out = curve.copy() + for r in range(1, radius_bins + 1): + out[r:] = np.maximum(out[r:], curve[:-r]) + out[:-r] = np.maximum(out[:-r], curve[r:]) + return out + + +def fuse_channels(channels: dict[str, np.ndarray], profile: ContentProfile) -> np.ndarray: + """Weighted sum of per-video-normalized channels, weights renormalized over what exists.""" + present = {k: v for k, v in channels.items() if profile.channel_weights.get(k, 0) > 0 and v.size} + if not present: + return np.zeros(next(iter(channels.values())).size if channels else 0) + total_w = sum(profile.channel_weights[k] for k in present) + fused = None + for k, curve in present.items(): + w = profile.channel_weights[k] / total_w + contrib = w * _robust_z(curve) + fused = contrib if fused is None else fused + contrib + return fused + + +def pick_peaks(curve: np.ndarray, height: float, min_gap_bins: int) -> list[int]: + """Local maxima above `height`, then greedy non-maximum suppression by min gap. + + Peaks are taken in descending value order and a lower peak is dropped if it falls + within min_gap_bins of an already-chosen higher one (1-D NMS). + """ + if curve.size == 0: + return [] + candidates = [ + i for i in range(1, len(curve) - 1) + if curve[i] >= curve[i - 1] and curve[i] >= curve[i + 1] and curve[i] >= height + ] + if curve[0] >= height and (len(curve) == 1 or curve[0] > curve[1]): + candidates.append(0) + if len(curve) > 1 and curve[-1] >= height and curve[-1] > curve[-2]: + candidates.append(len(curve) - 1) + candidates.sort(key=lambda i: curve[i], reverse=True) + chosen: list[int] = [] + for i in candidates: + if all(abs(i - j) >= min_gap_bins for j in chosen): + chosen.append(i) + return sorted(chosen) + + +def _snap_to_quiet(target_sec: float, energy_curve: np.ndarray, window_sec: float = 1.5) -> float: + """Nudge a boundary to the quietest second nearby, so cuts land in a lull not mid-action.""" + if energy_curve.size == 0: + return target_sec + center = int(round(target_sec * GRID_HZ)) + lo = max(0, center - int(window_sec * GRID_HZ)) + hi = min(len(energy_curve), center + int(window_sec * GRID_HZ) + 1) + if lo >= hi: + return target_sec + local = energy_curve[lo:hi] + return (lo + int(np.argmin(local))) / GRID_HZ + + +def _window_for_peak( + peak_sec: float, + reaction_level: float, + profile: ContentProfile, + duration: float, + energy_curve: np.ndarray, + min_dur: float, + max_dur: float, +) -> tuple[float, float, bool]: + """Clip window for a peak. A reaction peak expands backwards from the reaction onset.""" + is_reaction = reaction_level >= 0.15 + if is_reaction: + # The funny thing happens BEFORE the laugh, so keep the run-up: expand backwards + # from the reaction, snap only the end to a lull, and grow the start (not the + # end) if we're under the minimum so the payoff stays put. + start = max(0.0, peak_sec - profile.reaction_lookback_sec) + end = _snap_to_quiet(min(duration, peak_sec + profile.reaction_payoff_sec), energy_curve) + if end - start < min_dur: + start = max(0.0, end - min_dur) + elif end - start > max_dur: + start = end - max_dur + else: + half = min_dur / 2.0 + start = _snap_to_quiet(max(0.0, peak_sec - half), energy_curve) + end = _snap_to_quiet(min(duration, peak_sec + half), energy_curve) + if end - start < min_dur: + end = min(duration, start + min_dur) + elif end - start > max_dur: + end = start + max_dur + return round(max(0.0, start), 1), round(min(duration, end), 1), is_reaction + + +def detect_highlights( + video_path: str, + profile_name: str = "party", + top_n: int = 8, + min_dur: float = 8.0, + max_dur: float = 60.0, + height_z: float = 1.0, + progress_callback: Optional[Callable] = None, +) -> list[dict]: + """ + Generate highlight clips from a video's fused signal curve (no transcript needed). + + Returns clip dicts compatible with the render pipeline: + {title, start_second, end_second, duration, score, reasons, preview}. + """ + profile = get_profile(profile_name) + + if progress_callback: + progress_callback(10, "Analyzing audio energy...") + energy_data = extract_audio_energy(video_path) + + events_data = [] + if audio_events_available(): + if progress_callback: + progress_callback(40, "Detecting laughter and reactions...") + events_data = extract_audio_events(video_path) + + last_times = [e["time"] for e in energy_data] + [e["time"] for e in events_data] + if not last_times: + return [] + duration = max(last_times) + 1.0 + n_bins = int(duration * GRID_HZ) + 1 + + energy_curve = _energy_curve(energy_data, n_bins) + # Dilate reactions by ~2s so a single-frame laugh isn't suppressed by a louder + # energy neighbor and so the fused peak lands on the reaction, not next to it. + reaction_curve = _dilate(_reaction_curve(events_data, n_bins), int(2 * GRID_HZ)) + + fused = fuse_channels( + {"energy": energy_curve, "audio_event": reaction_curve}, profile + ) + if fused.size == 0: + return [] + + if progress_callback: + progress_callback(70, "Selecting highlight moments...") + min_gap_bins = max(1, int(profile.peak_min_gap_sec * GRID_HZ)) + + # Reaction moments are primary candidates — a detected laugh/cheer is almost always + # worth a clip regardless of loudness, so they aren't made to out-compete energy in + # the blended curve. Energy peaks then fill the rest, minus any that collide with a + # reaction. Reaction score is offset above energy so reactions rank first. + reaction_peaks = pick_peaks(reaction_curve, 0.15, min_gap_bins) + energy_peaks = pick_peaks(fused, height_z, min_gap_bins) + + candidates = [(i, float(reaction_curve[i]), True) for i in reaction_peaks] + reaction_bins = {i for i in reaction_peaks} + for i in energy_peaks: + if all(abs(i - j) >= min_gap_bins for j in reaction_bins): + candidates.append((i, float(fused[i]), False)) + + def rank_key(c): + i, val, is_reaction = c + return (1 if is_reaction else 0, val) + + candidates.sort(key=rank_key, reverse=True) + candidates = candidates[:top_n] + + clips = [] + for i, val, want_reaction in candidates: + peak_sec = i / GRID_HZ + reaction_level = float(reaction_curve[i]) if want_reaction else 0.0 + start, end, is_reaction = _window_for_peak( + peak_sec, reaction_level, profile, duration, energy_curve, min_dur, max_dur + ) + if end - start < min_dur * 0.75: + continue + kind = "laugh/cheer" if is_reaction else "high energy" + score = round(10.0 + reaction_level * 10.0, 2) if is_reaction else round(float(val), 2) + clips.append({ + "title": f"Highlight ({kind}) at {int(peak_sec // 60):d}:{int(peak_sec % 60):02d}", + "start_second": start, + "end_second": end, + "duration": round(end - start), + "score": score, + "reasons": ["reaction"] if is_reaction else ["energy_peak"], + "preview": "", + "content_type": "highlight", + }) + + clips.sort(key=lambda c: c["start_second"]) + if progress_callback: + progress_callback(100, f"Found {len(clips)} highlights") + return clips + + +def detect_highlights_pooled( + video_paths: list[str], + profile_name: str = "party", + top_n: int = 15, + min_dur: float = 8.0, + max_dur: float = 60.0, + progress_callback: Optional[Callable] = None, +) -> list[dict]: + """ + Detect highlights across many videos and rank them globally — "the best N bits + from tonight" across a folder of party clips. + + Each returned clip carries a `source_file`. Ranking is reaction-first, then by + score, so a genuine laugh in any file outranks a merely loud moment in another. + """ + pooled: list[dict] = [] + n = len(video_paths) or 1 + for idx, path in enumerate(video_paths): + clips = detect_highlights( + path, profile_name=profile_name, top_n=top_n, min_dur=min_dur, max_dur=max_dur + ) + for c in clips: + c["source_file"] = path + pooled.extend(clips) + if progress_callback: + progress_callback( + int((idx + 1) / n * 100), f"{path}: {len(clips)} highlights" + ) + + pooled.sort( + key=lambda c: (1 if "reaction" in c.get("reasons", []) else 0, c.get("score", 0)), + reverse=True, + ) + return pooled[:top_n] diff --git a/plans/moment-detection.md b/plans/moment-detection.md index 4665bd6..23f7aec 100644 --- a/plans/moment-detection.md +++ b/plans/moment-detection.md @@ -95,9 +95,9 @@ A new `profile` param threads the **same ~12 hops the `format` field did**: `sug ## Phasing -- **Phase 0 — profile scaffolding, zero behavior change.** Add `ContentProfile` abstraction; thread the `profile` param through the ~12 hops; `default = podcast` reproduces current selection exactly. **Gate: existing test suite green; same clips out for a fixed transcript.** -- **Phase 1 — audio-event channel (the isolated valuable core).** YAMNet-ONNX laughter/cheer/applause/scream computed in the detect-once hub. Feeds podcast ranking as a labeled signal (laughs already spike energy; now they're *named*) and lays the party foundation. **Gate: laughter timestamps validated on a sample clip; podcast output unchanged unless the channel is given weight.** -- **Phase 2 — fusion engine + saliency candidate source + party profile (audio-only).** `saliency.py` fusion + numpy peak-pick + reaction-expand (8 s) + boundary-snap. Party profile = energy + audio_event + prosody, no transcript, no motion. **Party videos auto-clip end to end. Gate: demo on real party footage.** +- **Phase 0 — profile scaffolding, zero behavior change.** [DONE, PR #43 / Phase 2 branch] `ContentProfile` abstraction added (`profiles.py`); `profile` param threaded through the CLI (`--profile`, config, selection signature). Python-side default `podcast` reproduces current selection. TS-side threading (MCP/web `profile` param) still open — see below. +- **Phase 1 — audio-event channel (the isolated valuable core).** [DONE, PR #43] YAMNet-ONNX laughter/cheer/applause/scream (`audio_events.py`) computed in the detect-once hub; surfaced in the packed transcript and CLI heuristic. Podcast output unchanged unless the channel is weighted. Validated on 32 real clips. +- **Phase 2 — fusion engine + saliency candidate source + party profile (audio-only).** [DONE, this branch] `saliency.py`: per-video robust-z normalization, weighted fusion, numpy peak-pick (NMS), reaction dilation, reaction-first candidate generation, 8 s backward expansion, quiet-point boundary snap. Wired to `--profile party|action` in `podcli process`. Deterministic on clean input; reactions detected with correct run-up windows. **Gate remaining: tune thresholds/weights on real party footage (synthetic podcast-clip concat is not representative).** - **Phase 3 — visual channels + action profile + multi-file pooling.** Optical flow (OpenCV) + face-reaction channels; action profile; pool peaks across a *folder* of clips and rank globally ("best 15 bits from tonight" across 80 phone videos). **Gate: catches a silent visual gag; folder-level ranking works.** - **Phase 4 (optional) — highlight reel renderer.** Ordering, pacing, optional music-bed ducking, transitions — a thin renderer atop the detected moments, reusing the clip-render stack. diff --git a/tests/test_saliency.py b/tests/test_saliency.py new file mode 100644 index 0000000..8734ec7 --- /dev/null +++ b/tests/test_saliency.py @@ -0,0 +1,147 @@ +"""Tests for backend.services.saliency pure signal functions.""" + +import os +import sys +import unittest + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +BACKEND_ROOT = os.path.join(ROOT, "backend") +if BACKEND_ROOT not in sys.path: + sys.path.insert(0, BACKEND_ROOT) + +import numpy as np + +from services import saliency as sal +from services.profiles import get_profile + + +class PickPeaksTests(unittest.TestCase): + def test_finds_local_maxima_above_height(self): + curve = np.array([0, 1, 5, 1, 0, 0, 4, 0, 0, 9, 0.0]) + self.assertEqual(sal.pick_peaks(curve, 2.0, 2), [2, 6, 9]) + + def test_height_filters_small_peaks(self): + curve = np.array([0, 3, 0, 0, 9, 0.0]) + self.assertEqual(sal.pick_peaks(curve, 5.0, 1), [4]) + + def test_min_gap_suppresses_nearby_lower_peak(self): + # two peaks 2 bins apart; min_gap 3 keeps only the taller + curve = np.array([0, 8, 0, 6, 0.0]) + self.assertEqual(sal.pick_peaks(curve, 1.0, 3), [1]) + + def test_empty_curve(self): + self.assertEqual(sal.pick_peaks(np.array([]), 1.0, 2), []) + + +class DilateTests(unittest.TestCase): + def test_spike_spreads_to_neighbors(self): + curve = np.zeros(9) + curve[4] = 1.0 + out = sal._dilate(curve, 2) + self.assertTrue(np.all(out[2:7] == 1.0)) + self.assertEqual(out[1], 0.0) + self.assertEqual(out[7], 0.0) + + def test_zero_radius_is_identity(self): + curve = np.array([0.0, 1.0, 0.0]) + self.assertTrue(np.array_equal(sal._dilate(curve, 0), curve)) + + +class RobustZTests(unittest.TestCase): + def test_constant_array_is_zero(self): + out = sal._robust_z(np.array([5.0, 5.0, 5.0])) + self.assertTrue(np.allclose(out, 0.0)) + + def test_outlier_gets_high_z(self): + out = sal._robust_z(np.array([1.0, 1.0, 1.0, 1.0, 10.0])) + self.assertEqual(int(np.argmax(out)), 4) + self.assertGreater(out[4], 1.0) + + +class FuseChannelsTests(unittest.TestCase): + def test_renormalizes_over_present_channels(self): + # party weights audio_event=0.4, energy=0.2; both present -> peak follows audio_event + channels = { + "energy": np.array([3.0, 2.0, 1.0]), + "audio_event": np.array([0.0, 0.0, 1.0]), + } + fused = sal.fuse_channels(channels, get_profile("party")) + self.assertEqual(int(np.argmax(fused)), 2) + + def test_zero_weight_channel_ignored(self): + # podcast gives motion weight 0, so a motion-only signal must not drive the curve + channels = {"motion": np.array([0.0, 9.0, 0.0])} + fused = sal.fuse_channels(channels, get_profile("podcast")) + self.assertTrue(np.allclose(fused, 0.0)) + + +class WindowForPeakTests(unittest.TestCase): + def setUp(self): + self.energy_flat = np.zeros(200) + self.party = get_profile("party") + + def test_reaction_expands_backwards_from_onset(self): + start, end, is_reaction = sal._window_for_peak( + 100.0, 0.5, self.party, 200.0, self.energy_flat, min_dur=8, max_dur=40 + ) + self.assertTrue(is_reaction) + # lookback 8s before, payoff 2s after + self.assertLess(start, 100.0) + self.assertGreaterEqual(100.0 - start, self.party.reaction_lookback_sec - 0.1) + self.assertLessEqual(end, 100.0 + self.party.reaction_payoff_sec + 0.1) + + def test_non_reaction_is_symmetric(self): + start, end, is_reaction = sal._window_for_peak( + 100.0, 0.0, self.party, 200.0, self.energy_flat, min_dur=8, max_dur=40 + ) + self.assertFalse(is_reaction) + self.assertAlmostEqual((start + end) / 2, 100.0, delta=1.0) + + def test_respects_min_duration(self): + start, end, _ = sal._window_for_peak( + 100.0, 0.5, self.party, 200.0, self.energy_flat, min_dur=12, max_dur=40 + ) + self.assertGreaterEqual(round(end - start, 1), 12.0) + + def test_clamps_to_video_bounds(self): + start, end, _ = sal._window_for_peak( + 2.0, 0.5, self.party, 200.0, self.energy_flat, min_dur=8, max_dur=40 + ) + self.assertGreaterEqual(start, 0.0) + + +class PooledTests(unittest.TestCase): + def test_pools_across_files_reaction_first_with_source(self): + fake = { + "a.mp4": [ + {"start_second": 10, "end_second": 18, "score": 30.0, "reasons": ["energy_peak"]}, + ], + "b.mp4": [ + {"start_second": 5, "end_second": 15, "score": 12.0, "reasons": ["reaction"]}, + ], + } + orig = sal.detect_highlights + sal.detect_highlights = lambda path, **kw: [dict(c) for c in fake[path]] + try: + pooled = sal.detect_highlights_pooled(["a.mp4", "b.mp4"], top_n=5) + finally: + sal.detect_highlights = orig + self.assertEqual(len(pooled), 2) + # reaction outranks the higher-scored energy peak across files + self.assertEqual(pooled[0]["reasons"], ["reaction"]) + self.assertEqual(pooled[0]["source_file"], "b.mp4") + self.assertTrue(all("source_file" in c for c in pooled)) + + def test_top_n_caps_pool(self): + fake = [{"start_second": i, "end_second": i + 8, "score": float(i), "reasons": ["energy_peak"]} for i in range(10)] + orig = sal.detect_highlights + sal.detect_highlights = lambda path, **kw: [dict(c) for c in fake] + try: + pooled = sal.detect_highlights_pooled(["a.mp4"], top_n=3) + finally: + sal.detect_highlights = orig + self.assertEqual(len(pooled), 3) + + +if __name__ == "__main__": + unittest.main()