From 747b47d99f5c4da135a97b9a693b80b1967efd3c Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 5 Jul 2026 18:49:18 +0400 Subject: [PATCH 01/10] Skip transcription for party/action profiles Saliency profiles select on audio/visual signals, not dialogue, so transcribing a long party video is wasted work. Detect the profile before Step 1 and skip transcription entirely (no words/segments needed), relax the no-segments gate, and stop the misleading 'heuristic mode' message when clips are already chosen. Verified end to end: 'process --profile party' skips transcription, detects highlights, and renders clips including a reaction clip. --- backend/cli.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/backend/cli.py b/backend/cli.py index 1524668..88d13ef 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -548,11 +548,21 @@ def cmd_process(args): segments = [] result = {} + # Saliency profiles (party/action) select on audio/visual signals, not dialogue, + # so transcribing a long video would be wasted work — skip it entirely. + from services.profiles import get_profile + + content_profile = get_profile(config.get("profile")) + skip_transcript = content_profile.candidate_source == "saliency" and not args.transcript + from services.transcript_packer import ( load_cached_transcript_for_video, save_cached_transcript_for_video, ) + if skip_transcript: + print(f" [1/4] Skipping transcription ({content_profile.name} profile uses audio/visual signals)") + if args.transcript: print(" [1/4] Loading transcript...") with open(args.transcript, "r", encoding="utf-8") as f: @@ -578,7 +588,7 @@ def cmd_process(args): words = parsed["words"] segments = parsed["segments"] print(f" Parsed: {len(segments)} segments, {len(words)} words") - else: + elif not skip_transcript: # Check cache first cached = load_cached_transcript_for_video(video_path) if cached and not config.get("no_cache", False): @@ -654,7 +664,7 @@ def _transcribe_progress(pct, msg): else: print(f" No speaker data (center crop fallback)") - if not segments: + if not segments and not skip_transcript: print(" Error: No transcript segments found.", file=sys.stderr) sys.exit(1) @@ -698,9 +708,6 @@ def _transcribe_progress(pct, msg): # Saliency profiles (party/action) pick moments from the fused laughter/energy # curve rather than the transcript, so they work on footage with no dialogue. - from services.profiles import get_profile - - content_profile = get_profile(config.get("profile")) if content_profile.candidate_source == "saliency" and not clips: from services.saliency import detect_highlights from services.formats import get_format @@ -725,7 +732,9 @@ def _transcribe_progress(pct, msg): from services.claude_suggest import suggest_initial_with_claude, _engine_label, _find_ai_cli ai_path, ai_engine = _find_ai_cli() - if not clips and ai_path and config.get("ai_select", True): + if clips: + pass # already selected (resumed cache or saliency profile) + elif ai_path and config.get("ai_select", True): ai_label = _engine_label(ai_engine) print(f" [3/4] Selecting moments with {ai_label} (PodStack)...") clips = suggest_initial_with_claude( From 0d599355a8ff05e53dfa335accebb6ff9571516e Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 5 Jul 2026 19:36:18 +0400 Subject: [PATCH 02/10] Snap highlight boundaries to sentences when a transcript exists Saliency detection now takes optional segments: clip windows are built from whole sentences (never cutting mid-thought) when a transcript is available, falling back to audio-lull snapping only for true no-dialogue footage. The process flow reuses a cached transcript for saliency profiles instead of skipping it, so podcast highlights are sentence-clean. Verified on a real cached 71-min episode: clips land on segment boundaries with coherent text. --- backend/cli.py | 17 ++++++-- backend/services/saliency.py | 81 ++++++++++++++++++++++++++++-------- 2 files changed, 78 insertions(+), 20 deletions(-) diff --git a/backend/cli.py b/backend/cli.py index 88d13ef..b5c4b41 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -561,7 +561,17 @@ def cmd_process(args): ) if skip_transcript: - print(f" [1/4] Skipping transcription ({content_profile.name} profile uses audio/visual signals)") + # Reuse an existing transcript so highlight boundaries snap to whole sentences; + # only skip transcription outright when there is none (true no-dialogue footage). + cached = load_cached_transcript_for_video(video_path) + if cached and not config.get("no_cache", False): + words = cached["words"] + segments = cached["segments"] + result = cached + skip_transcript = False + print(f" [1/4] Reusing cached transcript ({len(segments)} segments) for sentence-clean cuts") + else: + print(f" [1/4] Skipping transcription ({content_profile.name} profile uses audio/visual signals)") if args.transcript: print(" [1/4] Loading transcript...") @@ -718,8 +728,9 @@ def _transcribe_progress(pct, msg): video_path, profile_name=content_profile.name, top_n=top_n, - min_dur=min(8.0, float(spec.dur_min)), - max_dur=float(spec.dur_max), + min_dur=15.0, + max_dur=min(60.0, float(spec.dur_max)), + segments=segments or None, progress_callback=lambda pct, msg: print(f" {msg}") if msg else None, ) if clips: diff --git a/backend/services/saliency.py b/backend/services/saliency.py index bcedb8c..1fa0ff1 100644 --- a/backend/services/saliency.py +++ b/backend/services/saliency.py @@ -117,6 +117,46 @@ def _snap_to_quiet(target_sec: float, energy_curve: np.ndarray, window_sec: floa return (lo + int(np.argmin(local))) / GRID_HZ +def _seg_index_at(t: float, segments: list[dict]) -> int: + """Index of the segment covering t, else the nearest segment start.""" + for i, s in enumerate(segments): + if s.get("start", 0) <= t <= s.get("end", 0): + return i + return min(range(len(segments)), key=lambda i: abs(segments[i].get("start", 0) - t)) + + +def _sentence_window( + peak_sec: float, + back_sec: float, + fwd_sec: float, + segments: list[dict], + min_dur: float, + max_dur: float, +) -> tuple[float, float]: + """Build a clip out of whole segments (sentences) so it never cuts mid-thought. + + Starts at a sentence boundary, extends back ~back_sec and forward ~fwd_sec (and + enough to clear min_dur), always snapping to segment edges and never exceeding + max_dur. + """ + idx = _seg_index_at(peak_sec, segments) + start = segments[idx].get("start", peak_sec) + end = segments[idx].get("end", peak_sec) + i = idx + while i > 0 and (peak_sec - segments[i - 1]["start"]) <= back_sec and (end - segments[i - 1]["start"]) <= max_dur: + i -= 1 + start = segments[i]["start"] + j = idx + while ( + j < len(segments) - 1 + and (segments[j + 1]["end"] - start) <= max_dur + and ((end - start) < min_dur or (segments[j]["end"] - peak_sec) < fwd_sec) + ): + j += 1 + end = segments[j]["end"] + return start, end + + def _window_for_peak( peak_sec: float, reaction_level: float, @@ -125,27 +165,31 @@ def _window_for_peak( energy_curve: np.ndarray, min_dur: float, max_dur: float, + segments: Optional[list[dict]] = None, ) -> tuple[float, float, bool]: - """Clip window for a peak. A reaction peak expands backwards from the reaction onset.""" + """Clip window for a peak. A reaction peak expands backwards from the reaction onset. + + With a transcript, boundaries snap to whole sentences so each clip is a complete + thought; without one (party footage) they snap to audio lulls instead. + """ is_reaction = reaction_level >= 0.15 - if is_reaction: - # The funny thing happens BEFORE the laugh, so keep the run-up: expand backwards - # from the reaction, snap only the end to a lull, and grow the start (not the - # end) if we're under the minimum so the payoff stays put. - start = max(0.0, peak_sec - profile.reaction_lookback_sec) - end = _snap_to_quiet(min(duration, peak_sec + profile.reaction_payoff_sec), energy_curve) - if end - start < min_dur: - start = max(0.0, end - min_dur) - elif end - start > max_dur: - start = end - max_dur + back = profile.reaction_lookback_sec if is_reaction else min_dur / 2.0 + fwd = profile.reaction_payoff_sec if is_reaction else min_dur / 2.0 + + if segments: + start, end = _sentence_window(peak_sec, back, fwd, segments, min_dur, max_dur) else: - half = min_dur / 2.0 - start = _snap_to_quiet(max(0.0, peak_sec - half), energy_curve) - end = _snap_to_quiet(min(duration, peak_sec + half), energy_curve) + start = _snap_to_quiet(max(0.0, peak_sec - back), energy_curve) + end = _snap_to_quiet(min(duration, peak_sec + fwd), energy_curve) if end - start < min_dur: - end = min(duration, start + min_dur) + if is_reaction: + start = max(0.0, end - min_dur) + else: + end = min(duration, start + min_dur) elif end - start > max_dur: + start = end - max_dur if is_reaction else start end = start + max_dur + return round(max(0.0, start), 1), round(min(duration, end), 1), is_reaction @@ -156,11 +200,14 @@ def detect_highlights( min_dur: float = 8.0, max_dur: float = 60.0, height_z: float = 1.0, + segments: Optional[list[dict]] = None, progress_callback: Optional[Callable] = None, ) -> list[dict]: """ - Generate highlight clips from a video's fused signal curve (no transcript needed). + Generate highlight clips from a video's fused signal curve. + If `segments` (a transcript) is provided, clip boundaries snap to whole sentences + so each clip is a complete thought; without it, boundaries snap to audio lulls. Returns clip dicts compatible with the render pipeline: {title, start_second, end_second, duration, score, reasons, preview}. """ @@ -222,7 +269,7 @@ def rank_key(c): peak_sec = i / GRID_HZ reaction_level = float(reaction_curve[i]) if want_reaction else 0.0 start, end, is_reaction = _window_for_peak( - peak_sec, reaction_level, profile, duration, energy_curve, min_dur, max_dur + peak_sec, reaction_level, profile, duration, energy_curve, min_dur, max_dur, segments ) if end - start < min_dur * 0.75: continue From 3d476af69246121a283d603c77ed92279e929c91 Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 5 Jul 2026 19:59:28 +0400 Subject: [PATCH 03/10] Sentence-clean highlight boundaries + no auto-outro for highlight profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Snap saliency clip boundaries to real sentences (split words on .?! punctuation) so highlights never start or end mid-thought; falls back to segments then audio lulls when word punctuation is absent - Add --no-outro to process and stop auto-appending the outro for saliency (party/action) profiles — highlights are raw moments, not branded shorts --- backend/cli.py | 18 +++++++++++++----- backend/services/saliency.py | 34 +++++++++++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/backend/cli.py b/backend/cli.py index b5c4b41..e9f502f 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -462,7 +462,9 @@ def cmd_process(args): else: print(f" Warning: Logo '{args.logo}' not found (checked assets and filesystem)", file=sys.stderr) config["logo_path"] = args.logo # pass through anyway - if args.outro: + if getattr(args, "no_outro", False): + config["outro_path"] = "" + elif args.outro: from services.asset_store import resolve as resolve_asset_outro resolved = resolve_asset_outro(args.outro) if resolved: @@ -471,10 +473,14 @@ def cmd_process(args): print(f" Warning: Outro '{args.outro}' not found (checked assets and filesystem)", file=sys.stderr) config["outro_path"] = args.outro elif not config.get("outro_path"): - from services.asset_store import default_outro - auto_outro = default_outro() - if auto_outro: - config["outro_path"] = auto_outro + # Highlight profiles (party/action) are raw moments, not branded shorts, so they + # skip the auto-outro; the podcast flow keeps it. + from services.profiles import get_profile as _get_profile + if _get_profile(config.get("profile")).candidate_source != "saliency": + from services.asset_store import default_outro + auto_outro = default_outro() + if auto_outro: + config["outro_path"] = auto_outro if args.time_adjust is not None: config["time_adjust"] = args.time_adjust if args.no_energy: @@ -731,6 +737,7 @@ def _transcribe_progress(pct, msg): min_dur=15.0, max_dur=min(60.0, float(spec.dur_max)), segments=segments or None, + words=words or None, progress_callback=lambda pct, msg: print(f" {msg}") if msg else None, ) if clips: @@ -3388,6 +3395,7 @@ def main(): proc.add_argument("--profile", choices=["podcast", "party", "action"], help="Detection profile: podcast (transcript-first, default), party/action (laughter/energy highlights)") proc.add_argument("--logo", help="Logo image (asset name or path)") proc.add_argument("--outro", help="Outro video (asset name or path)") + proc.add_argument("--no-outro", action="store_true", help="Do not append an outro (default for highlight profiles)") proc.add_argument("--time-adjust", type=float, help="Timestamp offset in seconds") proc.add_argument("--no-energy", action="store_true", help="Skip audio energy analysis") proc.add_argument("--no-speakers", action="store_true", help="Skip speaker detection (faster, uses face detection only)") diff --git a/backend/services/saliency.py b/backend/services/saliency.py index 1fa0ff1..3771d59 100644 --- a/backend/services/saliency.py +++ b/backend/services/saliency.py @@ -117,6 +117,31 @@ def _snap_to_quiet(target_sec: float, energy_curve: np.ndarray, window_sec: floa return (lo + int(np.argmin(local))) / GRID_HZ +def sentences_from_words(words: list[dict]) -> list[dict]: + """Group word-level timestamps into sentences on terminal punctuation. + + Whisper *segments* often straddle a sentence boundary ("...was. But then..."), so + snapping to them still starts a clip mid-thought. Splitting on .?! gives real + sentence edges to snap to. + """ + sents: list[dict] = [] + start = None + end = None + for w in words: + text = str(w.get("word", "")).strip() + if not text: + continue + if start is None: + start = w.get("start", 0.0) + end = w.get("end", start) + if text.endswith((".", "?", "!")): + sents.append({"start": start, "end": end}) + start = None + if start is not None: + sents.append({"start": start, "end": end}) + return sents + + def _seg_index_at(t: float, segments: list[dict]) -> int: """Index of the segment covering t, else the nearest segment start.""" for i, s in enumerate(segments): @@ -201,17 +226,20 @@ def detect_highlights( max_dur: float = 60.0, height_z: float = 1.0, segments: Optional[list[dict]] = None, + words: Optional[list[dict]] = None, progress_callback: Optional[Callable] = None, ) -> list[dict]: """ Generate highlight clips from a video's fused signal curve. - If `segments` (a transcript) is provided, clip boundaries snap to whole sentences - so each clip is a complete thought; without it, boundaries snap to audio lulls. + If a transcript is provided, clip boundaries snap to whole sentences so each clip + is a complete thought; `words` (with punctuation) gives true sentence edges and is + preferred over `segments`. Without any transcript, boundaries snap to audio lulls. Returns clip dicts compatible with the render pipeline: {title, start_second, end_second, duration, score, reasons, preview}. """ profile = get_profile(profile_name) + snap_units = sentences_from_words(words) if words else segments if progress_callback: progress_callback(10, "Analyzing audio energy...") @@ -269,7 +297,7 @@ def rank_key(c): peak_sec = i / GRID_HZ reaction_level = float(reaction_curve[i]) if want_reaction else 0.0 start, end, is_reaction = _window_for_peak( - peak_sec, reaction_level, profile, duration, energy_curve, min_dur, max_dur, segments + peak_sec, reaction_level, profile, duration, energy_curve, min_dur, max_dur, snap_units ) if end - start < min_dur * 0.75: continue From 62cf71f29faa765db3dbdacb58e89a3fc43c31be Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 5 Jul 2026 22:32:31 +0400 Subject: [PATCH 04/10] Add editable highlight-reel sessions (CLI + MCP) Detect once, then iterate on moments fast. A persisted ReelSession stores the detected moments; editing one (longer/shorter/earlier/later/shift/drop/toggle) re-cuts only that moment straight from the source (~seconds) and rebuilds the reel, with no re-detection or heavy render. - services/reel.py: shared core (session persistence, edit ops, fast ffmpeg re-cut + concat) used by every surface - CLI: 'podcli reel new|show|edit|build' - MCP: 'manage_reel' task handler (new/show/edit/build) - 10 tests covering edit math, drop/toggle, and session round-trip Next: TS MCP tool definition and the web-studio reel panel. --- backend/cli.py | 67 +++++++++++++++ backend/main.py | 53 ++++++++++++ backend/services/reel.py | 177 +++++++++++++++++++++++++++++++++++++++ tests/test_reel.py | 94 +++++++++++++++++++++ 4 files changed, 391 insertions(+) create mode 100644 backend/services/reel.py create mode 100644 tests/test_reel.py diff --git a/backend/cli.py b/backend/cli.py index e9f502f..2a1e7a7 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -390,6 +390,53 @@ def cmd_studio(args): sys.exit(rc) +def cmd_reel(args): + """Create and iterate on a highlights reel — detect once, then edit moments fast.""" + from services.reel import ReelSession, seed_session, edit_moment, build_reel + from services.transcript_packer import compute_cache_hash, load_cached_transcript_for_video + + def _mmss(s): + return f"{int(s // 60)}:{int(s % 60):02d}" + + def _show(session): + for i, m in enumerate(session.moments, 1): + flag = "" if m.enabled else " (disabled)" + print(f" [{i}] {_mmss(m.start)}-{_mmss(m.end)} ({m.duration:.0f}s, {m.why}){flag}") + if m.text: + print(" " + (m.text[:150] + "…" if len(m.text) > 150 else m.text)) + + action = getattr(args, "reel_action", None) + if action == "new": + video = _clean_path(args.video) + sid = compute_cache_hash(video) + out_dir = args.output or os.path.join(os.getcwd(), f"reel_{sid[:8]}") + cached = load_cached_transcript_for_video(video) + words = cached.get("words") if cached else None + print(" Detecting moments (one-time)...") + session = seed_session( + sid, video, out_dir, profile=args.profile or "party", top_n=args.top or 10, + words=words, progress_callback=lambda p, m: print(f" {m}") if m else None, + ) + _show(session) + print(" Building reel...") + reel = build_reel(session) + print(f" ✓ {reel}") + print(f" session: {sid}") + print(f" edit with: podcli reel edit {sid} [secs]") + elif action == "show": + _show(ReelSession.load(args.session)) + elif action == "edit": + session = edit_moment(ReelSession.load(args.session), args.index, args.op, args.seconds) + reel = build_reel(session) + print(f" ✓ rebuilt {reel}") + _show(session) + elif action == "build": + reel = build_reel(ReelSession.load(args.session)) + print(f" ✓ {reel}") + else: + print(" Usage: podcli reel new