diff --git a/backend/cli.py b/backend/cli.py index f5c3f78..5d21292 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -395,6 +395,7 @@ def cmd_process(args): from services.clip_generator import generate_clip from services.transcript_parser import parse_speaker_transcript from services.audio_analyzer import get_energy_profile + from services.audio_events import get_event_profile, is_available as audio_events_available from services.encoder import get_encoder_info from presets import get_preset, DEFAULT_PRESET, MIN_CLIP_DURATION, MAX_CLIP_DURATION, TARGET_CLIP_DURATION_MIN, TARGET_CLIP_DURATION_MAX @@ -657,6 +658,7 @@ def _transcribe_progress(pct, msg): # ── Step 2: Analyze audio energy ── energy_scores = None + reaction_scores = None if config.get("energy_boost", True): print(" [2/4] Analyzing audio energy...") try: @@ -665,6 +667,15 @@ def _transcribe_progress(pct, msg): print(f" {len(profile['peak_times'])} peak moments found") except Exception as e: print(f" Skipped (error: {e})") + if audio_events_available(): + try: + reactions = get_event_profile(video_path, segments) + reaction_scores = reactions["segment_scores"] + n = len(reactions["reaction_times"]) + if n: + print(f" {n} laughter/reaction moments found") + except Exception as e: + print(f" Reactions skipped (error: {e})") else: print(" [2/4] Audio analysis skipped (--no-energy)") @@ -712,6 +723,7 @@ def _transcribe_progress(pct, msg): clips = _suggest_clips( segments=segments, energy_scores=energy_scores, + reaction_scores=reaction_scores, top_n=top_n, min_dur=config.get("min_clip_duration", MIN_CLIP_DURATION), max_dur=config.get("max_clip_duration", MAX_CLIP_DURATION), @@ -1607,6 +1619,7 @@ def _rerender_clip(r): def _suggest_clips( segments: list, energy_scores: list | None = None, + reaction_scores: list | None = None, top_n: int = 5, min_dur: float = MIN_CLIP_DURATION, max_dur: float = MAX_CLIP_DURATION, @@ -1804,6 +1817,15 @@ def _find_sentence_boundary_end(segs, idx, max_lookahead=3): if max_e > 7: reasons.append("high_energy") + # ── 6b. Laughter / reactions (0-6 pts) ── + if reaction_scores: + seg_reactions = reaction_scores[snap_start : snap_end + 1] + if seg_reactions: + max_r = max(seg_reactions) + score += min(max_r, 6) + if max_r > 3: + reasons.append("laughter") + # ── 7. Density check — penalize sparse/rambling segments ── words_per_sec = len(text.split()) / max(dur, 1) if words_per_sec < 1.5: diff --git a/backend/main.py b/backend/main.py index 468669a..9355357 100644 --- a/backend/main.py +++ b/backend/main.py @@ -87,11 +87,19 @@ def handle_transcribe(task_id: str, params: dict): except Exception: pass # energy is a nice-to-have + events_data = None + try: + from services.audio_events import extract_audio_events + events_data = extract_audio_events(file_path) + except Exception: + pass # reactions are a nice-to-have + packed_path, packed_md = write_packed( result, cache_hash, source_label=os.path.basename(file_path), energy_data=energy_data, + events_data=events_data, ) result["packed_path"] = packed_path result["packed_size_bytes"] = len(packed_md.encode("utf-8")) @@ -253,18 +261,27 @@ def handle_pack_transcript(task_id: str, params: dict): return energy_data = params.get("energy_data") - if energy_data is None and params.get("file_path"): - try: - from services.audio_analyzer import extract_audio_energy - energy_data = extract_audio_energy(params["file_path"]) - except Exception: - pass + events_data = params.get("events_data") + if params.get("file_path"): + if energy_data is None: + try: + from services.audio_analyzer import extract_audio_energy + energy_data = extract_audio_energy(params["file_path"]) + except Exception: + pass + if events_data is None: + try: + from services.audio_events import extract_audio_events + events_data = extract_audio_events(params["file_path"]) + except Exception: + pass path, md = write_packed( transcript, cache_hash, source_label=params.get("source_label"), energy_data=energy_data, + events_data=events_data, ) emit_result(task_id, "success", data={ "packed_path": path, diff --git a/backend/models/yamnet.onnx b/backend/models/yamnet.onnx new file mode 100644 index 0000000..a799a1f Binary files /dev/null and b/backend/models/yamnet.onnx differ diff --git a/backend/models/yamnet_class_map.csv b/backend/models/yamnet_class_map.csv new file mode 100644 index 0000000..9c07d81 --- /dev/null +++ b/backend/models/yamnet_class_map.csv @@ -0,0 +1,522 @@ +index,mid,display_name +0,/m/09x0r,Speech +1,/m/0ytgt,"Child speech, kid speaking" +2,/m/01h8n0,Conversation +3,/m/02qldy,"Narration, monologue" +4,/m/0261r1,Babbling +5,/m/0brhx,Speech synthesizer +6,/m/07p6fty,Shout +7,/m/07q4ntr,Bellow +8,/m/07rwj3x,Whoop +9,/m/07sr1lc,Yell +10,/t/dd00135,Children shouting +11,/m/03qc9zr,Screaming +12,/m/02rtxlg,Whispering +13,/m/01j3sz,Laughter +14,/t/dd00001,Baby laughter +15,/m/07r660_,Giggle +16,/m/07s04w4,Snicker +17,/m/07sq110,Belly laugh +18,/m/07rgt08,"Chuckle, chortle" +19,/m/0463cq4,"Crying, sobbing" +20,/t/dd00002,"Baby cry, infant cry" +21,/m/07qz6j3,Whimper +22,/m/07qw_06,"Wail, moan" +23,/m/07plz5l,Sigh +24,/m/015lz1,Singing +25,/m/0l14jd,Choir +26,/m/01swy6,Yodeling +27,/m/02bk07,Chant +28,/m/01c194,Mantra +29,/t/dd00005,Child singing +30,/t/dd00006,Synthetic singing +31,/m/06bxc,Rapping +32,/m/02fxyj,Humming +33,/m/07s2xch,Groan +34,/m/07r4k75,Grunt +35,/m/01w250,Whistling +36,/m/0lyf6,Breathing +37,/m/07mzm6,Wheeze +38,/m/01d3sd,Snoring +39,/m/07s0dtb,Gasp +40,/m/07pyy8b,Pant +41,/m/07q0yl5,Snort +42,/m/01b_21,Cough +43,/m/0dl9sf8,Throat clearing +44,/m/01hsr_,Sneeze +45,/m/07ppn3j,Sniff +46,/m/06h7j,Run +47,/m/07qv_x_,Shuffle +48,/m/07pbtc8,"Walk, footsteps" +49,/m/03cczk,"Chewing, mastication" +50,/m/07pdhp0,Biting +51,/m/0939n_,Gargling +52,/m/01g90h,Stomach rumble +53,/m/03q5_w,"Burping, eructation" +54,/m/02p3nc,Hiccup +55,/m/02_nn,Fart +56,/m/0k65p,Hands +57,/m/025_jnm,Finger snapping +58,/m/0l15bq,Clapping +59,/m/01jg02,"Heart sounds, heartbeat" +60,/m/01jg1z,Heart murmur +61,/m/053hz1,Cheering +62,/m/028ght,Applause +63,/m/07rkbfh,Chatter +64,/m/03qtwd,Crowd +65,/m/07qfr4h,"Hubbub, speech noise, speech babble" +66,/t/dd00013,Children playing +67,/m/0jbk,Animal +68,/m/068hy,"Domestic animals, pets" +69,/m/0bt9lr,Dog +70,/m/05tny_,Bark +71,/m/07r_k2n,Yip +72,/m/07qf0zm,Howl +73,/m/07rc7d9,Bow-wow +74,/m/0ghcn6,Growling +75,/t/dd00136,Whimper (dog) +76,/m/01yrx,Cat +77,/m/02yds9,Purr +78,/m/07qrkrw,Meow +79,/m/07rjwbb,Hiss +80,/m/07r81j2,Caterwaul +81,/m/0ch8v,"Livestock, farm animals, working animals" +82,/m/03k3r,Horse +83,/m/07rv9rh,Clip-clop +84,/m/07q5rw0,"Neigh, whinny" +85,/m/01xq0k1,"Cattle, bovinae" +86,/m/07rpkh9,Moo +87,/m/0239kh,Cowbell +88,/m/068zj,Pig +89,/t/dd00018,Oink +90,/m/03fwl,Goat +91,/m/07q0h5t,Bleat +92,/m/07bgp,Sheep +93,/m/025rv6n,Fowl +94,/m/09b5t,"Chicken, rooster" +95,/m/07st89h,Cluck +96,/m/07qn5dc,"Crowing, cock-a-doodle-doo" +97,/m/01rd7k,Turkey +98,/m/07svc2k,Gobble +99,/m/09ddx,Duck +100,/m/07qdb04,Quack +101,/m/0dbvp,Goose +102,/m/07qwf61,Honk +103,/m/01280g,Wild animals +104,/m/0cdnk,"Roaring cats (lions, tigers)" +105,/m/04cvmfc,Roar +106,/m/015p6,Bird +107,/m/020bb7,"Bird vocalization, bird call, bird song" +108,/m/07pggtn,"Chirp, tweet" +109,/m/07sx8x_,Squawk +110,/m/0h0rv,"Pigeon, dove" +111,/m/07r_25d,Coo +112,/m/04s8yn,Crow +113,/m/07r5c2p,Caw +114,/m/09d5_,Owl +115,/m/07r_80w,Hoot +116,/m/05_wcq,"Bird flight, flapping wings" +117,/m/01z5f,"Canidae, dogs, wolves" +118,/m/06hps,"Rodents, rats, mice" +119,/m/04rmv,Mouse +120,/m/07r4gkf,Patter +121,/m/03vt0,Insect +122,/m/09xqv,Cricket +123,/m/09f96,Mosquito +124,/m/0h2mp,"Fly, housefly" +125,/m/07pjwq1,Buzz +126,/m/01h3n,"Bee, wasp, etc." +127,/m/09ld4,Frog +128,/m/07st88b,Croak +129,/m/078jl,Snake +130,/m/07qn4z3,Rattle +131,/m/032n05,Whale vocalization +132,/m/04rlf,Music +133,/m/04szw,Musical instrument +134,/m/0fx80y,Plucked string instrument +135,/m/0342h,Guitar +136,/m/02sgy,Electric guitar +137,/m/018vs,Bass guitar +138,/m/042v_gx,Acoustic guitar +139,/m/06w87,"Steel guitar, slide guitar" +140,/m/01glhc,Tapping (guitar technique) +141,/m/07s0s5r,Strum +142,/m/018j2,Banjo +143,/m/0jtg0,Sitar +144,/m/04rzd,Mandolin +145,/m/01bns_,Zither +146,/m/07xzm,Ukulele +147,/m/05148p4,Keyboard (musical) +148,/m/05r5c,Piano +149,/m/01s0ps,Electric piano +150,/m/013y1f,Organ +151,/m/03xq_f,Electronic organ +152,/m/03gvt,Hammond organ +153,/m/0l14qv,Synthesizer +154,/m/01v1d8,Sampler +155,/m/03q5t,Harpsichord +156,/m/0l14md,Percussion +157,/m/02hnl,Drum kit +158,/m/0cfdd,Drum machine +159,/m/026t6,Drum +160,/m/06rvn,Snare drum +161,/m/03t3fj,Rimshot +162,/m/02k_mr,Drum roll +163,/m/0bm02,Bass drum +164,/m/011k_j,Timpani +165,/m/01p970,Tabla +166,/m/01qbl,Cymbal +167,/m/03qtq,Hi-hat +168,/m/01sm1g,Wood block +169,/m/07brj,Tambourine +170,/m/05r5wn,Rattle (instrument) +171,/m/0xzly,Maraca +172,/m/0mbct,Gong +173,/m/016622,Tubular bells +174,/m/0j45pbj,Mallet percussion +175,/m/0dwsp,"Marimba, xylophone" +176,/m/0dwtp,Glockenspiel +177,/m/0dwt5,Vibraphone +178,/m/0l156b,Steelpan +179,/m/05pd6,Orchestra +180,/m/01kcd,Brass instrument +181,/m/0319l,French horn +182,/m/07gql,Trumpet +183,/m/07c6l,Trombone +184,/m/0l14_3,Bowed string instrument +185,/m/02qmj0d,String section +186,/m/07y_7,"Violin, fiddle" +187,/m/0d8_n,Pizzicato +188,/m/01xqw,Cello +189,/m/02fsn,Double bass +190,/m/085jw,"Wind instrument, woodwind instrument" +191,/m/0l14j_,Flute +192,/m/06ncr,Saxophone +193,/m/01wy6,Clarinet +194,/m/03m5k,Harp +195,/m/0395lw,Bell +196,/m/03w41f,Church bell +197,/m/027m70_,Jingle bell +198,/m/0gy1t2s,Bicycle bell +199,/m/07n_g,Tuning fork +200,/m/0f8s22,Chime +201,/m/026fgl,Wind chime +202,/m/0150b9,Change ringing (campanology) +203,/m/03qjg,Harmonica +204,/m/0mkg,Accordion +205,/m/0192l,Bagpipes +206,/m/02bxd,Didgeridoo +207,/m/0l14l2,Shofar +208,/m/07kc_,Theremin +209,/m/0l14t7,Singing bowl +210,/m/01hgjl,Scratching (performance technique) +211,/m/064t9,Pop music +212,/m/0glt670,Hip hop music +213,/m/02cz_7,Beatboxing +214,/m/06by7,Rock music +215,/m/03lty,Heavy metal +216,/m/05r6t,Punk rock +217,/m/0dls3,Grunge +218,/m/0dl5d,Progressive rock +219,/m/07sbbz2,Rock and roll +220,/m/05w3f,Psychedelic rock +221,/m/06j6l,Rhythm and blues +222,/m/0gywn,Soul music +223,/m/06cqb,Reggae +224,/m/01lyv,Country +225,/m/015y_n,Swing music +226,/m/0gg8l,Bluegrass +227,/m/02x8m,Funk +228,/m/02w4v,Folk music +229,/m/06j64v,Middle Eastern music +230,/m/03_d0,Jazz +231,/m/026z9,Disco +232,/m/0ggq0m,Classical music +233,/m/05lls,Opera +234,/m/02lkt,Electronic music +235,/m/03mb9,House music +236,/m/07gxw,Techno +237,/m/07s72n,Dubstep +238,/m/0283d,Drum and bass +239,/m/0m0jc,Electronica +240,/m/08cyft,Electronic dance music +241,/m/0fd3y,Ambient music +242,/m/07lnk,Trance music +243,/m/0g293,Music of Latin America +244,/m/0ln16,Salsa music +245,/m/0326g,Flamenco +246,/m/0155w,Blues +247,/m/05fw6t,Music for children +248,/m/02v2lh,New-age music +249,/m/0y4f8,Vocal music +250,/m/0z9c,A capella +251,/m/0164x2,Music of Africa +252,/m/0145m,Afrobeat +253,/m/02mscn,Christian music +254,/m/016cjb,Gospel music +255,/m/028sqc,Music of Asia +256,/m/015vgc,Carnatic music +257,/m/0dq0md,Music of Bollywood +258,/m/06rqw,Ska +259,/m/02p0sh1,Traditional music +260,/m/05rwpb,Independent music +261,/m/074ft,Song +262,/m/025td0t,Background music +263,/m/02cjck,Theme music +264,/m/03r5q_,Jingle (music) +265,/m/0l14gg,Soundtrack music +266,/m/07pkxdp,Lullaby +267,/m/01z7dr,Video game music +268,/m/0140xf,Christmas music +269,/m/0ggx5q,Dance music +270,/m/04wptg,Wedding music +271,/t/dd00031,Happy music +272,/t/dd00033,Sad music +273,/t/dd00034,Tender music +274,/t/dd00035,Exciting music +275,/t/dd00036,Angry music +276,/t/dd00037,Scary music +277,/m/03m9d0z,Wind +278,/m/09t49,Rustling leaves +279,/t/dd00092,Wind noise (microphone) +280,/m/0jb2l,Thunderstorm +281,/m/0ngt1,Thunder +282,/m/0838f,Water +283,/m/06mb1,Rain +284,/m/07r10fb,Raindrop +285,/t/dd00038,Rain on surface +286,/m/0j6m2,Stream +287,/m/0j2kx,Waterfall +288,/m/05kq4,Ocean +289,/m/034srq,"Waves, surf" +290,/m/06wzb,Steam +291,/m/07swgks,Gurgling +292,/m/02_41,Fire +293,/m/07pzfmf,Crackle +294,/m/07yv9,Vehicle +295,/m/019jd,"Boat, Water vehicle" +296,/m/0hsrw,"Sailboat, sailing ship" +297,/m/056ks2,"Rowboat, canoe, kayak" +298,/m/02rlv9,"Motorboat, speedboat" +299,/m/06q74,Ship +300,/m/012f08,Motor vehicle (road) +301,/m/0k4j,Car +302,/m/0912c9,"Vehicle horn, car horn, honking" +303,/m/07qv_d5,Toot +304,/m/02mfyn,Car alarm +305,/m/04gxbd,"Power windows, electric windows" +306,/m/07rknqz,Skidding +307,/m/0h9mv,Tire squeal +308,/t/dd00134,Car passing by +309,/m/0ltv,"Race car, auto racing" +310,/m/07r04,Truck +311,/m/0gvgw0,Air brake +312,/m/05x_td,"Air horn, truck horn" +313,/m/02rhddq,Reversing beeps +314,/m/03cl9h,"Ice cream truck, ice cream van" +315,/m/01bjv,Bus +316,/m/03j1ly,Emergency vehicle +317,/m/04qvtq,Police car (siren) +318,/m/012n7d,Ambulance (siren) +319,/m/012ndj,"Fire engine, fire truck (siren)" +320,/m/04_sv,Motorcycle +321,/m/0btp2,"Traffic noise, roadway noise" +322,/m/06d_3,Rail transport +323,/m/07jdr,Train +324,/m/04zmvq,Train whistle +325,/m/0284vy3,Train horn +326,/m/01g50p,"Railroad car, train wagon" +327,/t/dd00048,Train wheels squealing +328,/m/0195fx,"Subway, metro, underground" +329,/m/0k5j,Aircraft +330,/m/014yck,Aircraft engine +331,/m/04229,Jet engine +332,/m/02l6bg,"Propeller, airscrew" +333,/m/09ct_,Helicopter +334,/m/0cmf2,"Fixed-wing aircraft, airplane" +335,/m/0199g,Bicycle +336,/m/06_fw,Skateboard +337,/m/02mk9,Engine +338,/t/dd00065,Light engine (high frequency) +339,/m/08j51y,"Dental drill, dentist's drill" +340,/m/01yg9g,Lawn mower +341,/m/01j4z9,Chainsaw +342,/t/dd00066,Medium engine (mid frequency) +343,/t/dd00067,Heavy engine (low frequency) +344,/m/01h82_,Engine knocking +345,/t/dd00130,Engine starting +346,/m/07pb8fc,Idling +347,/m/07q2z82,"Accelerating, revving, vroom" +348,/m/02dgv,Door +349,/m/03wwcy,Doorbell +350,/m/07r67yg,Ding-dong +351,/m/02y_763,Sliding door +352,/m/07rjzl8,Slam +353,/m/07r4wb8,Knock +354,/m/07qcpgn,Tap +355,/m/07q6cd_,Squeak +356,/m/0642b4,Cupboard open or close +357,/m/0fqfqc,Drawer open or close +358,/m/04brg2,"Dishes, pots, and pans" +359,/m/023pjk,"Cutlery, silverware" +360,/m/07pn_8q,Chopping (food) +361,/m/0dxrf,Frying (food) +362,/m/0fx9l,Microwave oven +363,/m/02pjr4,Blender +364,/m/02jz0l,"Water tap, faucet" +365,/m/0130jx,Sink (filling or washing) +366,/m/03dnzn,Bathtub (filling or washing) +367,/m/03wvsk,Hair dryer +368,/m/01jt3m,Toilet flush +369,/m/012xff,Toothbrush +370,/m/04fgwm,Electric toothbrush +371,/m/0d31p,Vacuum cleaner +372,/m/01s0vc,Zipper (clothing) +373,/m/03v3yw,Keys jangling +374,/m/0242l,Coin (dropping) +375,/m/01lsmm,Scissors +376,/m/02g901,"Electric shaver, electric razor" +377,/m/05rj2,Shuffling cards +378,/m/0316dw,Typing +379,/m/0c2wf,Typewriter +380,/m/01m2v,Computer keyboard +381,/m/081rb,Writing +382,/m/07pp_mv,Alarm +383,/m/07cx4,Telephone +384,/m/07pp8cl,Telephone bell ringing +385,/m/01hnzm,Ringtone +386,/m/02c8p,"Telephone dialing, DTMF" +387,/m/015jpf,Dial tone +388,/m/01z47d,Busy signal +389,/m/046dlr,Alarm clock +390,/m/03kmc9,Siren +391,/m/0dgbq,Civil defense siren +392,/m/030rvx,Buzzer +393,/m/01y3hg,"Smoke detector, smoke alarm" +394,/m/0c3f7m,Fire alarm +395,/m/04fq5q,Foghorn +396,/m/0l156k,Whistle +397,/m/06hck5,Steam whistle +398,/t/dd00077,Mechanisms +399,/m/02bm9n,"Ratchet, pawl" +400,/m/01x3z,Clock +401,/m/07qjznt,Tick +402,/m/07qjznl,Tick-tock +403,/m/0l7xg,Gears +404,/m/05zc1,Pulleys +405,/m/0llzx,Sewing machine +406,/m/02x984l,Mechanical fan +407,/m/025wky1,Air conditioning +408,/m/024dl,Cash register +409,/m/01m4t,Printer +410,/m/0dv5r,Camera +411,/m/07bjf,Single-lens reflex camera +412,/m/07k1x,Tools +413,/m/03l9g,Hammer +414,/m/03p19w,Jackhammer +415,/m/01b82r,Sawing +416,/m/02p01q,Filing (rasp) +417,/m/023vsd,Sanding +418,/m/0_ksk,Power tool +419,/m/01d380,Drill +420,/m/014zdl,Explosion +421,/m/032s66,"Gunshot, gunfire" +422,/m/04zjc,Machine gun +423,/m/02z32qm,Fusillade +424,/m/0_1c,Artillery fire +425,/m/073cg4,Cap gun +426,/m/0g6b5,Fireworks +427,/g/122z_qxw,Firecracker +428,/m/07qsvvw,"Burst, pop" +429,/m/07pxg6y,Eruption +430,/m/07qqyl4,Boom +431,/m/083vt,Wood +432,/m/07pczhz,Chop +433,/m/07pl1bw,Splinter +434,/m/07qs1cx,Crack +435,/m/039jq,Glass +436,/m/07q7njn,"Chink, clink" +437,/m/07rn7sz,Shatter +438,/m/04k94,Liquid +439,/m/07rrlb6,"Splash, splatter" +440,/m/07p6mqd,Slosh +441,/m/07qlwh6,Squish +442,/m/07r5v4s,Drip +443,/m/07prgkl,Pour +444,/m/07pqc89,"Trickle, dribble" +445,/t/dd00088,Gush +446,/m/07p7b8y,Fill (with liquid) +447,/m/07qlf79,Spray +448,/m/07ptzwd,Pump (liquid) +449,/m/07ptfmf,Stir +450,/m/0dv3j,Boiling +451,/m/0790c,Sonar +452,/m/0dl83,Arrow +453,/m/07rqsjt,"Whoosh, swoosh, swish" +454,/m/07qnq_y,"Thump, thud" +455,/m/07rrh0c,Thunk +456,/m/0b_fwt,Electronic tuner +457,/m/02rr_,Effects unit +458,/m/07m2kt,Chorus effect +459,/m/018w8,Basketball bounce +460,/m/07pws3f,Bang +461,/m/07ryjzk,"Slap, smack" +462,/m/07rdhzs,"Whack, thwack" +463,/m/07pjjrj,"Smash, crash" +464,/m/07pc8lb,Breaking +465,/m/07pqn27,Bouncing +466,/m/07rbp7_,Whip +467,/m/07pyf11,Flap +468,/m/07qb_dv,Scratch +469,/m/07qv4k0,Scrape +470,/m/07pdjhy,Rub +471,/m/07s8j8t,Roll +472,/m/07plct2,Crushing +473,/t/dd00112,"Crumpling, crinkling" +474,/m/07qcx4z,Tearing +475,/m/02fs_r,"Beep, bleep" +476,/m/07qwdck,Ping +477,/m/07phxs1,Ding +478,/m/07rv4dm,Clang +479,/m/07s02z0,Squeal +480,/m/07qh7jl,Creak +481,/m/07qwyj0,Rustle +482,/m/07s34ls,Whir +483,/m/07qmpdm,Clatter +484,/m/07p9k1k,Sizzle +485,/m/07qc9xj,Clicking +486,/m/07rwm0c,Clickety-clack +487,/m/07phhsh,Rumble +488,/m/07qyrcz,Plop +489,/m/07qfgpx,"Jingle, tinkle" +490,/m/07rcgpl,Hum +491,/m/07p78v5,Zing +492,/t/dd00121,Boing +493,/m/07s12q4,Crunch +494,/m/028v0c,Silence +495,/m/01v_m0,Sine wave +496,/m/0b9m1,Harmonic +497,/m/0hdsk,Chirp tone +498,/m/0c1dj,Sound effect +499,/m/07pt_g0,Pulse +500,/t/dd00125,"Inside, small room" +501,/t/dd00126,"Inside, large room or hall" +502,/t/dd00127,"Inside, public space" +503,/t/dd00128,"Outside, urban or manmade" +504,/t/dd00129,"Outside, rural or natural" +505,/m/01b9nn,Reverberation +506,/m/01jnbd,Echo +507,/m/096m7z,Noise +508,/m/06_y0by,Environmental noise +509,/m/07rgkc5,Static +510,/m/06xkwv,Mains hum +511,/m/0g12c5,Distortion +512,/m/08p9q4,Sidetone +513,/m/07szfh9,Cacophony +514,/m/0chx_,White noise +515,/m/0cj0r,Pink noise +516,/m/07p_0gm,Throbbing +517,/m/01jwx6,Vibration +518,/m/07c52,Television +519,/m/06bz3,Radio +520,/m/07hvw1,Field recording diff --git a/backend/requirements-runtime.txt b/backend/requirements-runtime.txt index 8112b0e..6e1b4f3 100644 --- a/backend/requirements-runtime.txt +++ b/backend/requirements-runtime.txt @@ -4,6 +4,9 @@ # via backend/requirements.txt. opencv-python-headless>=4.8.1.78 numpy>=1.24.0 +# Audio-event detection (YAMNet laughter/reaction channel) runs on ONNX Runtime — +# no torch/TF, so it stays on the hermetic native path. +onnxruntime>=1.16.0 Pillow>=10.0.0 questionary>=2.0.0 python-dotenv>=1.2.2 diff --git a/backend/requirements.txt b/backend/requirements.txt index bb6ae7b..7532eb6 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -12,6 +12,9 @@ openai-whisper>=20231117 opencv-python-headless>=4.8.1.78 numpy>=1.24.0 +# Audio-event detection (laughter/reaction channel via YAMNet ONNX — no torch/TF) +onnxruntime>=1.16.0 + # Thumbnails Pillow>=10.0.0 diff --git a/backend/services/audio_events.py b/backend/services/audio_events.py new file mode 100644 index 0000000..ce1a265 --- /dev/null +++ b/backend/services/audio_events.py @@ -0,0 +1,222 @@ +""" +Audio-event detection (laughter, cheering, applause, screaming) for reaction-based +clip scoring. + +Runs YAMNet (AudioSet, 521 classes) as a self-contained ONNX graph via onnxruntime, +which imports neither torch nor tensorflow — so this belongs on the native / +whisper.cpp runtime path. Audio is extracted at 16 kHz mono (the same format +whisper.cpp uses) and fed to the graph as a raw waveform; the log-mel front-end is +baked into the model, so there is no librosa dependency. + +A laugh is the strongest language-independent "something funny happened" signal, and +it anti-correlates with speech at the frame level, which makes it a reliable anchor +for extending a clip backwards to the moment that caused the reaction. +""" + +import csv +import subprocess +import tempfile +import wave +from pathlib import Path +from typing import Optional, Callable + +import numpy as np + +from utils.proc import run as proc_run + +try: + import onnxruntime as ort + _ORT_AVAILABLE = True +except ImportError: + _ORT_AVAILABLE = False + +_MODELS_DIR = Path(__file__).resolve().parents[1] / "models" +_MODEL_PATH = _MODELS_DIR / "yamnet.onnx" +_CLASS_MAP_PATH = _MODELS_DIR / "yamnet_class_map.csv" + +# AudioSet display names, grouped into the reaction channels we score on. Laughter is +# spread across fine-grained children in the ontology, so we collapse the whole family. +_LAUGHTER_KEYS = ("laugh", "giggle", "chuckle", "snicker", "chortle") +_CHEER_NAMES = ("Cheering", "Applause", "Whoop") +_SCREAM_NAMES = ("Screaming",) +_SPEECH_NAMES = ("Speech",) + +_session = None +_class_index: dict[str, list[int]] = {} + + +def _load_class_index() -> dict[str, list[int]]: + names: dict[int, str] = {} + with open(_CLASS_MAP_PATH, newline="") as f: + reader = csv.reader(f) + next(reader) # header: index,mid,display_name + for row in reader: + names[int(row[0])] = row[2] + + def by_substr(keys) -> list[int]: + return [i for i, n in names.items() if any(k in n.lower() for k in keys)] + + def by_name(wanted) -> list[int]: + return [i for i, n in names.items() if n in wanted] + + return { + "laughter": by_substr(_LAUGHTER_KEYS), + "cheering": by_name(_CHEER_NAMES), + "screaming": by_name(_SCREAM_NAMES), + "speech": by_name(_SPEECH_NAMES), + } + + +def _get_session(): + global _session, _class_index + if not _ORT_AVAILABLE or not _MODEL_PATH.exists(): + return None + if _session is None: + _session = ort.InferenceSession( + str(_MODEL_PATH), providers=["CPUExecutionProvider"] + ) + _class_index = _load_class_index() + return _session + + +def is_available() -> bool: + """True if onnxruntime and the YAMNet model are both present.""" + return _ORT_AVAILABLE and _MODEL_PATH.exists() + + +def _read_waveform_16k_mono(video_path: str) -> Optional[np.ndarray]: + """Extract audio as a float32 [-1, 1] mono waveform at 16 kHz via ffmpeg.""" + with tempfile.NamedTemporaryFile(suffix=".wav", delete=True) as tmp: + cmd = [ + "ffmpeg", "-y", "-i", video_path, + "-vn", "-ar", "16000", "-ac", "1", + "-f", "wav", tmp.name, "-loglevel", "error", + ] + result = proc_run(cmd, timeout=600, check=False) + if result.returncode != 0: + return None + with wave.open(tmp.name) as w: + frames = w.readframes(w.getnframes()) + if not frames: + return None + return np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0 + + +def extract_audio_events(video_path: str) -> list[dict]: + """ + Run YAMNet over a video's audio and return per-frame reaction probabilities. + + Returns a list of {time, laughter, cheering, screaming, speech} dicts, one per + YAMNet frame (~0.48 s hop). Empty list if the model/runtime is unavailable or the + audio can't be read, so callers degrade gracefully. + """ + session = _get_session() + if session is None: + return [] + + waveform = _read_waveform_16k_mono(video_path) + if waveform is None or waveform.size == 0: + return [] + + input_name = session.get_inputs()[0].name + scores = session.run(None, {input_name: waveform})[0] # [frames, 521] + n_frames = scores.shape[0] + if n_frames == 0: + return [] + + duration = waveform.size / 16000.0 + hop = duration / n_frames + + def channel_max(keys: list[int]) -> np.ndarray: + return scores[:, keys].max(axis=1) if keys else np.zeros(n_frames) + + laughter = channel_max(_class_index["laughter"]) + cheering = channel_max(_class_index["cheering"]) + screaming = channel_max(_class_index["screaming"]) + speech = channel_max(_class_index["speech"]) + + return [ + { + "time": round(i * hop, 2), + "laughter": round(float(laughter[i]), 3), + "cheering": round(float(cheering[i]), 3), + "screaming": round(float(screaming[i]), 3), + "speech": round(float(speech[i]), 3), + } + for i in range(n_frames) + ] + + +def _reaction_level(event: dict) -> float: + """Combined reaction strength for one frame: the loudest reaction channel.""" + return max(event["laughter"], event["cheering"], event["screaming"]) + + +def compute_event_scores( + events_data: list[dict], + segments: list[dict], +) -> list[float]: + """ + Per-segment reaction score (0-10) from the peak reaction within each segment. + + Unlike audio energy (RMS in dB, only meaningful relative to a video's own + baseline), YAMNet probabilities are absolute-calibrated in [0, 1], so the score is + a direct scaling of the peak rather than a z-score. A brief chuckle registers + ~0.25, a hearty laugh higher; peaks stand out sharply against a ~0 baseline. + """ + if not events_data: + return [0.0] * len(segments) + + scores = [] + for seg in segments: + seg_start = seg.get("start", 0) + seg_end = seg.get("end", 0) + levels = [ + _reaction_level(e) + for e in events_data + if seg_start <= e["time"] <= seg_end + ] + peak = max(levels) if levels else 0.0 + scores.append(round(min(10.0, peak * 12.0), 2)) + + return scores + + +def get_event_profile( + video_path: str, + segments: list[dict], + progress_callback: Optional[Callable] = None, + reaction_threshold: float = 0.15, +) -> dict: + """ + Full pipeline: detect audio events and score all segments. + + Returns {events_data, segment_scores, reaction_times} where reaction_times are the + timestamps of frames whose reaction level clears reaction_threshold — the anchors a + reaction-based detector expands backwards from. + """ + if not is_available(): + return {"events_data": [], "segment_scores": [0.0] * len(segments), "reaction_times": []} + + if progress_callback: + progress_callback(0, "Detecting laughter and reactions...") + + events_data = extract_audio_events(video_path) + + if progress_callback: + progress_callback(70, "Scoring segments by reaction...") + + segment_scores = compute_event_scores(events_data, segments) + + reaction_times = [ + e["time"] for e in events_data if _reaction_level(e) >= reaction_threshold + ] + + if progress_callback: + progress_callback(100, "Reaction analysis complete") + + return { + "events_data": events_data, + "segment_scores": segment_scores, + "reaction_times": reaction_times, + } diff --git a/backend/services/profiles.py b/backend/services/profiles.py new file mode 100644 index 0000000..c5450da --- /dev/null +++ b/backend/services/profiles.py @@ -0,0 +1,80 @@ +"""Content profiles — which signals drive moment detection, per content type. + +A ContentProfile is orthogonal to FormatSpec: FormatSpec is a render-time aspect-ratio +decision, a ContentProfile is a detection-time decision about which signal channels +matter and how candidate moments are generated. A podcast is selected transcript-first +(the LLM reads dialogue); a party video has no useful transcript and is driven by +laughter/energy peaks. Both are the same fusion engine with different channel weights. + +Weights are starting points, tuned with real footage. `candidate_source` picks how +moments are generated: "llm" keeps the existing transcript-first selector (podcast), +"saliency" peak-picks a fused signal curve (party/action, works with no transcript). +""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ContentProfile: + name: str + candidate_source: str # "llm" | "saliency" + channel_weights: dict[str, float] + # Reaction-based expansion: anchor on a laugh/cheer onset and extend backwards. + reaction_lookback_sec: float + reaction_payoff_sec: float + # Peak-pick tuning for the saliency candidate source (per-video normalized units). + peak_min_gap_sec: float + peak_top_percentile: float + + +_CHANNELS = ("transcript_semantic", "prosody", "audio_event", "energy", "motion", "face_reaction") + + +def _weights(**kw) -> dict[str, float]: + return {ch: float(kw.get(ch, 0.0)) for ch in _CHANNELS} + + +PROFILES = { + "podcast": ContentProfile( + name="podcast", + candidate_source="llm", + channel_weights=_weights( + transcript_semantic=0.5, prosody=0.2, audio_event=0.15, energy=0.1, face_reaction=0.05 + ), + reaction_lookback_sec=8.0, + reaction_payoff_sec=2.0, + peak_min_gap_sec=15.0, + peak_top_percentile=0.15, + ), + "party": ContentProfile( + name="party", + candidate_source="saliency", + channel_weights=_weights( + audio_event=0.4, prosody=0.2, energy=0.2, motion=0.1, face_reaction=0.1 + ), + reaction_lookback_sec=8.0, + reaction_payoff_sec=2.0, + peak_min_gap_sec=8.0, + peak_top_percentile=0.15, + ), + "action": ContentProfile( + name="action", + candidate_source="saliency", + channel_weights=_weights( + audio_event=0.3, motion=0.2, prosody=0.2, energy=0.2, transcript_semantic=0.1 + ), + reaction_lookback_sec=6.0, + reaction_payoff_sec=2.0, + peak_min_gap_sec=8.0, + peak_top_percentile=0.15, + ), +} + +DEFAULT_PROFILE = "podcast" + + +def get_profile(name: str | None) -> ContentProfile: + import sys + if name is not None and name not in PROFILES: + print(f"[profiles] unknown profile {name!r}; using {DEFAULT_PROFILE}", file=sys.stderr) + return PROFILES.get(name or DEFAULT_PROFILE, PROFILES[DEFAULT_PROFILE]) diff --git a/backend/services/transcript_packer.py b/backend/services/transcript_packer.py index f311496..cd88f48 100644 --- a/backend/services/transcript_packer.py +++ b/backend/services/transcript_packer.py @@ -38,6 +38,8 @@ def _packed_dir() -> str: PHRASE_MAX_SEC = 12.0 PHRASE_MAX_CHARS = 160 ENERGY_PEAKS_TO_REPORT = 20 +REACTIONS_TO_REPORT = 20 +REACTION_REPORT_THRESHOLD = 0.15 def compute_cache_hash(video_path: str) -> str: @@ -272,6 +274,7 @@ def pack_transcript( transcript: dict, source_label: str, energy_data: Optional[list[dict]] = None, + events_data: Optional[list[dict]] = None, ) -> str: """Render a packed markdown view of a transcription JSON.""" duration = float(transcript.get("duration", 0.0)) @@ -348,6 +351,31 @@ def pack_transcript( lines.append(f"[{_fmt_ts(t)}] {rms:.1f}dB{snippet}") lines.append("") + # Laughter & reactions (optional) — the moment BEFORE a laugh is usually the + # clippable one, so these timestamps flag where to look for the setup. + if events_data: + def level(e): + return max(e.get("laughter", 0), e.get("cheering", 0), e.get("screaming", 0)) + reactions = [e for e in events_data if level(e) >= REACTION_REPORT_THRESHOLD] + reactions.sort(key=level, reverse=True) + reactions = reactions[:REACTIONS_TO_REPORT] + reactions.sort(key=lambda e: e.get("time", 0)) + if reactions: + lines.append(f"## Laughter & reactions (top {len(reactions)})") + for e in reactions: + t = float(e.get("time", 0)) + kind = max( + ("laughter", "cheering", "screaming"), + key=lambda k: e.get(k, 0), + ) + near = _phrase_for_time(phrases, t) + snippet = "" + if near: + txt = near["text"] + snippet = f' near: "{txt[:70]}{"…" if len(txt) > 70 else ""}"' + lines.append(f"[{_fmt_ts(t)}] {kind} {e.get(kind, 0):.2f}{snippet}") + lines.append("") + return "\n".join(lines) @@ -372,10 +400,11 @@ def write_packed( cache_hash: str, source_label: Optional[str] = None, energy_data: Optional[list[dict]] = None, + events_data: Optional[list[dict]] = None, ) -> tuple[str, str]: """Pack a transcript dict and write to .podcli/packed/.md.""" label = source_label or cache_hash - md = pack_transcript(transcript, label, energy_data=energy_data) + md = pack_transcript(transcript, label, energy_data=energy_data, events_data=events_data) os.makedirs(_packed_dir(), exist_ok=True) out_path = packed_path_for(cache_hash) with open(out_path, "w", encoding="utf-8") as f: diff --git a/plans/moment-detection.md b/plans/moment-detection.md new file mode 100644 index 0000000..4665bd6 --- /dev/null +++ b/plans/moment-detection.md @@ -0,0 +1,110 @@ +# podcli → Generalized moment detection (multi-signal saliency engine) + +> Goal: one detector that works across content types. The podcast clip selector and a new "auto-cut the funny/interesting moments" highlighter (party videos, vlogs, action, streams) become **two profiles of one engine**, not two features. Anchor use case that surfaced the need: a folder of party clips, "cut me the funny bits" — any language, no labels. Podcast selection is preserved as the default profile. + +## North star + +```text +one long video (any content, any language) + → extract N signal channels (transcript, energy, laughter, prosody, motion, face) + → per-video-normalize each into a 0..1 saliency curve on a common time grid + → per-PROFILE weighted fusion → one interestingness curve + → peak-pick (NMS + prominence) → reaction-expand → snap to natural boundaries + → candidate moments → (render to formats via the existing fan-out) +``` + +## The one decision everything hangs on + +**Content profile is orthogonal to output format.** `FormatSpec` (9:16 / 16:9 / 1:1) is a *render-time* aspect-ratio decision. A **`ContentProfile`** (podcast / party / action / auto) is a *detection-time* decision about **which signals matter and how candidates are generated**. A moment is detected under a profile, then rendered to one or more formats. Do not overload `FormatSpec.score_key` for this — mirror its pattern in a separate `ContentProfile` spec. + +The existing transcript-LLM selector is **one channel** in this engine (the `transcript_semantic` channel), not the whole thing. That is the reframe: today the LLM *is* selection; tomorrow it is the dominant channel for the podcast profile and near-zero for party. + +## Why this generalizes (research-backed) + +Multi-signal highlight detection is a solved shape: compute a per-modality saliency curve, normalize **per-video** (never global), fuse (weighted sum with soft-OR bias, audio weighted heaviest for human-centric video), peak-pick, and for reaction-driven content extend **backwards from the reaction** to the cause. FunnyNet-W: a funny moment is "an n-second clip *followed by* laughter," audio carries >50% of the decision, optimal look-back ≈ 8s. Cai/Lu/Cai built laughter+applause+cheer detection *for home videos* at ~93% recall. All of it is unsupervised — no labeled data, only pretrained off-the-shelf models. See the audio-model and fusion research threads (this session) for citations. + +## Profiles as concrete weight vectors + +Channels each emit a per-video-normalized 0..1 curve; a profile is a weight vector + candidate-source + peak params. + +| Channel | podcast | party | action | Source status | +|---|---|---|---|---| +| `transcript_semantic` (LLM dialogue/quotability) | **0.5** | 0.0 | 0.1 | REUSE `claude_suggest.py` | +| `prosody` (pitch/energy/rate spikes) | 0.2 | 0.2 | 0.2 | NEW (numpy, no librosa) | +| `audio_event` (laughter/cheer/applause/scream) | 0.15 | **0.4** | **0.3** | NEW (YAMNet-ONNX, no torch) | +| `energy` (RMS, peak-weighted) | 0.1 | 0.2 | 0.2 | REUSE `audio_analyzer.py` | +| `motion` (optical flow + scene-cut density) | 0.0 | 0.1 | **0.2** | PARTIAL (cuts exist; flow NEW) | +| `face_reaction` (mouth activity / smile proxy) | 0.05 | 0.1 | 0.0 | REUSE `face_analysis.py` (crude) | +| **candidate source** | `llm` | `saliency` | `saliency` | — | + +Weights are starting points, tuned later. `auto` = a cheap first pass picks the profile: clean speech + 1-2 stable faces → podcast; sparse speech + high motion + crowd/laughter audio → party; sustained high motion → action. + +## Two candidate-generation modes (the key to not regressing podcast) + +- **`llm` (podcast):** the existing `claude_suggest` flow generates candidates over the whole transcript, exactly as today. The fused signal score blends in at the ranking seam. **When party channels have weight 0, output is byte-for-byte the current behavior.** This is the safety property. +- **`saliency` (party / action):** the fused curve drives candidate generation directly — `find_peaks` → reaction-expand → boundary-snap → candidate windows. Works with **no usable transcript**. The LLM is optional here, used only to title/caption the already-chosen windows. + +## Architecture — new modules + +1. **`backend/services/audio_events.py`** (NEW, no-torch) + - YAMNet exported to a **self-contained ONNX graph** (raw 16 kHz waveform in, log-mel baked into the graph → **no librosa needed**). ~15 MB, Apache-2.0. AudioSet classes `Laughter`, `Cheering`, `Applause`, `Screaming` are native outputs (+ Giggle/Belly-laugh/Whoop children). + - Reuse `transcription_whispercpp._extract_wav` (`-ar 16000 -ac 1`) for audio; **do not import `speaker_detection`** (drags in torch). + - Returns per-window (≈1 Hz, matching energy) class probabilities → one 0..1 curve per trigger class. + - Runtime: `onnxruntime`. **First test whether `cv2.dnn` (already loads YuNet `.onnx`) can run yamnet.onnx** — if yes, zero new deps. + - Ship `backend/models/yamnet.onnx` + a **dev-only** `scripts/export_yamnet.py` (uses tf2onnx; never shipped or imported at runtime). + +2. **`backend/services/saliency.py`** (NEW) + - `normalize_per_video(curve)` — robust z-score via numpy median/MAD, or percentile-rank. **Never global.** + - `fuse(channels, profile)` — weighted sum with soft-OR bias. + - `pick_peaks(curve, profile)` — **numpy reimplementation of `find_peaks`** (~40 lines: height floor `mean + kσ`, min-distance NMS by descending height, prominence filter, min-width). Avoids adding scipy. Keep peaks above the top ~10-15th percentile of the video's **own** prominence distribution (adaptive count). + - `reaction_expand(peak, events)` — if the peak's driver is an `audio_event` class, move anchor to reaction onset, extend ~8 s back, keep 1-3 s of payoff. + - `snap_boundaries(window)` — snap start/end to nearest {Whisper sentence-end, silence gap >300 ms, scene cut, motion-quiet local min} within ±500 ms. + +3. **`backend/services/profiles.py`** (NEW) — `ContentProfile` dataclass (name, `channel_weights`, `candidate_source`, peak params, reaction-expand params, boundary sources) mirroring `FormatSpec`; `get_profile(name)`; `auto_detect(signals)`. + +4. **`motion` channel** (Phase 3) — `cv2.calcOpticalFlowFarneback` on sampled frames (**OpenCV already present**) + reuse `local_reframe.count_scene_cuts` / `clip_generator._detect_scene_cuts`. + +5. **`prosody` channel** (Phase 2/3) — numpy F0 (autocorrelation) + short-term energy + speech-rate vs speaker baseline. No librosa. + +## Dependency deltas (no-torch native path) + +- **ADD** `onnxruntime` to `requirements-runtime.txt` — the only new hard dep. **Settled by spike:** OpenCV-DNN *cannot* run the YAMNet graph (rejects the dynamic-shape mel front-end: `dynamic 'zero' shapes are not supported`), so the zero-new-dep route is out. `onnxruntime` still pulls in no torch/TF. +- **NO** librosa (self-contained waveform-in graph, confirmed), **NO** torch/TF at runtime. +- `scipy` is present in the dev venv (so `find_peaks` works there) but is **not** in `requirements-runtime.txt` — either add it or keep the numpy peak-pick reimpl for the hermetic runtime. Decide at Phase 2. +- Optical flow + face already covered by `opencv-python-headless`. +- `yamnet.onnx` weight (~16 MB) committed to `backend/models/`; no export step needed — a self-contained waveform-in export already exists at HF `zeropointnine/yamnet-onnx` (input `waveform` [dynamic], outputs `[frames,521]` scores + `[frames,1024]` embeddings + `[frames,64]` mel). Verify its license before committing; keep a dev-only `scripts/export_yamnet.py` (tf2onnx) as the reproducible fallback. + +## Spike results (validated this session) + +Ran the self-contained YAMNet ONNX on real podcli audio (a 38 s Deeptech Decoded clip, extracted with podcli's exact `-ar 16000 -ac 1`): +- **No-torch confirmed:** `torch`/`tensorflow` absent from the process before and after inference (onnxruntime only). +- **Correct classification:** Speech 0.774 (dominant, correct for a talking clip), Silence 0.186, and Laughter/Cheering/Applause/Screaming all exactly 0.000 — correct negative control (serious monologue, no laughter). Mel front-end baked in the graph → featurization provably correct, no code to own. +- **Negligible cost:** 98 ms for 38 s of audio (~380× realtime); a 3 h video ≈ 28 s of audio-event analysis. +- **Positive reaction detection confirmed on podcast content.** Swept 32 real clips: the detector ranked the one comedic show + two chuckle moments above the dry technical majority (which scored exactly 0.00). Zooming in, laughter fires as a **sharp 1-2 frame spike anti-correlated with Speech** (e.g. laugh=0.23 while speech drops 1.00→0.11 at 13.5s, then back to speech) — the exact reaction signature the 8s-backward-expansion relies on. Absolute values are modest (0.23-0.25 for brief chuckles in pre-cut clips) but that is a non-issue: a spike on a 0.00 baseline is trivially caught by **prominence-based peak-picking**, which is why per-video normalization (not absolute thresholds) is mandated above. +- **Still worth a check (needs real party footage):** amplitude/robustness on noisy handheld party audio with music and overlapping speech — YAMNet's known soft spot. If weak, AST ONNX (`onnx-community`, pre-exported, mAP 0.485) is the drop-in fallback; needs a 128-bin log-mel numpy front-end. + +## Integration seams (from the code map) + +1. **Detect-once hub** — `backend/cli.py` ~658-720 (and MCP twin `backend/main.py:handle_suggest_clips` ~358). Energy is *already* computed once here (`get_energy_profile`) and `face_map` extracted. All new channels compute here; fusion produces the saliency curve. +2. **Scoring merge** — `claude_suggest.suggest_with_claude` normalization (~999) + `_select_top_by_score` (~687). Blend the LLM score in as the `transcript_semantic` channel; key the weights off the active `ContentProfile`. +3. **Transcript-packer co-location** — `transcript_packer.pack_transcript` (~274) already flags top-RMS moments for the in-chat MCP agent. Add laughter/event/motion flags so the conversational path also sees them. + +## Contract-change tax + +A new `profile` param threads the **same ~12 hops the `format` field did**: `suggest-clips.handler.ts` inputSchema, `src/models/index.ts` types, `src/server.ts` Zod (dual-declaration), `src/ui/web-server.ts` `styledClips` whitelist, `python-executor` stdin, `main.py` params, and `cli.py`'s `generate_clip` call sites — or it silently reverts to the default. Follow the `format` precedent exactly. + +## Phasing + +- **Phase 0 — profile scaffolding, zero behavior change.** Add `ContentProfile` abstraction; thread the `profile` param through the ~12 hops; `default = podcast` reproduces current selection exactly. **Gate: existing test suite green; same clips out for a fixed transcript.** +- **Phase 1 — audio-event channel (the isolated valuable core).** YAMNet-ONNX laughter/cheer/applause/scream computed in the detect-once hub. Feeds podcast ranking as a labeled signal (laughs already spike energy; now they're *named*) and lays the party foundation. **Gate: laughter timestamps validated on a sample clip; podcast output unchanged unless the channel is given weight.** +- **Phase 2 — fusion engine + saliency candidate source + party profile (audio-only).** `saliency.py` fusion + numpy peak-pick + reaction-expand (8 s) + boundary-snap. Party profile = energy + audio_event + prosody, no transcript, no motion. **Party videos auto-clip end to end. Gate: demo on real party footage.** +- **Phase 3 — visual channels + action profile + multi-file pooling.** Optical flow (OpenCV) + face-reaction channels; action profile; pool peaks across a *folder* of clips and rank globally ("best 15 bits from tonight" across 80 phone videos). **Gate: catches a silent visual gag; folder-level ranking works.** +- **Phase 4 (optional) — highlight reel renderer.** Ordering, pacing, optional music-bed ducking, transitions — a thin renderer atop the detected moments, reusing the clip-render stack. + +## Guardrails + +- **Per-video normalization is mandatory** — party clips vary wildly in level/room/camera; global normalization is wrong. +- **Podcast profile must reproduce current behavior** when party channels are weight 0. Ship behind the profile param; default is safe. This is the non-regression contract. +- **`find_peaks` `distance` param is the NMS / min-gap** — it deletes lower peaks within the gap automatically. Adaptive clip count = top percentile of the video's own prominence distribution, not an absolute N. +- **YAMNet ONNX must never import torch/TF at runtime.** The export step is dev/CI only; the native runtime installs `onnxruntime` (or reuses OpenCV-DNN) and feeds raw 16 kHz samples. +- **Reaction-expansion only fires for saliency candidates** whose trigger is an audio-event class — never rewrite LLM-chosen podcast windows. diff --git a/tests/test_audio_events.py b/tests/test_audio_events.py new file mode 100644 index 0000000..cea4e63 --- /dev/null +++ b/tests/test_audio_events.py @@ -0,0 +1,80 @@ +"""Tests for backend.services.audio_events scoring and profile scaffolding.""" + +import os +import sys +import unittest + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +BACKEND_ROOT = os.path.join(ROOT, "backend") +if BACKEND_ROOT not in sys.path: + sys.path.insert(0, BACKEND_ROOT) + +from services import audio_events as ae +from services import profiles + + +def _event(t, laughter=0.0, cheering=0.0, screaming=0.0, speech=0.0): + return { + "time": t, + "laughter": laughter, + "cheering": cheering, + "screaming": screaming, + "speech": speech, + } + + +class ComputeEventScoresTests(unittest.TestCase): + def test_empty_events_returns_zeros(self): + segments = [{"start": 0, "end": 10}, {"start": 10, "end": 20}] + self.assertEqual(ae.compute_event_scores([], segments), [0.0, 0.0]) + + def test_empty_segments_returns_empty(self): + self.assertEqual(ae.compute_event_scores([_event(1, laughter=0.5)], []), []) + + def test_segment_with_laugh_scores_higher_than_silent(self): + events = [_event(2, laughter=0.5, speech=0.1), _event(30, speech=1.0)] + scores = ae.compute_event_scores( + events, [{"start": 0, "end": 10}, {"start": 20, "end": 40}] + ) + self.assertGreater(scores[0], scores[1]) + self.assertEqual(scores[1], 0.0) + + def test_score_uses_loudest_reaction_channel(self): + # cheering should count even with zero laughter + events = [_event(1, cheering=0.5)] + scores = ae.compute_event_scores(events, [{"start": 0, "end": 5}]) + self.assertGreater(scores[0], 0.0) + + def test_score_is_clamped_to_ten(self): + events = [_event(1, laughter=1.0)] + scores = ae.compute_event_scores(events, [{"start": 0, "end": 5}]) + self.assertLessEqual(scores[0], 10.0) + + def test_speech_alone_is_not_a_reaction(self): + events = [_event(1, speech=1.0)] + scores = ae.compute_event_scores(events, [{"start": 0, "end": 5}]) + self.assertEqual(scores[0], 0.0) + + +class ProfileTests(unittest.TestCase): + def test_default_is_podcast_and_llm_sourced(self): + p = profiles.get_profile(None) + self.assertEqual(p.name, "podcast") + self.assertEqual(p.candidate_source, "llm") + + def test_unknown_profile_falls_back_to_default(self): + self.assertEqual(profiles.get_profile("nonsense").name, profiles.DEFAULT_PROFILE) + + def test_party_is_saliency_sourced_and_ignores_transcript(self): + p = profiles.get_profile("party") + self.assertEqual(p.candidate_source, "saliency") + self.assertEqual(p.channel_weights["transcript_semantic"], 0.0) + self.assertGreater(p.channel_weights["audio_event"], 0.0) + + def test_podcast_weights_are_transcript_dominant(self): + w = profiles.get_profile("podcast").channel_weights + self.assertEqual(max(w, key=w.get), "transcript_semantic") + + +if __name__ == "__main__": + unittest.main()