From ba3bc0c424bb10e170d490c262562f9e1712c3f0 Mon Sep 17 00:00:00 2001 From: glenn-chew Date: Sat, 17 Jan 2026 12:30:55 +0800 Subject: [PATCH 1/8] Extract captions from API --- content.js | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ manifest.json | 22 +++++++++++++- popup.js | 19 +++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 content.js create mode 100644 popup.js diff --git a/content.js b/content.js new file mode 100644 index 0000000..638100d --- /dev/null +++ b/content.js @@ -0,0 +1,79 @@ +async function getLanguage() { + try { + const url = 'https://mediaweb.ap.panopto.com/Panopto/Pages/Viewer/DeliveryInfo.aspx'; + const urlParams = new URLSearchParams(window.location.search); + const deliveryId = urlParams.get('id'); + const params = new URLSearchParams({ + deliveryId: deliveryId, + responseType: 'json' + }); + + + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: params.toString(), + credentials: 'include' + }); + + + const data = await response.json(); + console.log(data.Delivery.AvailableLanguages) + return data.Delivery.AvailableLanguages + + } catch (error) { + console.error('Error fetching DeliveryInfo:', error); + } +} + +async function getDeliveryInfo() { + try { + const languages = await getLanguage(); + + const url = 'https://mediaweb.ap.panopto.com/Panopto/Pages/Viewer/DeliveryInfo.aspx'; + const urlParams = new URLSearchParams(window.location.search); + const deliveryId = urlParams.get('id'); + const params = new URLSearchParams({ + deliveryId: deliveryId, + getCaptions: 'true', + language: languages[0], + responseType: 'json' + }); + + + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: params.toString(), + credentials: 'include' + }); + + const data = await response.json(); + + return data + + } catch (error) { + console.error('Error fetching DeliveryInfo:', error); + } +} + +chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { + if (request.action === "extractCaptions") { + (async () => { + try { + const data = await getDeliveryInfo(); + sendResponse({ success: true, data: data }); + + } catch (error) { + sendResponse({ success: false, error: error.message }); + } + })(); + return true; + } +}); + +// \ No newline at end of file diff --git a/manifest.json b/manifest.json index d201985..e2d19fa 100644 --- a/manifest.json +++ b/manifest.json @@ -6,5 +6,25 @@ "browser_action": { "default_icon": "icon.png", "default_popup": "window.html" - } + }, + "permissions": [ + "scripting", + "activeTab", + "cookies", + "storage" + ], + "host_permissions": [ + "https://mediaweb.ap.panopto.com/*" + ], + "content_scripts": [ + { + "matches": [ + "*://*.panopto.com/*" + ], + "js": [ + "content.js" + ], + "all_frames": false + } + ] } \ No newline at end of file diff --git a/popup.js b/popup.js new file mode 100644 index 0000000..5fd6602 --- /dev/null +++ b/popup.js @@ -0,0 +1,19 @@ +document.getElementById("getCaptions").addEventListener("click", async () => { + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }); + + chrome.scripting.executeScript( + { + target: { tabId: tab.id, allFrames: true }, + files: ["content.js"] + }, + () => { + chrome.tabs.sendMessage(tab.id, { action: "extractCaptions" }, (response) => { + if (chrome.runtime.lastError) { + console.error("Message failed:", chrome.runtime.lastError.message); + } else { + console.log("Captions:", response?.captions); + } + }); + } + ); +}); \ No newline at end of file From 8de2ddf9445f69f78aa351dbcd5edba2170b6259 Mon Sep 17 00:00:00 2001 From: glenn-chew Date: Sat, 17 Jan 2026 12:35:33 +0800 Subject: [PATCH 2/8] Add icon --- icon.png | Bin 0 -> 9336 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 icon.png diff --git a/icon.png b/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..9c5f56033fc2b341a9004bc264858d04d660d15a GIT binary patch literal 9336 zcmeHtc|6qL_y22Vn2}{DA|YhRuD9J-v(6~e*q2a}j6G}CnN*|&BZNX3Wv4_^GOdb` z$x_xy*|UZ0=6k2l=kxo0|NH*+{qOsD@R--VuXFCX_ndR@eV%h(S8c2HA%}Wd`&0}4;+Z2)Eh>}# z*eu%|?b{nu<7r=Us`BWhrGI9FX}_uQKlt6Dhp93=&J_NfZ^ynG8mLKQa&qRYH{(V+ zxAnv)Uhw`z{nS{ES1VZE9_-w9OT*wP|I=U76Im?DPRis^cL+oF@ZMY;MgBCRa1S@E zjZ{mXfOVqsb#05(G?3h%7DvQkXm}Dy9Knl-At$g!z4GSY}SD|e9n)^mqIBUtOsGSVIJ!`VQ32;1R zzu#WpQ*O&N*Qe39x&`o*ZrLK56}5DJ@>P9D-EJrP_kXf5r4GFrRl!B5J~yIa2NFk7 z)wFb3JfL4V?xOX52L=RJz~XUR61gz}Z^YQ8@i`4C;{iYUoj36!K-G3fO|-{6?5#!{ zAv}TS#`hm__I>KU5oQ#si_?|7jpbT{&;7pX)SuTUAtH4bL!_ZNz_3zkXwTZfqy&Yk zYLzBfV&<_s+r?2aHU)#9uFTY4v*=R~er0?JEZN;TMR&8O7@aX(ErBN#zbA>k<&EvN zT`}>DeVHMHCukwbR&RNudQ0nk^Os~wgALdV1LNQg80;l$KVbu5349qXdbIj{<+>kb z`OFoJe*67NsI`~CcupIglyLbR!SJR%0@Ij&;|fi8JCbYfo0Ld*!s0JJg=6q18!xVv z29!5pxfOMWKHjpE@qxOlLKq6PoUS9?S&|HAYUm+6VIWXuhuAw^@c{o^!;;dSWlueV ziI%9vRu5R~lv7fE+Csz@8M^Dlt?D1tcX_%+iMSLEJ~5+jPKk*L{M^v|O|+z3z8XaA zF^_F?vlWHK3u+69nG!^SuoQ2a2$+zMIf=<9W=v2Nf$Bz|F4WT1vn^0*0&T94%Lj=C z*}(MyQx)hy3K-{#W4OzQb?r5+9+4shF2xH`c=?4KmL$Ja-UY+FTNES-J{I?Za*3R4 zzKnBWeE97NZqj%y62+7efx8O>5!@nBd&{eP>kcddz{!`^izlJ7K^+!*4eWEQgBi+Z z%zclT5rJC>MpTxfeV^nj8w_E1z*_Dn;-*!6Aa$W??zY25{nO5bQ~r>8wsaS)TPxqM z)JPM`y_RUK^Zq^pH*E;Sj5)L3%&kc5{eUzI6~2ltd?}oQmMCSixV}$9(;|8LEG`P> zFTo=FKV#Y~!OU;aIpxhFL`l$r)c=q-v4%3d`Jd)pR+ASjp*!Iv7Z{uUz%}%}Rx4QN z?e&M$i`t07XQJ>3izJ(|h#w4G2v8}N#zqVHM80@YKs zHG!237=B%n-JzSL=S3!e&EgO%k0nLV)v`E#%RzNSp&C8xrReuPOv3j&Fq`tg>*&J# zbs6*yg(`R>&3*S4H8ZZ#g0R?;j4t#)mYz#mx}R905JMVyeMpx&dV$MmwjqC+A~7}- z&2S3iz%PO^bmQUNKNO9s(6FFaPnr1F!cg5jFBt*mkv>C`>XO9tBm%GN+$Wj8-T|klAZS61o@Pv`#hm0$PK4k0 z=uWpoHiTz`oK8bdh*+*5>7H)~7?&GP;-swywT7hX2A0*!L%7YFEYgTzCad`LuL-DV3uZ(k(qNgMl zh9~(h$7r!sDUo;8gU#`D0CDUmZch}KQI8WWmUmwYnvf>Pm>}QnI)WK{6$jZ}PC&+U zYO7#zM+{)Gl3<3hz-7v~VY;NTt|Oc^sZU0j>0k=KzaKQR_vUO+#hOYE-$(>nIU6Vy zAL5qb7drFMm_prghU=Mc8GU=3i1&(gRwXAJK2k7pQvB4* zeqD%jb9MKI5Gr`Wle}bcqtL2=S@H9D%BT`d=BhX@v~p>{@FxY+^M>awFIA^k|E{P% zo-+N~nba|T(HdQk!H1uhT0{`7$C?uG1gQ#4yiI$l^6am}M=5rOb6lTYW?Lq9q*re1 z!sO`>ix6ii20C$;fTM^ z+kOU7h+T^hBjxrBgZ&=*4JtvSO;IJKabUM-Wb=@w894CrcLT2MI%s%eutrwvXD(4& zzXC!*3si|*>a~A0-1mzWKKOfDg+AMzHtosUWTre;FvCPY7{(rWwwdeYM5fyn64!kb z;BuE;lM@Eq@6HT&@K$2*D^HNOW#ZE+V*)0ZA)pg2DH?hd)h9|cJUsmE6u2`DkmMV0 zc`I~!nrOj?P_7gzb{{JEJ)Pwiy%t78hs%d9OCUYyyK-#Ou%B<3q?fMXw&JtM9!}~zu5CR zI_P#Sq#~LDVX00m=u64*{s5>K3PLEq_w~e=z240zrxn0FcsuSR{pw&&Ty9)G%UsywKYDFg}e=V;-7puzs+w_+uQ!dS00^+n{WsukL}UNm*80q#JmrW7=o=791Ei z!Etp(ZG3HgiChiqoc0pL(+4{+0dr4y*8+nDHE2`&Np&sPJzPaBv9`=>LcTF}dZarvAc){7z!}Df1Er^l}3)+jrc9oL>WRc~q-GsKh&!uc&ts7et zG<=D(UT@JmP$VoH>O>iEizeTYDm7WeXN4Y{zr6J;rZnCx!BCIwgr zEblezjYSkG)tqft4SUq;C&N6V+|x9hhb4Ry>&Iw&7~g22xV(n;=!;=p z4dP$9ejyKFDka-qf8?R=3igA8Us?*mqf98V63U*Drd0avC6T;q=rn(O4ChMr^p557U zG4zd4S`=rv_gJZKW|=>Kk5eI~7^q}<+q?R8Rn$>u2OeHER5?z`*L!|BH*iqh*oBKP zF0HY2o3`ODe*4Nkm?nS1O~tmds=$Z-i8N$c!RUOw=7~3FXW^r~#~#+@sP_~-KVQ$ORGQ@8?6$a&l+C2lFqYitpxPhCELAT?vt)JKgjPZ*hJ4pYywF zuQ%`r@Ueyq$pcM4`8~Rgd{+_&7Ug zXjr)@X6I?p^xuj%H+HLQMD+)V^`HeZs4bFrrstb?x$$~BTUBiTtPLaz2a7DWR8)nV zOeEE7osD#v?)f3~g=#&o-yK{16YgfRg&KTZG2Dlq-X=Wxhk<>;;aePdPe$zrKXpQI zzIY`w=L?nhM>|z6K61W1^~j(&(5%ZUp;^ZBl6u{&>(PPoo8n7u-BSq-q*HX_yNPt< z$_d6pQUPQ^YNEZNCDOW5R7+xNSBEhgod}vAek4`u2IL5dvk9VnKK2yAvq5yTG${}VnY7fTmPW;v4jfAlTK@K#jADqZY)XE{ltzM zer0)=A+@okGCO^4elt0wF8sFU@KxKM{g6i6zNx%7I1oK%4j^I$*Fx#oO6%)sRm&a- zH?})N8E1&^5Hua?N(9ifO?j1+e!g6(Xbee-wt99aP9-?)ph1}nj@+a&TGoo}qzDC{) zXn082f97k8K}tgy8ou!Rcq7v2?r&cR-`XO{82R0;d=rf86={Y_!_@iT$`}?rhbXj; zlfsfPrAk2ti>P6^&BOPk1Q6=uiQhu#0i-wvHwhcb_f?E_BRq!DXL6WhD8fVQZ%)P( zJ3pb0oohU##>J*d2;_$$>y ztNZWb>nHkIup$2h7|ZW4Yc&BtUVktMlL-U)zJTs7K*Y*_(s&a4fu|jCDE+v|R^$y+#dXg5& zV5s1Ap6`SiPJyhB8hi@^aIdw;fH86V+s>+K2XxM$nH8pv{ff3$TLZy=?tpd;mk<+A z06Burg7pbGklJuFxcHYO^|sdn$_EOIE+@r2I&zEZ)*iFM8=u%$iX=111w8oHZ!I}K z3#jIgGj{WE85ZQh)+ecN8&ga}KX1S@A?h?nN*C4JPqaADpRyy+jRe!PnNt;PO6*N}ZIro}s+v z05>@`rCuOM=iW-R--ek`U!C3)_4swPz?y7nmrY$S{EJOq_8C4EFA|}+$oi1^=w#OSDo8+ab$`cHI>Pge0=9d&ACCu-^ zo^qUc)}+e$dd>DTgh-G^baTn-VfLG?Jatr&IQhgZp;EeguTp+Sy^_>UD>F>PYSq?J z`uUZ$&~wfXOh2KiUXNyQas#>{tI9lA!@aY@$od`z*PUt^{5AHth^FxD-3h~rZPN+* z=9g{XaXDRkDjIWg0QVzIIm?cGliC&@f(dX?VJ$E&T>JNLCWD;>6? zeX_C@V%E8&rU!D_)mNTocNje|kmY^%rE>8znRDZ-FHf?yTh32P`18-4AV}FvL~#Gg?RQ~$7qQfuug^z}7xIw$qg5IWeUK-cuy#ZI0?j1l><>_dK|8r#j+E7>L^7pP_!9B71|T2Fh-{ z6~8V@q%41r=IsgmN6&cwa?mS_4aGKMv*idp?&1@tnwukrTDGZARUaoWD{jS4uisB@ z)P?+Eb5h}wkA&_E6%Xyo|yV)M9n9Mly4GrOvfay|OdT z`V{c!p|ms9{d76>j^uXfb8aPvWj6CirKz_UWSUW@m~^4*I0FAMSZCptFYRRoForZM z?$&=KhDxYA2RY^)CD{7kKo`b>q#Pl1>m&&5K$@JA>*WKUymtHg9K;iL;;$?>xkvtRfAzbCQ(JVj|^Nxx5n+r2$)sO_9U z8yNk6GhqF1i2?O}Vn%x)TiUCfXi;rdTU#3cq3z*UIs1_vZ02Migcl80@Y}NW4iBkY z97P(US%Z9<&x1d()HLL#`#yL+mm>RnP(4$D96ohNrS6F0b82B}(Lua5>}$G=dGai6 z*kJpcEzLwN9P34s_k>3YFUS*WB}xxpBOL;1abaEbAnzc1EaIzWV_E0NJ)S%ZE}mjF zdYU(}VS)>uVH_+)6GtQ&el&B2H9c-q#u@nn7RB#-h4$|C1#oG)`a{%SQRpQ{?5+k8 zZNc0J3Gd0*NF$i`JtZ%}9V;jT`3Blj6}Fkyjr4+)8~lON!8*ZmHQL(OtU(T+WtxZm zh(V6ntgX3;)i;M11Q|reFNR`t3J<9t*hAl2H&|@pLTk?Gd~Yr=@y@2KQscb{j5v2Q$I4z!iR?*+K{>H`vQ z2C~96u~z-#2}X1JS+W>|8Y=`FFeNq-B@! zVHwA!6+B~%8YzJ*6BdCo88Euh^2bRD_I_*ZpGZ?w`f?aR(v2$61mnkvbTq8lJo>dU z!3sc-j|SXZ*ec8FL@(L^Db);>erpZusnHd}uw)#+4>IrKch)CR#jv=3DhPrd=oM&{X;{{1Ko~jQD#6rkki#WoPiFr(7}*DR)$X+8%c+ zHIXXAhzjE?H3jpyh$0;Yr4lJu(nu;=G82?~p5?)ZQwBE1e@R6t7)|SnKx1>fq)kAh zU_o95=@QFYjwn^^rS5zBRH4nNl@aa%($z^mVO_4s7j_~b$KO%)?o~6s>IPaO^Q|y6 zM%0>0nQ#i@TlplhkV>OZO5}j)2ZR7QjR#Q72Rg`DP3$E*P9SEI11(?`el!d!70`rQ zyBrsJ60GVno>V7NQ@%m679JqYVi~6o3d3+fe3r984SPv`rE(G??DB_h}d&7se`N|pmmlVZI+al+W};)yN5w5@8$sU7ciCeXw@V5c#UxhCh^p5VWh1mp(?rUXS$dW$`IIe^2pog9s zmedVQ+uBY6#WAkVTDyN(^YjIRFY}Q8Hpb0Zf;6KE?^rXw%BtKEKzjVrzu2Zh`;Jw9 zH;5Tp+hsvn1}yLQzn523*9EgHbp#1}*78KRiJ+c!mcxl(%vc_v)$_~po}fG>AEK-b z-P(AKG}74*DW`x6*OukKwb?TU_&@xZ10Myj9bkP6uuDX@^f;JBdCyvd)0Ya66a|(A z0mq#tu#D(wuml2L5Oy4-|FL0gYdQ>&O$`s=ixL2+7USgu<26sRF<=?noKy}O&j1X{>fv|0JfGV6Ed~%sgN??29TLBR zzIXFFA{)rE1f_)sQCxG91x2grf0)_bKEOp&U<@Sv$C=hQ+QDXF9Hn-ki_$~W1s>_$ zoT##s{<>jzTJAW<0)cUFQX;ocl_3sXA$yPm$MP~OHFzu*dr5G4x9C=}4Pd<=PV?@P zRDY;J3jzq<5*YkJBYG8CQZ54Q4<;0P16}}BXX|F}QXwaI8|<22H)tD^d>cYiw0Yo8 zx4=f7+Tjz|D8J^1@7y_B1`~lah6r?8k&^;$-apNbADjiRf)M;&lAMuZ27eCqfb+@n z@N=u0(|35?K$(~hJg`JyV)P=7BavF3fY$&)5G7Xh;l0gSRmLw6{IE_tiek&t&Xq45 z%CuWVaEcYLi+O6Z%2-LM+xR;0o__s@LgMBr?kGnPUxfm9ygHoZ2P^wyK#%xDAreW; zJ7J}mAGCz)=q^vMhNLz`x(Y+lkxDVB! z!K=0w@?SXSPK+Eqig#5mXTAit=z7jcw#g)m;>N>RqeM3Wu|gZyV5MyA;pe<0l^L5- zcOkLDBg3rjW+8CB<_19Z2#;j$gY3@S70Ku>NQ#y_$;aLgAKWcwd;uGBh!%-Rx_=Qz z7Hw!;l^;C$XygHjDJo{Xj(r6C%H<}!*OW8NA&i8KG7BTY zh^XDAGkWfEqH>pXPerseG0Q{mgf%nGXcHG}OHR2=_4861JA^2`$hDHf+_OPpq)$BN zsB?uT)cL1Auxy5p-QIH(Dsv{(I+$Zd#PJx7n%rd<`)cS`Z9+0SP3Ix?dFZ9Edv*L@ zIsK74T2nd+rK;DWmzlyCC2HAHoKeszyP{nqTLp!}9v0NbAh=y$Gc$hffBNW#^;ORP cQ6|FZV!7<8&ASocQz8g&W^Gz!?0NCO02o_ Date: Sat, 17 Jan 2026 16:38:05 +0800 Subject: [PATCH 3/8] Add ffprobe --- backend/src/services/transcription.service.ts | 78 ++++++++++++------- 1 file changed, 51 insertions(+), 27 deletions(-) diff --git a/backend/src/services/transcription.service.ts b/backend/src/services/transcription.service.ts index ebaf3c6..44dce5e 100644 --- a/backend/src/services/transcription.service.ts +++ b/backend/src/services/transcription.service.ts @@ -4,8 +4,10 @@ const require = createRequire(import.meta.url); const ffmpeg = require('fluent-ffmpeg'); import ffmpegStatic from 'ffmpeg-static'; // import { PassThrough } from 'stream'; -const videoUrl: string = 'https://s-cloudfront.cdn.ap.panopto.com/sessions/6022add8-4a07-47c6-a3d7-b3c6008a6bd2/fd4adc57-aaed-404b-ae6c-b3c6008a6bda-6283352b-6ff7-4cba-a309-b3c6008dd0be.hls/785609/fragmented.mp4https://s-cloudfront.cdn.ap.panopto.com/sessions/6022add8-4a07-47c6-a3d7-b3c6008a6bd2/fd4adc57-aaed-404b-ae6c-b3c6008a6bda-6283352b-6ff7-4cba-a309-b3c6008dd0be.hls/785609/fragmented.mp4'; -const outputFileName: string = 'output.mp3'; +const videoUrl: string = 'https://s-cloudfront.cdn.ap.panopto.com/sessions/c2405e48-0c2e-469e-a67f-b3ca00db4a0e/e14faa86-2c3f-4dff-953a-b3ca00db4a18-b5830cf8-3068-4665-89c2-b3d0006027d9.hls/310791/fragmented.mp4'; +const outputFileName: string = 'output.opus'; + + // Check if the path exists and tell fluent-ffmpeg where it is if (ffmpegStatic) { @@ -15,35 +17,57 @@ if (ffmpegStatic) { } export const createTranscriptionService = async (data: TranscriptionInput): Promise => { - await convertToAudio(data.url, outputFileName); + await convertToAudio(videoUrl, outputFileName); return []; }; const convertToAudio = async (videoUrl: string, outputFileName: string) => { - // const audioStream = new PassThrough(); - const command = ffmpeg(videoUrl) - .noVideo() - .audioCodec('libmp3lame') - .audioBitrate(64) - .format('mp3') - .on('start', (commandLine: string) => { - console.log(`Spawned FFmpeg with command: ${commandLine}`); - }) - .on('progress', (progress) => { - console.log(`Processing: ${progress.percent}% done`); - }) - .on('error', (err: Error) => { - console.error(`An error occurred: ${err.message}`); - }) - .on('end', () => { - console.log('Finished conversion!'); - }); - - // Option A: Save to a local file - command.save(outputFileName); - - // Option B: Pipe to a stream (Better for sending to an API) - // command.pipe(audioStream); + let totalDuration; + ffmpeg(videoUrl).ffprobe((err, metadata) => { + if (err) { + console.error("Error probing video:", err); + return; + } + totalDuration = metadata.format.duration; + console.log(`Video duration: ${totalDuration} seconds`); + }); + + await processInParallel(totalDuration, 900); +} + + +async function processInParallel(totalDuration, segmentDuration) { + const tasks = []; + + // Step 1: Create separate conversion tasks for each segment + for (let start = 0; start < totalDuration; start += segmentDuration) { + tasks.push(new Promise((resolve, reject) => { + const outputName = `temp_part_${start}.opus`; + ffmpeg(videoUrl) + .setStartTime(start) + .setDuration(segmentDuration) + .noVideo() + .audioCodec('libopus') + .audioBitrate('32k') + .audioFrequency(16000) + .audioChannels(1) + .format('opus') + .on('end', () => resolve(outputName)) + .on('error', reject) + .save(outputName); + })); + } + + // Step 2: Run all conversions in parallel + const tempFiles = await Promise.all(tasks); + + // Step 3: Combine all temp files into one final output + const mergedCommand = ffmpeg(); + tempFiles.forEach(file => mergedCommand.input(file)); + + mergedCommand + .on('end', () => console.log('Final audio combined!')) + .mergeToFile('final_output.mp3', './temp_dir/'); } \ No newline at end of file From 516f09f342c1311513a4106e4bad36e6ea1bc812 Mon Sep 17 00:00:00 2001 From: glenn-chew Date: Sat, 17 Jan 2026 12:30:55 +0800 Subject: [PATCH 4/8] Extract captions from API --- content.js | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ manifest.json | 30 +++++++++++++++++++ popup.js | 19 +++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 content.js create mode 100644 manifest.json create mode 100644 popup.js diff --git a/content.js b/content.js new file mode 100644 index 0000000..638100d --- /dev/null +++ b/content.js @@ -0,0 +1,79 @@ +async function getLanguage() { + try { + const url = 'https://mediaweb.ap.panopto.com/Panopto/Pages/Viewer/DeliveryInfo.aspx'; + const urlParams = new URLSearchParams(window.location.search); + const deliveryId = urlParams.get('id'); + const params = new URLSearchParams({ + deliveryId: deliveryId, + responseType: 'json' + }); + + + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: params.toString(), + credentials: 'include' + }); + + + const data = await response.json(); + console.log(data.Delivery.AvailableLanguages) + return data.Delivery.AvailableLanguages + + } catch (error) { + console.error('Error fetching DeliveryInfo:', error); + } +} + +async function getDeliveryInfo() { + try { + const languages = await getLanguage(); + + const url = 'https://mediaweb.ap.panopto.com/Panopto/Pages/Viewer/DeliveryInfo.aspx'; + const urlParams = new URLSearchParams(window.location.search); + const deliveryId = urlParams.get('id'); + const params = new URLSearchParams({ + deliveryId: deliveryId, + getCaptions: 'true', + language: languages[0], + responseType: 'json' + }); + + + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: params.toString(), + credentials: 'include' + }); + + const data = await response.json(); + + return data + + } catch (error) { + console.error('Error fetching DeliveryInfo:', error); + } +} + +chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { + if (request.action === "extractCaptions") { + (async () => { + try { + const data = await getDeliveryInfo(); + sendResponse({ success: true, data: data }); + + } catch (error) { + sendResponse({ success: false, error: error.message }); + } + })(); + return true; + } +}); + +// \ No newline at end of file diff --git a/manifest.json b/manifest.json new file mode 100644 index 0000000..e2d19fa --- /dev/null +++ b/manifest.json @@ -0,0 +1,30 @@ +{ + "manifest_version": 3, + "name": "Lecture Video Helper", + "description": "This extension, Lecture Video Helper, is designed to summarise key topics/points from a given Panopto lecture.", + "version": "1.0", + "browser_action": { + "default_icon": "icon.png", + "default_popup": "window.html" + }, + "permissions": [ + "scripting", + "activeTab", + "cookies", + "storage" + ], + "host_permissions": [ + "https://mediaweb.ap.panopto.com/*" + ], + "content_scripts": [ + { + "matches": [ + "*://*.panopto.com/*" + ], + "js": [ + "content.js" + ], + "all_frames": false + } + ] +} \ No newline at end of file diff --git a/popup.js b/popup.js new file mode 100644 index 0000000..5fd6602 --- /dev/null +++ b/popup.js @@ -0,0 +1,19 @@ +document.getElementById("getCaptions").addEventListener("click", async () => { + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }); + + chrome.scripting.executeScript( + { + target: { tabId: tab.id, allFrames: true }, + files: ["content.js"] + }, + () => { + chrome.tabs.sendMessage(tab.id, { action: "extractCaptions" }, (response) => { + if (chrome.runtime.lastError) { + console.error("Message failed:", chrome.runtime.lastError.message); + } else { + console.log("Captions:", response?.captions); + } + }); + } + ); +}); \ No newline at end of file From 6f68fbd45309b718ee93ac7056296777a142cc4f Mon Sep 17 00:00:00 2001 From: glenn-chew Date: Sat, 17 Jan 2026 12:35:33 +0800 Subject: [PATCH 5/8] Add icon --- icon.png | Bin 0 -> 9336 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 icon.png diff --git a/icon.png b/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..9c5f56033fc2b341a9004bc264858d04d660d15a GIT binary patch literal 9336 zcmeHtc|6qL_y22Vn2}{DA|YhRuD9J-v(6~e*q2a}j6G}CnN*|&BZNX3Wv4_^GOdb` z$x_xy*|UZ0=6k2l=kxo0|NH*+{qOsD@R--VuXFCX_ndR@eV%h(S8c2HA%}Wd`&0}4;+Z2)Eh>}# z*eu%|?b{nu<7r=Us`BWhrGI9FX}_uQKlt6Dhp93=&J_NfZ^ynG8mLKQa&qRYH{(V+ zxAnv)Uhw`z{nS{ES1VZE9_-w9OT*wP|I=U76Im?DPRis^cL+oF@ZMY;MgBCRa1S@E zjZ{mXfOVqsb#05(G?3h%7DvQkXm}Dy9Knl-At$g!z4GSY}SD|e9n)^mqIBUtOsGSVIJ!`VQ32;1R zzu#WpQ*O&N*Qe39x&`o*ZrLK56}5DJ@>P9D-EJrP_kXf5r4GFrRl!B5J~yIa2NFk7 z)wFb3JfL4V?xOX52L=RJz~XUR61gz}Z^YQ8@i`4C;{iYUoj36!K-G3fO|-{6?5#!{ zAv}TS#`hm__I>KU5oQ#si_?|7jpbT{&;7pX)SuTUAtH4bL!_ZNz_3zkXwTZfqy&Yk zYLzBfV&<_s+r?2aHU)#9uFTY4v*=R~er0?JEZN;TMR&8O7@aX(ErBN#zbA>k<&EvN zT`}>DeVHMHCukwbR&RNudQ0nk^Os~wgALdV1LNQg80;l$KVbu5349qXdbIj{<+>kb z`OFoJe*67NsI`~CcupIglyLbR!SJR%0@Ij&;|fi8JCbYfo0Ld*!s0JJg=6q18!xVv z29!5pxfOMWKHjpE@qxOlLKq6PoUS9?S&|HAYUm+6VIWXuhuAw^@c{o^!;;dSWlueV ziI%9vRu5R~lv7fE+Csz@8M^Dlt?D1tcX_%+iMSLEJ~5+jPKk*L{M^v|O|+z3z8XaA zF^_F?vlWHK3u+69nG!^SuoQ2a2$+zMIf=<9W=v2Nf$Bz|F4WT1vn^0*0&T94%Lj=C z*}(MyQx)hy3K-{#W4OzQb?r5+9+4shF2xH`c=?4KmL$Ja-UY+FTNES-J{I?Za*3R4 zzKnBWeE97NZqj%y62+7efx8O>5!@nBd&{eP>kcddz{!`^izlJ7K^+!*4eWEQgBi+Z z%zclT5rJC>MpTxfeV^nj8w_E1z*_Dn;-*!6Aa$W??zY25{nO5bQ~r>8wsaS)TPxqM z)JPM`y_RUK^Zq^pH*E;Sj5)L3%&kc5{eUzI6~2ltd?}oQmMCSixV}$9(;|8LEG`P> zFTo=FKV#Y~!OU;aIpxhFL`l$r)c=q-v4%3d`Jd)pR+ASjp*!Iv7Z{uUz%}%}Rx4QN z?e&M$i`t07XQJ>3izJ(|h#w4G2v8}N#zqVHM80@YKs zHG!237=B%n-JzSL=S3!e&EgO%k0nLV)v`E#%RzNSp&C8xrReuPOv3j&Fq`tg>*&J# zbs6*yg(`R>&3*S4H8ZZ#g0R?;j4t#)mYz#mx}R905JMVyeMpx&dV$MmwjqC+A~7}- z&2S3iz%PO^bmQUNKNO9s(6FFaPnr1F!cg5jFBt*mkv>C`>XO9tBm%GN+$Wj8-T|klAZS61o@Pv`#hm0$PK4k0 z=uWpoHiTz`oK8bdh*+*5>7H)~7?&GP;-swywT7hX2A0*!L%7YFEYgTzCad`LuL-DV3uZ(k(qNgMl zh9~(h$7r!sDUo;8gU#`D0CDUmZch}KQI8WWmUmwYnvf>Pm>}QnI)WK{6$jZ}PC&+U zYO7#zM+{)Gl3<3hz-7v~VY;NTt|Oc^sZU0j>0k=KzaKQR_vUO+#hOYE-$(>nIU6Vy zAL5qb7drFMm_prghU=Mc8GU=3i1&(gRwXAJK2k7pQvB4* zeqD%jb9MKI5Gr`Wle}bcqtL2=S@H9D%BT`d=BhX@v~p>{@FxY+^M>awFIA^k|E{P% zo-+N~nba|T(HdQk!H1uhT0{`7$C?uG1gQ#4yiI$l^6am}M=5rOb6lTYW?Lq9q*re1 z!sO`>ix6ii20C$;fTM^ z+kOU7h+T^hBjxrBgZ&=*4JtvSO;IJKabUM-Wb=@w894CrcLT2MI%s%eutrwvXD(4& zzXC!*3si|*>a~A0-1mzWKKOfDg+AMzHtosUWTre;FvCPY7{(rWwwdeYM5fyn64!kb z;BuE;lM@Eq@6HT&@K$2*D^HNOW#ZE+V*)0ZA)pg2DH?hd)h9|cJUsmE6u2`DkmMV0 zc`I~!nrOj?P_7gzb{{JEJ)Pwiy%t78hs%d9OCUYyyK-#Ou%B<3q?fMXw&JtM9!}~zu5CR zI_P#Sq#~LDVX00m=u64*{s5>K3PLEq_w~e=z240zrxn0FcsuSR{pw&&Ty9)G%UsywKYDFg}e=V;-7puzs+w_+uQ!dS00^+n{WsukL}UNm*80q#JmrW7=o=791Ei z!Etp(ZG3HgiChiqoc0pL(+4{+0dr4y*8+nDHE2`&Np&sPJzPaBv9`=>LcTF}dZarvAc){7z!}Df1Er^l}3)+jrc9oL>WRc~q-GsKh&!uc&ts7et zG<=D(UT@JmP$VoH>O>iEizeTYDm7WeXN4Y{zr6J;rZnCx!BCIwgr zEblezjYSkG)tqft4SUq;C&N6V+|x9hhb4Ry>&Iw&7~g22xV(n;=!;=p z4dP$9ejyKFDka-qf8?R=3igA8Us?*mqf98V63U*Drd0avC6T;q=rn(O4ChMr^p557U zG4zd4S`=rv_gJZKW|=>Kk5eI~7^q}<+q?R8Rn$>u2OeHER5?z`*L!|BH*iqh*oBKP zF0HY2o3`ODe*4Nkm?nS1O~tmds=$Z-i8N$c!RUOw=7~3FXW^r~#~#+@sP_~-KVQ$ORGQ@8?6$a&l+C2lFqYitpxPhCELAT?vt)JKgjPZ*hJ4pYywF zuQ%`r@Ueyq$pcM4`8~Rgd{+_&7Ug zXjr)@X6I?p^xuj%H+HLQMD+)V^`HeZs4bFrrstb?x$$~BTUBiTtPLaz2a7DWR8)nV zOeEE7osD#v?)f3~g=#&o-yK{16YgfRg&KTZG2Dlq-X=Wxhk<>;;aePdPe$zrKXpQI zzIY`w=L?nhM>|z6K61W1^~j(&(5%ZUp;^ZBl6u{&>(PPoo8n7u-BSq-q*HX_yNPt< z$_d6pQUPQ^YNEZNCDOW5R7+xNSBEhgod}vAek4`u2IL5dvk9VnKK2yAvq5yTG${}VnY7fTmPW;v4jfAlTK@K#jADqZY)XE{ltzM zer0)=A+@okGCO^4elt0wF8sFU@KxKM{g6i6zNx%7I1oK%4j^I$*Fx#oO6%)sRm&a- zH?})N8E1&^5Hua?N(9ifO?j1+e!g6(Xbee-wt99aP9-?)ph1}nj@+a&TGoo}qzDC{) zXn082f97k8K}tgy8ou!Rcq7v2?r&cR-`XO{82R0;d=rf86={Y_!_@iT$`}?rhbXj; zlfsfPrAk2ti>P6^&BOPk1Q6=uiQhu#0i-wvHwhcb_f?E_BRq!DXL6WhD8fVQZ%)P( zJ3pb0oohU##>J*d2;_$$>y ztNZWb>nHkIup$2h7|ZW4Yc&BtUVktMlL-U)zJTs7K*Y*_(s&a4fu|jCDE+v|R^$y+#dXg5& zV5s1Ap6`SiPJyhB8hi@^aIdw;fH86V+s>+K2XxM$nH8pv{ff3$TLZy=?tpd;mk<+A z06Burg7pbGklJuFxcHYO^|sdn$_EOIE+@r2I&zEZ)*iFM8=u%$iX=111w8oHZ!I}K z3#jIgGj{WE85ZQh)+ecN8&ga}KX1S@A?h?nN*C4JPqaADpRyy+jRe!PnNt;PO6*N}ZIro}s+v z05>@`rCuOM=iW-R--ek`U!C3)_4swPz?y7nmrY$S{EJOq_8C4EFA|}+$oi1^=w#OSDo8+ab$`cHI>Pge0=9d&ACCu-^ zo^qUc)}+e$dd>DTgh-G^baTn-VfLG?Jatr&IQhgZp;EeguTp+Sy^_>UD>F>PYSq?J z`uUZ$&~wfXOh2KiUXNyQas#>{tI9lA!@aY@$od`z*PUt^{5AHth^FxD-3h~rZPN+* z=9g{XaXDRkDjIWg0QVzIIm?cGliC&@f(dX?VJ$E&T>JNLCWD;>6? zeX_C@V%E8&rU!D_)mNTocNje|kmY^%rE>8znRDZ-FHf?yTh32P`18-4AV}FvL~#Gg?RQ~$7qQfuug^z}7xIw$qg5IWeUK-cuy#ZI0?j1l><>_dK|8r#j+E7>L^7pP_!9B71|T2Fh-{ z6~8V@q%41r=IsgmN6&cwa?mS_4aGKMv*idp?&1@tnwukrTDGZARUaoWD{jS4uisB@ z)P?+Eb5h}wkA&_E6%Xyo|yV)M9n9Mly4GrOvfay|OdT z`V{c!p|ms9{d76>j^uXfb8aPvWj6CirKz_UWSUW@m~^4*I0FAMSZCptFYRRoForZM z?$&=KhDxYA2RY^)CD{7kKo`b>q#Pl1>m&&5K$@JA>*WKUymtHg9K;iL;;$?>xkvtRfAzbCQ(JVj|^Nxx5n+r2$)sO_9U z8yNk6GhqF1i2?O}Vn%x)TiUCfXi;rdTU#3cq3z*UIs1_vZ02Migcl80@Y}NW4iBkY z97P(US%Z9<&x1d()HLL#`#yL+mm>RnP(4$D96ohNrS6F0b82B}(Lua5>}$G=dGai6 z*kJpcEzLwN9P34s_k>3YFUS*WB}xxpBOL;1abaEbAnzc1EaIzWV_E0NJ)S%ZE}mjF zdYU(}VS)>uVH_+)6GtQ&el&B2H9c-q#u@nn7RB#-h4$|C1#oG)`a{%SQRpQ{?5+k8 zZNc0J3Gd0*NF$i`JtZ%}9V;jT`3Blj6}Fkyjr4+)8~lON!8*ZmHQL(OtU(T+WtxZm zh(V6ntgX3;)i;M11Q|reFNR`t3J<9t*hAl2H&|@pLTk?Gd~Yr=@y@2KQscb{j5v2Q$I4z!iR?*+K{>H`vQ z2C~96u~z-#2}X1JS+W>|8Y=`FFeNq-B@! zVHwA!6+B~%8YzJ*6BdCo88Euh^2bRD_I_*ZpGZ?w`f?aR(v2$61mnkvbTq8lJo>dU z!3sc-j|SXZ*ec8FL@(L^Db);>erpZusnHd}uw)#+4>IrKch)CR#jv=3DhPrd=oM&{X;{{1Ko~jQD#6rkki#WoPiFr(7}*DR)$X+8%c+ zHIXXAhzjE?H3jpyh$0;Yr4lJu(nu;=G82?~p5?)ZQwBE1e@R6t7)|SnKx1>fq)kAh zU_o95=@QFYjwn^^rS5zBRH4nNl@aa%($z^mVO_4s7j_~b$KO%)?o~6s>IPaO^Q|y6 zM%0>0nQ#i@TlplhkV>OZO5}j)2ZR7QjR#Q72Rg`DP3$E*P9SEI11(?`el!d!70`rQ zyBrsJ60GVno>V7NQ@%m679JqYVi~6o3d3+fe3r984SPv`rE(G??DB_h}d&7se`N|pmmlVZI+al+W};)yN5w5@8$sU7ciCeXw@V5c#UxhCh^p5VWh1mp(?rUXS$dW$`IIe^2pog9s zmedVQ+uBY6#WAkVTDyN(^YjIRFY}Q8Hpb0Zf;6KE?^rXw%BtKEKzjVrzu2Zh`;Jw9 zH;5Tp+hsvn1}yLQzn523*9EgHbp#1}*78KRiJ+c!mcxl(%vc_v)$_~po}fG>AEK-b z-P(AKG}74*DW`x6*OukKwb?TU_&@xZ10Myj9bkP6uuDX@^f;JBdCyvd)0Ya66a|(A z0mq#tu#D(wuml2L5Oy4-|FL0gYdQ>&O$`s=ixL2+7USgu<26sRF<=?noKy}O&j1X{>fv|0JfGV6Ed~%sgN??29TLBR zzIXFFA{)rE1f_)sQCxG91x2grf0)_bKEOp&U<@Sv$C=hQ+QDXF9Hn-ki_$~W1s>_$ zoT##s{<>jzTJAW<0)cUFQX;ocl_3sXA$yPm$MP~OHFzu*dr5G4x9C=}4Pd<=PV?@P zRDY;J3jzq<5*YkJBYG8CQZ54Q4<;0P16}}BXX|F}QXwaI8|<22H)tD^d>cYiw0Yo8 zx4=f7+Tjz|D8J^1@7y_B1`~lah6r?8k&^;$-apNbADjiRf)M;&lAMuZ27eCqfb+@n z@N=u0(|35?K$(~hJg`JyV)P=7BavF3fY$&)5G7Xh;l0gSRmLw6{IE_tiek&t&Xq45 z%CuWVaEcYLi+O6Z%2-LM+xR;0o__s@LgMBr?kGnPUxfm9ygHoZ2P^wyK#%xDAreW; zJ7J}mAGCz)=q^vMhNLz`x(Y+lkxDVB! z!K=0w@?SXSPK+Eqig#5mXTAit=z7jcw#g)m;>N>RqeM3Wu|gZyV5MyA;pe<0l^L5- zcOkLDBg3rjW+8CB<_19Z2#;j$gY3@S70Ku>NQ#y_$;aLgAKWcwd;uGBh!%-Rx_=Qz z7Hw!;l^;C$XygHjDJo{Xj(r6C%H<}!*OW8NA&i8KG7BTY zh^XDAGkWfEqH>pXPerseG0Q{mgf%nGXcHG}OHR2=_4861JA^2`$hDHf+_OPpq)$BN zsB?uT)cL1Auxy5p-QIH(Dsv{(I+$Zd#PJx7n%rd<`)cS`Z9+0SP3Ix?dFZ9Edv*L@ zIsK74T2nd+rK;DWmzlyCC2HAHoKeszyP{nqTLp!}9v0NbAh=y$Gc$hffBNW#^;ORP cQ6|FZV!7<8&ASocQz8g&W^Gz!?0NCO02o_ Date: Sat, 17 Jan 2026 16:38:05 +0800 Subject: [PATCH 6/8] Add ffprobe --- backend/src/services/transcription.service.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/backend/src/services/transcription.service.ts b/backend/src/services/transcription.service.ts index 393725f..e2055b0 100644 --- a/backend/src/services/transcription.service.ts +++ b/backend/src/services/transcription.service.ts @@ -32,15 +32,16 @@ export const createTranscriptionService = async (data: TranscriptionInput): Prom const convertToAudio = async (videoUrl: string, outputFileName: string) => { - const metadata: any = await new Promise((resolve, reject) => { - ffmpeg(videoUrl).ffprobe((err, data) => { - if (err) reject(err); - else resolve(data); - }); + let totalDuration; + ffmpeg(videoUrl).ffprobe((err, metadata) => { + if (err) { + console.error("Error probing video:", err); + return; + } + totalDuration = metadata.format.duration; + console.log(`Video duration: ${totalDuration} seconds`); }); - const totalDuration = metadata.format.duration; - console.log(`Video duration: ${totalDuration} seconds`); await processInParallel(totalDuration, 900); } From 13bff08d5d249e285d0d6f7491d73b5bbeff3e16 Mon Sep 17 00:00:00 2001 From: glenn-chew Date: Sun, 18 Jan 2026 02:00:30 +0800 Subject: [PATCH 7/8] Update frontend based on caption availability --- frontend/src/App.tsx | 2 +- frontend/src/components/LoadingScreen.tsx | 91 +++++++++++++++++++++-- frontend/src/utils/summary.ts | 7 +- 3 files changed, 88 insertions(+), 12 deletions(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index c383079..9798d0c 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -134,7 +134,7 @@ function App() { {screen === 'loading' && ( { + return caps.map((cap, index, arr) => { + // Parse time string (format: "HH:MM:SS" or "MM:SS" or seconds) + const parseTime = (timeStr: string): number => { + // If it's already a number in string form + if (!isNaN(Number(timeStr))) { + return Number(timeStr); + } + // Parse HH:MM:SS or MM:SS format + const parts = timeStr.split(':').map(Number); + if (parts.length === 3) { + return parts[0] * 3600 + parts[1] * 60 + parts[2]; + } else if (parts.length === 2) { + return parts[0] * 60 + parts[1]; + } + return 0; + }; + + const start = parseTime(cap.time); + // End time is the start of the next caption, or start + 5 for the last one + const end = index < arr.length - 1 + ? parseTime(arr[index + 1].time) + : start + 5; + + return { + start, + end, + text: cap.caption + }; + }); + }; + // 2. Setup AbortController and Timeout const controller = new AbortController(); const timeoutId = setTimeout(() => { @@ -37,19 +70,61 @@ export function LoadingScreen({ isExtractingAudio, processMode, onSummaryGenerat const processLecture = async () => { try { + let summaries; + let transcriptionSegments; + if (processMode === 'summary') { - const resultingSummaries = await generateSummary(captions, streamUrl, { + // Generate summary (backend may auto-transcribe if no captions) + summaries = await generateSummary(captions, streamUrl, duration, { signal: controller.signal }); - clearTimeout(timeoutId); - onSummaryGenerated(resultingSummaries); + + // Also populate transcription tab from captions + if (captions && captions.length > 0) { + transcriptionSegments = convertCaptionsToSegments(captions); + } + // Note: If no captions, backend already transcribed, but we don't have segments here + // In this case, transcription tab will be empty - user can click transcribe if needed + } else { - const resultingSegments = await transcribeVideo(streamUrl, duration, { - signal: controller.signal - }); - clearTimeout(timeoutId); + // TRANSCRIPTION MODE + if (captions && captions.length > 0) { + // Use existing captions for both tabs + console.log('📝 Using existing captions for both tabs'); + transcriptionSegments = convertCaptionsToSegments(captions); + + // Also generate summary from captions + summaries = await generateSummary(captions, streamUrl, duration, { + signal: controller.signal + }); + } else { + // No captions - transcribe and then summarize + console.log('🎤 No captions found, extracting audio...'); + transcriptionSegments = await transcribeVideo(streamUrl, duration, { + signal: controller.signal + }); + + // Also generate summary from transcription + // Convert segments back to caption format for summary API + console.log('📊 Generating summary from transcription...'); + const captionsFromTranscription = transcriptionSegments.map(seg => ({ + caption: seg.text, + time: String(seg.start) + })); + summaries = await generateSummary(captionsFromTranscription, streamUrl, duration, { + signal: controller.signal + }); + } + } - onTranscriptionGenerated(resultingSegments); + clearTimeout(timeoutId); + + // Populate both tabs + if (summaries) { + onSummaryGenerated(summaries); + } + if (transcriptionSegments) { + onTranscriptionGenerated(transcriptionSegments); } } catch (err: any) { diff --git a/frontend/src/utils/summary.ts b/frontend/src/utils/summary.ts index 21acff9..7bf8330 100644 --- a/frontend/src/utils/summary.ts +++ b/frontend/src/utils/summary.ts @@ -38,8 +38,8 @@ const MOCK_SUMMARIES: Summary[] = [ } ] -export async function generateSummary(captions: Caption[], streamUrl: string, options: { signal: AbortSignal }): Promise { - console.log(`Calling backend API with ${captions.length} captions and streamUrl: ${streamUrl ? 'Present' : 'None'}`) +export async function generateSummary(captions: Caption[], streamUrl: string, duration: number, options: { signal: AbortSignal }): Promise { + console.log(`Calling backend API with ${captions.length} captions, streamUrl: ${streamUrl ? 'Present' : 'None'}, duration: ${duration}s`) // Map frontend 'caption' field to backend 'text' field const mappedCaptions = captions.map(c => ({ @@ -54,7 +54,8 @@ export async function generateSummary(captions: Caption[], streamUrl: string, op }, body: JSON.stringify({ captions: mappedCaptions, - streamUrl: streamUrl + streamUrl: streamUrl, + duration: duration }), signal: options.signal }); From 87adf8a55de36d38473db52bfeec345048033571 Mon Sep 17 00:00:00 2001 From: glenn-chew Date: Sun, 18 Jan 2026 02:05:06 +0800 Subject: [PATCH 8/8] Update frontend based on caption availability --- backend/src/utils/index.ts | 40 ++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/backend/src/utils/index.ts b/backend/src/utils/index.ts index d19802f..204c64d 100644 --- a/backend/src/utils/index.ts +++ b/backend/src/utils/index.ts @@ -21,17 +21,45 @@ app.use('/api/transcribe', transcriptionRoutes); app.post('/generate-summary', async (req: Request, res: Response) => { try { - console.log("Called") - const { captions } = req.body; + console.log("Called /generate-summary"); + const { captions, streamUrl, duration } = req.body; + let captionsToUse = captions; + + // Check if we have captions to work with if (!captions || !Array.isArray(captions) || captions.length === 0) { - res.status(400).json({ error: 'Captions array is required' }); - return; + // No captions - check if we can auto-transcribe + if (streamUrl && duration) { + console.log(`📝 No captions provided. Auto-transcribing from stream...`); + console.log(` URL: ${streamUrl.substring(0, 50)}...`); + console.log(` Duration: ${duration}s`); + + // Import and call transcription service + const { createTranscriptionService } = await import('../services/transcription.service.js'); + const transcriptionSegments = await createTranscriptionService({ + url: streamUrl, + duration: duration + }); + + console.log(`✅ Transcription complete: ${transcriptionSegments.length} segments`); + + // Convert transcription segments to caption format + captionsToUse = transcriptionSegments.map((segment: any) => ({ + text: segment.text, + start: segment.start, + end: segment.end + })); + } else { + res.status(400).json({ + error: 'Either captions array or streamUrl + duration are required' + }); + return; + } } - console.log(`Received ${captions.length} captions, generating summary...`); + console.log(`Generating summary from ${captionsToUse.length} captions...`); - const summaries = await generateLectureSummary(captions); + const summaries = await generateLectureSummary(captionsToUse); console.log(`Generated ${summaries.length} topic summaries`); res.json(summaries);