diff --git a/backend/src/utils/index.ts b/backend/src/utils/index.ts index d19802f..204c64d 100644 --- a/backend/src/utils/index.ts +++ b/backend/src/utils/index.ts @@ -21,17 +21,45 @@ app.use('/api/transcribe', transcriptionRoutes); app.post('/generate-summary', async (req: Request, res: Response) => { try { - console.log("Called") - const { captions } = req.body; + console.log("Called /generate-summary"); + const { captions, streamUrl, duration } = req.body; + let captionsToUse = captions; + + // Check if we have captions to work with if (!captions || !Array.isArray(captions) || captions.length === 0) { - res.status(400).json({ error: 'Captions array is required' }); - return; + // No captions - check if we can auto-transcribe + if (streamUrl && duration) { + console.log(`📝 No captions provided. Auto-transcribing from stream...`); + console.log(` URL: ${streamUrl.substring(0, 50)}...`); + console.log(` Duration: ${duration}s`); + + // Import and call transcription service + const { createTranscriptionService } = await import('../services/transcription.service.js'); + const transcriptionSegments = await createTranscriptionService({ + url: streamUrl, + duration: duration + }); + + console.log(`✅ Transcription complete: ${transcriptionSegments.length} segments`); + + // Convert transcription segments to caption format + captionsToUse = transcriptionSegments.map((segment: any) => ({ + text: segment.text, + start: segment.start, + end: segment.end + })); + } else { + res.status(400).json({ + error: 'Either captions array or streamUrl + duration are required' + }); + return; + } } - console.log(`Received ${captions.length} captions, generating summary...`); + console.log(`Generating summary from ${captionsToUse.length} captions...`); - const summaries = await generateLectureSummary(captions); + const summaries = await generateLectureSummary(captionsToUse); console.log(`Generated ${summaries.length} topic summaries`); res.json(summaries); diff --git a/content.js b/content.js new file mode 100644 index 0000000..638100d --- /dev/null +++ b/content.js @@ -0,0 +1,79 @@ +async function getLanguage() { + try { + const url = 'https://mediaweb.ap.panopto.com/Panopto/Pages/Viewer/DeliveryInfo.aspx'; + const urlParams = new URLSearchParams(window.location.search); + const deliveryId = urlParams.get('id'); + const params = new URLSearchParams({ + deliveryId: deliveryId, + responseType: 'json' + }); + + + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: params.toString(), + credentials: 'include' + }); + + + const data = await response.json(); + console.log(data.Delivery.AvailableLanguages) + return data.Delivery.AvailableLanguages + + } catch (error) { + console.error('Error fetching DeliveryInfo:', error); + } +} + +async function getDeliveryInfo() { + try { + const languages = await getLanguage(); + + const url = 'https://mediaweb.ap.panopto.com/Panopto/Pages/Viewer/DeliveryInfo.aspx'; + const urlParams = new URLSearchParams(window.location.search); + const deliveryId = urlParams.get('id'); + const params = new URLSearchParams({ + deliveryId: deliveryId, + getCaptions: 'true', + language: languages[0], + responseType: 'json' + }); + + + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: params.toString(), + credentials: 'include' + }); + + const data = await response.json(); + + return data + + } catch (error) { + console.error('Error fetching DeliveryInfo:', error); + } +} + +chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { + if (request.action === "extractCaptions") { + (async () => { + try { + const data = await getDeliveryInfo(); + sendResponse({ success: true, data: data }); + + } catch (error) { + sendResponse({ success: false, error: error.message }); + } + })(); + return true; + } +}); + +// \ No newline at end of file diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index c383079..9798d0c 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -134,7 +134,7 @@ function App() { {screen === 'loading' && ( { + return caps.map((cap, index, arr) => { + // Parse time string (format: "HH:MM:SS" or "MM:SS" or seconds) + const parseTime = (timeStr: string): number => { + // If it's already a number in string form + if (!isNaN(Number(timeStr))) { + return Number(timeStr); + } + // Parse HH:MM:SS or MM:SS format + const parts = timeStr.split(':').map(Number); + if (parts.length === 3) { + return parts[0] * 3600 + parts[1] * 60 + parts[2]; + } else if (parts.length === 2) { + return parts[0] * 60 + parts[1]; + } + return 0; + }; + + const start = parseTime(cap.time); + // End time is the start of the next caption, or start + 5 for the last one + const end = index < arr.length - 1 + ? parseTime(arr[index + 1].time) + : start + 5; + + return { + start, + end, + text: cap.caption + }; + }); + }; + // 2. Setup AbortController and Timeout const controller = new AbortController(); const timeoutId = setTimeout(() => { @@ -37,19 +70,61 @@ export function LoadingScreen({ isExtractingAudio, processMode, onSummaryGenerat const processLecture = async () => { try { + let summaries; + let transcriptionSegments; + if (processMode === 'summary') { - const resultingSummaries = await generateSummary(captions, streamUrl, { + // Generate summary (backend may auto-transcribe if no captions) + summaries = await generateSummary(captions, streamUrl, duration, { signal: controller.signal }); - clearTimeout(timeoutId); - onSummaryGenerated(resultingSummaries); + + // Also populate transcription tab from captions + if (captions && captions.length > 0) { + transcriptionSegments = convertCaptionsToSegments(captions); + } + // Note: If no captions, backend already transcribed, but we don't have segments here + // In this case, transcription tab will be empty - user can click transcribe if needed + } else { - const resultingSegments = await transcribeVideo(streamUrl, duration, { - signal: controller.signal - }); - clearTimeout(timeoutId); + // TRANSCRIPTION MODE + if (captions && captions.length > 0) { + // Use existing captions for both tabs + console.log('📝 Using existing captions for both tabs'); + transcriptionSegments = convertCaptionsToSegments(captions); + + // Also generate summary from captions + summaries = await generateSummary(captions, streamUrl, duration, { + signal: controller.signal + }); + } else { + // No captions - transcribe and then summarize + console.log('🎤 No captions found, extracting audio...'); + transcriptionSegments = await transcribeVideo(streamUrl, duration, { + signal: controller.signal + }); + + // Also generate summary from transcription + // Convert segments back to caption format for summary API + console.log('📊 Generating summary from transcription...'); + const captionsFromTranscription = transcriptionSegments.map(seg => ({ + caption: seg.text, + time: String(seg.start) + })); + summaries = await generateSummary(captionsFromTranscription, streamUrl, duration, { + signal: controller.signal + }); + } + } - onTranscriptionGenerated(resultingSegments); + clearTimeout(timeoutId); + + // Populate both tabs + if (summaries) { + onSummaryGenerated(summaries); + } + if (transcriptionSegments) { + onTranscriptionGenerated(transcriptionSegments); } } catch (err: any) { diff --git a/frontend/src/utils/summary.ts b/frontend/src/utils/summary.ts index 21acff9..7bf8330 100644 --- a/frontend/src/utils/summary.ts +++ b/frontend/src/utils/summary.ts @@ -38,8 +38,8 @@ const MOCK_SUMMARIES: Summary[] = [ } ] -export async function generateSummary(captions: Caption[], streamUrl: string, options: { signal: AbortSignal }): Promise { - console.log(`Calling backend API with ${captions.length} captions and streamUrl: ${streamUrl ? 'Present' : 'None'}`) +export async function generateSummary(captions: Caption[], streamUrl: string, duration: number, options: { signal: AbortSignal }): Promise { + console.log(`Calling backend API with ${captions.length} captions, streamUrl: ${streamUrl ? 'Present' : 'None'}, duration: ${duration}s`) // Map frontend 'caption' field to backend 'text' field const mappedCaptions = captions.map(c => ({ @@ -54,7 +54,8 @@ export async function generateSummary(captions: Caption[], streamUrl: string, op }, body: JSON.stringify({ captions: mappedCaptions, - streamUrl: streamUrl + streamUrl: streamUrl, + duration: duration }), signal: options.signal }); diff --git a/icon.png b/icon.png new file mode 100644 index 0000000..9c5f560 Binary files /dev/null and b/icon.png differ diff --git a/manifest.json b/manifest.json new file mode 100644 index 0000000..e2d19fa --- /dev/null +++ b/manifest.json @@ -0,0 +1,30 @@ +{ + "manifest_version": 3, + "name": "Lecture Video Helper", + "description": "This extension, Lecture Video Helper, is designed to summarise key topics/points from a given Panopto lecture.", + "version": "1.0", + "browser_action": { + "default_icon": "icon.png", + "default_popup": "window.html" + }, + "permissions": [ + "scripting", + "activeTab", + "cookies", + "storage" + ], + "host_permissions": [ + "https://mediaweb.ap.panopto.com/*" + ], + "content_scripts": [ + { + "matches": [ + "*://*.panopto.com/*" + ], + "js": [ + "content.js" + ], + "all_frames": false + } + ] +} \ No newline at end of file diff --git a/popup.js b/popup.js new file mode 100644 index 0000000..5fd6602 --- /dev/null +++ b/popup.js @@ -0,0 +1,19 @@ +document.getElementById("getCaptions").addEventListener("click", async () => { + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }); + + chrome.scripting.executeScript( + { + target: { tabId: tab.id, allFrames: true }, + files: ["content.js"] + }, + () => { + chrome.tabs.sendMessage(tab.id, { action: "extractCaptions" }, (response) => { + if (chrome.runtime.lastError) { + console.error("Message failed:", chrome.runtime.lastError.message); + } else { + console.log("Captions:", response?.captions); + } + }); + } + ); +}); \ No newline at end of file