Update frontend

backend/src/utils/index.ts

@@ -21,17 +21,45 @@ app.use('/api/transcribe', transcriptionRoutes);
 
 app.post('/generate-summary', async (req: Request, res: Response) => {
     try {
-        console.log("Called")
-        const { captions } = req.body;
+        console.log("Called /generate-summary");
+        const { captions, streamUrl, duration } = req.body;
 
+        let captionsToUse = captions;
+
+        // Check if we have captions to work with
         if (!captions || !Array.isArray(captions) || captions.length === 0) {
-            res.status(400).json({ error: 'Captions array is required' });
-            return;
+            // No captions - check if we can auto-transcribe
+            if (streamUrl && duration) {
+                console.log(`📝 No captions provided. Auto-transcribing from stream...`);
+                console.log(`   URL: ${streamUrl.substring(0, 50)}...`);
+                console.log(`   Duration: ${duration}s`);
+
+                // Import and call transcription service
+                const { createTranscriptionService } = await import('../services/transcription.service.js');
+                const transcriptionSegments = await createTranscriptionService({
+                    url: streamUrl,
+                    duration: duration
+                });
+
+                console.log(`✅ Transcription complete: ${transcriptionSegments.length} segments`);
+
+                // Convert transcription segments to caption format
+                captionsToUse = transcriptionSegments.map((segment: any) => ({
+                    text: segment.text,
+                    start: segment.start,
+                    end: segment.end
+                }));
+            } else {
+                res.status(400).json({
+                    error: 'Either captions array or streamUrl + duration are required'
+                });
+                return;
+            }
         }
 
-        console.log(`Received ${captions.length} captions, generating summary...`);
+        console.log(`Generating summary from ${captionsToUse.length} captions...`);
 
-        const summaries = await generateLectureSummary(captions);
+        const summaries = await generateLectureSummary(captionsToUse);
 
         console.log(`Generated ${summaries.length} topic summaries`);
         res.json(summaries);

content.js

@@ -0,0 +1,79 @@
+async function getLanguage() {
+    try {
+        const url = 'https://mediaweb.ap.panopto.com/Panopto/Pages/Viewer/DeliveryInfo.aspx';
+        const urlParams = new URLSearchParams(window.location.search);
+        const deliveryId = urlParams.get('id');
+        const params = new URLSearchParams({
+            deliveryId: deliveryId,
+            responseType: 'json'
+        });
+
+
+        const response = await fetch(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded',
+            },
+            body: params.toString(),
+            credentials: 'include'
+        });
+
+
+        const data = await response.json();
+        console.log(data.Delivery.AvailableLanguages)
+        return data.Delivery.AvailableLanguages
+
+    } catch (error) {
+        console.error('Error fetching DeliveryInfo:', error);
+    }
+}
+
+async function getDeliveryInfo() {
+    try {
+        const languages = await getLanguage();
+
+        const url = 'https://mediaweb.ap.panopto.com/Panopto/Pages/Viewer/DeliveryInfo.aspx';
+        const urlParams = new URLSearchParams(window.location.search);
+        const deliveryId = urlParams.get('id');
+        const params = new URLSearchParams({
+            deliveryId: deliveryId,
+            getCaptions: 'true',
+            language: languages[0],
+            responseType: 'json'
+        });
+
+
+        const response = await fetch(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded',
+            },
+            body: params.toString(),
+            credentials: 'include'
+        });
+
+        const data = await response.json();
+
+        return data
+
+    } catch (error) {
+        console.error('Error fetching DeliveryInfo:', error);
+    }
+}
+
+chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
+    if (request.action === "extractCaptions") {
+        (async () => {
+            try {
+                const data = await getDeliveryInfo();
+                sendResponse({ success: true, data: data });
+
+            } catch (error) {
+                sendResponse({ success: false, error: error.message });
+            }
+        })();
+        return true;
+    }
+});
+
+//

frontend/src/App.tsx

@@ -134,7 +134,7 @@ function App() {
 
                 {screen === 'loading' && (
                     <LoadingScreen
-                        isExtractingAudio={!hasCaptions || processMode === 'transcription'}
+                        isExtractingAudio={!hasCaptions}
                         processMode={processMode}
                         onSummaryGenerated={handleSummaryGenerated}
                         onTranscriptionGenerated={handleTranscriptionGenerated}

frontend/src/components/LoadingScreen.tsx

@@ -29,6 +29,39 @@ export function LoadingScreen({ isExtractingAudio, processMode, onSummaryGenerat
             return;
         }
 
+        // Helper function to convert Caption[] to TranscriptionSegment[]
+        const convertCaptionsToSegments = (caps: Caption[]): TranscriptionSegment[] => {
+            return caps.map((cap, index, arr) => {
+                // Parse time string (format: "HH:MM:SS" or "MM:SS" or seconds)
+                const parseTime = (timeStr: string): number => {
+                    // If it's already a number in string form
+                    if (!isNaN(Number(timeStr))) {
+                        return Number(timeStr);
+                    }
+                    // Parse HH:MM:SS or MM:SS format
+                    const parts = timeStr.split(':').map(Number);
+                    if (parts.length === 3) {
+                        return parts[0] * 3600 + parts[1] * 60 + parts[2];
+                    } else if (parts.length === 2) {
+                        return parts[0] * 60 + parts[1];
+                    }
+                    return 0;
+                };
+
+                const start = parseTime(cap.time);
+                // End time is the start of the next caption, or start + 5 for the last one
+                const end = index < arr.length - 1
+                    ? parseTime(arr[index + 1].time)
+                    : start + 5;
+
+                return {
+                    start,
+                    end,
+                    text: cap.caption
+                };
+            });
+        };
+
         // 2. Setup AbortController and Timeout
         const controller = new AbortController();
         const timeoutId = setTimeout(() => {
@@ -37,19 +70,61 @@ export function LoadingScreen({ isExtractingAudio, processMode, onSummaryGenerat
 
         const processLecture = async () => {
             try {
+                let summaries;
+                let transcriptionSegments;
+
                 if (processMode === 'summary') {
-                    const resultingSummaries = await generateSummary(captions, streamUrl, {
+                    // Generate summary (backend may auto-transcribe if no captions)
+                    summaries = await generateSummary(captions, streamUrl, duration, {
                         signal: controller.signal
                     });
-                    clearTimeout(timeoutId);
-                    onSummaryGenerated(resultingSummaries);
+
+                    // Also populate transcription tab from captions
+                    if (captions && captions.length > 0) {
+                        transcriptionSegments = convertCaptionsToSegments(captions);
+                    }
+                    // Note: If no captions, backend already transcribed, but we don't have segments here
+                    // In this case, transcription tab will be empty - user can click transcribe if needed
+
                 } else {
-                    const resultingSegments = await transcribeVideo(streamUrl, duration, {
-                        signal: controller.signal
-                    });
-                    clearTimeout(timeoutId);
+                    // TRANSCRIPTION MODE
+                    if (captions && captions.length > 0) {
+                        // Use existing captions for both tabs
+                        console.log('📝 Using existing captions for both tabs');
+                        transcriptionSegments = convertCaptionsToSegments(captions);
+
+                        // Also generate summary from captions
+                        summaries = await generateSummary(captions, streamUrl, duration, {
+                            signal: controller.signal
+                        });
+                    } else {
+                        // No captions - transcribe and then summarize
+                        console.log('🎤 No captions found, extracting audio...');
+                        transcriptionSegments = await transcribeVideo(streamUrl, duration, {
+                            signal: controller.signal
+                        });
+
+                        // Also generate summary from transcription
+                        // Convert segments back to caption format for summary API
+                        console.log('📊 Generating summary from transcription...');
+                        const captionsFromTranscription = transcriptionSegments.map(seg => ({
+                            caption: seg.text,
+                            time: String(seg.start)
+                        }));
+                        summaries = await generateSummary(captionsFromTranscription, streamUrl, duration, {
+                            signal: controller.signal
+                        });
+                    }
+                }
 
-                    onTranscriptionGenerated(resultingSegments);
+                clearTimeout(timeoutId);
+
+                // Populate both tabs
+                if (summaries) {
+                    onSummaryGenerated(summaries);
+                }
+                if (transcriptionSegments) {
+                    onTranscriptionGenerated(transcriptionSegments);
                 }
 
             } catch (err: any) {

frontend/src/utils/summary.ts

@@ -38,8 +38,8 @@ const MOCK_SUMMARIES: Summary[] = [
     }
 ]
 
-export async function generateSummary(captions: Caption[], streamUrl: string, options: { signal: AbortSignal }): Promise<Summary[]> {
-    console.log(`Calling backend API with ${captions.length} captions and streamUrl: ${streamUrl ? 'Present' : 'None'}`)
+export async function generateSummary(captions: Caption[], streamUrl: string, duration: number, options: { signal: AbortSignal }): Promise<Summary[]> {
+    console.log(`Calling backend API with ${captions.length} captions, streamUrl: ${streamUrl ? 'Present' : 'None'}, duration: ${duration}s`)
 
     // Map frontend 'caption' field to backend 'text' field
     const mappedCaptions = captions.map(c => ({
@@ -54,7 +54,8 @@ export async function generateSummary(captions: Caption[], streamUrl: string, op
         },
         body: JSON.stringify({
             captions: mappedCaptions,
-            streamUrl: streamUrl
+            streamUrl: streamUrl,
+            duration: duration
         }),
         signal: options.signal
     });

manifest.json

@@ -0,0 +1,30 @@
+{
+    "manifest_version": 3,
+    "name": "Lecture Video Helper",
+    "description": "This extension, Lecture Video Helper, is designed to summarise key topics/points from a given Panopto lecture.",
+    "version": "1.0",
+    "browser_action": {
+        "default_icon": "icon.png",
+        "default_popup": "window.html"
+    },
+    "permissions": [
+        "scripting",
+        "activeTab",
+        "cookies",
+        "storage"
+    ],
+    "host_permissions": [
+        "https://mediaweb.ap.panopto.com/*"
+    ],
+    "content_scripts": [
+        {
+            "matches": [
+                "*://*.panopto.com/*"
+            ],
+            "js": [
+                "content.js"
+            ],
+            "all_frames": false
+        }
+    ]
+}

popup.js

@@ -0,0 +1,19 @@
+document.getElementById("getCaptions").addEventListener("click", async () => {
+    const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });
+
+    chrome.scripting.executeScript(
+        {
+            target: { tabId: tab.id, allFrames: true },
+            files: ["content.js"]
+        },
+        () => {
+            chrome.tabs.sendMessage(tab.id, { action: "extractCaptions" }, (response) => {
+                if (chrome.runtime.lastError) {
+                    console.error("Message failed:", chrome.runtime.lastError.message);
+                } else {
+                    console.log("Captions:", response?.captions);
+                }
+            });
+        }
+    );
+});