fix: timeout issues

frontend/main.py

@@ -14,7 +14,7 @@
 # Page configuration
 st.set_page_config(
     page_title="OmniPDF",
-    page_icon="🦸",
+    page_icon="🔍",
     layout="wide",
     initial_sidebar_state="expanded",
 )
@@ -110,12 +110,6 @@
         icon="📄",
     )
 
-    settings_UI = st.Page(
-        page="my_pages/10_settings_UI.py",
-        title="Settings",
-        icon="⚙️"
-    )
-
     # To go between the different pages
     pg = st.navigation(
         pages=[
@@ -125,12 +119,10 @@
             wordcloud_UI,
             translate_UI,
             metadata_UI,
-            settings_UI,
         ]
     )
 
     # Add version number to sidebar footer
-    st.sidebar.markdown("---")
     st.sidebar.markdown(
         f"<div class='version-footer'>v{html.escape(__version__)}</div>",
         unsafe_allow_html=True

frontend/my_pages/1_upload_UI.py

@@ -18,8 +18,6 @@
 if "uploaded_files" not in st.session_state:
     st.session_state.uploaded_files = None
 
-client = httpx.AsyncClient(cookies=st.session_state.httpx_cookies, timeout=300.0)  # 5 minute timeout for long translations
-
 # Language options for translation
 LANGUAGES = {
     "auto": "Auto-detect",
@@ -38,7 +36,7 @@
 }
 
 
-async def check_job_status(doc_id: str, endpoint: str):
+async def check_job_status(doc_id: str, endpoint: str, client: httpx.AsyncClient):
     """Check the status of a specific job."""
     try:
         response = await client.get(f"{PDF_PROCESSOR_URL}/{endpoint}/{doc_id}")
@@ -56,7 +54,7 @@ async def check_job_status(doc_id: str, endpoint: str):
         return "error"
 
 
-async def poll_processing_status(doc_id: str, status_container, progress_text, source_lang="", target_lang="", max_attempts=120, delay=2):
+async def poll_processing_status(doc_id: str, status_container, progress_text, client: httpx.AsyncClient, source_lang="", target_lang="", delay=2):
     """
     Poll the backend for processing status and update the UI.
 
@@ -93,24 +91,24 @@ async def poll_processing_status(doc_id: str, status_container, progress_text, s
     translation_triggered = False
     renderer_triggered = False
 
-    for attempt in range(max_attempts):
+    while True:
         all_complete = True
         status_lines = []
 
         # STAGE 1: Check extraction status
-        extraction_status = await check_job_status(doc_id, "extractor")
+        extraction_status = await check_job_status(doc_id, "extractor", client)
         stages["extraction"]["status"] = extraction_status
 
         # STAGE 2: After extraction completes, check embedding and translation concurrently
         if extraction_status == "completed":
             # Check embedding status
-            embedding_status = await check_job_status(doc_id, "embed/sentence")
+            embedding_status = await check_job_status(doc_id, "embed/sentence", client)
             stages["embedding"]["status"] = embedding_status
 
             # Check/trigger translation (concurrent with embedding)
             if "translation" in stages:
                 if not translation_triggered:
-                    translation_status = await check_job_status(doc_id, "translation")
+                    translation_status = await check_job_status(doc_id, "translation", client)
 
                     # If translation not started, trigger it
                     if translation_status == "not_started":
@@ -143,21 +141,21 @@ async def poll_processing_status(doc_id: str, status_container, progress_text, s
                     stages["translation"]["status"] = translation_status
                 else:
                     # Already triggered, just check status
-                    translation_status = await check_job_status(doc_id, "translation")
+                    translation_status = await check_job_status(doc_id, "translation", client)
                     stages["translation"]["status"] = translation_status
 
             # STAGE 3: Concurrent processing after stage 2 completes
             # Metadata: Check after embedding completes (translation can still be running)
             if embedding_status == "completed":
-                metadata_status = await check_job_status(doc_id, "metadata")
+                metadata_status = await check_job_status(doc_id, "metadata", client)
                 stages["metadata"]["status"] = metadata_status
 
             # Renderer: Check/trigger after translation completes (embedding can still be running)
             if "renderer" in stages and "translation" in stages:
                 translation_status = stages["translation"]["status"]
                 if translation_status == "completed":
                     if not renderer_triggered:
-                        renderer_status = await check_job_status(doc_id, "renderer")
+                        renderer_status = await check_job_status(doc_id, "renderer", client)
 
                         # If renderer not started, trigger it
                         if renderer_status == "not_started":
@@ -189,7 +187,7 @@ async def poll_processing_status(doc_id: str, status_container, progress_text, s
                         stages["renderer"]["status"] = renderer_status
                     else:
                         # Already triggered, just check status
-                        renderer_status = await check_job_status(doc_id, "renderer")
+                        renderer_status = await check_job_status(doc_id, "renderer", client)
                         stages["renderer"]["status"] = renderer_status
 
         # Build status display grouped by stage
@@ -261,30 +259,37 @@ async def poll_processing_status(doc_id: str, status_container, progress_text, s
 
         await asyncio.sleep(delay)
 
-    status_container.warning("⚠️ Processing timeout. Some stages may still be running.")
-    progress_text.empty()
-    return False
-
 
 async def process_pdf(uploaded_file, file_expander, source_lang="", target_lang=""):
     """
     Uploads PDF to backend and stores document metadata in session state.
     Optionally triggers translation if languages are specified.
     """
-
     # Process pdf through PDF_processor endpoint
     try:
-        # Upload the PDF document
+        # Upload the PDF document with longer timeout for file upload
         logger.info(f"Uploading PDF: {uploaded_file}")
         bytes_data = uploaded_file.getvalue()  # bytes
         files = {"file": (uploaded_file.name, bytes_data, "application/pdf")}
 
-        upload_response = await client.post(
+        upload_client = httpx.AsyncClient(
+            cookies=st.session_state.httpx_cookies,
+            timeout=httpx.Timeout(connect=10.0, read=120.0, write=120.0, pool=10.0)
+        )
+        upload_response = await upload_client.post(
             f"{PDF_PROCESSOR_URL}/documents/", files=files
         )
+        await upload_client.aclose()
         if upload_response.cookies:
             st.session_state.httpx_cookies.update(upload_response.cookies)
 
+        # Create polling client AFTER upload to ensure it has the latest session cookies
+        # Use longer read timeout (120s) to handle translation/renderer trigger requests for large PDFs
+        poll_client = httpx.AsyncClient(
+            cookies=st.session_state.httpx_cookies,
+            timeout=httpx.Timeout(connect=10.0, read=120.0, write=10.0, pool=10.0)
+        )
+
         logger.info(f"Upload PDF response: {upload_response.text}")
 
         upload_data = upload_response.json()
@@ -312,17 +317,20 @@ async def process_pdf(uploaded_file, file_expander, source_lang="", target_lang=
 
             # Poll for processing status with spinner
             with st.spinner("Processing document..."):
-                await poll_processing_status(doc_id, status_container, progress_text, source_lang, target_lang)
+                await poll_processing_status(doc_id, status_container, progress_text, poll_client, source_lang, target_lang)
 
     except Exception as e:
         with file_expander:
             st.error(f"❌ Error processing PDF: {uploaded_file.name}")
         logger.error(f"Error processing PDF: {e}")
+    finally:
+        # Close the client after processing
+        await poll_client.aclose()
 
 
 
 
-st.markdown('<h1 class="main-header">🦸 OmniPDF</h1>', unsafe_allow_html=True)
+st.markdown('<h1 class="main-header">🔍 OmniPDF</h1>', unsafe_allow_html=True)
 st.header("📁 Upload PDF")
 uploaded_files = st.file_uploader(
     "Choose a PDF file",

pdf_extraction_service/main.py

@@ -8,6 +8,10 @@
     level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s"
 )
 
+# Enable DEBUG logging for docling to see more detailed processing information
+logging.getLogger("docling").setLevel(logging.DEBUG)
+logging.getLogger("docling_core").setLevel(logging.DEBUG)
+
 app = FastAPI(root_path="/pdf_extraction")
 
 # Initialize Prometheus metrics

pdf_extraction_service/routers/extractor.py

@@ -1,5 +1,5 @@
 import asyncio
-from fastapi import APIRouter, BackgroundTasks, HTTPException
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Response
 import logging
 import time
 import io
@@ -30,7 +30,7 @@
 USE_GPU = os.getenv("USE_GPU", "false").lower() == "true"
 logger.info(f"GPU acceleration enabled: {USE_GPU}")
 
-async def process_pdf(doc_id: str, presign_url: str, img_scale: float = 2.0):
+async def process_pdf(doc_id: str, presign_url: str, img_scale: float = 1.0):
     start_time = time.time()
 
     opts = PdfPipelineOptions()
@@ -164,18 +164,24 @@ async def submit_pdf(doc_id: str, download_url: str, background_tasks: Backgroun
 
 
 @router.get("/{doc_id}", response_model=ExtractResponse)
-async def get_status(doc_id: str):
+async def get_status(doc_id: str, response: Response):
     job = load_job(doc_id=doc_id, job_type=JobType.EXTRACTION)
     if not job:
         raise HTTPException(status_code=404, detail="Document ID not found")
 
-    if job.get("status") == "failed":
+    job_status = job.get("status", "unknown")
+
+    if job_status == "failed":
         error_message = job.get("data", {}).get("message", "Processing failed")
         raise HTTPException(status_code=500, detail=error_message)
 
     job_data = job.get("data", {})
     result = job_data.get("result", None)
 
+    # Return 202 Accepted if still processing, 200 OK if completed
+    if job_status == "processing":
+        response.status_code = 202
+
     return ExtractResponse(
-        doc_id=doc_id, status=job.get("status", "unknown"), result=result
+        doc_id=doc_id, status=job_status, result=result
     )

pdf_extraction_service/utils/caption.py

@@ -13,14 +13,14 @@ async def get_caption(doc_id: str, image_id: str, image_url: str) -> str:
         "image_url": image_url
     }
 
-    async with httpx.AsyncClient() as client:
+    async with httpx.AsyncClient(timeout=httpx.Timeout(connect=60.0, read=120.0, write=60.0, pool=60.0)) as client:
         try:
             response = await client.post(f"{IMAGE_CAPTIONER_URL}/caption/", json=payload)
             response.raise_for_status()
             data = response.json()
             return data.get("caption", "")
         except (httpx.HTTPStatusError, httpx.RequestError) as e:
-            logger.error(f"HTTP error retrieving caption: {e}")
+            logger.error(f"HTTP error retrieving caption for image {image_id}: {type(e).__name__}: {e}", exc_info=True)
             return ""
         except Exception as e:
             logger.error(f"Unexpected error retrieving caption: {e}", exc_info=True)