source_scraper.py

#!/usr/bin/env python3
"""
Source Scraper — continuously pulls knowledge from free public APIs
and feeds it into the knowledge engine database.

Sources:
  1. Project Gutenberg (via Gutendex)
  2. arXiv papers
  3. openFDA drug labels
  4. Internet Archive (military field manuals)
  5. PubMed Central open access
  6. OpenStax textbooks (metadata)

Runs in a continuous loop with 30s cooldown between fetches.
"""

import hashlib
import json
import logging
import os
import subprocess
import sys
import time
import xml.etree.ElementTree as ET
from datetime import datetime

import requests

# Add project dir to path so we can import db
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import db

# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------

CYCLE_FILE = "/tmp/source-scraper-cycle"
OFFSET_DIR = "/tmp"
LOG_FILE = "/tmp/source-scraper.log"
COOLDOWN = 30          # seconds between fetches
RATE_LIMIT = 3         # seconds between HTTP requests
MAX_TEXT_LEN = 30000   # trim full texts to this
MIN_IMAGE_SIZE = 2000  # skip tiny images (icons, tracking pixels)
IMAGE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images")
HIVEMIND_SCRIPT = os.environ.get("KE_HIVEMIND_SCRIPT", "")

# Source definitions
SOURCES = [
    "gutenberg",
    "arxiv",
    "openfda",
    "archive-military",
    "pubmed",
    "openstax",
]

GUTENBERG_TOPICS = [
    "science", "history", "philosophy", "medicine", "mathematics",
    "military", "agriculture", "engineering", "biology", "chemistry",
    "physics", "astronomy", "geology", "botany", "zoology", "anatomy",
]

ARXIV_CATEGORIES = [
    "cs.AI", "cs.CL", "q-bio.GN", "physics.gen-ph", "math.HO",
    "q-bio.PE", "astro-ph.EP", "cond-mat.mtrl-sci",
]

PUBMED_QUERIES = [
    "survival medicine", "infectious disease", "nutrition deficiency",
    "herbal medicine", "emergency trauma", "vaccine development",
    "antibiotic resistance", "cancer treatment", "genetics CRISPR",
    "protein folding",
]

OPENSTAX_BOOKS = [
    {"title": "Biology 2e", "url": "https://openstax.org/details/books/biology-2e", "subject": "biology"},
    {"title": "Chemistry 2e", "url": "https://openstax.org/details/books/chemistry-2e", "subject": "chemistry"},
    {"title": "University Physics Volume 1", "url": "https://openstax.org/details/books/university-physics-volume-1", "subject": "physics"},
    {"title": "Anatomy and Physiology 2e", "url": "https://openstax.org/details/books/anatomy-and-physiology-2e", "subject": "anatomy"},
    {"title": "Astronomy 2e", "url": "https://openstax.org/details/books/astronomy-2e", "subject": "astronomy"},
    {"title": "Psychology 2e", "url": "https://openstax.org/details/books/psychology-2e", "subject": "psychology"},
    {"title": "Principles of Economics 3e", "url": "https://openstax.org/details/books/principles-economics-3e", "subject": "economics"},
    {"title": "Microbiology", "url": "https://openstax.org/details/books/microbiology", "subject": "microbiology"},
    {"title": "Concepts of Biology", "url": "https://openstax.org/details/books/concepts-biology", "subject": "biology"},
    {"title": "Introduction to Sociology 3e", "url": "https://openstax.org/details/books/introduction-sociology-3e", "subject": "sociology"},
]

# ---------------------------------------------------------------------------
# Logging
# ---------------------------------------------------------------------------

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    handlers=[
        logging.FileHandler(LOG_FILE),
        logging.StreamHandler(sys.stdout),
    ],
)
log = logging.getLogger("source_scraper")

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

session = requests.Session()
session.headers.update({"User-Agent": "KnowledgeEngine/1.0"})
_last_request_time = 0


def rate_limited_get(url, **kwargs):
    """GET with rate limiting."""
    global _last_request_time
    elapsed = time.time() - _last_request_time
    if elapsed < RATE_LIMIT:
        time.sleep(RATE_LIMIT - elapsed)
    _last_request_time = time.time()
    kwargs.setdefault("timeout", 30)
    return session.get(url, **kwargs)


def make_hash(text):
    return hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest()


def get_offset(source):
    path = os.path.join(OFFSET_DIR, f"source-scraper-{source}-offset")
    try:
        with open(path) as f:
            return int(f.read().strip())
    except (FileNotFoundError, ValueError):
        return 0


def set_offset(source, val):
    path = os.path.join(OFFSET_DIR, f"source-scraper-{source}-offset")
    with open(path, "w") as f:
        f.write(str(val))


def get_cycle():
    try:
        with open(CYCLE_FILE) as f:
            return int(f.read().strip())
    except (FileNotFoundError, ValueError):
        return 0


def set_cycle(val):
    with open(CYCLE_FILE, "w") as f:
        f.write(str(val))


def ensure_source(name, source_type="api", url=None):
    """Make sure a source row exists; return its id."""
    conn = db.get_conn()
    row = conn.execute("SELECT id FROM sources WHERE name = ?", (name,)).fetchone()
    if row:
        conn.close()
        return row[0]
    conn.close()
    db.add_source(source_type, name, url=url)
    conn = db.get_conn()
    row = conn.execute("SELECT id FROM sources WHERE name = ?", (name,)).fetchone()
    conn.close()
    return row[0] if row else None


def hash_exists(doc_hash):
    """Check if a document hash already exists."""
    conn = db.get_conn()
    row = conn.execute("SELECT 1 FROM documents WHERE hash = ?", (doc_hash,)).fetchone()
    conn.close()
    return row is not None


def title_exists(title):
    """Check if a document with this exact title already exists (cross-pipeline dedup)."""
    conn = db.get_conn()
    row = conn.execute("SELECT 1 FROM documents WHERE title = ?", (title,)).fetchone()
    conn.close()
    return row is not None


def hivemind_log(msg):
    """Log to hive-mind ledger."""
    if os.path.isfile(HIVEMIND_SCRIPT):
        try:
            subprocess.run(
                [HIVEMIND_SCRIPT, "claude-code", "deploy", msg],
                timeout=10, capture_output=True,
            )
        except Exception:
            pass


def download_image(url, prefix="img"):
    """Download an image and save to IMAGE_DIR. Returns local path or None."""
    os.makedirs(IMAGE_DIR, exist_ok=True)
    try:
        # Determine extension
        ext = ".jpg"
        for e in (".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg"):
            if e in url.lower():
                ext = e
                break

        # Hash the URL for a unique filename
        fname = hashlib.md5(url.encode()).hexdigest()[:12] + ext
        local_path = os.path.join(IMAGE_DIR, fname)

        if os.path.exists(local_path):
            return f"/images/{fname}"

        resp = rate_limited_get(url, timeout=15)
        if resp.status_code != 200:
            return None
        if len(resp.content) < MIN_IMAGE_SIZE:
            return None

        # Check content type
        ctype = resp.headers.get("content-type", "")
        if ctype and "image" not in ctype and "octet" not in ctype:
            return None

        with open(local_path, "wb") as f:
            f.write(resp.content)
        log.info(f"[image] saved {fname} ({len(resp.content)} bytes)")
        return f"/images/{fname}"
    except Exception as e:
        log.warning(f"[image] failed to download {url[:80]}: {e}")
        return None


JUNK_IMG_PATTERNS = [
    "logo", "icon", "avatar", "banner", "button", "arrow", "pixel",
    "tracking", "spacer", "blank", "spinner", "loading", "badge",
    "favicon", "widget", "ad-", "advert", "social", "share",
    "1x1", "transparent",
]


def is_junk_image(url):
    """Filter out junk images (logos, icons, tracking pixels)."""
    url_lower = url.lower()
    return any(p in url_lower for p in JUNK_IMG_PATTERNS)


def extract_images_from_html(html, base_url, max_images=3):
    """Extract meaningful image URLs from HTML content."""
    import re as _re
    img_urls = _re.findall(r'<img[^>]+src=["\']([^"\']+)["\']', html, _re.IGNORECASE)
    results = []
    for src in img_urls:
        if is_junk_image(src):
            continue
        # Make absolute
        if src.startswith("//"):
            src = "https:" + src
        elif src.startswith("/"):
            from urllib.parse import urlparse
            parsed = urlparse(base_url)
            src = f"{parsed.scheme}://{parsed.netloc}{src}"
        elif not src.startswith("http"):
            continue
        # Must be an actual image extension or likely image
        if any(ext in src.lower() for ext in [".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg"]):
            results.append(src)
        if len(results) >= max_images:
            break
    return results


# ---------------------------------------------------------------------------
# Source scrapers
# ---------------------------------------------------------------------------

def scrape_gutenberg():
    """Fetch books from Project Gutenberg via Gutendex API."""
    offset = get_offset("gutenberg")
    topic_idx = offset % len(GUTENBERG_TOPICS)
    topic = GUTENBERG_TOPICS[topic_idx]
    page = (offset // len(GUTENBERG_TOPICS)) + 1

    log.info(f"[gutenberg] topic={topic} page={page}")
    source_id = ensure_source("gutenberg", "api", "https://gutendex.com")

    try:
        resp = rate_limited_get(f"https://gutendex.com/books?topic={topic}&page={page}")
        resp.raise_for_status()
        data = resp.json()
    except Exception as e:
        log.error(f"[gutenberg] API error: {e}")
        set_offset("gutenberg", offset + 1)
        return 0

    count = 0
    books = data.get("results", [])
    if not books:
        # Reset to next topic
        set_offset("gutenberg", ((topic_idx + 1) % len(GUTENBERG_TOPICS)) * len(GUTENBERG_TOPICS))
        return 0

    for book in books[:3]:  # limit per cycle
        title = book.get("title", "Unknown")
        authors = ", ".join(a.get("name", "") for a in book.get("authors", []))
        book_id = book.get("id", "")
        url = f"https://www.gutenberg.org/ebooks/{book_id}"

        doc_hash = make_hash(f"gutenberg-{book_id}")
        if hash_exists(doc_hash) or title_exists(title):
            log.info(f"[gutenberg] skip duplicate: {title}")
            continue

        # Try to fetch full text
        formats = book.get("formats", {})
        text_url = (
            formats.get("text/plain; charset=utf-8")
            or formats.get("text/plain")
            or formats.get("text/plain; charset=us-ascii")
        )

        content = ""
        if text_url:
            try:
                text_resp = rate_limited_get(text_url)
                if text_resp.status_code == 200:
                    content = text_resp.text[:MAX_TEXT_LEN]
            except Exception as e:
                log.warning(f"[gutenberg] failed to fetch text for {title}: {e}")

        if not content:
            # Store metadata only
            subjects = book.get("subjects", [])
            content = f"Title: {title}\nAuthor: {authors}\nSubjects: {', '.join(subjects)}"

        # Download cover image if available
        image_refs = []
        cover_url = formats.get("image/jpeg")
        if cover_url:
            local = download_image(cover_url, "gutenberg")
            if local:
                image_refs.append(local)

        if image_refs:
            content += "\n\n## Images\n" + "\n".join(f"![image]({p})" for p in image_refs)

        keywords = [topic, "gutenberg", "book"]
        keywords.extend(book.get("subjects", [])[:5])

        result = db.insert_document(
            source_id=source_id,
            title=title,
            content=content,
            url=url,
            author=authors,
            keywords=keywords,
            doc_hash=doc_hash,
        )
        if result:
            count += 1
            log.info(f"[gutenberg] inserted: {title}")

    set_offset("gutenberg", offset + 1)
    return count


def scrape_arxiv():
    """Fetch papers from arXiv API."""
    offset = get_offset("arxiv")
    cat_idx = offset % len(ARXIV_CATEGORIES)
    category = ARXIV_CATEGORIES[cat_idx]
    start = (offset // len(ARXIV_CATEGORIES)) * 5

    log.info(f"[arxiv] category={category} start={start}")
    source_id = ensure_source("arxiv", "api", "https://arxiv.org")

    try:
        resp = rate_limited_get(
            f"http://export.arxiv.org/api/query?search_query=cat:{category}"
            f"&start={start}&max_results=5&sortBy=relevance"
        )
        resp.raise_for_status()
    except Exception as e:
        log.error(f"[arxiv] API error: {e}")
        set_offset("arxiv", offset + 1)
        return 0

    count = 0
    ns = {"atom": "http://www.w3.org/2005/Atom"}

    try:
        root = ET.fromstring(resp.text)
    except ET.ParseError as e:
        log.error(f"[arxiv] XML parse error: {e}")
        set_offset("arxiv", offset + 1)
        return 0

    entries = root.findall("atom:entry", ns)
    if not entries:
        set_offset("arxiv", ((cat_idx + 1) % len(ARXIV_CATEGORIES)) * len(ARXIV_CATEGORIES))
        return 0

    for entry in entries:
        title_el = entry.find("atom:title", ns)
        summary_el = entry.find("atom:summary", ns)
        title = title_el.text.strip() if title_el is not None else "Unknown"
        summary = summary_el.text.strip() if summary_el is not None else ""

        # Get authors
        author_els = entry.findall("atom:author/atom:name", ns)
        authors = ", ".join(a.text for a in author_els if a.text)

        # Get link
        link_el = entry.find("atom:id", ns)
        url = link_el.text.strip() if link_el is not None else ""

        doc_hash = make_hash(f"arxiv-{url}")
        if hash_exists(doc_hash) or title_exists(title):
            continue

        # Try to fetch full paper text from arXiv HTML version
        content = ""
        image_refs = []
        import re as _re
        arxiv_id_match = _re.search(r'/abs/(.+)$', url)
        if arxiv_id_match:
            arxiv_id = arxiv_id_match.group(1)
            html_url = f"https://arxiv.org/html/{arxiv_id}"
            try:
                html_resp = rate_limited_get(html_url, timeout=30)
                if html_resp.status_code == 200:
                    html_text = html_resp.text
                    # Extract images before stripping tags
                    img_urls = extract_images_from_html(html_text, html_url, max_images=3)
                    for img_url in img_urls:
                        local = download_image(img_url, "arxiv")
                        if local:
                            image_refs.append(local)
                    # Strip HTML tags to get clean body text
                    from html.parser import HTMLParser

                    class _ArxivTextExtractor(HTMLParser):
                        def __init__(self):
                            super().__init__()
                            self.texts = []
                            self._skip = False

                        def handle_starttag(self, tag, attrs):
                            if tag in ("script", "style", "nav", "header", "footer", "head"):
                                self._skip = True

                        def handle_endtag(self, tag):
                            if tag in ("script", "style", "nav", "header", "footer", "head"):
                                self._skip = False

                        def handle_data(self, data):
                            if not self._skip:
                                text = data.strip()
                                if text:
                                    self.texts.append(text)

                    parser = _ArxivTextExtractor()
                    parser.feed(html_text)
                    full_text = "\n".join(parser.texts).strip()
                    if len(full_text) > 200:
                        content = full_text[:MAX_TEXT_LEN]
                        log.info(f"[arxiv] got full HTML text for {arxiv_id} ({len(content)} chars)")
            except Exception as e:
                log.warning(f"[arxiv] HTML fetch failed for {arxiv_id}: {e}")

        # Fall back to title + abstract if full text unavailable
        if not content:
            content = f"{title}\n\n{summary}"

        if image_refs:
            content += "\n\n## Images\n" + "\n".join(f"![figure]({p})" for p in image_refs)

        keywords = [category, "arxiv", "paper", "research"]

        result = db.insert_document(
            source_id=source_id,
            title=title,
            content=content,
            url=url,
            author=authors,
            keywords=keywords,
            doc_hash=doc_hash,
        )
        if result:
            count += 1
            log.info(f"[arxiv] inserted: {title[:80]}")

    set_offset("arxiv", offset + 1)
    return count


def scrape_openfda():
    """Fetch drug label data from openFDA."""
    offset = get_offset("openfda")
    skip = offset * 5

    log.info(f"[openfda] skip={skip}")
    source_id = ensure_source("openfda", "api", "https://api.fda.gov")

    try:
        resp = rate_limited_get(
            f"https://api.fda.gov/drug/label.json"
            f"?search=_exists_:description&limit=5&skip={skip}"
        )
        resp.raise_for_status()
        data = resp.json()
    except Exception as e:
        log.error(f"[openfda] API error: {e}")
        set_offset("openfda", offset + 1)
        return 0

    count = 0
    results = data.get("results", [])
    if not results:
        set_offset("openfda", 0)  # wrap around
        return 0

    for drug in results:
        openfda = drug.get("openfda", {})
        brand_names = openfda.get("brand_name", [])
        generic_names = openfda.get("generic_name", [])
        name = (brand_names[0] if brand_names
                else generic_names[0] if generic_names
                else None)

        # If no brand/generic name, try to extract from description
        if not name:
            desc = drug.get("description", [])
            if desc:
                desc_text = desc[0] if isinstance(desc, list) else str(desc)
                import re as _re
                # Strip "DESCRIPTION" prefix and leading numbers
                clean = _re.sub(r'^[\d\s]*DESCRIPTION:?\s*', '', desc_text, flags=_re.IGNORECASE).strip()
                # Take the first drug name before "is/are/contains/for/tablets"
                m = _re.match(r'(.+?)\s+(?:is|are|contains|for\s|tablets|capsules|solution|injection)', clean, _re.IGNORECASE)
                if m:
                    name = m.group(1).strip().rstrip(",.").split("(")[0].strip()[:60]
            if not name or name.lower() in ("each tablet", "each capsule", ""):
                name = f"Drug-{drug.get('id', skip)}"

        title = f"FDA Drug Label: {name}"
        doc_hash = make_hash(f"openfda-{drug.get('id', skip)}")
        if hash_exists(doc_hash) or title_exists(title):
            continue

        # Build content from available fields
        sections = []
        for field in ["description", "indications_and_usage", "warnings",
                       "dosage_and_administration", "adverse_reactions",
                       "contraindications"]:
            val = drug.get(field, [])
            if val:
                text = val[0] if isinstance(val, list) else str(val)
                sections.append(f"## {field.replace('_', ' ').title()}\n{text}")

        if brand_names:
            sections.insert(0, f"Brand Name: {', '.join(brand_names)}")
        if generic_names:
            sections.insert(1 if brand_names else 0, f"Generic Name: {', '.join(generic_names)}")

        content = "\n\n".join(sections)
        if not content.strip():
            continue

        keywords = ["fda", "drug", "pharmaceutical", "medicine"]
        keywords.extend(brand_names[:3])
        keywords.extend(generic_names[:3])

        result = db.insert_document(
            source_id=source_id,
            title=title,
            content=content[:MAX_TEXT_LEN],
            url="https://api.fda.gov/drug/label.json",
            author="U.S. Food and Drug Administration",
            keywords=keywords,
            doc_hash=doc_hash,
        )
        if result:
            count += 1
            log.info(f"[openfda] inserted: {title}")

    set_offset("openfda", offset + 1)
    return count


def scrape_archive_military():
    """Fetch military field manuals from Internet Archive."""
    offset = get_offset("archive-military")
    page = offset + 1

    log.info(f"[archive-military] page={page}")
    source_id = ensure_source("archive-military", "api", "https://archive.org")

    try:
        resp = rate_limited_get(
            f"https://archive.org/advancedsearch.php"
            f"?q=subject:military+manual"
            f"&output=json&rows=5&page={page}"
            f"&fl[]=identifier&fl[]=title&fl[]=description&fl[]=date&fl[]=creator"
        )
        resp.raise_for_status()
        data = resp.json()
    except Exception as e:
        log.error(f"[archive-military] API error: {e}")
        set_offset("archive-military", offset + 1)
        return 0

    count = 0
    docs = data.get("response", {}).get("docs", [])
    if not docs:
        set_offset("archive-military", 0)
        return 0

    for doc in docs:
        identifier = doc.get("identifier", "")
        title = doc.get("title", identifier)
        description = doc.get("description", "")
        creator = doc.get("creator", "")
        date = doc.get("date", "")
        url = f"https://archive.org/details/{identifier}"

        doc_hash = make_hash(f"archive-military-{identifier}")
        if hash_exists(doc_hash) or title_exists(title):
            continue

        # Try to fetch text content
        content = ""
        try:
            # First get the file list
            files_resp = rate_limited_get(
                f"https://archive.org/metadata/{identifier}/files"
            )
            if files_resp.status_code == 200:
                files = files_resp.json().get("result", [])
                # Look for a text file
                text_file = None
                image_files = []
                for f in files:
                    fname = f.get("name", "")
                    if fname.endswith(".txt") or fname.endswith("_djvu.txt"):
                        text_file = fname
                    # Collect image files (cover pages, diagrams)
                    if any(fname.lower().endswith(e) for e in [".jpg", ".jpeg", ".png", ".gif"]):
                        fsize = int(f.get("size", 0) or 0)
                        if fsize > MIN_IMAGE_SIZE and not is_junk_image(fname):
                            image_files.append(fname)

                if text_file:
                    text_resp = rate_limited_get(
                        f"https://archive.org/download/{identifier}/{text_file}"
                    )
                    if text_resp.status_code == 200:
                        content = text_resp.text[:MAX_TEXT_LEN]
        except Exception as e:
            log.warning(f"[archive-military] text fetch failed for {identifier}: {e}")

        if not content:
            # Store metadata only
            if isinstance(description, list):
                description = " ".join(description)
            content = f"Title: {title}\nCreator: {creator}\nDate: {date}\nDescription: {description}"

        # Download images (cover, diagrams) from archive files
        image_refs = []
        for img_file in image_files[:3]:
            img_url = f"https://archive.org/download/{identifier}/{img_file}"
            local = download_image(img_url, "archive")
            if local:
                image_refs.append(local)
        # Also try the standard cover image
        if not image_refs:
            cover_url = f"https://archive.org/services/img/{identifier}"
            local = download_image(cover_url, "archive")
            if local:
                image_refs.append(local)

        if image_refs:
            content += "\n\n## Images\n" + "\n".join(f"![image]({p})" for p in image_refs)

        keywords = ["military", "field manual", "archive.org", "defense"]

        result = db.insert_document(
            source_id=source_id,
            title=title,
            content=content,
            url=url,
            author=creator if isinstance(creator, str) else ", ".join(creator) if isinstance(creator, list) else "",
            keywords=keywords,
            doc_hash=doc_hash,
        )
        if result:
            count += 1
            log.info(f"[archive-military] inserted: {title}")

    set_offset("archive-military", offset + 1)
    return count


def scrape_pubmed():
    """Fetch open access abstracts from PubMed Central."""
    offset = get_offset("pubmed")
    query_idx = offset % len(PUBMED_QUERIES)
    query = PUBMED_QUERIES[query_idx]
    retstart = (offset // len(PUBMED_QUERIES)) * 5

    log.info(f"[pubmed] query='{query}' retstart={retstart}")
    source_id = ensure_source("pubmed", "api", "https://pubmed.ncbi.nlm.nih.gov")

    try:
        # Search for PMC IDs
        search_resp = rate_limited_get(
            f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
            f"?db=pmc&term={requests.utils.quote(query)}"
            f"&retmax=5&retstart={retstart}&retmode=json"
        )
        search_resp.raise_for_status()
        search_data = search_resp.json()
    except Exception as e:
        log.error(f"[pubmed] search error: {e}")
        set_offset("pubmed", offset + 1)
        return 0

    id_list = search_data.get("esearchresult", {}).get("idlist", [])
    if not id_list:
        set_offset("pubmed", ((query_idx + 1) % len(PUBMED_QUERIES)) * len(PUBMED_QUERIES))
        return 0

    count = 0
    for pmcid in id_list:
        doc_hash = make_hash(f"pubmed-{pmcid}")
        if hash_exists(doc_hash):
            continue

        try:
            # Fetch article XML from PMC
            abs_resp = rate_limited_get(
                f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
                f"?db=pmc&id={pmcid}&retmode=xml"
            )
            if abs_resp.status_code != 200:
                continue
            raw = abs_resp.text.strip()
            if not raw or len(raw) < 50:
                continue
        except Exception as e:
            log.warning(f"[pubmed] fetch error for {pmcid}: {e}")
            continue

        # Parse XML and extract clean text
        title = f"PMC Article {pmcid}"
        authors = ""
        content_parts = []
        try:
            root = ET.fromstring(raw)
            # Extract title
            title_el = root.find(".//article-title")
            if title_el is not None:
                title = "".join(title_el.itertext()).strip()[:200]
            # Skip if title already exists in DB (cross-pipeline dedup)
            if title_exists(title):
                log.info(f"[pubmed] skip duplicate title: {title[:60]}")
                continue
            # Extract authors
            author_names = []
            for contrib in root.findall(".//contrib[@contrib-type='author']"):
                surname = contrib.findtext(".//surname", "")
                given = contrib.findtext(".//given-names", "")
                if surname:
                    author_names.append(f"{given} {surname}".strip())
            authors = ", ".join(author_names)
            # Extract abstract
            for abstract in root.findall(".//abstract"):
                for p in abstract.iter("p"):
                    text = "".join(p.itertext()).strip()
                    if text:
                        content_parts.append(text)
            # Extract body text
            for body in root.findall(".//body"):
                for sec in body.findall(".//sec"):
                    sec_title = sec.findtext("title", "")
                    if sec_title:
                        content_parts.append(f"\n## {sec_title}")
                    for p in sec.findall("p"):
                        text = "".join(p.itertext()).strip()
                        if text:
                            content_parts.append(text)
            # Extract figure images from XML
            image_refs = []
            for graphic in root.findall(".//{http://www.w3.org/1999/xlink}href/.."):
                href = graphic.get("{http://www.w3.org/1999/xlink}href", "")
                if href:
                    if not href.startswith("http"):
                        href = f"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC{pmcid}/bin/{href}"
                    if any(ext in href.lower() for ext in [".jpg", ".jpeg", ".png", ".gif", ".webp"]):
                        if not is_junk_image(href):
                            local = download_image(href, "pubmed")
                            if local:
                                image_refs.append(local)
                            if len(image_refs) >= 3:
                                break
            # Also try self-uri PDF figures
            for graphic in root.iter("graphic"):
                href = graphic.get("{http://www.w3.org/1999/xlink}href", "")
                if href and not href.startswith("http"):
                    href = f"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC{pmcid}/bin/{href}.jpg"
                    if not is_junk_image(href):
                        local = download_image(href, "pubmed")
                        if local:
                            image_refs.append(local)
                        if len(image_refs) >= 3:
                            break

        except ET.ParseError:
            image_refs = []
            # Fall back to stripping tags
            import re as _re
            content_parts = [_re.sub(r"<[^>]+>", " ", raw)]

        content = "\n\n".join(content_parts).strip()
        if not content or len(content) < 100:
            continue

        if image_refs:
            content += "\n\n## Images\n" + "\n".join(f"![figure]({p})" for p in image_refs)

        url = f"https://www.ncbi.nlm.nih.gov/pmc/articles/PMC{pmcid}/"
        keywords = [query, "pubmed", "medical", "research"]

        result = db.insert_document(
            source_id=source_id,
            title=title,
            content=content[:MAX_TEXT_LEN],
            url=url,
            author=authors,
            keywords=keywords,
            doc_hash=doc_hash,
        )
        if result:
            count += 1
            log.info(f"[pubmed] inserted: {title[:80]}")

    set_offset("pubmed", offset + 1)
    return count


def scrape_openstax():
    """Store OpenStax textbook metadata and try to fetch descriptions."""
    offset = get_offset("openstax")
    book_idx = offset % len(OPENSTAX_BOOKS)
    book = OPENSTAX_BOOKS[book_idx]

    log.info(f"[openstax] book={book['title']}")
    source_id = ensure_source("openstax", "web", "https://openstax.org")

    doc_hash = make_hash(f"openstax-{book['title']}")
    if hash_exists(doc_hash):
        log.info(f"[openstax] skip duplicate: {book['title']}")
        set_offset("openstax", offset + 1)
        return 0

    content = ""
    try:
        resp = rate_limited_get(book["url"])
        if resp.status_code == 200:
            # Try to extract useful text from the page
            from html.parser import HTMLParser

            class TextExtractor(HTMLParser):
                def __init__(self):
                    super().__init__()
                    self.texts = []
                    self._skip = False

                def handle_starttag(self, tag, attrs):
                    if tag in ("script", "style", "nav", "header", "footer"):
                        self._skip = True

                def handle_endtag(self, tag):
                    if tag in ("script", "style", "nav", "header", "footer"):
                        self._skip = False

                def handle_data(self, data):
                    if not self._skip:
                        text = data.strip()
                        if text:
                            self.texts.append(text)

            parser = TextExtractor()
            parser.feed(resp.text)
            page_text = "\n".join(parser.texts)

            if len(page_text) > 200:
                content = page_text[:MAX_TEXT_LEN]

            # Extract images from the page (textbook covers, diagrams)
            image_urls = extract_images_from_html(resp.text, book["url"], max_images=3)
            image_refs = []
            for img_url in image_urls:
                local = download_image(img_url, "openstax")
                if local:
                    image_refs.append(local)
            if image_refs:
                content += "\n\n## Images\n" + "\n".join(f"![image]({p})" for p in image_refs)
    except Exception as e:
        log.warning(f"[openstax] page fetch failed for {book['title']}: {e}")

    if not content:
        content = (
            f"OpenStax Free Textbook: {book['title']}\n"
            f"Subject: {book['subject']}\n"
            f"URL: {book['url']}\n\n"
            f"This is a free, peer-reviewed, openly licensed textbook "
            f"available from OpenStax (openstax.org)."
        )

    keywords = [book["subject"], "openstax", "textbook", "education", "free"]

    result = db.insert_document(
        source_id=source_id,
        title=f"OpenStax: {book['title']}",
        content=content,
        url=book["url"],
        author="OpenStax",
        keywords=keywords,
        doc_hash=doc_hash,
    )
    inserted = 1 if result else 0
    if inserted:
        log.info(f"[openstax] inserted: {book['title']}")

    set_offset("openstax", offset + 1)
    return inserted


# ---------------------------------------------------------------------------
# Dispatcher map
# ---------------------------------------------------------------------------

SCRAPERS = {
    "gutenberg": scrape_gutenberg,
    "arxiv": scrape_arxiv,
    "openfda": scrape_openfda,
    "archive-military": scrape_archive_military,
    "pubmed": scrape_pubmed,
    "openstax": scrape_openstax,
}

# ---------------------------------------------------------------------------
# Main loop
# ---------------------------------------------------------------------------

def run_once():
    """Run one scrape cycle (single source)."""
    cycle = get_cycle()
    source_idx = cycle % len(SOURCES)
    source = SOURCES[source_idx]

    log.info(f"=== Cycle {cycle}: source={source} ===")
    scraper = SCRAPERS[source]

    try:
        inserted = scraper()
        log.info(f"[{source}] inserted {inserted} documents")
        if inserted > 0:
            hivemind_log(f"source_scraper: {source} +{inserted} docs (cycle {cycle})")
    except Exception as e:
        log.error(f"[{source}] FAILED: {e}", exc_info=True)
        hivemind_log(f"source_scraper: {source} ERROR: {e}")

    set_cycle(cycle + 1)


def main():
    """Continuous scraping loop."""
    log.info("Source scraper starting up")
    db.init_db()

    # Log startup
    hivemind_log("source_scraper started — cycling through 6 public API sources")

    total_inserted = 0
    while True:
        try:
            run_once()
        except KeyboardInterrupt:
            log.info("Shutting down (keyboard interrupt)")
            break
        except Exception as e:
            log.error(f"Unhandled error in main loop: {e}", exc_info=True)

        log.info(f"Sleeping {COOLDOWN}s before next fetch...")
        try:
            time.sleep(COOLDOWN)
        except KeyboardInterrupt:
            log.info("Shutting down (keyboard interrupt)")
            break

    hivemind_log("source_scraper stopped")


if __name__ == "__main__":
    main()