driver.py

"""Reddit User Scrapper — driver that runs all subtools for one username.

Calls each subtool's in-memory entry point (no temp files between tools),
holds the results in memory, and writes a single combined JSON:

    {
      "username", "generated_at", "counts", "errors",
      "profile":  { ... profile fields ... },
      "activity": [ {subreddit, author, score, created_iso, permalink, type}, ... ]
    }

The activity section merges the user's comments (Playwright SPA + old.reddit,
deduped by comment id) and posts (old.reddit), all sharing the 5-field schema
plus a `type` field = "comment" or "post". Sorted newest-first.

Usage:
    python driver.py <username> [options]

Options:
    --out PATH         output JSON (default: <username>_full.json)
    --skip-spa         skip the slow Playwright comment scraper
    --scrolls N        SPA scroll iterations (default: 30)
    --pages N          old.reddit pages for comments AND posts (default: 10)
"""

import argparse
import sys
import time
from datetime import datetime, timezone

from reddit_common import write_json, setup_stdout
from new_comment_scraper import scrape_user_comments
from old_comment_scraper import scrape_user_comments_oldreddit
from old_post_scraper import scrape_user_posts
from user_profile_scraper import scrape_user_profile
from merge_comments import merge, FIELDS

NAME = "Reddit User Scrapper"


def _sort_key(rec):
    iso = rec.get("created_iso")
    try:
        return datetime.fromisoformat(iso)
    except (TypeError, ValueError):
        return datetime.min.replace(tzinfo=timezone.utc)


def _run(label, fn, errors):
    """Call a subtool, timing it; on failure record the error and return None."""
    t0 = time.time()
    try:
        result = fn()
        n = len(result) if isinstance(result, list) else 1
        print(f"  {label:22} {n} ({time.time() - t0:.1f}s)", file=sys.stderr)
        return result
    except Exception as e:  # one failing source must not sink the whole run
        msg = f"{type(e).__name__}: {e}"
        print(f"  {label:22} FAILED — {msg}", file=sys.stderr)
        errors.append({"source": label, "error": msg})
        return None


def scrape_user(username, skip_spa=False, scrolls=30, pages=10):
    """Run every subtool for `username` and assemble the combined record."""
    errors = []
    print(f"scraping {username} ...", file=sys.stderr)

    profile = _run("profile", lambda: scrape_user_profile(username), errors)
    posts = _run("posts (old.reddit)", lambda: scrape_user_posts(username, pages=pages), errors)
    old_comments = _run(
        "comments (old.reddit)",
        lambda: scrape_user_comments_oldreddit(username, pages=pages),
        errors,
    )
    spa_comments = None
    if not skip_spa:
        spa_comments = _run(
            "comments (SPA)", lambda: scrape_user_comments(username, scrolls=scrolls), errors
        )

    # Merge comment sources by comment id, then tag type.
    comment_sources = [
        (lbl, rows)
        for lbl, rows in (("oldreddit", old_comments), ("spa", spa_comments))
        if rows
    ]
    merged_comments, _stats = merge(comment_sources) if comment_sources else ([], {})
    for c in merged_comments:
        c["type"] = "comment"

    tagged_posts = []
    for p in posts or []:
        rec = {k: p.get(k) for k in FIELDS}
        rec["type"] = "post"
        tagged_posts.append(rec)

    activity = sorted(merged_comments + tagged_posts, key=_sort_key, reverse=True)

    canonical = (profile or {}).get("username") or username
    return {
        "username": canonical,
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "counts": {
            "comments": len(merged_comments),
            "posts": len(tagged_posts),
            "activity_total": len(activity),
        },
        "errors": errors,
        "profile": profile,
        "activity": activity,
    }


def main():
    setup_stdout()
    ap = argparse.ArgumentParser(description="Run all Reddit subtools for one username.")
    ap.add_argument("username", nargs="?", default="spez")
    ap.add_argument("--out")
    ap.add_argument("--skip-spa", action="store_true")
    ap.add_argument("--scrolls", type=int, default=30)
    ap.add_argument("--pages", type=int, default=10)
    args = ap.parse_args()

    t0 = time.time()
    data = scrape_user(
        args.username, skip_spa=args.skip_spa, scrolls=args.scrolls, pages=args.pages
    )
    out = args.out or f"{args.username}_full.json"
    write_json(data, out)
    c = data["counts"]
    print(
        f"done in {time.time() - t0:.1f}s -> {out}  "
        f"(profile {'ok' if data['profile'] else 'MISSING'}, "
        f"{c['comments']} comments + {c['posts']} posts = {c['activity_total']})",
        file=sys.stderr,
    )


if __name__ == "__main__":
    main()