scraper.py

"""
Web scraper -> clean structured data.
Crawls a catalog (sandbox site books.toscrape.com), extracts typed records,
dedupes, and writes JSON ready for a database or spreadsheet. Static HTML uses
requests+BeautifulSoup; JS-rendered sites swap in Playwright with the same schema.
"""
import re, json, statistics, time
from collections import Counter
import requests
from bs4 import BeautifulSoup

BASE = "https://books.toscrape.com/catalogue/page-{}.html"
RATING = {"One":1,"Two":2,"Three":3,"Four":4,"Five":5}

def scrape(pages=3):
    out=[]
    for p in range(1, pages+1):
        r=requests.get(BASE.format(p), timeout=20); r.raise_for_status()
        soup=BeautifulSoup(r.text, "html.parser")
        for pod in soup.select("article.product_pod"):
            a=pod.h3.a
            price=re.search(r'([\d.]+)', pod.select_one("p.price_color").get_text()).group(1)
            out.append({
                "title": a["title"].strip(),
                "price_gbp": float(price),
                "rating": RATING.get(pod.select_one("p.star-rating")["class"][1], 0),
                "in_stock": "in stock" in pod.select_one("p.instock.availability").get_text().lower(),
                "url": "https://books.toscrape.com/catalogue/" + a["href"].replace("../../../","")
            })
        time.sleep(0.4)  # polite
    # dedupe by url
    seen={}; 
    for b in out: seen[b["url"]]=b
    return list(seen.values())

if __name__=="__main__":
    books=scrape(3)
    json.dump(books, open(__file__.replace("scraper.py","books.json"),"w"), indent=2, ensure_ascii=False)
    prices=[b["price_gbp"] for b in books]
    rc=Counter(b["rating"] for b in books)
    print(f"scraped {len(books)} records from 3 pages")
    print(f"price: £{min(prices):.2f}–£{max(prices):.2f}  avg £{statistics.mean(prices):.2f}")
    print("rating distribution:", dict(sorted(rc.items())))
    cheap=min(books,key=lambda b:b["price_gbp"])
    print(f"cheapest: {cheap['title']} (£{cheap['price_gbp']:.2f})")
    print("sample record:", json.dumps(books[0], ensure_ascii=False))