Automate menu-spending PDF pipeline (fetch → extract → validate → geocode) + 2024–2026 data

data/geocode/ungeocoded.txt

@@ -0,0 +1,14 @@
+3500 W NORMAN R BOBINS PL
+ON E 42ND PL FROM S DR MARTIN LUTHER KING JR DR E (400 E) TO S VINCENNES AVE  (499 E)
+ON E LAKE SHORE DR FROM N LAKE SHORE DR RAMP  (330 N) TO N MICHIGAN AVE  (1000 N)
+ON N HARDING AVE FROM W WAVELAND AVE  (3700 N) TO N AVONDALE AVE  (3800 N)
+ON N KERCHEVAL AVE FROM N KERBS AVE  (5730 N) TO N FOREST GLEN AVE  (5800 N)
+ON N MICHIGAN LOWER AVE FROM E NORTH WATER LOWER ST  (420 N) TO E GRAND AVE (600 N)
+ON N NORTHWEST HWY FROM N LONG AVE  (5000 N) TO W CARMEN AVE  (5099 N)
+ON N TAHOMA AVE FROM N MCALPIN AVE  (6970 N) TO W ESTES AVE  (7100 N)
+ON N WOLCOTT AVE FROM N WICKER PARK AVE  (1370 N) TO N MILWAUKEE AVE  (1400 N)
+ON S DR MARTIN LUTHER KING JR DR E FROM E 45TH ST  (4500 S) TO E 45TH PL  (4530 S)
+ON S MCDERMOTT ST FROM 2984 S TO S ARCHER AVE  (2990 S)
+ON W DICKENS AVE FROM N STOCKTON DR  (280 W) TO N LINCOLN PARK WEST  (300 W)
+ON W MIDWAY PARK FROM 5709 W TO N MAYFIELD AVE  (5900 W)
+W FULTON MARKET &  N UNION AVE

scripts_automation/fetch_menu_pdfs.py

@@ -0,0 +1,132 @@
+"""Auto-fetch Chicago aldermanic menu-spending PDFs from the city's CIP asset paths.
+
+The city's index PAGES are WAF-gated to scripts and the PDF filenames are inconsistent (varying date
+suffixes, "Menu Posting" vs quarterly "Aldermanic Menu Program Report" vs "AMR-YYYY"), so we can't guess
+URLs. But the PDF ASSET files (under /content/dam/.../) are NOT gated, and the Wayback Machine keeps
+un-gated snapshots of the index pages that LIST those asset URLs. So we discover URLs from Wayback
+snapshots of the index pages + a manifest of verified ones, then download the assets directly.
+
+  python -m scripts_automation.fetch_menu_pdfs            # discover + download to data/pdf/
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+from pathlib import Path
+from urllib.parse import quote, unquote, urlsplit, urlunsplit
+from urllib.request import Request, urlopen
+
+UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120 Safari/537.36"
+DEST = Path("data/pdf")
+DAM = "https://www.chicago.gov/content/dam/city/depts/obm"
+# Index pages the city has used for menu spending (Wayback has un-gated snapshots that list the assets).
+INDEX_PAGES = [
+    "https://www.chicago.gov/city/en/depts/obm/provdrs/cap_improve/svcs/cip-archive.html",
+    "https://www.chicago.gov/city/en/depts/obm/provdrs/budget/svcs/CapitalPublications.html",
+]
+# Verified-working asset URLs (a floor in case Wayback misses one).
+MANIFEST = [
+    f"{DAM}/general/CIP/CIPDocs/AldermanicMenuPostings/{n}"
+    for n in ("2012Menu.pdf", "2013Menu.pdf", "2014Menu.pdf", "2015Menu.pdf", "2016Menu.pdf",
+              "2017OBMMenu50WardDetailsRpt3Dec2018.pdf", "2018OBMrpt_MenuWarddetailsprojects2DEC2019.pdf",
+              "2019 Menu Posting - 22-10-02.pdf", "2020 Menu Posting - 22-10-02.pdf",
+              "2021 Menu Posting - 22-10-02.pdf")
+] + [
+    f"{DAM}/supp_info/CIP_Archive/Aldermanic Menu/2022 Menu - 2-9-23.pdf",
+    f"{DAM}/supp_info/CIP_Archive/Aldermanic Menu/AMR-2024-(Q1-Q3).pdf",
+    f"{DAM}/supp_info/CIP_Archive/Q1 2025 Aldermanic Menu Program Report.pdf",
+]
+_ASSET_RE = re.compile(r"https://www\.chicago\.gov/content/dam[^\"'\\ )]*?(?:Menu|AMR)[^\"'\\ )]*?\.pdf",
+                       re.IGNORECASE)
+
+
+def _norm(url: str) -> str:
+    """Percent-encode spaces in the path (urllib rejects them) while preserving (), /, and existing %."""
+    parts = urlsplit(url)
+    return urlunsplit(parts._replace(path=quote(parts.path, safe="/()%")))
+
+
+def _get(url, timeout=60):
+    return urlopen(Request(_norm(url), headers={"User-Agent": UA}), timeout=timeout)
+
+
+def _exists(url: str) -> bool:
+    """True if the asset returns 200 (GET headers, don't download the body)."""
+    try:
+        resp = _get(url, 15)
+        ok = getattr(resp, "status", 200) == 200
+        resp.close()
+        return ok
+    except Exception:  # noqa: BLE001 — 404/network -> treat as absent
+        return False
+
+
+def probe_recent_quarters(through_year=None) -> set:
+    """Directly probe recent quarterly reports. Filenames are inconsistent (three observed styles), so
+    try each style per (year, quarter). Removes the Wayback-snapshot lag for current-term data."""
+    import datetime
+
+    last = through_year or datetime.date.today().year
+    cip = f"{DAM}/supp_info/CIP_Archive"
+    found = set()
+    for y in range(2024, last + 2):  # +2 so a just-started next year is covered
+        for q in (1, 2, 3, 4):
+            for cand in (f"{cip}/Aldermanic Menu/{y} Q{q} Menu Report.pdf",
+                         f"{cip}/Aldermanic Menu/Menu Report {y} Q{q}.pdf",
+                         f"{cip}/Q{q} {y} Aldermanic Menu Program Report.pdf"):
+                if _exists(cand):
+                    found.add(cand)
+                time.sleep(0.15)
+    return found
+
+
+def discover_from_wayback() -> set:
+    """Menu-PDF asset URLs found in the latest Wayback snapshot of each index page."""
+    found = set()
+    for page in INDEX_PAGES:
+        try:
+            avail = json.loads(_get(f"http://archive.org/wayback/available?url={quote(page)}", 30).read())
+            snap = avail.get("archived_snapshots", {}).get("closest", {}).get("url")
+            if not snap:
+                continue
+            html = _get(snap).read().decode("utf-8", "replace")
+            found |= {m for m in _ASSET_RE.findall(html) if "author.chicago" not in m}
+        except Exception as exc:  # noqa: BLE001
+            print(f"  wayback discovery failed for {page}: {exc}")
+    return found
+
+
+def download(url: str) -> str | None:
+    DEST.mkdir(parents=True, exist_ok=True)
+    name = unquote(url.rsplit("/", 1)[-1])
+    dest = DEST / name
+    if dest.exists() and dest.stat().st_size > 1000:
+        return f"have    {name}"
+    try:
+        data = _get(url, 120).read()
+    except Exception as exc:  # noqa: BLE001 — 404s + network blips are expected; skip
+        return f"MISS    {name} ({getattr(exc, 'code', exc)})"
+    if not data.startswith(b"%PDF"):
+        return f"NOTPDF  {name}"
+    dest.write_bytes(data)
+    return f"got     {name} ({len(data)//1024} KB)"
+
+
+def main():
+    print("probing recent quarters + Wayback discovery …")
+    urls = sorted({_norm(u) for u in set(MANIFEST) | discover_from_wayback() | probe_recent_quarters()})
+    print(f"resolved {len(urls)} menu-PDF URLs (manifest + Wayback + recent-quarter probe); downloading to {DEST}/ …")
+    got = miss = 0
+    for url in urls:
+        result = download(url)
+        print("  " + result)
+        got += result.startswith(("got", "have"))
+        miss += result.startswith(("MISS", "NOTPDF"))
+        time.sleep(0.3)  # be polite to the city's server
+    print(f"\n{got} PDFs available locally, {miss} missing/unreadable.")
+
+
+if __name__ == "__main__":
+    main()

scripts_automation/geocode_gap_years.py

@@ -0,0 +1,131 @@
+"""Geocode the 2024-2026 gap-year locations via Sean's API geocoder, with a resumable JSONL cache.
+
+Dedups locations first (many repeat), caches each location -> WKT geometry to a JSONL file so a re-run
+costs nothing and an interrupted run resumes (always-be-caching). Emits a geojson matching Sean's
+2019-2022 schema so spread_metric.py runs on it unchanged.
+
+  python -m scripts_automation.geocode_gap_years
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+import pandas as pd
+from shapely import wkt as shapely_wkt
+from shapely.geometry import mapping
+
+from src.chicago_participatory_urbanism.geocoder_api import GeoCoderAPI
+from src.chicago_participatory_urbanism.ward_spending.location_geocoding import LocationGeocoder
+
+DATA = Path("data/output/menu_2024_2026_processed.csv")
+CACHE = Path("data/geocode/gap_years_cache.jsonl")
+OUT = Path("data/output/menu_2024_2026_geocoded.geojson")
+ANNUAL = {2024: "Q3", 2025: "Q4", 2026: "Q1"}  # latest cumulative snapshot per year
+RESIDUAL = Path("data/geocode/ungeocoded.txt")  # documented leftovers after repair
+
+
+def normalize_location(loc: str) -> str:
+    """Repair location strings the geocoder rejects. Applied ONLY to already-failed locations (the repair
+    pass), so it can never regress a good geocode. Recovers ~62% of failures; the rest are the verbose
+    'ON STREET FROM CROSS TO CROSS (range)' segment format the geocoder doesn't parse."""
+    loc = re.sub(r"JEAN BAPTISTE POINTE DUSABLE LAKE SHORE DR", "LAKE SHORE DR", loc)  # LSD rename
+    loc = re.sub(r"\s+(SB|NB)(\b|;|$)", r"\2", loc)                      # drop bound suffix
+    loc = re.sub(r"\bBROADWAY\b(?!\s+(AVE|ST|BLVD))", "BROADWAY ST", loc)  # Broadway has no type
+    return re.sub(r"\s{2,}", " ", loc).strip()
+
+
+def load_cache() -> dict:
+    cache = {}
+    if CACHE.exists():
+        for line in CACHE.read_text().splitlines():
+            if line.strip():
+                rec = json.loads(line)
+                cache[rec["loc"]] = rec["wkt"]
+    return cache
+
+
+def main():
+    CACHE.parent.mkdir(parents=True, exist_ok=True)
+    OUT.parent.mkdir(parents=True, exist_ok=True)
+    df = pd.read_csv(DATA)
+    locs = sorted({str(x) for x in df["location"].dropna().unique() if str(x).strip()})
+    cache = load_cache()
+    todo = [l for l in locs if l not in cache]
+    print(f"{len(locs)} unique locations; {len(cache)} cached; geocoding {len(todo)} new …", flush=True)
+
+    lg = LocationGeocoder(GeoCoderAPI())  # stateless wrappers -> safe to share across threads
+
+    def geocode_one(loc):
+        try:
+            geom = lg.process_location_text(loc)
+            return loc, (geom.wkt if geom is not None and not geom.is_empty else None)
+        except Exception:  # noqa: BLE001 — geocoder fails on some location formats; store null, continue
+            return loc, None
+
+    # I/O-bound (API waits) -> thread pool. Main thread is the sole writer, so the JSONL cache stays
+    # consistent; the resumable cache means a kill mid-run just continues next time.
+    t0 = time.time()
+    with CACHE.open("a") as fh, ThreadPoolExecutor(max_workers=8) as pool:
+        for i, fut in enumerate(as_completed(pool.submit(geocode_one, l) for l in todo), 1):
+            loc, w = fut.result()
+            cache[loc] = w
+            fh.write(json.dumps({"loc": loc, "wkt": w}) + "\n")
+            fh.flush()
+            if i % 200 == 0:
+                rate = i / (time.time() - t0)
+                print(f"  {i}/{len(todo)}  ({rate:.1f}/s, ~{(len(todo)-i)/rate/60:.0f} min left)", flush=True)
+
+    # repair pass: retry failed locations with targeted normalization (only touches nulls -> no regression)
+    repairable = [l for l in locs if cache.get(l) is None and normalize_location(l) != l]
+    if repairable:
+        print(f"repair: retrying {len(repairable)} failed locations with normalization …", flush=True)
+        recovered = 0
+        with CACHE.open("a") as fh, ThreadPoolExecutor(max_workers=8) as pool:
+            def repair_one(loc):
+                try:
+                    geom = lg.process_location_text(normalize_location(loc))
+                    return loc, (geom.wkt if geom is not None and not geom.is_empty else None)
+                except Exception:  # noqa: BLE001
+                    return loc, None
+            for fut in as_completed(pool.submit(repair_one, l) for l in repairable):
+                loc, w = fut.result()
+                if w:
+                    cache[loc] = w
+                    fh.write(json.dumps({"loc": loc, "wkt": w}) + "\n")
+                    recovered += 1
+        print(f"repair recovered {recovered}/{len(repairable)}", flush=True)
+
+    # document the residual ungeocodable locations so they're visible, not silently dropped
+    residual = sorted(l for l in locs if cache.get(l) is None)
+    RESIDUAL.write_text("\n".join(residual) + ("\n" if residual else ""))
+    print(f"{len(residual)} locations remain ungeocodable -> {RESIDUAL} "
+          f"(mostly verbose 'ON X FROM Y TO Z (range)' segment descriptions)", flush=True)
+
+    # build geojson for the annual snapshots
+    feats = []
+    for _, r in df.iterrows():
+        if ANNUAL.get(r["year"]) != r["period"]:
+            continue
+        w = cache.get(str(r["location"]))
+        geom = None
+        if w:
+            try:
+                geom = mapping(shapely_wkt.loads(w))
+            except Exception:  # noqa: BLE001
+                geom = None
+        feats.append({"type": "Feature", "geometry": geom,
+                      "properties": {k: (None if pd.isna(r[k]) else r[k])
+                                     for k in ("ward", "item", "location", "cost", "year", "category")}})
+    OUT.write_text(json.dumps({"type": "FeatureCollection", "features": feats}))
+    got = sum(1 for f in feats if f["geometry"])
+    print(f"\nwrote {len(feats)} annual-snapshot features ({got} geocoded, "
+          f"{got/len(feats)*100:.0f}%) -> {OUT}", flush=True)
+
+
+if __name__ == "__main__":
+    main()