diff --git a/orpheus_engine/defs/ysws_true_spend_site/site.py b/orpheus_engine/defs/ysws_true_spend_site/site.py index 185f2b2..0abc5d5 100644 --- a/orpheus_engine/defs/ysws_true_spend_site/site.py +++ b/orpheus_engine/defs/ysws_true_spend_site/site.py @@ -868,11 +868,14 @@ def render_program_page(document: Dict[str, Any]) -> str: def render_llms_txt(index_document: Dict[str, Any], example_slug: str) -> str: """ - Barebones map for machines, per the llms.txt convention: what this is, and - where the JSON is. The JSON is self-describing, so this does not restate its - fields -- a field list here would be one more thing to drift. + Map for machines, per the llms.txt convention: what this is, how to query + it, and where the files are. DuckDB comes first because an agent that can + run SQL gets every answer from one 8 MB file instead of walking hundreds of + JSON documents. The JSON is self-describing, so its fields are not restated + here -- a field list would be one more thing to drift. """ meta = index_document["metadata"] + db_url = f"https://{CUSTOM_DOMAIN}/{DUCKDB_FILENAME}" return f"""# YSWS true spend What each Hack Club YSWS program actually spent, published from the Hack Club @@ -885,6 +888,64 @@ def render_llms_txt(index_document: Dict[str, Any], example_slug: str) -> str: files with redacted payment identities. Amounts are US dollars, dates ISO-8601, timestamps UTC. +## How to query this (agents: read this first) + +If you can run DuckDB (the `duckdb` CLI, the Python `duckdb` package, or any +DuckDB client, 1.0 or newer), use the database and skip the JSON. It is the +same data as every page and JSON file, already flattened into tables, and one +query answers questions that would otherwise take hundreds of HTTP fetches. +The HTML and JSON are for people and for environments with no DuckDB. + +Query it in place over HTTPS (the host supports range requests, so only the +pages you touch are fetched): + + INSTALL httpfs; LOAD httpfs; + ATTACH '{db_url}' AS ts (READ_ONLY); + SELECT name, true_spend_dollars, cost_per_weighted_hour + FROM ts.programs ORDER BY true_spend_dollars DESC LIMIT 10; + +Or download it once (about 8 MB) and open it locally, which is faster for +anything beyond a couple of queries: + + curl -sLO {db_url} + duckdb {DUCKDB_FILENAME} "SELECT name, true_spend_dollars FROM programs ORDER BY 2 DESC LIMIT 10" + +In Python: `duckdb.connect("{DUCKDB_FILENAME}", read_only=True).sql("...")`. + +Tables, one row per: + +- metadata: the single row of freshness timestamps (hcb_data_pulled, + spend_recalculated, page_built) and the transaction-detail notice. +- programs: program. true_spend_dollars is the headline number; the other + *_dollars columns are the buckets it was separated from (author_fund, + returned_to_hq, intra_tree, internal_cost, ...). gross_outflow_dollars is + what HCB would report. weighted_hours and cost_per_weighted_hour come from + the Unified YSWS DB. is_ysws_program is false for marketing-style rows. +- program_orgs: HCB org in a program's tree, with parent_org_slug and depth. +- spend_transactions: outflow across all programs. category (A-D, I, X) and + bucket classify it; counted_as_spend is whether it is in true_spend_dollars. + Filter on counted_as_spend for spend questions. +- revenue_transactions: inflow; is_intra_tree marks money from inside the + same program tree, which is not external revenue. +- withheld_orgs: orgs whose detail is summarised, not listed. +- unmatched_orgs: HCB orgs no program claims. +- unlinked_programs: YSWS programs with no HCB link. +- budgets: one person's YSWS individual budget (see the rule below). +- budget_transactions: its inflows and outflows, with counted_as_personal_spend. +- people_without_budget: people the roster expects a budget for but none links. + +Rules that matter for correct SQL: + +- Never add budgets.personal_spend_dollars to programs.true_spend_dollars. + A program funding a budget is not the program's spend, and a budget sending + money back to an org is not the person's spend. +- date columns are ISO-8601 strings; CAST(date AS DATE) to do arithmetic. +- programs.true_spend_dollars = sum of spend_transactions.amount_dollars + where counted_as_spend, plus withheld_orgs.spend_dollars for that program + (orgs outside HCB transparency mode are summarised, not listed row by row). +- Use describe / information_schema.columns for the full column list; the + tables are the source of truth, not this file. + ## Data The home page is rendered from /index.json, which holds the metadata (when HCB @@ -906,14 +967,9 @@ def render_llms_txt(index_document: Dict[str, Any], example_slug: str) -> str: together. Money a budget sends back to an HCB org is likewise not counted as that person's spend, because the receiving org's ledger already counts it. -The same data is also published as a DuckDB database at /{DUCKDB_FILENAME}, for -querying rather than walking the JSON, with tables programs, program_orgs, -spend_transactions, revenue_transactions, withheld_orgs, unmatched_orgs, -unlinked_programs, budgets, budget_transactions, people_without_budget and -metadata: - - duckdb {DUCKDB_FILENAME} - SELECT name, true_spend_dollars FROM programs ORDER BY 2 DESC LIMIT 10; +The DuckDB database at /{DUCKDB_FILENAME} is a third rendering of these same +documents: nothing is in it that is not on a page, and nothing on a page is +missing from it. ## Mapping contract @@ -942,7 +998,7 @@ def _machine_readable(json_path: str = "index.json", base: str = "") -> str: f'{_link(base + json_path, json_path)} · ' f'{_link(base + "llms.txt", "llms.txt")} · ' f'{_link(base + DUCKDB_FILENAME, DUCKDB_FILENAME)} ' - "(use duckdb if you can get access in your environment)

" + "(prefer the DuckDB file if you can run duckdb; llms.txt shows how)

" ) @@ -966,7 +1022,10 @@ def render_readme(index_document: Dict[str, Any]) -> str: breakdown, HCB org tree, and every transaction counted. - `budgets/.json` / `.html` — one person's YSWS individual budget: totals, bucket breakdown, and every transaction behind them. -- `llms.txt` — the JSON layout and what the numbers mean. +- `ysws-true-spend.duckdb` — every table above in one file; query it + in place with DuckDB's httpfs or download it. Preferred for agents. +- `llms.txt` — how to query the DuckDB, the JSON layout, and what the + numbers mean. {len(linked)} programs · true spend {money(spend)} · {len(index_document["ysws_individual_budgets"]) + len(index_document["ysws_individual_budgets_with_no_linked_person"])} individual budgets · built diff --git a/orpheus_engine_tests/test_ysws_true_spend_site.py b/orpheus_engine_tests/test_ysws_true_spend_site.py index 2715fb9..3b43878 100644 --- a/orpheus_engine_tests/test_ysws_true_spend_site.py +++ b/orpheus_engine_tests/test_ysws_true_spend_site.py @@ -740,16 +740,23 @@ def test_duckdb_file_holds_the_same_rows_as_the_documents(): con.close() -def test_llms_txt_points_at_the_json_without_restating_it(): - """Barebones: the JSON is self-describing, a field list here would drift.""" +def test_llms_txt_steers_agents_to_duckdb_before_the_json(): + """DuckDB first, with a runnable recipe; the JSON is self-describing so its + fields are not restated (a field list here would drift).""" files = _render() llms = files["llms.txt"] assert llms.startswith("# YSWS true spend") - assert "/index.json" in llms - assert "/programs/{program_name}.json" in llms - assert "/programs/fallout.json" in llms # a real example path - assert "ysws-true-spend.duckdb" in llms # and the database - assert "spend_transactions" not in llms.split("## Data")[0] # no field dump + how, data = llms.split("## Data") + assert "## How to query this" in how + assert "ATTACH 'https://ysws-true-spend.hackclub.com/ysws-true-spend.duckdb'" in how + assert "INSTALL httpfs; LOAD httpfs;" in how + assert "curl -sLO https://ysws-true-spend.hackclub.com/ysws-true-spend.duckdb" in how + assert "counted_as_spend" in how # the filter that matters + assert "Never add budgets.personal_spend_dollars" in how + assert "/index.json" in data + assert "/programs/{program_name}.json" in data + assert "/programs/fallout.json" in data # a real example path + assert "ysws_programs_with_linked_hcbs" not in how # no JSON field dump up top def test_machine_readable_line_appears_top_and_bottom(): @@ -763,7 +770,7 @@ def test_machine_readable_line_appears_top_and_bottom(): for href in ('href="index.json"', 'href="llms.txt"', 'href="ysws-true-spend.duckdb"'): assert href in index, href - assert "(use duckdb if you can get access in your environment)" in index + assert "(prefer the DuckDB file if you can run duckdb; llms.txt shows how)" in index assert "
" not in index
 
     # program pages point at their own document