From b2ab0797fb582ff4ae9cf9538b2ed59a83bae0f1 Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:27:06 -0700 Subject: [PATCH 01/15] edit no-code examples --- custom-nocode-examples/quantiles.toml | 61 ++++++++++++--------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/custom-nocode-examples/quantiles.toml b/custom-nocode-examples/quantiles.toml index d757f61..c9b6f0f 100644 --- a/custom-nocode-examples/quantiles.toml +++ b/custom-nocode-examples/quantiles.toml @@ -1,26 +1,24 @@ -# This file contains configuration to run several popular, public benchmarks as -# custom no-code evaluations. Each benchmark below is configured to use the 'random' -# model, which just outputs random text or randomly selects the right answer -# from a uniform distribution of the choices, for exact_match and multiple_choice -# benchmarks respectively. +# This file contains complete local configurations for several popular public +# benchmarks. These configurations allow you to customize settings such as the +# hosted AI model or number of samples to run. # -# If you want to customize the model for any of these benchmarks, pass the `--input` -# flag to the `qt run ...` command. For example, the below command shows how to run -# the MMLU-pro benchmark as configured in this file, using OpenAI's gpt-5.6-luna model. -# Make sure to run this command from the same directory as this file. +# When you run a benchmark, Quantiles first checks for a matching local +# configuration in `quantiles.tom` or `.quantiles.toml`. If none is found, +# it queries the remote Quantiles benchmark registry at `https://api.quantiles.io` +# for a built-in benchmark with the requested name. # -# qt run mmlu-pro --input '{"model": "openai:gpt-5.6-luna"}' +# Each benchmark below uses the `random` demo model. For exact-match +# evaluations, the model generates random text. For multiple-choice +# evaluations, it selects uniformly from the available answer choices. # -# To read more about custom no-code evaluations, see the following documentation: -# -# https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations - +# Each benchmark is limited to 10 samples. Set `samples` to any positive +# integer up to the number of available samples, or omit it to run the +# full benchmark. -# The SimpleQA-verified becnhmark configured to use the random model. -# +# SimpleQA Verified # Read more about the SimpleQA-verified benchmark here: -# # https://arxiv.org/abs/2509.07968 + [benchmarks.simpleqa-verified] type = "custom_nocode" style = { type = "exact_match", golden_column = "answer" } @@ -29,11 +27,10 @@ model = "random" prompt_template_file = "prompts/qa.txt" limit = 10 -# The MedQA benchmark configured to use the random model. -# +# MedQA # Read more about the MedQA benchmark here: -# # https://arxiv.org/abs/2009.13081 + [benchmarks.medqa] type = "custom_nocode" dataset = { name = "quantiles/MedQA-USMLE-4-options", config_name = "default", split = "test" } @@ -47,17 +44,16 @@ choices = { column = "options" } choice_labels = ["A", "B", "C", "D"] answer = { label_column = "answer_idx" } -# The MedMCQA benchmark configured as a custom_nocode benchmark, using the random model. -# +# MedMCQA # Read more about the MedMCQA benchmark here: -# # https://medmcqa.github.io/ + [benchmarks.medmcqa] type = "custom_nocode" dataset = { name = "quantiles/medmcqa", config_name = "default", split = "validation" } model = "random" prompt_template_file = "prompts/medmcqa.txt" -limit = 100 +limit = 10 [benchmarks.medmcqa.style] type = "multiple_choice" @@ -65,17 +61,16 @@ choices = { columns = ["opa", "opb", "opc", "opd"] } choice_labels = ["A", "B", "C", "D"] answer = { index_column = "cop", index_base = 0 } -# The MMLU-pro benchmark configured to use the random model by default. -# -# Read more about the MMLU-pro benchmark here: -# +# MMLU-Pro +# Read more about the MMLU-Pro benchmark here: # https://arxiv.org/abs/2406.01574 + [benchmarks.mmlu-pro] type = "custom_nocode" dataset = { name = "quantiles/MMLU-Pro", config_name = "default", split = "test" } model = "random" prompt_template_file = "prompts/mmlu-pro.txt" -limit = 1000 +limit = 10 [benchmarks.mmlu-pro.style] type = "multiple_choice" @@ -83,16 +78,16 @@ choices = { column = "options" } choice_labels = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"] answer = { label_column = "answer" } -# The GPQA benchmark configured to use the random model -# +# GPQA Diamond # Read more about GPQA here: -# # https://arxiv.org/abs/2311.12022 + [benchmarks.gpqa] type = "custom_nocode" +dataset = { name = "quantiles/gpqa", config_name = "gpqa_diamond", split = "train" } model = "random" prompt_template_file = "prompts/gpqa.txt" -dataset = { name = "quantiles/gpqa", config_name = "gpqa_diamond", split = "train" } +limit=10 # This key shows F1 aggregate metrics whether or not the `--json` flag is passed to `qt show`, # and shows a confusion matrix only when the `--json` flag is passed. metrics = [{ name = "f1", show = "all" }, { name = "confusion", show = "json" }] From ef3b1496427dfb3c45be0937db77d3b563320ef4 Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:29:57 -0700 Subject: [PATCH 02/15] edit security --- SECURITY.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index a85a361..f52fc11 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,11 +4,11 @@ Quantiles provides security fixes for the latest released versions of actively maintained open source project components. -| Component | Supported | -| --------------------------------------------- | --------------------------------------------- | -| Latest `qt` CLI release | Yes | -| Latest Python SDK release | Yes | -| Older releases | No, unless explicitly stated in release notes | +| Component | Supported | +| ------------------------- | --------------------------------------------- | +| Latest `qt` CLI release | Yes | +| Latest Python SDK release | Yes | +| Older releases | No, unless explicitly stated in release notes | ## Reporting a Vulnerability @@ -20,7 +20,7 @@ Please include as much of the information below as you can. This will help us un - A description of the vulnerability and likely impact - Steps to reproduce, proof of concept, or affected command/API route -- Affected component(s), such as `qt`, the Python SDK, local `qt` server API, dataset loader, or export path +- Affected component(s), such as `qt`, the Python SDK, local `qt` server API, remote benchmark registry, dataset loader, or export path - Affected versions, commits, packages, or deployment environment - Whether the issue involves secrets, authentication, local `.quantiles/` artifacts, or model outputs From bd03233d130400f1cfef1d304629b2de517df5ba Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:59:03 -0700 Subject: [PATCH 03/15] edit cli readme --- cli/README.md | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/cli/README.md b/cli/README.md index 7e8d299..2a17d70 100644 --- a/cli/README.md +++ b/cli/README.md @@ -13,7 +13,7 @@ curl -fsSL https://cli.quantiles.io/install.sh | bash A few commands to see `qt` in action: ```bash -# 1. Run a built-in evaluation using a demo model that does +# 1. Run a built-in benchmark using a demo model that does # not incur any usage charges. # # You can also build and run custom evaluations. @@ -27,6 +27,8 @@ qt list qt show ``` +Running built-in benchmarks requires access to `https://api.quantiles.io`, where qt retrieves their configuration. + See the [CLI reference](https://quantiles.io/documentation/reference/cli) for a detailed list of `qt` commands. > Note: Quantiles is designed for high-throughput execution and may issue many parallel requests to your LLM provider. Depending on your provider, model, and account limits, benchmark runs can hit API rate limits or concurrency quotas. Reduce request concurrency or use a model or provider with higher throughput limits. The example below shows how to adjust `max_workers` if you encounter throttling. @@ -35,33 +37,43 @@ See the [CLI reference](https://quantiles.io/documentation/reference/cli) for a The CLI supports three evaluation types: -- [Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) run predefined datasets and scoring methods. They work without configuration, but you can override settings such as the model, sample count, and concurrency. +- [Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) are ready-to-run evaluations, with optional configuration for custom settings such as a hosted AI model. - [`custom_nocode` evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) define the dataset, prompt template, model, and scoring method entirely in configuration. - [`custom_code` evaluations](https://quantiles.io/documentation/custom-evaluations) run your own Python evaluation through the Quantiles Python SDK. -Add a `quantiles.toml` or `.quantiles.toml` file to configure an evaluation. For example: +Add a `quantiles.toml` or `.quantiles.toml` config file to configure an evaluation. When you run a benchmark, Quantiles first checks this file for a matching configuration. If none is found, it queries the remote Quantiles benchmark registry at `https://api.quantiles.io` for a built-in benchmark with that name. + +The following example configures the built-in PubMedQA benchmark to use an OpenAI model and limit the number of samples: ```toml +# Define a local configuration for the PubMedQA benchmark. [benchmarks.pubmedqa] -dataset = "hf://quantiles/PubMedQA" -samples = 50 -model = "openai:gpt-5.6" -max_workers = 100 -``` -See the [configuration guide](https://quantiles.io/documentation/configuration) for file location, supported fields, validation behavior, and examples. See the [model configuration guide](https://quantiles.io/documentation/model-configuration) for guidance on setting up provider models, managing credentials, and troubleshooting configuration issues. Additional runnable configurations are available in [CLI configuration examples](./examples/configs) and [custom no-code examples](../custom-nocode-examples/quantiles.toml). +# Use the configurable no-code evaluation framework. +type = "custom-nocode" -### Custom evaluations and the remote benchmark service +# Use the same dataset as the built-in PubMedQA benchmark. +dataset = "hf://quantiles/PubMedQA" -When you run `qt run `, the CLI first looks in the local configuration file for an evaluation called `eval_name`. If one is found, the CLI runs it immediately. If none is found, `qt` looks in the Quantiles remote benchmark service for an evaluation of the same name. If a match is found, the CLI downloads the benchmark definition and runs it. +# Run the evaluation on 50 samples. +# Omit this field to evaluate the full dataset. +samples = 50 ->If you want to override the location of the remote benchmark service, use the `--remote-url` flag or the `QUANTILES_REMOTE_URL` environment variable. +# Replace the default demo model with a hosted OpenAI model. +model = "openai:gpt-5.6-luna" +``` -When `qt` uses the remote benchmark service, downloaded remote definitions and prompt templates are verified and kept in memory for the run. They will not be cached on disk. +For additional guidance, see: + +- [Configuration guide](https://quantiles.io/documentation/configuration) for file location, supported fields, validation behavior, and examples. +- [Model configuration guide](https://quantiles.io/documentation/model-configuration) for guidance on setting up hosted AI models, managing credentials, and troubleshooting configuration issues. +- [CLI configuration examples](./examples/configs) and [custom no-code examples](../custom-nocode-examples/quantiles.toml) for additional runnable examples. ## Architecture -The Quantiles CLI, `qt`, keeps execution simple: your code runs locally, while `qt` handles durability and observability. +ASK AARON IF THIS NEEDS TO CHANGE + +The Quantiles CLI, `qt` runs code locally, while `qt` handles durability and observability. ``` +--------------------------------------+ From a0d137ef9a6a151e6ea1f35401370b7dd48eb34f Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:35:18 -0700 Subject: [PATCH 04/15] update readmes --- README.md | 37 ++++++++++++++++++++----------------- cli/README.md | 2 +- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 36ead5c..940e2ed 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,11 @@ Quantiles centralizes its components in this monorepo so developers, researchers ## ![New](./docs/assets/new-badge.svg) What's New +**[2026.08.08]** Added built-in benchmark support for `gpqa`, `medmcqa`, `medqa`, `mmlu-pro`, and `pubmedqa`. The new `qt add` command adds any built-in benchmark to a configuration file to easily customize it. See [Built-in benchmarks](https://quantiles.io/documentation/builtin-benchmarks) for details. + **[2026.07.27]** Published the [model configuration guide](https://quantiles.io/documentation/model-configuration), covering the built-in demo model, supported model providers, credentials, request concurrency, cost and data handling, and troubleshooting. -**[2026.07.19]** Added `custom_nocode` evaluations, which let users configure custom evals in `quantiles.toml` without writing or maintaining custom code. See the [custom no-code evaluation documentation](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) for details. +**[2026.07.19]** Added `custom_configuration` evaluations, which let users configure custom evals in `quantiles.toml` without writing or maintaining custom code. See the [custom configuration evaluation documentation](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) for details. ## Why use Quantiles? @@ -37,9 +39,9 @@ Evaluation workflows quickly outgrow one-off scripts once teams need caching, re - Analyze individual samples using recorded step status, outputs, and metrics - Inspect and compare evaluation runs directly from the same `qt` CLI - Write standard Python with familiar Pythonic patterns -- Resume interrupted or failed runs without repeating completed work +- Resume failed or interrupted evaluation runs without repeating completed work -Quantiles borrows concepts from durable workflow execution systems to make evaluation runs resilient to crashes and restarts, while adding a high-throughput execution engine, rich observability, metrics, and eval reproducibility. Use it to run custom eval code or built-in benchmarks, then inspect what changed across runs without requiring notebooks, pipelines, manual comparisons, or a hosted evaluation service. +Quantiles borrows concepts from durable workflow execution systems to make supported evaluation runs resilient to crashes and restarts, while adding a high-throughput execution engine, rich observability, metrics, and eval reproducibility. Use it to build and run custom evals or built-in benchmarks, then inspect what changed across runs without requiring notebooks, pipelines, or manual comparisons. ## Quickstart @@ -55,6 +57,8 @@ Run the [SimpleQA Verified](https://quantiles.io/benchmark-hub/benchmark/simpleq qt run simpleqa-verified ``` +Running a built-in benchmark requires an internet connection so that `qt` can retrieve its configuration from `https://api.quantiles.io`. + > The command above runs [`simpleqa-verified`](https://quantiles.io/benchmark-hub/benchmark/simpleqa-verified) with a demo model that generates random text. It validates the evaluation workflow without requiring provider API keys or incurring inference costs. Do not use its results to draw conclusions about model quality. Inspect the recorded run: @@ -86,21 +90,20 @@ Common commands: ```bash qt --version -qt run +ASK AARON - IS THIS REMOTE-URL NEEDED? +qt run [--input ] [--remote-url ] qt list qt show qt compare ``` -When `qt run ` does not find the requested name in the local configuration, it queries the Quantiles remote benchmark service for a benchmark that has the given name. See the [CLI's README file](./cli/README.md#remote-benchmark-fallback) for details. - > Note: Pass `--json` to any of these commands to output machine- and agent-friendly JSON instead of human-formatted output. See the [CLI reference](https://quantiles.io/documentation/reference/cli) for available commands, options, and usage details. ### Configuration and customization -You can customize how the CLI executes [built-in-benchmarks](https://quantiles.io/documentation/built-in-benchmarks), [custom no-code evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations), and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory or a parent directory. The CLI uses this configuration each time you run the benchmark with `qt run`. +You can customize how the CLI executes [built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks), [custom configuration evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations), and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory or a parent directory. When a benchmark or evaluation is run, Quantiles first checks the configuration file for a matching configuration. If none is found, it queries the hosted Quantiles benchmark registry at `https://api.quantiles.io` for a built-in benchmark with that name. See the following resources for more details: @@ -110,9 +113,11 @@ See the following resources for more details: #### Built-in benchmarks -[Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) are ready-to-run evaluations with predefined datasets, scoring methods, and metrics. Configuration is optional and can override execution settings such as the model and sample count. Use them to get started quickly or establish a repeatable baseline. +[Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) are ready-to-run evaluations with predefined datasets, scoring methods, and metrics. Use them to start evaluating quickly or establish a repeatable baseline. Configuration is optional and can override execution settings such as the model and sample count. -The [benchmark hub](https://quantiles.io/benchmark-hub) describes available benchmarks, their evaluation setup, and common metrics used across AI evaluation workflows. +When a built-in benchmark is run, Quantiles downloads its definition and prompt files into memory and verifies each file against its declared size and SHA-256 hash. These files are not cached on disk. Benchmark datasets are downloaded separately and may be cached locally. + +The [Quantiles Benchmark Hub](https://quantiles.io/benchmark-hub) describes available benchmarks, their evaluation setup, and common metrics used across AI evaluation workflows. > To request another open-source built-in benchmark, [file an issue](https://github.com/quantiles-evals/quantiles/issues) with its name, source dataset or repository, and any available reference implementation. @@ -120,12 +125,12 @@ The [benchmark hub](https://quantiles.io/benchmark-hub) describes available benc Custom evaluations measure behavior specific to your product, workflow, prompt, dataset, rubric, or release process. Quantiles provides two ways to build them: -- [`custom_nocode`](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations): define a custom evaluation entirely in configuration. +- [`custom_configuration`](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations): define a custom evaluation entirely in configuration without writing or maintining Python. - [`custom_code`](https://quantiles.io/documentation/custom-evaluations): build specialized evaluation logic with [Python](https://quantiles.io/documentation/reference/python-sdk). -Prefer to use `custom_nocode` evaluations wherever possible, since they're easier for humans and agents to create and maintain. When required, fall back to `custom_code` evaluations. +Prefer to use `custom_configuration` evaluations wherever possible, since they're easier for humans and agents to create and maintain. When required, fall back to `custom_code` evaluations. -#### Python SDK for `custom_code` evaluations +##### Python SDK for `custom_code` evaluations Use the [official Quantiles Python SDK](https://quantiles.io/documentation/reference/python-sdk) to build `custom_code` evaluations. The SDK provides Python-native APIs for resilient, efficient evaluations, including durable steps, structured inputs and outputs, and high-performance metrics emission. @@ -133,13 +138,11 @@ The SDK integrates tightly with the `qt` CLI’s local API for running, recordin The [Python SDK source code](./python) is available in this repository, and the [Python SDK reference](https://quantiles.io/documentation/reference/python-sdk) has usage instructions and API documentation. -## Local-First and Offline by Default +## Local-First Execution and Offline Workflows Quantiles is a [local-first system that supports offline workflows](https://quantiles.io/documentation/local-first-offline) and stores evaluation metadata, outputs, and metrics on your computer by default. -The entire Quantiles toolchain, including the `qt` CLI, SDKs, on-disk data formats, and REST API, is optimized to use your local computing power instead of relying on cloud or other non-local resources. - -The CLI and Python SDK support offline evaluation workflows when the required code, datasets, and models are available locally: +Quantiles supports fully offline evaluation through both the CLI and Python SDK when all required evaluation configurations, prompts, and datasets are stored locally: - Quantiles scoring and metric aggregation are computed locally. - Run metadata, inputs, outputs, steps, and events are stored in a local [SQLite](https://sqlite.org/) database. @@ -147,7 +150,7 @@ The CLI and Python SDK support offline evaluation workflows when the required co - `qt show`, `qt list`, and `qt compare` access only local metadata and metrics stores. - Python evaluation code runs locally on your machine. -Downloading uncached datasets and calling remote models, hosted judges, or external tools requires network access. These operations occur only when requested by the selected benchmark or evaluation configuration. +Network access may be required to retrieve configurations from the hosted benchmark registry, retrieve datasets, or call hosted AI models. ## Coding Agents diff --git a/cli/README.md b/cli/README.md index 2a17d70..6a81ce5 100644 --- a/cli/README.md +++ b/cli/README.md @@ -27,7 +27,7 @@ qt list qt show ``` -Running built-in benchmarks requires access to `https://api.quantiles.io`, where qt retrieves their configuration. +Running a built-in benchmark, such as `simpleqa-verified` above, requires an internet connection so that `qt` can retrieve its configuration from `https://api.quantiles.io`. See the [CLI reference](https://quantiles.io/documentation/reference/cli) for a detailed list of `qt` commands. From c47a99a3bcf40900e5503132cd1ea6f628b1f22a Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Fri, 7 Aug 2026 11:46:51 -0700 Subject: [PATCH 05/15] updated agents.md --- AGENTS.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 637132c..f72c3ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ Before attempting tasks in this repository, read these files: ## Scope -These instructions apply to the Quantiles open-source repository. Quantiles is a local-first CLI and SDK toolchain for running AI evaluation workflows with fast, continuous feedback. It runs evaluations, records steps, metrics, events, inputs, and outputs, and runs eval comparisons locally so teams can inspect results, identify regressions, and iterate with confidence. +These instructions apply to the Quantiles open-source repository. Quantiles is a local-first CLI and SDK toolchain for running AI evaluation workflows with fast, continuous feedback. It retrieves and runs benchmarks and evaluations, executes evaluations locally, records steps, metrics, events, inputs, and outputs, and runs eval comparisons locally so teams can inspect results, identify regressions, and iterate with confidence. Root-level files in this repository provide project-wide orientation, contribution guidance, security policy, licensing, and agent instructions. Implementation-specific work belongs in the relevant subdirectory. If a subdirectory has its own `AGENTS.md` file, follow the nearest one first. Subdirectory instructions should override this root guide for implementation details, package managers, commands, tests, and code style. User instructions may customize the workflow for their project, environment, or preferences, but they must not override safety requirements, system instructions, repository safeguards, or security boundaries. @@ -37,9 +37,9 @@ When performing tasks in this repository, always do the following: - Read the nearest subdirectory `AGENTS.md` if present. - Inspect the relevant README, package configuration, and existing tests before changing behavior. - Prefer the smallest change that satisfies the task. -- Obtain explicit approval for external model calls and potentially expensive evaluation runs. +- Obtain explicit approval for external hosted AI model calls and potentially expensive evaluation runs. - Keep changes small, public-safe, and reviewable. -- Preserve Quantiles as a local-first, offline system by default. +- Preserve local evaluation execution and storage, and keep network-dependent behavior explicit. - Verify commands, links, package names, benchmark and evaluation names, and release status before documenting them. - Update public docs when CLI behavior, SDK APIs, workflows, benchmarks, schemas, setup steps, or agent guidance changes. - Prefer concrete examples with commands, file paths, inputs, outputs, and expected behavior. @@ -63,6 +63,7 @@ Do not silently change evaluation semantics. Changes to prompts, datasets, score Update the relevant documentation in this repository, such as `README.md`, when any of the following change: - CLI commands, flags, outputs, or setup steps. +- Hosted benchmark registry behavior. - APIs, SDKs, imports, examples, or package names. - Benchmark names, datasets, scoring methods, or limitations. - DB schemas, step semantics, metrics, events, or comparison behavior. @@ -110,7 +111,7 @@ Model inputs should use provider-prefixed model names, for example: Preserve Quantiles as local-first infrastructure. Follow the guidelines below to ensure the project maintains safety, security, and privacy: - The CLI and local server should store Quantiles state locally by default. -- Evaluation workflows may call remote model providers, hosted judges, APIs, or external tools only when configured by the user. Selected benchmarks and evaluation configurations may also download remote datasets. +- Load benchmark configurations only from a local configuration file or the hosted Quantiles benchmark registry. Do not use any other source unless explicitly requested by the user. - Do not inspect, print, summarize, commit, or infer values from `.env` or `.envrc` files, secrets, tokens, private datasets, PHI, customer data, or local Quantiles databases unless the user explicitly asks and the data is safe to inspect. - Never commit the `.quantiles/` directory, SQLite databases, Parquet metrics, local traces, benchmark outputs, provider credentials, or temporary run artifacts. - Use placeholder names such as `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or `QUANTILES_API_KEY` when examples need credentials. @@ -128,16 +129,13 @@ Use these terms consistently in public docs: - `@quantiles/sdk`: the unreleased TypeScript SDK package. - `evaluation`: user-authored evaluation or agent-loop code. - `benchmark`: a repeatable evaluation harness with a defined dataset, scoring method, and result shape. +- `built-in benchmark`: a benchmark from the hosted Quantiles benchmark registry. - `run`: one recorded execution of an evaluation or benchmark. - `step`: a durable recorded unit of an evaluation execution. - `metric`: a measured value emitted during a run. - `event`: recorded observability data from an evaluation. - `.quantiles/`: local Quantiles workspace state, including a SQLite database and metrics Parquet files. -Prefer `local-first` and `offline by default` for open-source behavior. - -When remote model calls, hosted judges, external tools, provider APIs, or network datasets are involved, state that those calls are user-configured exceptions to the local-first default. - ## Output Style For Coding Agents When adding documentation to files in this repository, follow the guidelines below: @@ -158,6 +156,7 @@ After running, inspecting, comparing, or resuming Quantiles evaluations, report: - Run ID or run IDs. - Evaluation or benchmark name. - Model, including whether it was a demo model. +- Hosted built-in benchmark version and manifest hash, when applicable. - Input and output JSON. - Status and success or failure. - Key metrics. From e0fb5fe79a16eb11272795cd08c91b26dc62cfa0 Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:49:46 -0700 Subject: [PATCH 06/15] edit cli agent.md --- cli/AGENTS.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cli/AGENTS.md b/cli/AGENTS.md index b78b237..e2045e3 100644 --- a/cli/AGENTS.md +++ b/cli/AGENTS.md @@ -2,12 +2,15 @@ ## Project Overview -`qt`, the Quantiles CLI, is a local-first Rust CLI for running and analyzing AI evaluations, benchmarks, and agent loops. It stores run data in `.quantiles/quantiles.sqlite`, stores metrics as Parquet files, and provides commands for inspecting recorded evaluation runs. Dataset downloads may occur when a user selects a benchmark, while model-provider calls occur only when configured. The long-term goal is a developer tool for reliable AI/ML evaluation workflows where evaluation runs, execution history, aggregrate and sample-level outputs, and events are queryable from the start. +`qt`, the Quantiles CLI, is a local-first Rust CLI for running and analyzing AI evaluations, benchmarks, and agent loops. It stores run data in `.quantiles/quantiles.sqlite`, stores metrics as Parquet files, and provides commands for inspecting recorded evaluation runs. The CLI can retrieve ready-to-run benchmark configurations from the hosted Quantiles registry and call hosted AI model providers when configured. Its goal is to provide a developer tool for reliable AI/ML evaluation workflows where evaluation runs, execution history, aggregate and sample-level outputs, and events are queryable from the start. ## Working in This Repository - Prefer focused changes that fit the current CLI and library structure. -- Preserve local-first, offline-by-default behavior. Do not introduce implicit network or cloud behavior. When a task explicitly adds remote behavior, make it user-configured, document it clearly, and call it out to the user. +- Preserve local evaluation execution and storage. Keep all network boundaries explicit and documented. +- Do not add telemetry or background uploads. +- Any non-local network activity must be directly triggered by a user-invoked command or explicitly configured workflow, be necessary for that workflow, and be clearly documented. +- Preserve supported network-dependent workflows such as resolving built-in benchmarks, downloading datasets, and calling configured model providers. - Preserve existing SQLite data model assumptions unless the change includes a deliberate schema migration or initialization update. - Use idiomatic Rust and keep error handling clear. This project uses [`anyhow`](https://docs.rs/anyhow) to create and propagate application-level errors. - Avoid broad refactors while implementing narrow behavior changes. From 57a66bc7477ba15251a07c875c586b1ce36d3db1 Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:17:15 -0700 Subject: [PATCH 07/15] update readme --- README.md | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 940e2ed..f387b75 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,11 @@ Quantiles centralizes its components in this monorepo so developers, researchers ## ![New](./docs/assets/new-badge.svg) What's New -**[2026.08.08]** Added built-in benchmark support for `gpqa`, `medmcqa`, `medqa`, `mmlu-pro`, and `pubmedqa`. The new `qt add` command adds any built-in benchmark to a configuration file to easily customize it. See [Built-in benchmarks](https://quantiles.io/documentation/builtin-benchmarks) for details. +**[2026.08.08]** Added built-in benchmark support for `gpqa`, `medmcqa`, `medqa`, `mmlu-pro`, and `pubmedqa`. The new `qt add ` command downloads a built-in benchmark’s configuration and prompt from the hosted registry, adds them to the local project, and makes the benchmark easy to customize. See [Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) for details. **[2026.07.27]** Published the [model configuration guide](https://quantiles.io/documentation/model-configuration), covering the built-in demo model, supported model providers, credentials, request concurrency, cost and data handling, and troubleshooting. -**[2026.07.19]** Added `custom_configuration` evaluations, which let users configure custom evals in `quantiles.toml` without writing or maintaining custom code. See the [custom configuration evaluation documentation](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) for details. +**[2026.07.19]** Added custom no-code (`custom_nocode`) evaluations, which let users configure custom evals in `quantiles.toml` without writing or maintaining custom code. See the [custom no-code evaluation documentation](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) for details. ## Why use Quantiles? @@ -41,7 +41,7 @@ Evaluation workflows quickly outgrow one-off scripts once teams need caching, re - Write standard Python with familiar Pythonic patterns - Resume failed or interrupted evaluation runs without repeating completed work -Quantiles borrows concepts from durable workflow execution systems to make supported evaluation runs resilient to crashes and restarts, while adding a high-throughput execution engine, rich observability, metrics, and eval reproducibility. Use it to build and run custom evals or built-in benchmarks, then inspect what changed across runs without requiring notebooks, pipelines, or manual comparisons. +Quantiles borrows concepts from durable workflow execution systems to make evaluation runs resilient to crashes and restarts, while adding a high-throughput execution engine, rich observability, metrics, and eval reproducibility. Use it to build and run custom evals or built-in benchmarks, then inspect what changed across runs without requiring notebooks, pipelines, or manual comparisons. ## Quickstart @@ -57,9 +57,9 @@ Run the [SimpleQA Verified](https://quantiles.io/benchmark-hub/benchmark/simpleq qt run simpleqa-verified ``` -Running a built-in benchmark requires an internet connection so that `qt` can retrieve its configuration from `https://api.quantiles.io`. +The command above downloads the [`simpleqa-verified`](https://quantiles.io/benchmark-hub/benchmark/simpleqa-verified) definition from the hosted Quantiles benchmark registry and runs it locally with a demo model that generates random text. Fetching the benchmark definition and an uncached dataset requires network access, but no provider API key or paid model inference is required. -> The command above runs [`simpleqa-verified`](https://quantiles.io/benchmark-hub/benchmark/simpleqa-verified) with a demo model that generates random text. It validates the evaluation workflow without requiring provider API keys or incurring inference costs. Do not use its results to draw conclusions about model quality. +> The demo model validates the evaluation workflow. Do not use its results to draw conclusions about model quality. Inspect the recorded run: @@ -90,26 +90,28 @@ Common commands: ```bash qt --version -ASK AARON - IS THIS REMOTE-URL NEEDED? -qt run [--input ] [--remote-url ] +qt add +qt run [--input ] qt list qt show qt compare +qt resume ``` -> Note: Pass `--json` to any of these commands to output machine- and agent-friendly JSON instead of human-formatted output. +> Note: Pass `--json` to `qt add`, `qt run`, `qt list`, `qt show`, `qt compare`, or `qt resume` to request machine- and agent-friendly output. See the [CLI reference](https://quantiles.io/documentation/reference/cli) for available commands, options, and usage details. ### Configuration and customization -You can customize how the CLI executes [built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks), [custom configuration evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations), and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory or a parent directory. When a benchmark or evaluation is run, Quantiles first checks the configuration file for a matching configuration. If none is found, it queries the hosted Quantiles benchmark registry at `https://api.quantiles.io` for a built-in benchmark with that name. +You can customize how the CLI executes [built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks), [custom no-code evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations), and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory. When you run a benchmark or evaluation, Quantiles first checks the configuration file for a matching local definition. If none is found, it queries the hosted Quantiles benchmark registry at `https://api.quantiles.io` for a built-in benchmark with that name. See the following resources for more details: - [Configuration guide](https://quantiles.io/documentation/configuration) - Detailed configuration instructions and reference documentation for supported fields, validation rules, and examples. - [Model configuration guide](https://quantiles.io/documentation/model-configuration) - Configure provider models and credentials, and troubleshoot common setup issues. -- [Configuration examples](./cli/examples/configs) - Complete examples, including a [custom-code evaluation](./cli/examples/configs/custom_code/quantiles.toml) +- [Custom-code configuration example](./cli/examples/configs/custom_code/quantiles.toml) - A complete Python SDK evaluation configuration. +- [Custom no-code examples](./custom-nocode-examples/quantiles.toml) - Complete dataset, prompt, model, and scoring configurations. #### Built-in benchmarks @@ -117,6 +119,14 @@ See the following resources for more details: When a built-in benchmark is run, Quantiles downloads its definition and prompt files into memory and verifies each file against its declared size and SHA-256 hash. These files are not cached on disk. Benchmark datasets are downloaded separately and may be cached locally. +To save and customize a built-in benchmark locally, add it by name: + +```bash +qt add simpleqa-verified +``` + +This command downloads the benchmark definition and prompt template, appends the benchmark to an existing `quantiles.toml` or `.quantiles.toml`, or creates `quantiles.toml` in the current directory. The prompt template is stored beside the configuration at `-prompt/prompt.txt`. The command returns an error if the benchmark is already configured or is not present in the registry. Pass `--json` for machine-readable output. + The [Quantiles Benchmark Hub](https://quantiles.io/benchmark-hub) describes available benchmarks, their evaluation setup, and common metrics used across AI evaluation workflows. > To request another open-source built-in benchmark, [file an issue](https://github.com/quantiles-evals/quantiles/issues) with its name, source dataset or repository, and any available reference implementation. @@ -125,10 +135,10 @@ The [Quantiles Benchmark Hub](https://quantiles.io/benchmark-hub) describes avai Custom evaluations measure behavior specific to your product, workflow, prompt, dataset, rubric, or release process. Quantiles provides two ways to build them: -- [`custom_configuration`](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations): define a custom evaluation entirely in configuration without writing or maintining Python. -- [`custom_code`](https://quantiles.io/documentation/custom-evaluations): build specialized evaluation logic with [Python](https://quantiles.io/documentation/reference/python-sdk). +- [Custom no-code (`custom_nocode`) evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations): define a custom evaluation entirely in configuration without writing or maintaining Python. +- [Custom code (`custom_code`) evaluations](https://quantiles.io/documentation/custom-evaluations): build specialized evaluation logic with [Python](https://quantiles.io/documentation/reference/python-sdk). -Prefer to use `custom_configuration` evaluations wherever possible, since they're easier for humans and agents to create and maintain. When required, fall back to `custom_code` evaluations. +Prefer custom no-code evaluations wherever possible because they are easier for humans and agents to create and maintain. Use a custom code evaluation when the required behavior cannot be expressed in configuration. ##### Python SDK for `custom_code` evaluations @@ -142,7 +152,7 @@ The [Python SDK source code](./python) is available in this repository, and the Quantiles is a [local-first system that supports offline workflows](https://quantiles.io/documentation/local-first-offline) and stores evaluation metadata, outputs, and metrics on your computer by default. -Quantiles supports fully offline evaluation through both the CLI and Python SDK when all required evaluation configurations, prompts, and datasets are stored locally: +Quantiles supports fully offline evaluation through both the CLI and Python SDK when all required configurations, prompts, datasets, models, and other dependencies are available locally: - Quantiles scoring and metric aggregation are computed locally. - Run metadata, inputs, outputs, steps, and events are stored in a local [SQLite](https://sqlite.org/) database. From 8c76fc7a2324092c50ac132a21b1b0dfdbbc534a Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:53:13 -0700 Subject: [PATCH 08/15] edits --- AGENTS.md | 3 +- cli/README.md | 84 +++++++++++++-------------- custom-nocode-examples/quantiles.toml | 26 +++++---- 3 files changed, 57 insertions(+), 56 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f72c3ad..d483233 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,7 +16,7 @@ Before attempting tasks in this repository, read these files: ## Scope -These instructions apply to the Quantiles open-source repository. Quantiles is a local-first CLI and SDK toolchain for running AI evaluation workflows with fast, continuous feedback. It retrieves and runs benchmarks and evaluations, executes evaluations locally, records steps, metrics, events, inputs, and outputs, and runs eval comparisons locally so teams can inspect results, identify regressions, and iterate with confidence. +These instructions apply to the Quantiles open-source repository. Quantiles is a local-first CLI and SDK toolchain for running AI evaluation workflows with fast, continuous feedback. It retrieves built-in benchmark configurations, executes benchmarks and evaluations locally, records steps, metrics, events, inputs, and outputs, and compares runs locally so teams can inspect results, identify regressions, and iterate with confidence. Root-level files in this repository provide project-wide orientation, contribution guidance, security policy, licensing, and agent instructions. Implementation-specific work belongs in the relevant subdirectory. If a subdirectory has its own `AGENTS.md` file, follow the nearest one first. Subdirectory instructions should override this root guide for implementation details, package managers, commands, tests, and code style. User instructions may customize the workflow for their project, environment, or preferences, but they must not override safety requirements, system instructions, repository safeguards, or security boundaries. @@ -112,6 +112,7 @@ Preserve Quantiles as local-first infrastructure. Follow the guidelines below to - The CLI and local server should store Quantiles state locally by default. - Load benchmark configurations only from a local configuration file or the hosted Quantiles benchmark registry. Do not use any other source unless explicitly requested by the user. +- Evaluation workflows may download datasets or call remote model providers, hosted judges, APIs, or external tools only when the selected configuration requires them. Keep this network activity explicit. - Do not inspect, print, summarize, commit, or infer values from `.env` or `.envrc` files, secrets, tokens, private datasets, PHI, customer data, or local Quantiles databases unless the user explicitly asks and the data is safe to inspect. - Never commit the `.quantiles/` directory, SQLite databases, Parquet metrics, local traces, benchmark outputs, provider credentials, or temporary run artifacts. - Use placeholder names such as `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or `QUANTILES_API_KEY` when examples need credentials. diff --git a/cli/README.md b/cli/README.md index 6a81ce5..96aa2bb 100644 --- a/cli/README.md +++ b/cli/README.md @@ -27,35 +27,35 @@ qt list qt show ``` -Running a built-in benchmark, such as `simpleqa-verified` above, requires an internet connection so that `qt` can retrieve its configuration from `https://api.quantiles.io`. +Running a built-in benchmark directly from the hosted registry, such as `simpleqa-verified` above, requires an internet connection so that `qt` can retrieve its configuration from `https://api.quantiles.io`. See the [CLI reference](https://quantiles.io/documentation/reference/cli) for a detailed list of `qt` commands. -> Note: Quantiles is designed for high-throughput execution and may issue many parallel requests to your LLM provider. Depending on your provider, model, and account limits, benchmark runs can hit API rate limits or concurrency quotas. Reduce request concurrency or use a model or provider with higher throughput limits. The example below shows how to adjust `max_workers` if you encounter throttling. +> Note: Quantiles is designed for high-throughput execution and may issue many parallel requests to your model provider. Depending on your provider, model, and account limits, benchmark runs can hit API rate limits or concurrency quotas. Reduce `max_workers` or use a model or provider with higher throughput limits if you encounter throttling. ## Configure evaluations -The CLI supports three evaluation types: +The CLI supports built-in benchmarks and two custom evaluation approaches: -- [Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) are ready-to-run evaluations, with optional configuration for custom settings such as a hosted AI model. -- [`custom_nocode` evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) define the dataset, prompt template, model, and scoring method entirely in configuration. -- [`custom_code` evaluations](https://quantiles.io/documentation/custom-evaluations) run your own Python evaluation through the Quantiles Python SDK. +- [Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) are ready-to-run evaluations retrieved by name from the hosted Quantiles benchmark registry and executed locally. +- [Custom configuration evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) define the dataset, prompt template, model, and scoring method entirely in configuration using `type = "custom_nocode"`. Supported scoring styles include exact match, multiple choice, and text similarity. +- [Custom code evaluations](https://quantiles.io/documentation/custom-evaluations) run your own Python evaluation through the Quantiles Python SDK using `type = "custom_code"`. -Add a `quantiles.toml` or `.quantiles.toml` config file to configure an evaluation. When you run a benchmark, Quantiles first checks this file for a matching configuration. If none is found, it queries the remote Quantiles benchmark registry at `https://api.quantiles.io` for a built-in benchmark with that name. +Add a `quantiles.toml` or `.quantiles.toml` config file to configure an evaluation. When you run a evaluation, Quantiles first checks this file for a matching configuration. If none is found, it queries the hosted Quantiles benchmark registry at for a built-in benchmark with that name. -The following example configures the built-in PubMedQA benchmark to use an OpenAI model and limit the number of samples: +The following `quantiles.toml` example configures PubMedQA to use an OpenAI model and evaluate a limited number of samples. Use `qt add ` to add any built-in benchmark for local customization: -```toml +````toml # Define a local configuration for the PubMedQA benchmark. [benchmarks.pubmedqa] -# Use the configurable no-code evaluation framework. -type = "custom-nocode" +# Type must be specified. +type = "custom_nocode" # Use the same dataset as the built-in PubMedQA benchmark. dataset = "hf://quantiles/PubMedQA" -# Run the evaluation on 50 samples. +# Run 50 samples of the benchmark. # Omit this field to evaluate the full dataset. samples = 50 @@ -67,43 +67,37 @@ For additional guidance, see: - [Configuration guide](https://quantiles.io/documentation/configuration) for file location, supported fields, validation behavior, and examples. - [Model configuration guide](https://quantiles.io/documentation/model-configuration) for guidance on setting up hosted AI models, managing credentials, and troubleshooting configuration issues. -- [CLI configuration examples](./examples/configs) and [custom no-code examples](../custom-nocode-examples/quantiles.toml) for additional runnable examples. +- [Custom-code example](./examples/configs/custom_code/quantiles.toml) and [custom configuration examples](../custom-nocode-examples/quantiles.toml) for additional runnable examples. -## Architecture +ASK AARON IS THIS SECTION IS RIGHT/NEED -ASK AARON IF THIS NEEDS TO CHANGE +## Architecture -The Quantiles CLI, `qt` runs code locally, while `qt` handles durability and observability. +The Quantiles CLI, `qt`, resolves configuration, orchestrates evaluation runs, and stores results locally: ``` -+--------------------------------------+ -| Benchmark / Custom Evaluation | -+-------------------+------------------+ - │ - │ HTTP / JSON - │ - ▼ -+--------------------------------------+ -| Quantiles Server | -+-------------------+------------------+ - │ - │ SQLite / Parquet - │ - ▼ -+------------------------------------------------+ -| .quantiles/ | -| quantiles.sqlite metrics/*.parquet | -+-------------------+----------------------------+ - │ - │ - │ - ▼ -+--------------------------------------+ -| CLI | -| (list, show, compare) | -+--------------------------------------+ ++------------------------------+ +| qt CLI | ++---------------+--------------+ + | + +---------+---------+ + | | + v v ++----------------+ +-------------------+ +| Built-in and | | custom_code child | +| custom_nocode | | process + local | +| runtime | | API server | ++--------+-------+ +---------+---------+ + | | + +---------+----------+ + v ++---------------------------------------+ +| .quantiles/ | +| quantiles.sqlite metrics/*.parquet | ++---------------------------------------+ ``` -- **Server** owns durability decisions for run state, and metrics. -- **Client** (your script) owns code execution; the server never runs your evaluation logic. -- **CLI** reads run data from SQLite and metrics from Parquet. +- **Built-in and `custom_nocode` evaluations** execute inside the local CLI process. +- **`custom_code` evaluations** execute as user-configured child processes and record workflow data through the local API server. +- **CLI inspection commands** read run data from SQLite and metrics from Parquet. +```` diff --git a/custom-nocode-examples/quantiles.toml b/custom-nocode-examples/quantiles.toml index c9b6f0f..1b860de 100644 --- a/custom-nocode-examples/quantiles.toml +++ b/custom-nocode-examples/quantiles.toml @@ -3,25 +3,31 @@ # hosted AI model or number of samples to run. # # When you run a benchmark, Quantiles first checks for a matching local -# configuration in `quantiles.tom` or `.quantiles.toml`. If none is found, -# it queries the remote Quantiles benchmark registry at `https://api.quantiles.io` +# configuration in `quantiles.toml` or `.quantiles.toml`. If none is found, +# it queries the hosted Quantiles benchmark registry at `https://api.quantiles.io` # for a built-in benchmark with the requested name. # -# Each benchmark below uses the `random` demo model. For exact-match -# evaluations, the model generates random text. For multiple-choice -# evaluations, it selects uniformly from the available answer choices. +# Each benchmark below uses the `random` demo model. For exact-match and +# similarity evaluations, the model generates random text; similarity scoring +# then applies the configured local metric. For multiple-choice evaluations, +# the model selects uniformly from the available answer choices. # -# Each benchmark is limited to 10 samples. Set `samples` to any positive +# Each benchmark is limited to 10 samples. Set `limit` to any positive # integer up to the number of available samples, or omit it to run the # full benchmark. +# +# To learn more about custom configuration evaluations, see: +# https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations # SimpleQA Verified -# Read more about the SimpleQA-verified benchmark here: +# This example uses cosine similarity with the local FastEmbed model. The first +# run may download the embedding model if it is not already cached. +# Read more about the SimpleQA Verified benchmark here: # https://arxiv.org/abs/2509.07968 [benchmarks.simpleqa-verified] type = "custom_nocode" -style = { type = "exact_match", golden_column = "answer" } +style = { type = "similarity", golden_column = "answer", metric = { type = "cosine", embedding_model = "fastembed" } } dataset = { name = "quantiles/simpleqa-verified" } model = "random" prompt_template_file = "prompts/qa.txt" @@ -78,7 +84,7 @@ choices = { column = "options" } choice_labels = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"] answer = { label_column = "answer" } -# GPQA Diamond +# GPQA Diamond # Read more about GPQA here: # https://arxiv.org/abs/2311.12022 @@ -87,7 +93,7 @@ type = "custom_nocode" dataset = { name = "quantiles/gpqa", config_name = "gpqa_diamond", split = "train" } model = "random" prompt_template_file = "prompts/gpqa.txt" -limit=10 +limit = 10 # This key shows F1 aggregate metrics whether or not the `--json` flag is passed to `qt show`, # and shows a confusion matrix only when the `--json` flag is passed. metrics = [{ name = "f1", show = "all" }, { name = "confusion", show = "json" }] From 620dd07e223a0ccfe22758b162bb463249ad785b Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:42:34 -0700 Subject: [PATCH 09/15] small fixups --- README.md | 10 +++++++--- SECURITY.md | 10 +++++----- cli/README.md | 6 ++---- custom-nocode-examples/quantiles.toml | 4 ++-- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index f387b75..098b69a 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,11 @@ Common commands: ```bash qt --version -qt add +# download an eval from the Quantiles hosted benchmark registry +# and save it to the local quantiles.toml configuration file +qt add +# run the eval that was just downloaded and saved with the previous +# `qt add` command. qt run [--input ] qt list qt show @@ -104,11 +108,11 @@ See the [CLI reference](https://quantiles.io/documentation/reference/cli) for av ### Configuration and customization -You can customize how the CLI executes [built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks), [custom no-code evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations), and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory. When you run a benchmark or evaluation, Quantiles first checks the configuration file for a matching local definition. If none is found, it queries the hosted Quantiles benchmark registry at `https://api.quantiles.io` for a built-in benchmark with that name. +You can customize how the CLI executes [built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks), [custom no-code evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations), and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory. When you run a benchmark or evaluation, Quantiles first checks the configuration file for a matching local definition. If none is found, it queries the hosted Quantiles benchmark registry (hosted at `api.quantiles.io`) for a built-in benchmark with that name. See the following resources for more details: -- [Configuration guide](https://quantiles.io/documentation/configuration) - Detailed configuration instructions and reference documentation for supported fields, validation rules, and examples. +- [Configuration documentation](https://quantiles.io/documentation/configuration) - Detailed configuration instructions and reference documentation for supported fields, validation rules, and examples. - [Model configuration guide](https://quantiles.io/documentation/model-configuration) - Configure provider models and credentials, and troubleshoot common setup issues. - [Custom-code configuration example](./cli/examples/configs/custom_code/quantiles.toml) - A complete Python SDK evaluation configuration. - [Custom no-code examples](./custom-nocode-examples/quantiles.toml) - Complete dataset, prompt, model, and scoring configurations. diff --git a/SECURITY.md b/SECURITY.md index f52fc11..3d40556 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,11 +4,11 @@ Quantiles provides security fixes for the latest released versions of actively maintained open source project components. -| Component | Supported | -| ------------------------- | --------------------------------------------- | -| Latest `qt` CLI release | Yes | -| Latest Python SDK release | Yes | -| Older releases | No, unless explicitly stated in release notes | +| Component | Supported | +| --- | --- | +| Latest `qt` CLI release | Yes | +| Latest Python SDK release | Yes | +| Older releases | No, unless explicitly stated in release notes | ## Reporting a Vulnerability diff --git a/cli/README.md b/cli/README.md index 96aa2bb..776f47c 100644 --- a/cli/README.md +++ b/cli/README.md @@ -27,7 +27,7 @@ qt list qt show ``` -Running a built-in benchmark directly from the hosted registry, such as `simpleqa-verified` above, requires an internet connection so that `qt` can retrieve its configuration from `https://api.quantiles.io`. +Running a built-in benchmark directly from the hosted registry, such as `simpleqa-verified` above, requires an internet connection so that `qt` can retrieve its configuration from the Quantiles hosted benchmark registry (hosted at `api.quantiles.io`). See the [CLI reference](https://quantiles.io/documentation/reference/cli) for a detailed list of `qt` commands. @@ -65,12 +65,10 @@ model = "openai:gpt-5.6-luna" For additional guidance, see: -- [Configuration guide](https://quantiles.io/documentation/configuration) for file location, supported fields, validation behavior, and examples. +- [Configuration documentation](https://quantiles.io/documentation/configuration) for file location, supported fields, validation behavior, and examples. - [Model configuration guide](https://quantiles.io/documentation/model-configuration) for guidance on setting up hosted AI models, managing credentials, and troubleshooting configuration issues. - [Custom-code example](./examples/configs/custom_code/quantiles.toml) and [custom configuration examples](../custom-nocode-examples/quantiles.toml) for additional runnable examples. -ASK AARON IS THIS SECTION IS RIGHT/NEED - ## Architecture The Quantiles CLI, `qt`, resolves configuration, orchestrates evaluation runs, and stores results locally: diff --git a/custom-nocode-examples/quantiles.toml b/custom-nocode-examples/quantiles.toml index 1b860de..39bd366 100644 --- a/custom-nocode-examples/quantiles.toml +++ b/custom-nocode-examples/quantiles.toml @@ -4,8 +4,8 @@ # # When you run a benchmark, Quantiles first checks for a matching local # configuration in `quantiles.toml` or `.quantiles.toml`. If none is found, -# it queries the hosted Quantiles benchmark registry at `https://api.quantiles.io` -# for a built-in benchmark with the requested name. +# it queries the hosted Quantiles benchmark registry, which is available +# at `api.quantiles.io`, for a built-in benchmark with the requested name. # # Each benchmark below uses the `random` demo model. For exact-match and # similarity evaluations, the model generates random text; similarity scoring From cdf3ee6fa1ac70c567e3b22d955050d7722c9412 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:59:16 -0700 Subject: [PATCH 10/15] s/no-code/configuration --- README.md | 10 +++++----- cli/README.md | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index af9efad..95f3b4a 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Quantiles centralizes its components in this monorepo so developers, researchers **[2026.07.27]** Published the [model configuration guide](https://quantiles.io/documentation/model-configuration), covering the built-in demo model, supported model providers, credentials, request concurrency, cost and data handling, and troubleshooting. -**[2026.07.19]** Added custom no-code (`custom_nocode`) evaluations, which let users configure custom evals in `quantiles.toml` without writing or maintaining custom code. See the [custom no-code evaluation documentation](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) for details. +**[2026.07.19]** Added custom configuration (`custom_nocode`) evaluations, which let users configure custom evals in `quantiles.toml` without writing or maintaining custom code. See the [custom configuration evaluation documentation](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) for details. ## Why use Quantiles? @@ -109,14 +109,14 @@ See the [CLI reference](https://quantiles.io/documentation/reference/cli) for av ### Configuration and customization -You can customize how the CLI executes [built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks), [custom no-code evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations), and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory. When you run a benchmark or evaluation, Quantiles first checks the configuration file for a matching local definition. If none is found, it queries the hosted Quantiles benchmark registry (hosted at `api.quantiles.io`) for a built-in benchmark with that name. +You can customize how the CLI executes [built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks), [custom configuration evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations), and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory. When you run a benchmark or evaluation, Quantiles first checks the configuration file for a matching local definition. If none is found, it queries the hosted Quantiles benchmark registry (hosted at `api.quantiles.io`) for a built-in benchmark with that name. See the following resources for more details: - [Configuration documentation](https://quantiles.io/documentation/configuration) - Detailed configuration instructions and reference documentation for supported fields, validation rules, and examples. - [Model configuration guide](https://quantiles.io/documentation/model-configuration) - Configure provider models and credentials, and troubleshoot common setup issues. - [Custom-code configuration example](./cli/examples/configs/custom_code/quantiles.toml) - A complete Python SDK evaluation configuration. -- [Custom no-code examples](./custom-nocode-examples/quantiles.toml) - Complete dataset, prompt, model, and scoring configurations. +- [Custom configuration examples](./custom-nocode-examples/quantiles.toml) - Complete dataset, prompt, model, and scoring configurations. #### Registry benchmarks @@ -140,10 +140,10 @@ The [Quantiles Benchmark Hub](https://quantiles.io/benchmark-hub) describes avai Custom evaluations measure behavior specific to your product, workflow, prompt, dataset, rubric, or release process. Quantiles provides two ways to build them: -- [Custom no-code (`custom_nocode`) evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations): define a custom evaluation entirely in configuration without writing or maintaining Python. +- [Custom configuration (`custom_nocode`) evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations): define a custom evaluation entirely in configuration without writing or maintaining Python. - [Custom code (`custom_code`) evaluations](https://quantiles.io/documentation/custom-evaluations): build specialized evaluation logic with [Python](https://quantiles.io/documentation/reference/python-sdk). -Prefer custom no-code evaluations wherever possible because they are easier for humans and agents to create and maintain. Use a custom code evaluation when the required behavior cannot be expressed in configuration. +Prefer custom configuration evaluations wherever possible because they are easier for humans and agents to create and maintain. Use a custom code evaluation when the required behavior cannot be expressed in configuration. ##### Python SDK for `custom_code` evaluations diff --git a/cli/README.md b/cli/README.md index 8757008..33d8bb4 100644 --- a/cli/README.md +++ b/cli/README.md @@ -39,7 +39,7 @@ See the [CLI reference](https://quantiles.io/documentation/reference/cli) for a The CLI supports built-in benchmarks and two locally configured evaluation types: - [Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) are ready-to-run evaluations retrieved by name from the hosted Quantiles benchmark registry and executed locally using the native `custom_nocode` runtime. -- [Custom no-code (`custom_nocode`) evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) define the dataset, prompt template, model, and scoring method entirely in configuration. Supported scoring styles include exact match, multiple choice, and text similarity. +- [Custom configuration (`custom_nocode`) evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) define the dataset, prompt template, model, and scoring method entirely in configuration. Supported scoring styles include exact match, multiple choice, and text similarity. - [Custom code (`custom_code`) evaluations](https://quantiles.io/documentation/custom-evaluations) run your own Python evaluation through the Quantiles Python SDK. Add a `quantiles.toml` or `.quantiles.toml` file to configure a custom evaluation. For example: @@ -53,7 +53,7 @@ command = ["uv", "run", "eval.py"] model = "openai:gpt-5.6" ``` -Custom no-code similarity evaluations support Levenshtein distance and cosine similarity. The following configuration uses Levenshtein distance: +Custom configuration similarity evaluations support Levenshtein distance and cosine similarity. The following configuration uses Levenshtein distance: ```toml [benchmarks.simpleqa-levenshtein] @@ -77,7 +77,7 @@ For additional guidance, see: - [Configuration documentation](https://quantiles.io/documentation/configuration) for file location, supported fields, validation behavior, and examples. - [Model configuration guide](https://quantiles.io/documentation/model-configuration) for guidance on setting up provider models, managing credentials, and troubleshooting configuration issues. -- [Custom-code example](./examples/configs/custom_code/quantiles.toml) and [custom no-code examples](../custom-nocode-examples/quantiles.toml) for additional runnable examples. +- [Custom-code example](./examples/configs/custom_code/quantiles.toml) and [custom configuration examples](../custom-nocode-examples/quantiles.toml) for additional runnable examples. ### Hosted benchmark registry From a5aeff397f40c02578d73f8955d1fa41eb190105 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:01:29 -0700 Subject: [PATCH 11/15] small fixups --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 95f3b4a..22b4b2d 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ See the [CLI reference](https://quantiles.io/documentation/reference/cli) for av ### Configuration and customization -You can customize how the CLI executes [built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks), [custom configuration evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations), and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory. When you run a benchmark or evaluation, Quantiles first checks the configuration file for a matching local definition. If none is found, it queries the hosted Quantiles benchmark registry (hosted at `api.quantiles.io`) for a built-in benchmark with that name. +You can define [custom configuration evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory. When you run a benchmark or evaluation, Quantiles first checks the configuration file for a matching local definition. If none is found, it queries the hosted Quantiles benchmark registry (hosted at `api.quantiles.io`) for a built-in benchmark with that name. See the following resources for more details: @@ -120,7 +120,7 @@ See the following resources for more details: #### Registry benchmarks -[Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) are ready-to-run evaluations with predefined datasets, scoring methods, and metrics. Use them to start evaluating quickly or establish a repeatable baseline. Configuration is optional and can override execution settings such as the model and sample count. +[Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) are ready-to-run evaluations with predefined datasets, scoring methods, and metrics. Run them directly from the hosted registry without a local configuration, and apply supported run-specific overrides such as the model and sample limit with `--input`. To save and persistently customize a built-in benchmark, use `qt add` as described below. When a built-in benchmark is run, Quantiles downloads its definition and prompt files into memory and verifies each file against its declared size and SHA-256 hash. These files are not cached on disk. Benchmark datasets are downloaded separately and may be cached locally. From 27a0ab79cb3e6cf4d0f71bee9196d9891863a911 Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:19:09 -0700 Subject: [PATCH 12/15] edits --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 22b4b2d..e45cc22 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Quantiles centralizes its components in this monorepo so developers, researchers ## ![New](./docs/assets/new-badge.svg) What's New -**[2026.08.08]** Added built-in benchmark support for `gpqa`, `medmcqa`, `medqa`, `mmlu-pro`, and `pubmedqa`. The new `qt add ` command downloads a built-in benchmark’s configuration and prompt from the hosted registry, adds them to the local project, and makes the benchmark easy to customize. See [Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) for details. +**[2026.08.12]** Added built-in benchmark support for `gpqa`, `medmcqa`, `medqa`, `mmlu-pro`, and `pubmedqa`. The new `qt add ` command downloads a built-in benchmark’s configuration and prompt from the hosted registry, adds them to the local project, and makes the benchmark easy to customize. See [Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) for details and the [Quantiles Benchmark Hub](https://quantiles.io/benchmark-hub#built-in) for detailed information on each benchmark. **[2026.07.27]** Published the [model configuration guide](https://quantiles.io/documentation/model-configuration), covering the built-in demo model, supported model providers, credentials, request concurrency, cost and data handling, and troubleshooting. @@ -108,7 +108,6 @@ See the [CLI reference](https://quantiles.io/documentation/reference/cli) for av ### Configuration and customization - You can define [custom configuration evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory. When you run a benchmark or evaluation, Quantiles first checks the configuration file for a matching local definition. If none is found, it queries the hosted Quantiles benchmark registry (hosted at `api.quantiles.io`) for a built-in benchmark with that name. See the following resources for more details: From 744279789896e01151bac5ce8c6b6db684a42c09 Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:52:38 -0700 Subject: [PATCH 13/15] edits --- README.md | 45 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index e45cc22..69e26fe 100644 --- a/README.md +++ b/README.md @@ -64,13 +64,15 @@ The command above downloads the [`simpleqa-verified`](https://quantiles.io/bench Inspect the recorded run: ```bash -# If you've run `qt run` before, you might need to pass a different integer to `qt show` +# If you have run `qt run` previously, replace the value passed to `qt show` +# with the ID of the evaluation run you want to inspect. # -# See all your runs with `qt list`. +# Use `qt list` to view all evaluation runs and their IDs. + qt show 1 ``` -Or output machine- and agent-readable JSON: +To output machine- and agent-readable JSON: ```bash qt show 1 --json @@ -84,21 +86,42 @@ qt --help ## CLI -Use `qt show` to inspect a single run, `qt list` to see a list of all runs, and `qt compare` to compare behavior across runs. +The `qt` CLI runs local evaluation workflows. It stores run metadata in the local workspace, starts a local HTTP server when needed, records workflow steps and metrics, and compares runs from the command line. Common commands: ```bash -qt --version -# download an eval from the Quantiles hosted benchmark registry -# and save it to the local quantiles.toml configuration file +# Import a built-in benchmark configuration and prompt from the hosted Quantiles benchmark registry into quantiles.toml: qt add -# run the eval that was just downloaded and saved with the previous -# `qt add` command. +``` + +```bash +# Run a built-in benchmark, custom configuration evaluation, or custom code evaluation +qt run +``` + +```bash +# Add a one-time override to the evaluation run qt run [--input ] +``` + +```bash +# List all evaluation runs qt list +``` + +```bash +# Show details of a given evaluation run qt show +``` + +```bash +# Compare two evaluation runs qt compare +``` + +```bash +# Resume a failed or interrupted evaluation run qt resume ``` @@ -108,7 +131,7 @@ See the [CLI reference](https://quantiles.io/documentation/reference/cli) for av ### Configuration and customization -You can define [custom configuration evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations) and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory. When you run a benchmark or evaluation, Quantiles first checks the configuration file for a matching local definition. If none is found, it queries the hosted Quantiles benchmark registry (hosted at `api.quantiles.io`) for a built-in benchmark with that name. +You can customize how the CLI executes [built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks), [custom configuration evaluations](https://quantiles.io/documentation/custom-evaluations/custom-nocode-evaluations), and [custom code evaluations](https://quantiles.io/documentation/custom-evaluations) using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory. When you run a benchmark or evaluation, Quantiles first checks the configuration file for a matching local definition. If none is found, it queries the Quantiles benchmark registry (hosted at `https://api.quantiles.io`) for a built-in benchmark with that name. See the following resources for more details: @@ -126,7 +149,7 @@ When a built-in benchmark is run, Quantiles downloads its definition and prompt To save and customize a built-in benchmark locally, add it by name: ```bash -qt add simpleqa-verified +qt add ``` This command downloads the benchmark definition and prompt template, appends the benchmark to an existing `quantiles.toml` or `.quantiles.toml`, or creates `quantiles.toml` in the current directory. The prompt template is stored beside the configuration at `-prompt/prompt.txt`. The command returns an error if the benchmark is already configured or is not present in the registry. Pass `--json` for machine-readable output. From 5e1858cb4122705b616bbbf447c0c3722d9668f6 Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:41:16 -0700 Subject: [PATCH 14/15] edits --- README.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 69e26fe..80e6bc4 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ qt --help ## CLI -The `qt` CLI runs local evaluation workflows. It stores run metadata in the local workspace, starts a local HTTP server when needed, records workflow steps and metrics, and compares runs from the command line. +The `qt` CLI starts a local HTTP server when needed, runs evaluation workflows, stores run metadata in the local workspace, records and analyzes workflow steps and metrics, compares runs from the command line, and resumes failed or interrupted evaluation runs. Common commands: @@ -96,7 +96,7 @@ qt add ``` ```bash -# Run a built-in benchmark, custom configuration evaluation, or custom code evaluation +# Run a benchmark or evaluation qt run ``` @@ -136,23 +136,24 @@ You can customize how the CLI executes [built-in benchmarks](https://quantiles.i See the following resources for more details: - [Configuration documentation](https://quantiles.io/documentation/configuration) - Detailed configuration instructions and reference documentation for supported fields, validation rules, and examples. -- [Model configuration guide](https://quantiles.io/documentation/model-configuration) - Configure provider models and credentials, and troubleshoot common setup issues. -- [Custom-code configuration example](./cli/examples/configs/custom_code/quantiles.toml) - A complete Python SDK evaluation configuration. - [Custom configuration examples](./custom-nocode-examples/quantiles.toml) - Complete dataset, prompt, model, and scoring configurations. +- [Custom-code configuration examples](./cli/examples/configs/custom_code/quantiles.toml) - A complete Python SDK evaluation configuration. -#### Registry benchmarks +#### Built-in benchmarks -[Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) are ready-to-run evaluations with predefined datasets, scoring methods, and metrics. Run them directly from the hosted registry without a local configuration, and apply supported run-specific overrides such as the model and sample limit with `--input`. To save and persistently customize a built-in benchmark, use `qt add` as described below. +[Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) are ready-to-run evaluations with predefined datasets, scoring methods, and metrics. Run them directly from the hosted Quantiles benchmark registry with their default configuration, or configure their settings in one of two ways: -When a built-in benchmark is run, Quantiles downloads its definition and prompt files into memory and verifies each file against its declared size and SHA-256 hash. These files are not cached on disk. Benchmark datasets are downloaded separately and may be cached locally. - -To save and customize a built-in benchmark locally, add it by name: +- [Apply a one-time override](https://quantiles.io/documentation/built-in-benchmarks#apply-one-time-configuration-overrides), such as the AI model or sample limit, with `--input`. For example: ```bash -qt add +qt run gpqa --input '{"model":"openai:gpt-5.6-luna","limit":10}' ``` -This command downloads the benchmark definition and prompt template, appends the benchmark to an existing `quantiles.toml` or `.quantiles.toml`, or creates `quantiles.toml` in the current directory. The prompt template is stored beside the configuration at `-prompt/prompt.txt`. The command returns an error if the benchmark is already configured or is not present in the registry. Pass `--json` for machine-readable output. +- [Add a customized built-in benchmark](https://quantiles.io/documentation/built-in-benchmarks#apply-persistent-configuration-settings) to a config file to apply the same settings in future runs. For example: + +```bash +qt add gpqa +``` The [Quantiles Benchmark Hub](https://quantiles.io/benchmark-hub) describes available benchmarks, their evaluation setup, and common metrics used across AI evaluation workflows. From d24b684aaf3e3094ad3de779a0b7c0876295ded7 Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:42:11 -0700 Subject: [PATCH 15/15] edit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 80e6bc4..2ef44cb 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Quantiles centralizes its components in this monorepo so developers, researchers ## ![New](./docs/assets/new-badge.svg) What's New -**[2026.08.12]** Added built-in benchmark support for `gpqa`, `medmcqa`, `medqa`, `mmlu-pro`, and `pubmedqa`. The new `qt add ` command downloads a built-in benchmark’s configuration and prompt from the hosted registry, adds them to the local project, and makes the benchmark easy to customize. See [Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) for details and the [Quantiles Benchmark Hub](https://quantiles.io/benchmark-hub#built-in) for detailed information on each benchmark. +**[2026.08.12]** Added built-in benchmark support for `gpqa`, `medmcqa`, `medqa`, `mmlu-pro`, and `pubmedqa`. The new `qt add ` command downloads a built-in benchmark’s configuration and prompt from the hosted registry, adds them to the local project, and makes the benchmark easy to customize. See [Built-in benchmarks](https://quantiles.io/documentation/built-in-benchmarks) for details and the [Quantiles Benchmark Hub](https://quantiles.io/benchmark-hub#built-in) for detailed information on each benchmark. **[2026.07.27]** Published the [model configuration guide](https://quantiles.io/documentation/model-configuration), covering the built-in demo model, supported model providers, credentials, request concurrency, cost and data handling, and troubleshooting.