Making runs started from the remote benchmark registry resumable with strict version provenance

cli/README.md

@@ -79,6 +79,10 @@ When you run `qt run <eval_name>`, the CLI first looks in the local configuratio
 
 When `qt` uses the remote benchmark service, downloaded remote definitions and prompt templates are verified and kept in memory for the run. They will not be cached on disk.
 
+If you run an eval from the remote benchmark registry, the CLI will persist the registry endpoint, immutable benchmark version, and manifest hash. If that run needs to be resumed later with `qt resume`, the CLI will re-download that exact benchmark version again and reject it if the manifest hash changed (the registry guarantees that versions are immutable once published). This behavior means that resuming runs that were started from the benchmark registry requires internet access.
+
+When `--input` overrides a remote benchmark's `prompt_template_file` with a local file, `qt` also persists the file's SHA-256 hash. On resume, `qt` reads the local prompt into memory and rejects the resume before changing the run status if its contents no longer match the stored hash.
+
 ## Architecture
 
 The Quantiles CLI, `qt`, keeps execution simple: your code runs locally, while `qt` handles durability and observability.

cli/proto/quantiles/benchmark/v1/benchmark_registry.proto

@@ -11,15 +11,24 @@ message ResolveBenchmarkRequest {
   // Stable benchmark name, such as "simpleqa-verified".
   string benchmark_name = 1;
 
-  // Immutable version to resolve. An empty value requests the latest published version.
+  // Immutable version to resolve.
+  //
+  // Versions that were previously published are immutable, so
+  // any valid version request should return a valid response.
+  //
+  // If you pass an empty value for a valid benchmark, you'll
+  // get the latest published version.
   string version = 2;
 }
 
 message ResolveBenchmarkResponse {
   string benchmark_name = 1;
+
+  // Immutable published version resolved by the registry.
   string version = 2;
 
-  // SHA-256 digest of the canonical resource manifest.
+  // SHA-256 digest of the canonical resource manifest. After a
+  // version is published, this value must be stable.
   string manifest_sha256 = 3;
 
   repeated BenchmarkResource resources = 4;

cli/src/benchmark_registry/benchmark.rs

@@ -8,6 +8,7 @@ use super::proto::v1::{ResolveBenchmarkResponse, ResourceKind};
 use crate::config::{BenchmarkConfig, CustomNoCodeBenchmarkConfig, WorkspaceConfig};
 
 /// A downloaded benchmark ready to execute without materializing its resources on disk.
+#[derive(Debug)]
 pub struct RemoteBenchmark {
     pub config: CustomNoCodeBenchmarkConfig,
     pub prompt_template: String,

cli/src/benchmark_registry/client.rs

@@ -8,6 +8,8 @@ use connectrpc::client::{ClientConfig, HttpClient};
 use reqwest::Url;
 use rustls_platform_verifier::ConfigVerifierExt as _;
 
+use crate::benchmark_registry::version::Version;
+
 use super::proto::v1::{
     BenchmarkRegistryServiceClient, ResolveBenchmarkRequest, ResolveBenchmarkResponse,
 };
@@ -61,6 +63,7 @@ pub(super) fn validate_remote_url(remote_url: &str) -> Result<Url> {
 /// Resolve benchmark metadata from the remote `ConnectRPC` service.
 pub(super) async fn resolve_manifest(
     benchmark_name: &str,
+    version: Option<Version>,
     endpoint: &Url,
 ) -> Result<Option<ResolveBenchmarkResponse>> {
     let uri = endpoint
@@ -80,7 +83,13 @@ pub(super) async fn resolve_manifest(
     let client = BenchmarkRegistryServiceClient::new(transport, config);
     let request = ResolveBenchmarkRequest {
         benchmark_name: benchmark_name.to_owned(),
-        version: String::new(),
+        // If the version is passed as `None` to this function, send the empty string
+        // over RPC
+        version: if let Some(ver) = version {
+            ver.to_string()
+        } else {
+            String::new()
+        },
         ..Default::default()
     };
 
@@ -135,7 +144,7 @@ mod tests {
 
         let endpoint = validate_remote_url(&server.uri()).unwrap();
         assert!(
-            resolve_manifest("missing", &endpoint)
+            resolve_manifest("missing", None, &endpoint)
                 .await
                 .unwrap()
                 .is_none()

cli/src/benchmark_registry/manifest.rs

@@ -4,6 +4,8 @@ use std::path::{Component, Path, PathBuf};
 use anyhow::{Context, Result, bail};
 use reqwest::Url;
 
+use crate::benchmark_registry::version::Version;
+
 use super::proto::v1::{BenchmarkResource, ResolveBenchmarkResponse, ResourceKind};
 
 /// Maximum number of resources allowed in a remote benchmark manifest.
@@ -16,6 +18,7 @@ const MAX_BUNDLE_BYTES: u64 = 50 * 1024 * 1024;
 /// Validate that a response identifies the requested immutable benchmark manifest.
 pub(super) fn validate_response_identity(
     benchmark_name: &str,
+    requested_version: Option<Version>,
     response: &ResolveBenchmarkResponse,
 ) -> Result<()> {
     if response.benchmark_name != benchmark_name {
@@ -27,6 +30,14 @@ pub(super) fn validate_response_identity(
     if response.version.is_empty() {
         bail!("remote benchmark response is missing an immutable version");
     }
+    if let Some(version) = requested_version
+        && response.version != version.as_str()
+    {
+        bail!(
+            "remote benchmark response version `{}` does not match requested version `{version}`",
+            response.version
+        );
+    }
     validate_sha256("manifest", &response.manifest_sha256)?;
     Ok(())
 }

cli/src/benchmark_registry/mod.rs

@@ -3,10 +3,12 @@
 pub use self::benchmark::RemoteBenchmark;
 pub use self::client::select_remote_url;
 pub use self::resolver::resolve_and_download;
+pub use self::version::Version;
 
 mod benchmark;
 mod client;
 mod download;
 mod manifest;
 mod proto;
 mod resolver;
+mod version;

cli/src/benchmark_registry/resolver.rs

@@ -1,29 +1,36 @@
 use anyhow::Result;
 
+use crate::benchmark_registry::version::Version;
+
 use super::RemoteBenchmark;
 use super::client::{resolve_manifest, validate_remote_url};
 use super::download::download_resources;
 use super::manifest::{validate_resources, validate_response_identity};
 
-/// Resolve a benchmark and download all of its resources into memory.
+/// Resolve a benchmark, optionally with a version, and download all of its
+/// resources into memory.
+///
+/// If you pass `None` for `version`, this function returns the latest latest
+/// published version for that benchmark.
 ///
-/// `Ok(None)` means the registry returned Connect's `not_found` status. Other transport and
-/// service failures are returned to the caller rather than treated as absence.
+/// A return value of `Ok(None)` means the registry did not find that benchmark
+/// name and/or version.
 ///
 /// # Errors
 ///
 /// Returns an error for invalid endpoints, RPC failures, malformed manifests, failed downloads,
 /// digest mismatches, invalid UTF-8, or invalid no-code benchmark definitions.
 pub async fn resolve_and_download(
     benchmark_name: &str,
+    version: Option<Version>,
     remote_url: &str,
 ) -> Result<Option<RemoteBenchmark>> {
     let endpoint = validate_remote_url(remote_url)?;
-    let Some(response) = resolve_manifest(benchmark_name, &endpoint).await? else {
+    let Some(response) = resolve_manifest(benchmark_name, version.clone(), &endpoint).await? else {
         return Ok(None);
     };
 
-    validate_response_identity(benchmark_name, &response)?;
+    validate_response_identity(benchmark_name, version, &response)?;
     let resources = validate_resources(&response.resources, endpoint.scheme() == "http")?;
     let downloaded = download_resources(&resources).await?;
     let remote = RemoteBenchmark::new(benchmark_name, response, downloaded)?;
@@ -35,9 +42,11 @@ mod tests {
     use buffa::Message as _;
     use sha2::{Digest as _, Sha256};
     use wiremock::matchers::{method, path};
-    use wiremock::{Mock, MockServer, ResponseTemplate};
+    use wiremock::{Match, Mock, MockServer, Request, ResponseTemplate};
 
-    use super::super::proto::v1::{BenchmarkResource, ResolveBenchmarkResponse, ResourceKind};
+    use super::super::proto::v1::{
+        BenchmarkResource, ResolveBenchmarkRequest, ResolveBenchmarkResponse, ResourceKind,
+    };
     use super::*;
 
     #[tokio::test]
@@ -80,6 +89,7 @@ mod tests {
             .and(path(
                 "/quantiles.benchmark.v1.BenchmarkRegistryService/ResolveBenchmark",
             ))
+            .and(RequestedVersion(""))
             .respond_with(
                 ResponseTemplate::new(200)
                     .insert_header("content-type", "application/proto")
@@ -98,7 +108,7 @@ mod tests {
             .mount(&server)
             .await;
 
-        let benchmark = resolve_and_download("remote-test", &server.uri())
+        let benchmark = resolve_and_download("remote-test", None, &server.uri())
             .await
             .unwrap()
             .unwrap();
@@ -114,6 +124,52 @@ mod tests {
         ));
     }
 
+    #[tokio::test]
+    async fn rejects_a_response_for_a_different_version() {
+        let server = MockServer::start().await;
+        let response = ResolveBenchmarkResponse {
+            benchmark_name: "remote-test".to_owned(),
+            version: "v2".to_owned(),
+            manifest_sha256: "a".repeat(64),
+            ..Default::default()
+        };
+        Mock::given(method("POST"))
+            .and(path(
+                "/quantiles.benchmark.v1.BenchmarkRegistryService/ResolveBenchmark",
+            ))
+            .and(RequestedVersion("v1"))
+            .respond_with(
+                ResponseTemplate::new(200)
+                    .insert_header("content-type", "application/proto")
+                    .set_body_bytes(response.encode_to_vec()),
+            )
+            .mount(&server)
+            .await;
+
+        let error = resolve_and_download(
+            "remote-test",
+            Some(Version::new("v1").unwrap()),
+            &server.uri(),
+        )
+        .await
+        .unwrap_err();
+
+        assert!(
+            error
+                .to_string()
+                .contains("does not match requested version")
+        );
+    }
+
+    struct RequestedVersion(&'static str);
+
+    impl Match for RequestedVersion {
+        fn matches(&self, request: &Request) -> bool {
+            ResolveBenchmarkRequest::decode_from_slice(&request.body)
+                .is_ok_and(|request| request.version == self.0)
+        }
+    }
+
     fn resource(
         id: &str,
         logical_path: &str,

cli/src/benchmark_registry/version.rs

@@ -0,0 +1,56 @@
+use anyhow::{Result, bail};
+
+#[derive(Clone, Debug)]
+pub struct Version(String);
+
+impl Version {
+    /// Create a new Version from a version string.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if the given `ver` string is empty.
+    pub fn new(ver: &str) -> Result<Self> {
+        if ver.is_empty() {
+            bail!("version must not be empty");
+        }
+        Ok(Self(ver.to_string()))
+    }
+
+    #[must_use]
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+impl std::fmt::Display for Version {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str(&self.0)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn rejects_an_empty_version() {
+        let error = Version::new("").unwrap_err();
+
+        assert!(error.to_string().contains("version must not be empty"));
+    }
+
+    #[test]
+    fn preserves_and_displays_a_nonempty_version() {
+        let version = Version::new("v1.2.3").unwrap();
+
+        assert_eq!(format!("{version}"), "v1.2.3");
+        assert_eq!(version.to_string(), "v1.2.3");
+    }
+
+    #[test]
+    fn clone_preserves_the_version() {
+        let version = Version::new("release-42").unwrap();
+
+        assert_eq!(version.clone().to_string(), version.to_string());
+    }
+}