diff --git a/src/build.rs b/src/build.rs
index 7ad17a7..dcaa84d 100644
--- a/src/build.rs
+++ b/src/build.rs
@@ -135,6 +135,8 @@ pub fn build(
     cargo_args: &[&str],
     extra_args: &[String],
 ) -> (bool, Vec<PathBuf>, String) {
+    let want_test = cargo_args.first() == Some(&"test");
+
     let mut child = Command::new("cargo")
         .args(cargo_args)
         .args(["--message-format=json", "--color=always"])
@@ -176,7 +178,9 @@ pub fn build(
         match message {
             Message::CompilerArtifact(artifact) => {
                 let fresh = artifact.fresh;
-                if let Some(path) = artifact.executable {
+                if artifact.profile.test == want_test
+                    && let Some(path) = artifact.executable
+                {
                     executables.push(path.into_std_path_buf());
                 }
                 if let Some(tx) = tx {
diff --git a/src/test.rs b/src/test.rs
index 2546ad4..95218d3 100644
--- a/src/test.rs
+++ b/src/test.rs
@@ -1,7 +1,8 @@
 use std::collections::HashMap;
-use std::io::{BufRead, BufReader};
+use std::io::{self, BufRead, BufReader};
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
+use std::thread;
 
 use nobubbles::effects::Emitter;
 use serde_json::Value;
@@ -41,9 +42,15 @@ fn run_one(tx: &Emitter<Event>, path: &Path, harness_args: &[String]) -> bool {
         .args(harness_args)
         .env("RUSTC_BOOTSTRAP", "1")
         .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
         .spawn()
         .expect("failed to spawn test binary");
 
+    // Anything that the test binary writes to stderr would mangle the terminal
+    // capture it so it doesn't
+    let stderr = child.stderr.take().unwrap();
+    let stderr_thread = thread::spawn(move || io::read_to_string(stderr));
+
     let reader = BufReader::new(child.stdout.take().unwrap());
     let mut ok = true;
 
@@ -80,7 +87,20 @@ fn run_one(tx: &Emitter<Event>, path: &Path, harness_args: &[String]) -> bool {
         }
     }
 
-    let _ = child.wait();
+    let status = child.wait();
+    let stderr_text = stderr_thread
+        .join()
+        .ok()
+        .and_then(Result::ok)
+        .unwrap_or_default();
+
+    if ok && !status.is_ok_and(|s| s.success()) {
+        ok = false;
+        tx.send(Event::Error(format!(
+            "{} exited without reporting a failing test\n{stderr_text}",
+            path.display()
+        )));
+    }
     ok
 }
 
@@ -125,6 +145,7 @@ fn retry_one(path: &Path, name: &str) -> Option<(f32, Outcome)> {
         ])
         .env("RUSTC_BOOTSTRAP", "1")
         .stdout(Stdio::piped())
+        .stderr(Stdio::null())
         .spawn()
         .expect("failed to spawn test binary");