From 979c9746cfe2db3bb51978b99957453db80c1b60 Mon Sep 17 00:00:00 2001 From: DavidKoleczek <45405824+DavidKoleczek@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:14:04 -0400 Subject: [PATCH] fix(eval): sanitize jobbench DTU names for Incus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Incus instance names accept only alphanumerics and hyphens. JobBench occupation slugs come from the dataset directory names and are underscore-separated (`civil_engineers`, `financial_analysts`), so `_dtu_name` was emitting names Incus rejects outright -- the container never launches and the trial fails before the agent runs. Most occupations in the dataset contain an underscore, so this affects the majority of the suite. Transliterate any non-alphanumeric character to a hyphen for both the occupation slug and the agent name before assembling `jb---t-`. The agent name is sanitized too because it is caller-supplied and can carry dots or slashes. Scope: eval harness only. No engine, CLI, HTTP, protocol, or wrapper surface is touched. 🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> --- .../evaluation/jobbench/src/jobbench/trial.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.amplifier/evaluation/jobbench/src/jobbench/trial.py b/.amplifier/evaluation/jobbench/src/jobbench/trial.py index 61e691bf..f0bc0af5 100644 --- a/.amplifier/evaluation/jobbench/src/jobbench/trial.py +++ b/.amplifier/evaluation/jobbench/src/jobbench/trial.py @@ -250,16 +250,27 @@ def _flatten_pulled_deliverables(deliverables_dir: Path) -> None: nested.rmdir() +def _incus_safe(value: str) -> str: + """Incus instance names accept only alphanumerics and hyphens. + + Occupation slugs are underscore-separated (`civil_engineers`), so they + must be transliterated before they can appear in a container name -- + Incus rejects the launch outright otherwise. + """ + safe = "".join(ch if ch.isalnum() else "-" for ch in value) + return safe.strip("-") + + def _dtu_name(agent_name: str, task: Task) -> str: """`jb---t-`, kept to Incus's naming budget and unique per (agent, task) so concurrent trials never collide on the container name. """ uuid6 = uuid.uuid4().hex[:6] - occupation = task.occupation[:12] + occupation = _incus_safe(task.occupation[:12]) tail = f"-t{task.task_num}-{uuid6}" fixed = len("jb-") + len("-") + len(occupation) + len(tail) - agent_short = agent_name[: max(60 - fixed, 1)] + agent_short = _incus_safe(agent_name[: max(60 - fixed, 1)]) return f"jb-{agent_short}-{occupation}{tail}"[:60]