From 75f279cfd943dfbb050ab2d8f3829f13687d471e Mon Sep 17 00:00:00 2001 From: Basit Mustafa Date: Tue, 1 Sep 2026 16:38:22 -0600 Subject: [PATCH 1/8] fix: keep private config off shellm argv Co-authored-by: Basit Mustafa --- bin/shellm | 7 ++--- tests/test_thinker_env_fallback.sh | 42 ++++++++++++++++++++++++++++++ tests/test_var_secrets.sh | 16 +++++++----- thinkers/_lib/common.sh | 18 +++++++++++-- 4 files changed, 72 insertions(+), 11 deletions(-) diff --git a/bin/shellm b/bin/shellm index 90f6c9ba..2224bc0a 100755 --- a/bin/shellm +++ b/bin/shellm @@ -2980,8 +2980,9 @@ exit \$__shellm_rc # The command line as recorded in the shellm-run trajectory row. Trajectories # get rendered into prompts, exported and shared, so `--var NAME=VALUE` values -# whose NAME looks like a credential are masked. (Passing secrets as a bare -# `--var NAME` keeps them off the command line altogether.) +# whose NAME looks like a credential or endpoint are masked. (Passing private +# configuration as a bare `--var NAME` keeps it off the process command line +# altogether.) _redacted_cmdline() { local out="shellm" prev="" a n nu for a in "$@"; do @@ -2992,7 +2993,7 @@ _redacted_cmdline() { # substitution" that kills the whole run (every shellm run exited # rc=1 with no output on macOS). nu=$(printf '%s' "$n" | tr '[:lower:]' '[:upper:]') - if [[ "$nu" =~ (KEY|TOKEN|SECRET|PASSW|CREDENTIAL) ]]; then + if [[ "$nu" =~ (KEY|TOKEN|SECRET|PASSW|CREDENTIAL|URL|URI|ENDPOINT) ]]; then a="$n=" fi fi diff --git a/tests/test_thinker_env_fallback.sh b/tests/test_thinker_env_fallback.sh index 3705e8e4..fe8f27a9 100755 --- a/tests/test_thinker_env_fallback.sh +++ b/tests/test_thinker_env_fallback.sh @@ -125,6 +125,48 @@ case "$out" in *) bad "the key is forwarded to the nested shellm" "no bare --var for it" ;; esac +# Endpoints and skill-declared environment values are also private +# configuration. They must be forwarded by bare NAME, never embedded in the +# shellm command line. This also proves values loaded from .env were exported +# before the bare-name handoff. +out=$( + H=$(mktemp -d); trap 'rm -rf "$H"' EXIT; export HOME="$H" + unset HEADLONG_HOME SHELLM_HOME + mkdir -p "$H/id/memories" "$H/id/skills/probe" "$H/id/kernel" "$H/id/trajectories" "$H/wd" + printf 'name=probe\n' > "$H/id/info.txt" + cat > "$H/id/skills/probe/SKILL.md" <<'EOF' +--- +name: probe +description: Test fixture. +metadata: + shelllm: + requires: + env: ["PROBE_SERVICE_CONFIG"] +--- +EOF + export IDENTITY_DIR="$H/id" TRAJ_DIR="$H/id/trajectories" TRAJ_ID=t1 MEM_DIR="$H/id/memories" + export SHELLM_API_URL="https://example.invalid/v1/responses" + export LLM_API_URL="https://example.invalid/v1/responses" + export PROBE_SERVICE_CONFIG="private-config-canary" + cd "$H/wd" || exit 1 + # shellcheck disable=SC1090 # the library under test + source "$REPO/thinkers/_lib/common.sh" + _require_env >/dev/null 2>&1 + _build_shellm_flags "$IDENTITY_DIR" 2>/dev/null | tr '\n' ' ' +) +case "$out" in + *"example.invalid"*|*"private-config-canary"*) bad "private config values stay out of thinker shellm flags" ;; + *) + if [[ "$out" == *"--var SHELLM_API_URL "* \ + && "$out" == *"--var LLM_API_URL "* \ + && "$out" == *"--var PROBE_SERVICE_CONFIG "* ]]; then + ok "endpoints and skill vars are forwarded by bare name" + else + bad "endpoints and skill vars are forwarded by bare name" "missing bare forwarding names" + fi + ;; +esac + echo echo "$pass passed, $fail failed" [[ $fail -eq 0 ]] diff --git a/tests/test_var_secrets.sh b/tests/test_var_secrets.sh index c9dc7541..78c0a8c6 100755 --- a/tests/test_var_secrets.sh +++ b/tests/test_var_secrets.sh @@ -9,8 +9,8 @@ # 2. While the generated code runs, the secret's value appears in no # process's argv (`ps`): not shellm's, not `env`'s, not bash's. # 3. The shellm-run trajectory row records the command with legacy -# `--var SOME_KEY=value` values masked, and the literal value is -# nowhere under the state home. +# credential and endpoint `--var NAME=value` values masked, and the +# literal values are nowhere under the state home. # # `llm` is stubbed (canned fenced blocks, no network), same pattern as # tests/test_inactivity_beacon.sh. Local execution is pinned: shellm would @@ -64,6 +64,7 @@ run_shellm() { # (this script's own text included) unless something actually leaks them. SECRET="sk-test-secret-$$-$RANDOM$RANDOM" LEGACY="sk-legacy-literal-$$-$RANDOM$RANDOM" +PRIVATE_URL="https://example.invalid/private/$$?opaque=$RANDOM" # --- 1. bare --var with the variable unset is an error ------------------------ unset SECRET_PROBE @@ -79,7 +80,8 @@ export SECRET_PROBE="$SECRET" fence "printf 'got=%s plain=%s\\n' \"\$SECRET_PROBE\" \"\$PLAIN\" > '$WORK/probe.txt' ps -axo args= > '$WORK/ps.txt' 2>/dev/null || ps -eo args= > '$WORK/ps.txt'" > "$WORK/script/1" fence 'FINAL=done' > "$WORK/script/last" -run_shellm --var SECRET_PROBE --var PLAIN=1 --var "OPENROUTER_API_KEY=$LEGACY" "task" +run_shellm --var SECRET_PROBE --var PLAIN=1 --var "OPENROUTER_API_KEY=$LEGACY" \ + --var "SERVICE_ENDPOINT=$PRIVATE_URL" "task" if grep -qx "got=$SECRET plain=1" "$WORK/probe.txt" 2>/dev/null; then ok "bare --var NAME forwards the value into the generated code" else @@ -109,10 +111,12 @@ fi # --- 3. recorded command is redacted; literal value nowhere in state ---------- row=$(grep -rh '"type":"shellm-run"' "$HEADLONG_HOME" 2>/dev/null | tail -1) -if [[ -n "$row" ]] && grep -qF 'OPENROUTER_API_KEY=' <<<"$row" && ! grep -qF "$LEGACY" <<<"$row"; then - ok "shellm-run row masks credential-looking --var values" +if [[ -n "$row" ]] && grep -qF 'OPENROUTER_API_KEY=' <<<"$row" \ + && grep -qF 'SERVICE_ENDPOINT=' <<<"$row" \ + && ! grep -qF "$LEGACY" <<<"$row" && ! grep -qF "$PRIVATE_URL" <<<"$row"; then + ok "shellm-run row masks credential and endpoint --var values" else - bad "shellm-run row masks credential-looking --var values" "$(printf '%s' "$row" | cut -c1-200)" + bad "shellm-run row masks credential and endpoint --var values" "redacted fields missing" fi if grep -qF -- '--var SECRET_PROBE --var PLAIN=1' <<<"$row"; then ok "shellm-run row keeps the bare name and non-secret vars readable" diff --git a/thinkers/_lib/common.sh b/thinkers/_lib/common.sh index a9f0f844..71889916 100755 --- a/thinkers/_lib/common.sh +++ b/thinkers/_lib/common.sh @@ -423,8 +423,16 @@ _build_shellm_flags() { [[ -n "${SHELLM_MODEL:-}" ]] && printf '%s\n' "--var" "SHELLM_MODEL=$SHELLM_MODEL" # The generic openai-compatible provider is env-configured and never # auto-detected, so nested calls need the provider name (routing, not a - # secret) and its key (bare name, like the vendor keys below). + # secret), endpoint, and key. Endpoints go by bare NAME just like keys: + # private hosts and URL credentials must not appear in ps or trajectory + # command rows. [[ -n "${LLM_PROVIDER:-}" ]] && printf '%s\n' "--var" "LLM_PROVIDER=$LLM_PROVIDER" + for _cfg in SHELLM_API_URL LLM_API_URL; do + if [[ -n "${!_cfg:-}" ]]; then + export "${_cfg?}" + printf '%s\n' "--var" "$_cfg" + fi + done for _ak in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY OPENROUTER_API_KEY \ OPENCODE_API_KEY LLM_API_KEY; do if [[ -n "${!_ak:-}" ]]; then @@ -437,7 +445,13 @@ _build_shellm_flags() { while IFS= read -r vname; do [[ -z "$vname" ]] && continue local vval="${!vname:-}" - [[ -n "$vval" ]] && printf '%s\n' "--var" "$vname=$vval" + if [[ -n "$vval" ]]; then + # Skills commonly declare credentials and service endpoints. Keep + # every declared value off argv rather than trying to infer which + # names are sensitive. + export "${vname?}" + printf '%s\n' "--var" "$vname" + fi done < <(collect_skill_vars "$identity_dir") # Standard binaries. Keep this in sync with the tools promised to the From e90f4fd10546f130bb98e9891800c9a87c29debc Mon Sep 17 00:00:00 2001 From: Amp Date: Wed, 2 Sep 2026 00:15:42 +0000 Subject: [PATCH 2/8] fix: harden sensitive config forwarding Co-authored-by: Basit Mustafa --- bin/shellm | 17 ++++---- tests/test_persona_bugreport.sh | 17 +++++++- tests/test_thinker_env_fallback.sh | 27 ++++++++----- tests/test_var_secrets.sh | 39 +++++++++++++----- thinkers/_lib/common.sh | 24 ++++++----- thinkers/monolith/step | 1 + tools/persona | 64 +++++++++++++++++++++--------- 7 files changed, 128 insertions(+), 61 deletions(-) diff --git a/bin/shellm b/bin/shellm index 2224bc0a..5cbcbe1b 100755 --- a/bin/shellm +++ b/bin/shellm @@ -70,7 +70,7 @@ SHELLM_MAX_ITERATIONS="${SHELLM_MAX_ITERATIONS:-}" SHELLM_MAX_CONSECUTIVE_FAILURES="${SHELLM_MAX_CONSECUTIVE_FAILURES:-10}" SHELLM_MAX_REPEAT_FAILURES="${SHELLM_MAX_REPEAT_FAILURES:-3}" SHELLM_TRUNCATE="${SHELLM_TRUNCATE:-2000}" -SHELLM_API_URL="${SHELLM_API_URL:-}" +SHELLM_API_URL="${SHELLM_API_URL:-${LLM_API_URL:-}}" # Clear any inherited LLM_API_URL so llm uses per-provider defaults unset LLM_API_URL 2>/dev/null || true if [[ -n "$SHELLM_API_URL" ]]; then @@ -2983,17 +2983,18 @@ exit \$__shellm_rc # whose NAME looks like a credential or endpoint are masked. (Passing private # configuration as a bare `--var NAME` keeps it off the process command line # altogether.) +_shellm_private_var_name() { + local u + u=$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]') + [[ "$u" =~ (^|_)(KEY|TOKEN|SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?|URL|URI|ENDPOINT|DSN)(_|$) ]] +} + _redacted_cmdline() { - local out="shellm" prev="" a n nu + local out="shellm" prev="" a n for a in "$@"; do if [[ "$prev" == "--var" && "$a" == *=* ]]; then n="${a%%=*}" - # Uppercase via tr, not ${n^^}: the latter is a bash-4 expansion - # and macOS's /bin/bash is 3.2, where it is a fatal "bad - # substitution" that kills the whole run (every shellm run exited - # rc=1 with no output on macOS). - nu=$(printf '%s' "$n" | tr '[:lower:]' '[:upper:]') - if [[ "$nu" =~ (KEY|TOKEN|SECRET|PASSW|CREDENTIAL|URL|URI|ENDPOINT) ]]; then + if _shellm_private_var_name "$n"; then a="$n=" fi fi diff --git a/tests/test_persona_bugreport.sh b/tests/test_persona_bugreport.sh index dbc3caef..7f0acd2a 100755 --- a/tests/test_persona_bugreport.sh +++ b/tests/test_persona_bugreport.sh @@ -40,7 +40,8 @@ check_not() { local label="$1"; shift; if "$@" >/dev/null 2>&1; then bad "$label export HOME="$WORK/home" export HEADLONG_HOME="$WORK/home/.headlong" export HEADLONG_APP_DIR="$WORK/app" -mkdir -p "$HOME" "$HEADLONG_HOME/logs" "$HEADLONG_APP_DIR" +export TMPDIR="$WORK/tmp" +mkdir -p "$HOME" "$HEADLONG_HOME/logs" "$HEADLONG_APP_DIR" "$TMPDIR" ln -s "$REPO/bin" "$HEADLONG_APP_DIR/bin" ln -s "$REPO/tools" "$HEADLONG_APP_DIR/tools" ln -s "$REPO/thinkers" "$HEADLONG_APP_DIR/thinkers" @@ -49,9 +50,15 @@ export PATH="$REPO/bin:$REPO/tools:$PATH" KEY="sk-or-v1-0123456789abcdef0123456789abcdef0123456789abcdef" PW="hunter2hunter2-very-secret" +DSN="postgresql://bugreport-user:bugreport-pass@example.invalid/private-db" +# This produces a malformed one-line sed program. Its isolated literal pass +# may fail, but it must not prevent the other literals or fixed patterns from +# being scrubbed. +export BROKEN_SECRET=$'first-line\nsecond-line' cat > "$HEADLONG_HOME/.env" < "$HEADLONG_HOME/logs/web.log" @@ -70,11 +77,12 @@ cat >> "$TJ/trajectory.jsonl" < "$ID/run/logs/monolith.log" -printf -- '---\ntitle: db\n---\nthe db password is %s\n' "$PW" > "$ID/memories/db.md" +printf -- '---\ntitle: db\n---\nthe db password is %s\nthe dsn is %s\n' "$PW" "$DSN" > "$ID/memories/db.md" printf 'scratch file\n' > "$ID/workdir/scratch.txt" head -c 2048 /dev/urandom > "$TJ/blobs/bin.dat" @@ -115,17 +123,21 @@ check "binary blob carried intact" cmp -s "$TJ/blobs/bin.dat" "$TOP/ident # 2. scrubbing check_not "API key value nowhere in bundle" grep -rqF "$KEY" "$TOP" check_not "DB password nowhere in bundle" grep -rqF "$PW" "$TOP" +check_not "DSN literal nowhere in bundle" grep -rqF "$DSN" "$TOP" check_not "no sk-... shaped string survives" grep -rqE 'sk-[A-Za-z0-9_-]{8,}' "$TOP" check_not "other token value nowhere in bundle" grep -rqF 'ghp_OTHERTOKEN0123456789abcdefghijkl' "$TOP" check_not "other password nowhere in bundle" grep -rqF 'correcthorsebatterystaple' "$TOP" check "legacy --var KEY=value row masked, 4+4 hint kept" grep -q -- '--var OPENROUTER_API_KEY= think' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "token not in .env still masked by pattern (hint kept)" grep -q -- '--var GH_TOKEN= ' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "password on argv masked whole" grep -q -- '--var PG_PASSWORD= --var SHELLM_ENV=local' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" +check "DSN on argv masked whole" grep -q -- '--var DATABASE_DSN= --var CURL_OPTS=--retry=2' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" +check "CURL_OPTS is not a URL false positive" grep -q -- '--var CURL_OPTS=--retry=2 run' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check_not "no dangling hint tails" grep -q -- ' [^ ]*>' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "bare --var KEY row untouched" grep -q -- '--var OPENROUTER_API_KEY think' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "non-secret --var value kept" grep -q -- '--var SHELLM_MODEL=anthropic/claude-sonnet-4.5' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "thinker log key scrubbed (hint kept)" grep -q 'OPENROUTER_API_KEY=$' "$TOP/identity/run/logs/monolith.log" check "memory password scrubbed" grep -q 'password is ' "$TOP/identity/memories/db.md" +check "memory DSN literal scrubbed" grep -q 'dsn is ' "$TOP/identity/memories/db.md" check "plain thought text kept" grep -q 'all is well' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "source trajectory untouched" grep -qF "$KEY" "$TJ/trajectory.jsonl" @@ -139,6 +151,7 @@ check "--help exits 0 and mentions Usage" bash -c 'persona alpha bugreport --hel check_not "unknown option is an error" persona alpha bugreport --bogus check "appears in persona --help" bash -c 'persona alpha --help | grep -q bugreport' check_not "no stray staging dirs left" bash -c 'ls -d "${TMPDIR:-/tmp}"/headlong-bugreport.* 2>/dev/null | grep -q .' +check_not "private sed scripts are removed" bash -c 'ls -d "${TMPDIR:-/tmp}"/headlong-redact.* 2>/dev/null | grep -q .' printf '\n%d passed, %d failed\n' "$pass" "$fail" [[ "$fail" -eq 0 ]] diff --git a/tests/test_thinker_env_fallback.sh b/tests/test_thinker_env_fallback.sh index fe8f27a9..9b438c22 100755 --- a/tests/test_thinker_env_fallback.sh +++ b/tests/test_thinker_env_fallback.sh @@ -125,10 +125,10 @@ case "$out" in *) bad "the key is forwarded to the nested shellm" "no bare --var for it" ;; esac -# Endpoints and skill-declared environment values are also private -# configuration. They must be forwarded by bare NAME, never embedded in the -# shellm command line. This also proves values loaded from .env were exported -# before the bare-name handoff. +# Endpoints are inherited rather than repeated as --var, because shellm owns +# their Docker rewrite. Skill-declared values still use bare --var NAME, and an +# existing-but-unexported skill value must be exported in the parent before +# flag assembly runs in process substitution. out=$( H=$(mktemp -d); trap 'rm -rf "$H"' EXIT; export HOME="$H" unset HEADLONG_HOME SHELLM_HOME @@ -145,24 +145,29 @@ metadata: --- EOF export IDENTITY_DIR="$H/id" TRAJ_DIR="$H/id/trajectories" TRAJ_ID=t1 MEM_DIR="$H/id/memories" - export SHELLM_API_URL="https://example.invalid/v1/responses" export LLM_API_URL="https://example.invalid/v1/responses" - export PROBE_SERVICE_CONFIG="private-config-canary" + unset SHELLM_API_URL + PROBE_SERVICE_CONFIG="private-config-canary" cd "$H/wd" || exit 1 # shellcheck disable=SC1090 # the library under test source "$REPO/thinkers/_lib/common.sh" _require_env >/dev/null 2>&1 + _export_skill_vars "$IDENTITY_DIR" _build_shellm_flags "$IDENTITY_DIR" 2>/dev/null | tr '\n' ' ' + if bash -c '[[ -n "${LLM_API_URL:-}" && -n "${PROBE_SERVICE_CONFIG:-}" ]]'; then + printf 'parent-export-ok ' + fi ) case "$out" in *"example.invalid"*|*"private-config-canary"*) bad "private config values stay out of thinker shellm flags" ;; *) - if [[ "$out" == *"--var SHELLM_API_URL "* \ - && "$out" == *"--var LLM_API_URL "* \ - && "$out" == *"--var PROBE_SERVICE_CONFIG "* ]]; then - ok "endpoints and skill vars are forwarded by bare name" + if [[ "$out" != *"--var SHELLM_API_URL "* \ + && "$out" != *"--var LLM_API_URL "* \ + && "$out" == *"--var PROBE_SERVICE_CONFIG "* \ + && "$out" == *"parent-export-ok "* ]]; then + ok "endpoint is inherited and an unexported skill var is parent-exported" else - bad "endpoints and skill vars are forwarded by bare name" "missing bare forwarding names" + bad "endpoint is inherited and an unexported skill var is parent-exported" "unexpected flags or missing parent export" fi ;; esac diff --git a/tests/test_var_secrets.sh b/tests/test_var_secrets.sh index 78c0a8c6..7519e546 100755 --- a/tests/test_var_secrets.sh +++ b/tests/test_var_secrets.sh @@ -65,6 +65,9 @@ run_shellm() { SECRET="sk-test-secret-$$-$RANDOM$RANDOM" LEGACY="sk-legacy-literal-$$-$RANDOM$RANDOM" PRIVATE_URL="https://example.invalid/private/$$?opaque=$RANDOM" +PRIVATE_DSN="postgresql://user:password@example.invalid/private_$$" +INHERITED_URL="http://127.0.0.1:9/private/$RANDOM" +CURL_VALUE="--retry=2" # --- 1. bare --var with the variable unset is an error ------------------------ unset SECRET_PROBE @@ -77,16 +80,24 @@ fi # --- 2. forwarded value reaches the code; nothing carries it in argv ---------- export SECRET_PROBE="$SECRET" -fence "printf 'got=%s plain=%s\\n' \"\$SECRET_PROBE\" \"\$PLAIN\" > '$WORK/probe.txt' +unset SHELLM_API_URL +export LLM_API_URL="$INHERITED_URL" +fence "printf 'got=%s plain=%s api=%s shellm=%s\\n' \"\$SECRET_PROBE\" \"\$PLAIN\" \"\$LLM_API_URL\" \"\$SHELLM_API_URL\" > '$WORK/probe.txt' ps -axo args= > '$WORK/ps.txt' 2>/dev/null || ps -eo args= > '$WORK/ps.txt'" > "$WORK/script/1" fence 'FINAL=done' > "$WORK/script/last" run_shellm --var SECRET_PROBE --var PLAIN=1 --var "OPENROUTER_API_KEY=$LEGACY" \ - --var "SERVICE_ENDPOINT=$PRIVATE_URL" "task" -if grep -qx "got=$SECRET plain=1" "$WORK/probe.txt" 2>/dev/null; then + --var "SERVICE_URL=$PRIVATE_URL" --var "DATABASE_DSN=$PRIVATE_DSN" \ + --var "CURL_OPTS=$CURL_VALUE" "task" +if grep -qx "got=$SECRET plain=1 api=$INHERITED_URL shellm=$INHERITED_URL" "$WORK/probe.txt" 2>/dev/null; then ok "bare --var NAME forwards the value into the generated code" else bad "bare --var NAME forwards the value into the generated code" "probe: $(cat "$WORK/probe.txt" 2>/dev/null) err: $(tail -2 "$WORK/err")" fi +if [[ -z "${SHELLM_API_URL+x}" ]]; then + ok "the shellm run started with LLM_API_URL only" +else + bad "the shellm run started with LLM_API_URL only" +fi if [[ -s "$WORK/ps.txt" ]]; then if ! grep -qF "$SECRET" "$WORK/ps.txt"; then ok "forwarded secret is in no process's argv while the code runs" @@ -112,16 +123,24 @@ fi # --- 3. recorded command is redacted; literal value nowhere in state ---------- row=$(grep -rh '"type":"shellm-run"' "$HEADLONG_HOME" 2>/dev/null | tail -1) if [[ -n "$row" ]] && grep -qF 'OPENROUTER_API_KEY=' <<<"$row" \ - && grep -qF 'SERVICE_ENDPOINT=' <<<"$row" \ - && ! grep -qF "$LEGACY" <<<"$row" && ! grep -qF "$PRIVATE_URL" <<<"$row"; then - ok "shellm-run row masks credential and endpoint --var values" + && grep -qF 'SERVICE_URL=' <<<"$row" \ + && grep -qF 'DATABASE_DSN=' <<<"$row" \ + && ! grep -qF "$LEGACY" <<<"$row" && ! grep -qF "$PRIVATE_URL" <<<"$row" \ + && ! grep -qF "$PRIVATE_DSN" <<<"$row"; then + ok "shellm-run row masks credential, URL, and DSN --var values" +else + bad "shellm-run row masks credential, URL, and DSN --var values" "redacted fields missing" +fi +if grep -qF -- '--var SECRET_PROBE --var PLAIN=1' <<<"$row" \ + && grep -qF -- "--var CURL_OPTS=$CURL_VALUE" <<<"$row"; then + ok "shellm-run row keeps bare names and CURL_OPTS readable" else - bad "shellm-run row masks credential and endpoint --var values" "redacted fields missing" + bad "shellm-run row keeps bare names and CURL_OPTS readable" "$(printf '%s' "$row" | cut -c1-200)" fi -if grep -qF -- '--var SECRET_PROBE --var PLAIN=1' <<<"$row"; then - ok "shellm-run row keeps the bare name and non-secret vars readable" +if ! grep -qF "$INHERITED_URL" <<<"$row"; then + ok "inherited endpoint is absent from the shellm command record" else - bad "shellm-run row keeps the bare name and non-secret vars readable" "$(printf '%s' "$row" | cut -c1-200)" + bad "inherited endpoint is absent from the shellm command record" fi hits=$(grep -rlF "$LEGACY" "$HEADLONG_HOME" "$WORK/wd" 2>/dev/null | wc -l | tr -d ' ') if [[ "$hits" -eq 0 ]]; then diff --git a/thinkers/_lib/common.sh b/thinkers/_lib/common.sh index 71889916..04930125 100755 --- a/thinkers/_lib/common.sh +++ b/thinkers/_lib/common.sh @@ -377,6 +377,16 @@ collect_skill_vars() { fi } +# Bare `--var NAME` is resolved from shellm's inherited environment. Export +# skill values in the caller before flag assembly runs in process substitution, +# whose subshell cannot export anything back to its parent. +_export_skill_vars() { + local identity_dir="$1" vname + while IFS= read -r vname; do + [[ -n "$vname" && -n "${!vname:-}" ]] && export "$vname" + done < <(collect_skill_vars "$identity_dir") +} + # --------------------------------------------------------------------------- # Path resolution # --------------------------------------------------------------------------- @@ -423,20 +433,13 @@ _build_shellm_flags() { [[ -n "${SHELLM_MODEL:-}" ]] && printf '%s\n' "--var" "SHELLM_MODEL=$SHELLM_MODEL" # The generic openai-compatible provider is env-configured and never # auto-detected, so nested calls need the provider name (routing, not a - # secret), endpoint, and key. Endpoints go by bare NAME just like keys: - # private hosts and URL credentials must not appear in ps or trajectory - # command rows. + # secret) and key. Endpoint variables are inherited rather than repeated + # as --var: shellm rewrites their values for Docker, and a duplicate extra + # var would overwrite that rewritten value in generated code. [[ -n "${LLM_PROVIDER:-}" ]] && printf '%s\n' "--var" "LLM_PROVIDER=$LLM_PROVIDER" - for _cfg in SHELLM_API_URL LLM_API_URL; do - if [[ -n "${!_cfg:-}" ]]; then - export "${_cfg?}" - printf '%s\n' "--var" "$_cfg" - fi - done for _ak in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY OPENROUTER_API_KEY \ OPENCODE_API_KEY LLM_API_KEY; do if [[ -n "${!_ak:-}" ]]; then - export "${_ak?}" printf '%s\n' "--var" "$_ak" fi done @@ -449,7 +452,6 @@ _build_shellm_flags() { # Skills commonly declare credentials and service endpoints. Keep # every declared value off argv rather than trying to infer which # names are sensitive. - export "${vname?}" printf '%s\n' "--var" "$vname" fi done < <(collect_skill_vars "$identity_dir") diff --git a/thinkers/monolith/step b/thinkers/monolith/step index 524ce43c..08b80be3 100755 --- a/thinkers/monolith/step +++ b/thinkers/monolith/step @@ -247,6 +247,7 @@ goals=$(get_goals "$MEM_DIR") think_prompt=$(load_prompt "$prompt_file" "$IDENTITY_NAME" "$goals") # Build shellm flags (includes traj/mem/chat/etc. in --bin). +_export_skill_vars "$IDENTITY_DIR" shellm_flags=() while IFS= read -r _flag; do [[ -n "$_flag" ]] && shellm_flags+=("$_flag") diff --git a/tools/persona b/tools/persona index d2b5af90..0450adf4 100755 --- a/tools/persona +++ b/tools/persona @@ -563,8 +563,8 @@ _bugreport_text() { # anything under 20 characters are masked whole. _redact_tree() { local root="$1" - local -a pairs=() files=() # pairs: "valuereplacement" - # Literal values: credential-looking names from the env files and the + local -a values=() replacements=() files=() + # Literal values: private-looking names from the env files and the # current environment (the env files were loaded into it at startup). local n v e for e in "$SHELLM_HOME/.env" "$APP_DIR/.env"; do @@ -574,13 +574,19 @@ _redact_tree() { _is_secret_name "$n" || continue # shellcheck disable=SC1090 v=$(set -a; . "$e" 2>/dev/null; printf '%s' "${!n}") || continue - [[ "${#v}" -ge 8 ]] && pairs+=("$v $(_redact_hint "$n" "$v")") + if [[ "${#v}" -ge 8 ]]; then + values+=("$v") + replacements+=("$(_redact_hint "$n" "$v")") + fi done < <(sed -n 's/^[[:space:]]*\(export[[:space:]]\{1,\}\)\{0,1\}\([A-Za-z_][A-Za-z0-9_]*\)[[:space:]]*=.*/\2/p' "$e") done while IFS= read -r n; do _is_secret_name "$n" || continue v="${!n:-}" - [[ "${#v}" -ge 8 ]] && pairs+=("$v $(_redact_hint "$n" "$v")") + if [[ "${#v}" -ge 8 ]]; then + values+=("$v") + replacements+=("$(_redact_hint "$n" "$v")") + fi done < <(compgen -e) # Text files only (tar'd dirs can carry binaries in blobs/). while IFS= read -r -d '' f; do files+=("$f"); done \ @@ -588,29 +594,47 @@ _redact_tree() { -o -name '*.txt' -o -name '*.md' -o -name '*.sh' -o -name '*.yaml' -o -name '*.yml' \ -o -name '*.toml' -o -name '*.env' -o -name '*.cfg' -o -name '*.ini' \) -print0 2>/dev/null) [[ "${#files[@]}" -gt 0 ]] || return 0 - # One sed script for everything, ERE (`sed -E` is in both BSD and GNU sed; - # BSD sed has no usable BRE alternation). Write via temp file because - # `sed -i` differs between the two. - local script="" pr - for pr in "${pairs[@]+"${pairs[@]}"}"; do - script+="s/$(_sed_escape "${pr%% *}")/$(_sed_escape "${pr#* }")/g;" + # Keep literals out of sed's argv and isolate each one in its own pass, so + # an unusual ambient value that sed cannot parse does not disable every + # other replacement. mktemp creates mode 0600; chmod makes that contract + # explicit on implementations with a surprising umask policy. + local sed_script f tmp i escaped_value escaped_replacement + sed_script=$(mktemp "${TMPDIR:-/tmp}/headlong-redact.XXXXXX") || return 1 + chmod 600 "$sed_script" || { rm -f "$sed_script"; return 1; } + for ((i=0; i<${#values[@]}; i++)); do + escaped_value=$(_sed_escape "${values[$i]}") + escaped_replacement=$(_sed_escape "${replacements[$i]}") + printf 's/%s/%s/g\n' "$escaped_value" "$escaped_replacement" > "$sed_script" + for f in "${files[@]}"; do + tmp="$f.redact.$$" + if sed -E -f "$sed_script" "$f" > "$tmp" 2>/dev/null; then + mv -f "$tmp" "$f" + else + rm -f "$tmp" + fi + done done + # --var NAME=VALUE: KEY/TOKEN names keep a 4+4 hint when long enough, - # other credential names are masked whole. + # other private names are masked whole. Token boundaries are underscores + # or the ends of the name, so CURL_OPTS is not treated as a URL variable. # (a value starting with "<" is one an earlier pass already replaced) - script+='s/--var ([A-Za-z_]*(KEY|TOKEN)[A-Za-z0-9_]*)=([^ "<\\][^ "\\]{3})[^ "\\]{12,}([^ "\\]{4})/--var \1=/g;' - script+='s/--var ([A-Za-z_]*(KEY|TOKEN|SECRET|PASSW|CREDENTIAL)[A-Za-z0-9_]*)=[^ "<\\][^ "\\]*/--var \1=/g;' - script+='s/sk-([A-Za-z0-9_-]{4})[A-Za-z0-9_-]{12,}([A-Za-z0-9_-]{4})//g;' - script+='s/sk-[A-Za-z0-9_-]{8,}//g;' - local f tmp + cat > "$sed_script" <<'SED' +s/--var (([A-Za-z_][A-Za-z0-9_]*_)?(SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?|URL|URI|ENDPOINT|DSN)(_[A-Za-z0-9_]+)*)=[^ "<\\][^ "\\]*/--var \1=/g +s/--var (([A-Za-z_][A-Za-z0-9_]*_)?(KEY|TOKEN)(_[A-Za-z0-9_]+)*)=([^ "<\\]{4})[^ "\\]{12,}([^ "\\]{4})/--var \1=/g +s/--var (([A-Za-z_][A-Za-z0-9_]*_)?(KEY|TOKEN)(_[A-Za-z0-9_]+)*)=[^ "<\\][^ "\\]*/--var \1=/g +s/sk-([A-Za-z0-9_-]{4})[A-Za-z0-9_-]{12,}([A-Za-z0-9_-]{4})//g +s/sk-[A-Za-z0-9_-]{8,}//g +SED for f in "${files[@]}"; do tmp="$f.redact.$$" - if sed -E "$script" "$f" > "$tmp" 2>/dev/null; then + if sed -E -f "$sed_script" "$f" > "$tmp" 2>/dev/null; then mv -f "$tmp" "$f" else rm -f "$tmp" fi done + rm -f "$sed_script" } # _redact_hint NAME VALUE — the replacement for VALUE: first/last 4 chars for @@ -622,7 +646,9 @@ _upper() { printf '%s' "$1" | tr '[:lower:]' '[:upper:]'; } _redact_hint() { local u v="$2" u=$(_upper "$1") - if [[ "$u" =~ (KEY|TOKEN) && ! "$u" =~ (SECRET|PASSW|CREDENTIAL) && "${#v}" -ge 20 ]]; then + if [[ "$u" =~ (^|_)(KEY|TOKEN)(_|$) \ + && ! "$u" =~ (^|_)(SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?|URL|URI|ENDPOINT|DSN)(_|$) \ + && "${#v}" -ge 20 ]]; then printf '' "${v:0:4}" "${v: -4}" else printf '' @@ -632,7 +658,7 @@ _redact_hint() { _is_secret_name() { # same notion as shellm's trajectory masking local u u=$(_upper "$1") - [[ "$u" =~ (KEY|TOKEN|SECRET|PASSW|CREDENTIAL) ]] + [[ "$u" =~ (^|_)(KEY|TOKEN|SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?|URL|URI|ENDPOINT|DSN)(_|$) ]] } _sed_escape() { # escape a literal for use in an ERE sed s/// pattern or replacement From 990239173161ece33ac974be27b9fd52a52f15ac Mon Sep 17 00:00:00 2001 From: Amp Date: Wed, 2 Sep 2026 00:25:57 +0000 Subject: [PATCH 3/8] fix: export thinker provider keys in parent Co-authored-by: Basit Mustafa --- tests/test_thinker_env_fallback.sh | 40 ++++++++++++++++++++++++++++++ thinkers/_lib/common.sh | 9 +++++++ thinkers/monolith/step | 1 + 3 files changed, 50 insertions(+) diff --git a/tests/test_thinker_env_fallback.sh b/tests/test_thinker_env_fallback.sh index 9b438c22..9b4774f7 100755 --- a/tests/test_thinker_env_fallback.sh +++ b/tests/test_thinker_env_fallback.sh @@ -125,6 +125,46 @@ case "$out" in *) bad "the key is forwarded to the nested shellm" "no bare --var for it" ;; esac +# A shell-local provider key must be exported before flags are assembled in a +# process substitution. Exercise a real nested shellm: without the parent +# export, its bare --var parser fails before generated code can see the key. +out=$( + H=$(mktemp -d); trap 'rm -rf "$H"' EXIT; export HOME="$H" + unset ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY OPENROUTER_API_KEY \ + OPENCODE_API_KEY LLM_API_KEY HEADLONG_HOME SHELLM_HOME + mkdir -p "$H/id/memories" "$H/id/skills" "$H/id/kernel" "$H/id/trajectories" "$H/wd" "$H/bin" + printf 'name=probe\n' > "$H/id/info.txt" + cat > "$H/bin/llm" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' '```bash' 'if [[ -n "${OPENROUTER_API_KEY:-}" ]]; then FINAL=provider-key-present; else FINAL=provider-key-missing; fi' '```' +EOF + chmod +x "$H/bin/llm" + export PATH="$H/bin:$REPO/bin:$REPO/tools:$PATH" + export IDENTITY_DIR="$H/id" TRAJ_DIR="$H/id/trajectories" TRAJ_ID=t1 MEM_DIR="$H/id/memories" + export SHELLM_MODEL=test-model SHELLM_THINKER_ENV=local + OPENROUTER_API_KEY=synthetic-provider-key + cd "$H/wd" || exit 1 + # shellcheck disable=SC1090 # the library under test + source "$REPO/thinkers/_lib/common.sh" + _require_env >/dev/null 2>&1 + _export_provider_keys + flags=() + while IFS= read -r flag; do + [[ -n "$flag" ]] && flags+=("$flag") + done < <(_build_shellm_flags "$IDENTITY_DIR" "$H/wd") + nested_out=$("$REPO/bin/shellm" "${flags[@]}" --max-iterations 1 key-probe 2>/dev/null) + printf '%s\n' "$nested_out" + row=$(grep -rh '"type":"shellm-run"' "$H/id/trajectories" 2>/dev/null | tail -1) + if [[ "$row" == *"--var OPENROUTER_API_KEY "* && "$row" != *"synthetic-provider-key"* ]]; then + printf 'provider-key-command-safe\n' + fi +) +if [[ "$out" == *provider-key-present* && "$out" == *provider-key-command-safe* ]]; then + ok "an unexported provider key reaches an actual nested shellm" +else + bad "an unexported provider key reaches an actual nested shellm" "got $out" +fi + # Endpoints are inherited rather than repeated as --var, because shellm owns # their Docker rewrite. Skill-declared values still use bare --var NAME, and an # existing-but-unexported skill value must be exported in the parent before diff --git a/thinkers/_lib/common.sh b/thinkers/_lib/common.sh index 04930125..de299a32 100755 --- a/thinkers/_lib/common.sh +++ b/thinkers/_lib/common.sh @@ -387,6 +387,15 @@ _export_skill_vars() { done < <(collect_skill_vars "$identity_dir") } +_export_provider_keys() { + local vname + for vname in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY OPENROUTER_API_KEY \ + OPENCODE_API_KEY LLM_API_KEY; do + [[ -n "${!vname:-}" ]] && export "$vname" + done + return 0 +} + # --------------------------------------------------------------------------- # Path resolution # --------------------------------------------------------------------------- diff --git a/thinkers/monolith/step b/thinkers/monolith/step index 08b80be3..3233698a 100755 --- a/thinkers/monolith/step +++ b/thinkers/monolith/step @@ -247,6 +247,7 @@ goals=$(get_goals "$MEM_DIR") think_prompt=$(load_prompt "$prompt_file" "$IDENTITY_NAME" "$goals") # Build shellm flags (includes traj/mem/chat/etc. in --bin). +_export_provider_keys _export_skill_vars "$IDENTITY_DIR" shellm_flags=() while IFS= read -r _flag; do From 5c11344d50aa5ae800e54b8ef6a26690b9b7c80f Mon Sep 17 00:00:00 2001 From: Amp Date: Wed, 2 Sep 2026 00:46:54 +0000 Subject: [PATCH 4/8] fix: keep bugreport sed programs anonymous Co-authored-by: Basit Mustafa --- tests/test_persona_bugreport.sh | 72 ++++++++++++++++++++++++++++++++- tools/persona | 45 ++++++++++++--------- 2 files changed, 95 insertions(+), 22 deletions(-) diff --git a/tests/test_persona_bugreport.sh b/tests/test_persona_bugreport.sh index 7f0acd2a..eeec80bb 100755 --- a/tests/test_persona_bugreport.sh +++ b/tests/test_persona_bugreport.sh @@ -141,7 +141,74 @@ check "memory DSN literal scrubbed" grep -q 'dsn is ' "$TOP/ check "plain thought text kept" grep -q 'all is well' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "source trajectory untouched" grep -qF "$KEY" "$TJ/trajectory.jsonl" -# 3. options +# 3. anonymous sed programs and output-temp cleanup +REAL_SED=$(command -v sed) +mkdir -p "$WORK/failbin" "$WORK/redact-state" +cat > "$WORK/failbin/sed" <> "\$REDACT_STATE/program-args" + if [[ -f "\$REDACT_STATE/previous-temp" ]]; then + old=\$(cat "\$REDACT_STATE/previous-temp") + [[ ! -e "\$old" ]] || touch "\$REDACT_STATE/leaked-temp" + fi + current=\$(find "\$TMPDIR" -name '.redact-output.*' -print -quit 2>/dev/null) + [[ -z "\$current" ]] || printf '%s\n' "\$current" > "\$REDACT_STATE/previous-temp" + if [[ "\${REDACT_SED_MODE:-}" == error && ! -f "\$REDACT_STATE/failed-once" ]]; then + touch "\$REDACT_STATE/failed-once" + exit 42 + fi + if [[ "\${REDACT_SED_MODE:-}" == signal && ! -f "\$REDACT_STATE/signalled" ]]; then + touch "\$REDACT_STATE/signalled" + kill -TERM "\$PPID" + sleep 1 + exit 143 + fi +fi +exec "\$real_sed" "\$@" +EOF +chmod +x "$WORK/failbin/sed" + +rm -rf "$WORK/redact-state"; mkdir "$WORK/redact-state" +if PATH="$WORK/failbin:$PATH" REDACT_STATE="$WORK/redact-state" REDACT_SED_MODE=error \ + persona alpha bugreport --out "$WORK/error-pass.tgz" >/dev/null 2>&1; then + ok "one sed error does not abort later redaction passes" +else + bad "one sed error does not abort later redaction passes" +fi +check_not "failed sed output temp is removed before the next pass" test -e "$WORK/redact-state/leaked-temp" +if [[ -s "$WORK/redact-state/program-args" ]] \ + && ! grep -qF "$KEY" "$WORK/redact-state/program-args" \ + && ! grep -qF "$PW" "$WORK/redact-state/program-args" \ + && ! grep -qF "$DSN" "$WORK/redact-state/program-args"; then + ok "sed program argv contains no private literals" +else + bad "sed program argv contains no private literals" +fi +check_not "implementation has no secret sed script path" grep -q 'headlong-redact' "$REPO/tools/persona" +rm -f "$WORK/error-pass.tgz" + +rm -rf "$WORK/redact-state"; mkdir "$WORK/redact-state" +if PATH="$WORK/failbin:$PATH" REDACT_STATE="$WORK/redact-state" REDACT_SED_MODE=signal \ + persona alpha bugreport --out "$WORK/signal-pass.tgz" >/dev/null 2>&1; then + bad "TERM interrupts redaction" +else + ok "TERM interrupts redaction" +fi +check_not "signal leaves no output redaction temp" bash -c \ + 'find "$1" -name ".redact-output.*" -print -quit | grep -q .' _ "$WORK" +check_not "signal leaves no secret-bearing sed script" bash -c \ + 'find "$1" -name "headlong-redact.*" -print -quit | grep -q .' _ "$WORK" + +# 4. options check "--include-workdir adds workdir" bash -c ' persona alpha bugreport --out "$1" --include-workdir >/dev/null 2>&1 && tar -tzf "$1" | grep -q "identity/workdir/scratch.txt"' _ "$WORK/b2.tgz" @@ -151,7 +218,8 @@ check "--help exits 0 and mentions Usage" bash -c 'persona alpha bugreport --hel check_not "unknown option is an error" persona alpha bugreport --bogus check "appears in persona --help" bash -c 'persona alpha --help | grep -q bugreport' check_not "no stray staging dirs left" bash -c 'ls -d "${TMPDIR:-/tmp}"/headlong-bugreport.* 2>/dev/null | grep -q .' -check_not "private sed scripts are removed" bash -c 'ls -d "${TMPDIR:-/tmp}"/headlong-redact.* 2>/dev/null | grep -q .' +check_not "no secret-bearing sed scripts exist" bash -c \ + 'find "$1" -name "headlong-redact.*" -print -quit | grep -q .' _ "$WORK" printf '\n%d passed, %d failed\n' "$pass" "$fail" [[ "$fail" -eq 0 ]] diff --git a/tools/persona b/tools/persona index 0450adf4..3671244a 100755 --- a/tools/persona +++ b/tools/persona @@ -561,9 +561,12 @@ _bugreport_text() { # API keys and tokens keep their first and last 4 characters # (``) so two keys can be told apart; passwords and # anything under 20 characters are masked whole. -_redact_tree() { +_redact_tree() ( local root="$1" local -a values=() replacements=() files=() + local tmp="" + trap '[[ -z "$tmp" ]] || rm -f "$tmp"' EXIT + trap 'exit 1' HUP INT TERM # Literal values: private-looking names from the env files and the # current environment (the env files were loaded into it at startup). local n v e @@ -596,21 +599,23 @@ _redact_tree() { [[ "${#files[@]}" -gt 0 ]] || return 0 # Keep literals out of sed's argv and isolate each one in its own pass, so # an unusual ambient value that sed cannot parse does not disable every - # other replacement. mktemp creates mode 0600; chmod makes that contract - # explicit on implementations with a surprising umask policy. - local sed_script f tmp i escaped_value escaped_replacement - sed_script=$(mktemp "${TMPDIR:-/tmp}/headlong-redact.XXXXXX") || return 1 - chmod 600 "$sed_script" || { rm -f "$sed_script"; return 1; } + # other replacement. Process substitution gives sed an anonymous program + # FD: no literal enters argv or a filesystem script. Output temps live only + # under the staging root and the subshell-local traps clean them without + # disturbing cmd_bugreport's traps. + local f i escaped_value escaped_replacement for ((i=0; i<${#values[@]}; i++)); do escaped_value=$(_sed_escape "${values[$i]}") escaped_replacement=$(_sed_escape "${replacements[$i]}") - printf 's/%s/%s/g\n' "$escaped_value" "$escaped_replacement" > "$sed_script" for f in "${files[@]}"; do - tmp="$f.redact.$$" - if sed -E -f "$sed_script" "$f" > "$tmp" 2>/dev/null; then + tmp=$(mktemp "$root/.redact-output.XXXXXX") || return 1 + if sed -E -f <(printf 's/%s/%s/g\n' "$escaped_value" "$escaped_replacement") \ + "$f" > "$tmp" 2>/dev/null; then mv -f "$tmp" "$f" + tmp="" else rm -f "$tmp" + tmp="" fi done done @@ -619,23 +624,23 @@ _redact_tree() { # other private names are masked whole. Token boundaries are underscores # or the ends of the name, so CURL_OPTS is not treated as a URL variable. # (a value starting with "<" is one an earlier pass already replaced) - cat > "$sed_script" <<'SED' -s/--var (([A-Za-z_][A-Za-z0-9_]*_)?(SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?|URL|URI|ENDPOINT|DSN)(_[A-Za-z0-9_]+)*)=[^ "<\\][^ "\\]*/--var \1=/g -s/--var (([A-Za-z_][A-Za-z0-9_]*_)?(KEY|TOKEN)(_[A-Za-z0-9_]+)*)=([^ "<\\]{4})[^ "\\]{12,}([^ "\\]{4})/--var \1=/g -s/--var (([A-Za-z_][A-Za-z0-9_]*_)?(KEY|TOKEN)(_[A-Za-z0-9_]+)*)=[^ "<\\][^ "\\]*/--var \1=/g -s/sk-([A-Za-z0-9_-]{4})[A-Za-z0-9_-]{12,}([A-Za-z0-9_-]{4})//g -s/sk-[A-Za-z0-9_-]{8,}//g -SED for f in "${files[@]}"; do - tmp="$f.redact.$$" - if sed -E -f "$sed_script" "$f" > "$tmp" 2>/dev/null; then + tmp=$(mktemp "$root/.redact-output.XXXXXX") || return 1 + if sed -E -f <(printf '%s\n' \ + 's/--var (([A-Za-z_][A-Za-z0-9_]*_)?(SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?|URL|URI|ENDPOINT|DSN)(_[A-Za-z0-9_]+)*)=[^ "<\\][^ "\\]*/--var \1=/g' \ + 's/--var (([A-Za-z_][A-Za-z0-9_]*_)?(KEY|TOKEN)(_[A-Za-z0-9_]+)*)=([^ "<\\]{4})[^ "\\]{12,}([^ "\\]{4})/--var \1=/g' \ + 's/--var (([A-Za-z_][A-Za-z0-9_]*_)?(KEY|TOKEN)(_[A-Za-z0-9_]+)*)=[^ "<\\][^ "\\]*/--var \1=/g' \ + 's/sk-([A-Za-z0-9_-]{4})[A-Za-z0-9_-]{12,}([A-Za-z0-9_-]{4})//g' \ + 's/sk-[A-Za-z0-9_-]{8,}//g') \ + "$f" > "$tmp" 2>/dev/null; then mv -f "$tmp" "$f" + tmp="" else rm -f "$tmp" + tmp="" fi done - rm -f "$sed_script" -} +) # _redact_hint NAME VALUE — the replacement for VALUE: first/last 4 chars for # keys and tokens of 20+ chars, a plain otherwise. From 9308f8157b9a6a6641ab3f54b3254db6eaaa8eba Mon Sep 17 00:00:00 2001 From: Amp Date: Thu, 3 Sep 2026 04:34:41 +0000 Subject: [PATCH 5/8] fix: quiet shellcheck on indirect exports SC2163 wants ${var?} for export "$vname"; SC2034 flagged the intentionally unexported skill-var canary in the parent-export test. Amp-Thread-ID: https://ampcode.com/threads/T-01a06587-820b-734b-a505-b10d58712e10 Co-authored-by: Basit Mustafa --- tests/test_thinker_env_fallback.sh | 2 ++ thinkers/_lib/common.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_thinker_env_fallback.sh b/tests/test_thinker_env_fallback.sh index 9b4774f7..e00d170d 100755 --- a/tests/test_thinker_env_fallback.sh +++ b/tests/test_thinker_env_fallback.sh @@ -187,6 +187,8 @@ EOF export IDENTITY_DIR="$H/id" TRAJ_DIR="$H/id/trajectories" TRAJ_ID=t1 MEM_DIR="$H/id/memories" export LLM_API_URL="https://example.invalid/v1/responses" unset SHELLM_API_URL + # Intentionally unexported: _export_skill_vars must promote it for --var NAME. + # shellcheck disable=SC2034 PROBE_SERVICE_CONFIG="private-config-canary" cd "$H/wd" || exit 1 # shellcheck disable=SC1090 # the library under test diff --git a/thinkers/_lib/common.sh b/thinkers/_lib/common.sh index de299a32..f7c1f999 100755 --- a/thinkers/_lib/common.sh +++ b/thinkers/_lib/common.sh @@ -383,7 +383,7 @@ collect_skill_vars() { _export_skill_vars() { local identity_dir="$1" vname while IFS= read -r vname; do - [[ -n "$vname" && -n "${!vname:-}" ]] && export "$vname" + [[ -n "$vname" && -n "${!vname:-}" ]] && export "${vname?}" done < <(collect_skill_vars "$identity_dir") } @@ -391,7 +391,7 @@ _export_provider_keys() { local vname for vname in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY OPENROUTER_API_KEY \ OPENCODE_API_KEY LLM_API_KEY; do - [[ -n "${!vname:-}" ]] && export "$vname" + [[ -n "${!vname:-}" ]] && export "${vname?}" done return 0 } From c556614de45e9e8c044c0e4b23ab09765b900f98 Mon Sep 17 00:00:00 2001 From: Amp Date: Thu, 3 Sep 2026 14:28:50 +0000 Subject: [PATCH 6/8] fix: preserve endpoint precedence and compact secrets Preserve process endpoint aliases before loading env files, and restore broad credential-name matching without reintroducing CURL_OPTS false positives. Add regressions for conflicting process/file endpoint aliases and compact APIKEY, ACCESSTOKEN, and PGPASSWORD names. Co-authored-by: Basit Mustafa Amp-Thread-ID: https://ampcode.com/threads/T-01a0679e-60c1-72ca-a36f-2428be671374 --- bin/shellm | 27 +++++++++++++++++++++++++-- tests/test_persona_bugreport.sh | 4 ++++ tests/test_var_secrets.sh | 30 ++++++++++++++++++++++++++---- tools/persona | 20 ++++++++++++-------- 4 files changed, 67 insertions(+), 14 deletions(-) diff --git a/bin/shellm b/bin/shellm index 5cbcbe1b..142a3206 100755 --- a/bin/shellm +++ b/bin/shellm @@ -11,6 +11,20 @@ set -euo pipefail # a parent shellm run) always wins over files — same semantics as # thinkers/_lib/common.sh:_load_env_defaults. Values are extracted by # sourcing the file in a subshell so quoting behaves like a plain `source`. +# Preserve endpoint values from the original process separately: the two +# endpoint names are aliases, so a process LLM_API_URL must also beat a +# SHELLM_API_URL subsequently loaded from a file (and vice versa). +_shellm_process_shellm_api_url_set=0 +_shellm_process_llm_api_url_set=0 +if [[ -n "${SHELLM_API_URL+x}" ]]; then + _shellm_process_shellm_api_url_set=1 + _shellm_process_shellm_api_url="$SHELLM_API_URL" +fi +if [[ -n "${LLM_API_URL+x}" ]]; then + _shellm_process_llm_api_url_set=1 + _shellm_process_llm_api_url="$LLM_API_URL" +fi + _shellm_load_env() { local envfile="$1" [[ -f "$envfile" ]] || return 1 @@ -70,7 +84,15 @@ SHELLM_MAX_ITERATIONS="${SHELLM_MAX_ITERATIONS:-}" SHELLM_MAX_CONSECUTIVE_FAILURES="${SHELLM_MAX_CONSECUTIVE_FAILURES:-10}" SHELLM_MAX_REPEAT_FAILURES="${SHELLM_MAX_REPEAT_FAILURES:-3}" SHELLM_TRUNCATE="${SHELLM_TRUNCATE:-2000}" -SHELLM_API_URL="${SHELLM_API_URL:-${LLM_API_URL:-}}" +if [[ "$_shellm_process_shellm_api_url_set" -eq 1 ]]; then + SHELLM_API_URL="$_shellm_process_shellm_api_url" +elif [[ "$_shellm_process_llm_api_url_set" -eq 1 ]]; then + SHELLM_API_URL="$_shellm_process_llm_api_url" +else + SHELLM_API_URL="${SHELLM_API_URL:-${LLM_API_URL:-}}" +fi +unset _shellm_process_shellm_api_url_set _shellm_process_shellm_api_url \ + _shellm_process_llm_api_url_set _shellm_process_llm_api_url 2>/dev/null || true # Clear any inherited LLM_API_URL so llm uses per-provider defaults unset LLM_API_URL 2>/dev/null || true if [[ -n "$SHELLM_API_URL" ]]; then @@ -2986,7 +3008,8 @@ exit \$__shellm_rc _shellm_private_var_name() { local u u=$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]') - [[ "$u" =~ (^|_)(KEY|TOKEN|SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?|URL|URI|ENDPOINT|DSN)(_|$) ]] + [[ "$u" =~ (KEY|TOKEN|SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?) \ + || "$u" =~ (^|_)(URL|URI|ENDPOINT|DSN)(_|$) ]] } _redacted_cmdline() { diff --git a/tests/test_persona_bugreport.sh b/tests/test_persona_bugreport.sh index eeec80bb..1b819a0c 100755 --- a/tests/test_persona_bugreport.sh +++ b/tests/test_persona_bugreport.sh @@ -78,6 +78,7 @@ cat >> "$TJ/trajectory.jsonl" < think' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "token not in .env still masked by pattern (hint kept)" grep -q -- '--var GH_TOKEN= ' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "password on argv masked whole" grep -q -- '--var PG_PASSWORD= --var SHELLM_ENV=local' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "DSN on argv masked whole" grep -q -- '--var DATABASE_DSN= --var CURL_OPTS=--retry=2' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" +check "compact credential names are masked" grep -q -- '--var SERVICE_APIKEY= --var APIKEY= --var ACCESSTOKEN= --var PGPASSWORD= run' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "CURL_OPTS is not a URL false positive" grep -q -- '--var CURL_OPTS=--retry=2 run' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check_not "no dangling hint tails" grep -q -- ' [^ ]*>' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "bare --var KEY row untouched" grep -q -- '--var OPENROUTER_API_KEY think' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" diff --git a/tests/test_var_secrets.sh b/tests/test_var_secrets.sh index 7519e546..068ccf65 100755 --- a/tests/test_var_secrets.sh +++ b/tests/test_var_secrets.sh @@ -67,6 +67,11 @@ LEGACY="sk-legacy-literal-$$-$RANDOM$RANDOM" PRIVATE_URL="https://example.invalid/private/$$?opaque=$RANDOM" PRIVATE_DSN="postgresql://user:password@example.invalid/private_$$" INHERITED_URL="http://127.0.0.1:9/private/$RANDOM" +FILE_URL="http://file.example.invalid/v1/$RANDOM" +COMPACT_APIKEY="api-compact-secret-$$-$RANDOM$RANDOM" +COMPACT_BARE_KEY="bare-key-compact-secret-$$-$RANDOM$RANDOM" +COMPACT_TOKEN="token-compact-secret-$$-$RANDOM$RANDOM" +COMPACT_PASSWORD="password-compact-secret-$$-$RANDOM$RANDOM" CURL_VALUE="--retry=2" # --- 1. bare --var with the variable unset is an error ------------------------ @@ -82,17 +87,26 @@ fi export SECRET_PROBE="$SECRET" unset SHELLM_API_URL export LLM_API_URL="$INHERITED_URL" +printf 'SHELLM_API_URL=%s\n' "$FILE_URL" > "$WORK/wd/.env" fence "printf 'got=%s plain=%s api=%s shellm=%s\\n' \"\$SECRET_PROBE\" \"\$PLAIN\" \"\$LLM_API_URL\" \"\$SHELLM_API_URL\" > '$WORK/probe.txt' ps -axo args= > '$WORK/ps.txt' 2>/dev/null || ps -eo args= > '$WORK/ps.txt'" > "$WORK/script/1" fence 'FINAL=done' > "$WORK/script/last" run_shellm --var SECRET_PROBE --var PLAIN=1 --var "OPENROUTER_API_KEY=$LEGACY" \ --var "SERVICE_URL=$PRIVATE_URL" --var "DATABASE_DSN=$PRIVATE_DSN" \ + --var "SERVICE_APIKEY=$COMPACT_APIKEY" --var "APIKEY=$COMPACT_BARE_KEY" \ + --var "ACCESSTOKEN=$COMPACT_TOKEN" \ + --var "PGPASSWORD=$COMPACT_PASSWORD" \ --var "CURL_OPTS=$CURL_VALUE" "task" -if grep -qx "got=$SECRET plain=1 api=$INHERITED_URL shellm=$INHERITED_URL" "$WORK/probe.txt" 2>/dev/null; then +if grep -q "^got=$SECRET plain=1 " "$WORK/probe.txt" 2>/dev/null; then ok "bare --var NAME forwards the value into the generated code" else bad "bare --var NAME forwards the value into the generated code" "probe: $(cat "$WORK/probe.txt" 2>/dev/null) err: $(tail -2 "$WORK/err")" fi +if grep -q " api=$INHERITED_URL shellm=$INHERITED_URL$" "$WORK/probe.txt" 2>/dev/null; then + ok "process LLM_API_URL wins over file SHELLM_API_URL for both aliases" +else + bad "process LLM_API_URL wins over file SHELLM_API_URL for both aliases" "probe: $(cat "$WORK/probe.txt" 2>/dev/null)" +fi if [[ -z "${SHELLM_API_URL+x}" ]]; then ok "the shellm run started with LLM_API_URL only" else @@ -125,11 +139,19 @@ row=$(grep -rh '"type":"shellm-run"' "$HEADLONG_HOME" 2>/dev/null | tail -1) if [[ -n "$row" ]] && grep -qF 'OPENROUTER_API_KEY=' <<<"$row" \ && grep -qF 'SERVICE_URL=' <<<"$row" \ && grep -qF 'DATABASE_DSN=' <<<"$row" \ + && grep -qF 'SERVICE_APIKEY=' <<<"$row" \ + && grep -qF 'APIKEY=' <<<"$row" \ + && grep -qF 'ACCESSTOKEN=' <<<"$row" \ + && grep -qF 'PGPASSWORD=' <<<"$row" \ && ! grep -qF "$LEGACY" <<<"$row" && ! grep -qF "$PRIVATE_URL" <<<"$row" \ - && ! grep -qF "$PRIVATE_DSN" <<<"$row"; then - ok "shellm-run row masks credential, URL, and DSN --var values" + && ! grep -qF "$PRIVATE_DSN" <<<"$row" \ + && ! grep -qF "$COMPACT_APIKEY" <<<"$row" \ + && ! grep -qF "$COMPACT_BARE_KEY" <<<"$row" \ + && ! grep -qF "$COMPACT_TOKEN" <<<"$row" \ + && ! grep -qF "$COMPACT_PASSWORD" <<<"$row"; then + ok "shellm-run row masks compact credentials, URL, and DSN --var values" else - bad "shellm-run row masks credential, URL, and DSN --var values" "redacted fields missing" + bad "shellm-run row masks compact credentials, URL, and DSN --var values" "redacted fields missing" fi if grep -qF -- '--var SECRET_PROBE --var PLAIN=1' <<<"$row" \ && grep -qF -- "--var CURL_OPTS=$CURL_VALUE" <<<"$row"; then diff --git a/tools/persona b/tools/persona index 3671244a..cc8e55b2 100755 --- a/tools/persona +++ b/tools/persona @@ -621,15 +621,17 @@ _redact_tree() ( done # --var NAME=VALUE: KEY/TOKEN names keep a 4+4 hint when long enough, - # other private names are masked whole. Token boundaries are underscores - # or the ends of the name, so CURL_OPTS is not treated as a URL variable. + # other private names are masked whole. Credential terms match anywhere + # in the name (including APIKEY and PGPASSWORD); endpoint terms require + # underscore/name boundaries so CURL_OPTS is not treated as a URL variable. # (a value starting with "<" is one an earlier pass already replaced) for f in "${files[@]}"; do tmp=$(mktemp "$root/.redact-output.XXXXXX") || return 1 if sed -E -f <(printf '%s\n' \ - 's/--var (([A-Za-z_][A-Za-z0-9_]*_)?(SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?|URL|URI|ENDPOINT|DSN)(_[A-Za-z0-9_]+)*)=[^ "<\\][^ "\\]*/--var \1=/g' \ - 's/--var (([A-Za-z_][A-Za-z0-9_]*_)?(KEY|TOKEN)(_[A-Za-z0-9_]+)*)=([^ "<\\]{4})[^ "\\]{12,}([^ "\\]{4})/--var \1=/g' \ - 's/--var (([A-Za-z_][A-Za-z0-9_]*_)?(KEY|TOKEN)(_[A-Za-z0-9_]+)*)=[^ "<\\][^ "\\]*/--var \1=/g' \ + 's/--var ([A-Za-z_][A-Za-z0-9_]*(SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?)[A-Za-z0-9_]*)=[^ "<\\][^ "\\]*/--var \1=/g' \ + 's/--var (([A-Za-z_][A-Za-z0-9_]*_)?(URL|URI|ENDPOINT|DSN)(_[A-Za-z0-9_]+)*)=[^ "<\\][^ "\\]*/--var \1=/g' \ + 's/--var ([A-Za-z_][A-Za-z0-9_]*(KEY|TOKEN)[A-Za-z0-9_]*)=([^ "<\\]{4})[^ "\\]{12,}([^ "\\]{4})/--var \1=/g' \ + 's/--var ([A-Za-z_][A-Za-z0-9_]*(KEY|TOKEN)[A-Za-z0-9_]*)=[^ "<\\][^ "\\]*/--var \1=/g' \ 's/sk-([A-Za-z0-9_-]{4})[A-Za-z0-9_-]{12,}([A-Za-z0-9_-]{4})//g' \ 's/sk-[A-Za-z0-9_-]{8,}//g') \ "$f" > "$tmp" 2>/dev/null; then @@ -651,8 +653,9 @@ _upper() { printf '%s' "$1" | tr '[:lower:]' '[:upper:]'; } _redact_hint() { local u v="$2" u=$(_upper "$1") - if [[ "$u" =~ (^|_)(KEY|TOKEN)(_|$) \ - && ! "$u" =~ (^|_)(SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?|URL|URI|ENDPOINT|DSN)(_|$) \ + if [[ "$u" =~ (KEY|TOKEN) \ + && ! "$u" =~ (SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?) \ + && ! "$u" =~ (^|_)(URL|URI|ENDPOINT|DSN)(_|$) \ && "${#v}" -ge 20 ]]; then printf '' "${v:0:4}" "${v: -4}" else @@ -663,7 +666,8 @@ _redact_hint() { _is_secret_name() { # same notion as shellm's trajectory masking local u u=$(_upper "$1") - [[ "$u" =~ (^|_)(KEY|TOKEN|SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?|URL|URI|ENDPOINT|DSN)(_|$) ]] + [[ "$u" =~ (KEY|TOKEN|SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?) \ + || "$u" =~ (^|_)(URL|URI|ENDPOINT|DSN)(_|$) ]] } _sed_escape() { # escape a literal for use in an ERE sed s/// pattern or replacement From 0244ea67a273d79db8201389af6f139bd1604e5b Mon Sep 17 00:00:00 2001 From: Amp Date: Thu, 3 Sep 2026 15:47:25 +0000 Subject: [PATCH 7/8] fix: fail closed when scrubbing bug reports Amp-Thread-ID: https://ampcode.com/threads/T-01a0679e-60c1-72ca-a36f-2428be671374 Co-authored-by: Basit Mustafa --- bin/shellm | 6 ++ tests/test_persona_bugreport.sh | 89 ++++++++----------------- tests/test_thinker_env_fallback.sh | 2 +- tests/test_var_secrets.sh | 5 +- thinkers/_lib/common.sh | 3 + tools/persona | 100 +++++++++++++++++------------ 6 files changed, 99 insertions(+), 106 deletions(-) diff --git a/bin/shellm b/bin/shellm index 142a3206..570ae903 100755 --- a/bin/shellm +++ b/bin/shellm @@ -2691,6 +2691,12 @@ exit \$__shellm_rc for _ev in "${_SHELLM_EXTRA_VARS[@]+"${_SHELLM_EXTRA_VARS[@]}"}"; do env_vars+=("$_ev") done + # Endpoint aliases are canonical shellm state, not ordinary extras. + # Keep the Docker-rewritten value last so a duplicate --var cannot + # restore a host-only URL inside generated code. + if [[ -n "$execution_api_url" ]]; then + env_vars+=(SHELLM_API_URL="$execution_api_url" LLM_API_URL="$execution_api_url") + fi if [[ "$SHELLM_DOCKER_ACCESS" == "broker" && -n "${_SHELLM_DOCKER_BROKER_DIR:-}" ]]; then env_vars+=(SHELLM_DOCKER_BROKER="$_SHELLM_DOCKER_BROKER_DIR") env_vars+=(SHELLM_DOCKER_TRANSPORT="$_SHELLM_DOCKER_BROKER_TRANSPORT") diff --git a/tests/test_persona_bugreport.sh b/tests/test_persona_bugreport.sh index 1b819a0c..77765d6c 100755 --- a/tests/test_persona_bugreport.sh +++ b/tests/test_persona_bugreport.sh @@ -51,9 +51,6 @@ export PATH="$REPO/bin:$REPO/tools:$PATH" KEY="sk-or-v1-0123456789abcdef0123456789abcdef0123456789abcdef" PW="hunter2hunter2-very-secret" DSN="postgresql://bugreport-user:bugreport-pass@example.invalid/private-db" -# This produces a malformed one-line sed program. Its isolated literal pass -# may fail, but it must not prevent the other literals or fixed patterns from -# being scrubbed. export BROKEN_SECRET=$'first-line\nsecond-line' cat > "$HEADLONG_HOME/.env" <> "$TJ/trajectory.jsonl" < "$ID/run/logs/monolith.log" +printf 'env: OPENROUTER_API_KEY=%s\nmultiline: %s\n' "$KEY" "$BROKEN_SECRET" > "$ID/run/logs/monolith.log" printf -- '---\ntitle: db\n---\nthe db password is %s\nthe dsn is %s\n' "$PW" "$DSN" > "$ID/memories/db.md" printf 'scratch file\n' > "$ID/workdir/scratch.txt" head -c 2048 /dev/urandom > "$TJ/blobs/bin.dat" @@ -130,11 +128,13 @@ check_not "other token value nowhere in bundle" grep -rqF 'ghp_OTHERTOKEN0123456 check_not "other password nowhere in bundle" grep -rqF 'correcthorsebatterystaple' "$TOP" check_not "compact API key values nowhere in bundle" grep -rqE '(svc|api)_compact_0123456789abcdef' "$TOP" check_not "compact access token nowhere in bundle" grep -rqF 'tok_compact_0123456789abcdef' "$TOP" +check_not "multiline literal nowhere in bundle" grep -rqF "$BROKEN_SECRET" "$TOP" +check_not "spaced password suffix nowhere in bundle" grep -rqF 'horse battery staple' "$TOP" check "legacy --var KEY=value row masked, 4+4 hint kept" grep -q -- '--var OPENROUTER_API_KEY= think' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "token not in .env still masked by pattern (hint kept)" grep -q -- '--var GH_TOKEN= ' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "password on argv masked whole" grep -q -- '--var PG_PASSWORD= --var SHELLM_ENV=local' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "DSN on argv masked whole" grep -q -- '--var DATABASE_DSN= --var CURL_OPTS=--retry=2' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" -check "compact credential names are masked" grep -q -- '--var SERVICE_APIKEY= --var APIKEY= --var ACCESSTOKEN= --var PGPASSWORD= run' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" +check "compact credential names are masked" grep -q -- '--var SERVICE_APIKEY= --var APIKEY= --var ACCESSTOKEN= --var PGPASSWORD=' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "CURL_OPTS is not a URL false positive" grep -q -- '--var CURL_OPTS=--retry=2 run' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check_not "no dangling hint tails" grep -q -- ' [^ ]*>' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "bare --var KEY row untouched" grep -q -- '--var OPENROUTER_API_KEY think' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" @@ -145,72 +145,37 @@ check "memory DSN literal scrubbed" grep -q 'dsn is ' "$TOP/ check "plain thought text kept" grep -q 'all is well' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "source trajectory untouched" grep -qF "$KEY" "$TJ/trajectory.jsonl" -# 3. anonymous sed programs and output-temp cleanup -REAL_SED=$(command -v sed) +# 3. helper failure is fail-closed; literals stay out of argv and temp files +REAL_PYTHON=$(command -v python3) mkdir -p "$WORK/failbin" "$WORK/redact-state" -cat > "$WORK/failbin/sed" < "$WORK/failbin/python3" <> "\$REDACT_STATE/program-args" - if [[ -f "\$REDACT_STATE/previous-temp" ]]; then - old=\$(cat "\$REDACT_STATE/previous-temp") - [[ ! -e "\$old" ]] || touch "\$REDACT_STATE/leaked-temp" - fi - current=\$(find "\$TMPDIR" -name '.redact-output.*' -print -quit 2>/dev/null) - [[ -z "\$current" ]] || printf '%s\n' "\$current" > "\$REDACT_STATE/previous-temp" - if [[ "\${REDACT_SED_MODE:-}" == error && ! -f "\$REDACT_STATE/failed-once" ]]; then - touch "\$REDACT_STATE/failed-once" - exit 42 - fi - if [[ "\${REDACT_SED_MODE:-}" == signal && ! -f "\$REDACT_STATE/signalled" ]]; then - touch "\$REDACT_STATE/signalled" - kill -TERM "\$PPID" - sleep 1 - exit 143 - fi -fi -exec "\$real_sed" "\$@" +printf '%s\n' "\$@" >> "\$REDACT_STATE/python-args" +[[ "\${REDACT_PYTHON_MODE:-}" != error ]] || exit 42 +exec '$REAL_PYTHON' "\$@" EOF -chmod +x "$WORK/failbin/sed" +chmod +x "$WORK/failbin/python3" rm -rf "$WORK/redact-state"; mkdir "$WORK/redact-state" -if PATH="$WORK/failbin:$PATH" REDACT_STATE="$WORK/redact-state" REDACT_SED_MODE=error \ - persona alpha bugreport --out "$WORK/error-pass.tgz" >/dev/null 2>&1; then - ok "one sed error does not abort later redaction passes" -else - bad "one sed error does not abort later redaction passes" -fi -check_not "failed sed output temp is removed before the next pass" test -e "$WORK/redact-state/leaked-temp" -if [[ -s "$WORK/redact-state/program-args" ]] \ - && ! grep -qF "$KEY" "$WORK/redact-state/program-args" \ - && ! grep -qF "$PW" "$WORK/redact-state/program-args" \ - && ! grep -qF "$DSN" "$WORK/redact-state/program-args"; then - ok "sed program argv contains no private literals" +if PATH="$WORK/failbin:$PATH" REDACT_STATE="$WORK/redact-state" REDACT_PYTHON_MODE=error \ + persona alpha bugreport --out "$WORK/error-pass.tgz" >/dev/null 2>"$WORK/error-stderr"; then + bad "redactor failure aborts the bugreport" else - bad "sed program argv contains no private literals" + ok "redactor failure aborts the bugreport" fi -check_not "implementation has no secret sed script path" grep -q 'headlong-redact' "$REPO/tools/persona" -rm -f "$WORK/error-pass.tgz" - -rm -rf "$WORK/redact-state"; mkdir "$WORK/redact-state" -if PATH="$WORK/failbin:$PATH" REDACT_STATE="$WORK/redact-state" REDACT_SED_MODE=signal \ - persona alpha bugreport --out "$WORK/signal-pass.tgz" >/dev/null 2>&1; then - bad "TERM interrupts redaction" +check_not "redactor failure writes no archive" test -e "$WORK/error-pass.tgz" +check "redactor failure is explained" grep -q 'secret scrubbing failed; no archive was written' "$WORK/error-stderr" +check_not "failed helper output temp is removed" bash -c \ + 'find "$1" -name ".redact-output.*" -print -quit | grep -q .' _ "$WORK" +if [[ -s "$WORK/redact-state/python-args" ]] \ + && ! grep -qF "$KEY" "$WORK/redact-state/python-args" \ + && ! grep -qF "$PW" "$WORK/redact-state/python-args" \ + && ! grep -qF "$DSN" "$WORK/redact-state/python-args"; then + ok "redactor argv contains no private literals" else - ok "TERM interrupts redaction" + bad "redactor argv contains no private literals" fi -check_not "signal leaves no output redaction temp" bash -c \ - 'find "$1" -name ".redact-output.*" -print -quit | grep -q .' _ "$WORK" -check_not "signal leaves no secret-bearing sed script" bash -c \ - 'find "$1" -name "headlong-redact.*" -print -quit | grep -q .' _ "$WORK" +check_not "implementation writes no secret helper script" grep -q 'headlong-redact' "$REPO/tools/persona" # 4. options check "--include-workdir adds workdir" bash -c ' diff --git a/tests/test_thinker_env_fallback.sh b/tests/test_thinker_env_fallback.sh index e00d170d..45758f09 100755 --- a/tests/test_thinker_env_fallback.sh +++ b/tests/test_thinker_env_fallback.sh @@ -181,7 +181,7 @@ description: Test fixture. metadata: shelllm: requires: - env: ["PROBE_SERVICE_CONFIG"] + env: ["PROBE_SERVICE_CONFIG", "LLM_API_URL", "SHELLM_API_URL"] --- EOF export IDENTITY_DIR="$H/id" TRAJ_DIR="$H/id/trajectories" TRAJ_ID=t1 MEM_DIR="$H/id/memories" diff --git a/tests/test_var_secrets.sh b/tests/test_var_secrets.sh index 068ccf65..9236f604 100755 --- a/tests/test_var_secrets.sh +++ b/tests/test_var_secrets.sh @@ -93,6 +93,7 @@ ps -axo args= > '$WORK/ps.txt' 2>/dev/null || ps -eo args= > '$WORK/ps.txt'" > " fence 'FINAL=done' > "$WORK/script/last" run_shellm --var SECRET_PROBE --var PLAIN=1 --var "OPENROUTER_API_KEY=$LEGACY" \ --var "SERVICE_URL=$PRIVATE_URL" --var "DATABASE_DSN=$PRIVATE_DSN" \ + --var "SHELLM_API_URL=$FILE_URL" \ --var "SERVICE_APIKEY=$COMPACT_APIKEY" --var "APIKEY=$COMPACT_BARE_KEY" \ --var "ACCESSTOKEN=$COMPACT_TOKEN" \ --var "PGPASSWORD=$COMPACT_PASSWORD" \ @@ -103,9 +104,9 @@ else bad "bare --var NAME forwards the value into the generated code" "probe: $(cat "$WORK/probe.txt" 2>/dev/null) err: $(tail -2 "$WORK/err")" fi if grep -q " api=$INHERITED_URL shellm=$INHERITED_URL$" "$WORK/probe.txt" 2>/dev/null; then - ok "process LLM_API_URL wins over file SHELLM_API_URL for both aliases" + ok "process LLM_API_URL wins over file and duplicate extra endpoint aliases" else - bad "process LLM_API_URL wins over file SHELLM_API_URL for both aliases" "probe: $(cat "$WORK/probe.txt" 2>/dev/null)" + bad "process LLM_API_URL wins over file and duplicate extra endpoint aliases" "probe: $(cat "$WORK/probe.txt" 2>/dev/null)" fi if [[ -z "${SHELLM_API_URL+x}" ]]; then ok "the shellm run started with LLM_API_URL only" diff --git a/thinkers/_lib/common.sh b/thinkers/_lib/common.sh index f7c1f999..9d80e844 100755 --- a/thinkers/_lib/common.sh +++ b/thinkers/_lib/common.sh @@ -456,6 +456,9 @@ _build_shellm_flags() { # Skill-declared vars while IFS= read -r vname; do [[ -z "$vname" ]] && continue + # shellm owns endpoint alias resolution and Docker rewriting. Emitting + # either alias again as an extra var would overwrite its canonical URL. + case "$vname" in LLM_API_URL|SHELLM_API_URL) continue ;; esac local vval="${!vname:-}" if [[ -n "$vval" ]]; then # Skills commonly declare credentials and service endpoints. Keep diff --git a/tools/persona b/tools/persona index cc8e55b2..614f4061 100755 --- a/tools/persona +++ b/tools/persona @@ -467,7 +467,7 @@ EOF ( set +e; set +o pipefail; _bugreport_text "$iddir" "${big[@]+"${big[@]}"}" ) > "$top/report.txt" 2>&1 || true echo "Scrubbing secrets..." >&2 - _redact_tree "$top" + _redact_tree "$top" || die "bugreport: secret scrubbing failed; no archive was written" tar "${tarq[@]+"${tarq[@]}"}" -czf "$out" -C "$stage" "$(basename "$top")" cat >&2 </dev/null) [[ "${#files[@]}" -gt 0 ]] || return 0 - # Keep literals out of sed's argv and isolate each one in its own pass, so - # an unusual ambient value that sed cannot parse does not disable every - # other replacement. Process substitution gives sed an anonymous program - # FD: no literal enters argv or a filesystem script. Output temps live only - # under the staging root and the subshell-local traps clean them without - # disturbing cmd_bugreport's traps. - local f i escaped_value escaped_replacement - for ((i=0; i<${#values[@]}; i++)); do - escaped_value=$(_sed_escape "${values[$i]}") - escaped_replacement=$(_sed_escape "${replacements[$i]}") - for f in "${files[@]}"; do - tmp=$(mktemp "$root/.redact-output.XXXXXX") || return 1 - if sed -E -f <(printf 's/%s/%s/g\n' "$escaped_value" "$escaped_replacement") \ - "$f" > "$tmp" 2>/dev/null; then - mv -f "$tmp" "$f" - tmp="" - else - rm -f "$tmp" - tmp="" - fi - done - done - - # --var NAME=VALUE: KEY/TOKEN names keep a 4+4 hint when long enough, - # other private names are masked whole. Credential terms match anywhere - # in the name (including APIKEY and PGPASSWORD); endpoint terms require - # underscore/name boundaries so CURL_OPTS is not treated as a URL variable. - # (a value starting with "<" is one an earlier pass already replaced) + # A length-framed stdin stream keeps literals out of argv and the + # filesystem. Python's byte replacement handles embedded newlines and + # regex metacharacters literally; any helper failure aborts the report. + # The --var pass masks through the next --var (or JSON quote/end of line), + # because old unquoted command records cannot distinguish a value with + # spaces from the command words after it. Hiding too much is safer than + # retaining a password suffix. + local f i for f in "${files[@]}"; do tmp=$(mktemp "$root/.redact-output.XXXXXX") || return 1 - if sed -E -f <(printf '%s\n' \ - 's/--var ([A-Za-z_][A-Za-z0-9_]*(SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?)[A-Za-z0-9_]*)=[^ "<\\][^ "\\]*/--var \1=/g' \ - 's/--var (([A-Za-z_][A-Za-z0-9_]*_)?(URL|URI|ENDPOINT|DSN)(_[A-Za-z0-9_]+)*)=[^ "<\\][^ "\\]*/--var \1=/g' \ - 's/--var ([A-Za-z_][A-Za-z0-9_]*(KEY|TOKEN)[A-Za-z0-9_]*)=([^ "<\\]{4})[^ "\\]{12,}([^ "\\]{4})/--var \1=/g' \ - 's/--var ([A-Za-z_][A-Za-z0-9_]*(KEY|TOKEN)[A-Za-z0-9_]*)=[^ "<\\][^ "\\]*/--var \1=/g' \ - 's/sk-([A-Za-z0-9_-]{4})[A-Za-z0-9_-]{12,}([A-Za-z0-9_-]{4})//g' \ - 's/sk-[A-Za-z0-9_-]{8,}//g') \ - "$f" > "$tmp" 2>/dev/null; then + if { + printf '%s\n' "${#values[@]}" + for ((i=0; i<${#values[@]}; i++)); do + LC_ALL=C + printf '%s\n' "${#values[$i]}" + printf '%s\n' "${values[$i]}" + printf '%s\n' "${#replacements[$i]}" + printf '%s\n' "${replacements[$i]}" + done + cat "$f" + } | python3 -c ' +import re, sys + +source = sys.stdin.buffer +def framed(): + size = int(source.readline()) + value = source.read(size) + if source.read(1) != b"\n": + raise ValueError("invalid redaction input") + return value + +pairs = [(framed(), framed()) for _ in range(int(source.readline()))] +data = source.read() +for needle, replacement in pairs: + if needle: + data = data.replace(needle, replacement) + +credential = re.compile(br"(?:SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?|KEY|TOKEN)") +endpoint = re.compile(br"(?:^|_)(?:URL|URI|ENDPOINT|DSN)(?:_|$)") +var = re.compile(br"(--var ([A-Za-z_][A-Za-z0-9_]*)=)(.*?)(?= --var |\"|$)", re.M) +def mask(match): + name = match.group(2).upper() + value = match.group(3) + if value.startswith(b"<") or not (credential.search(name) or endpoint.search(name)): + return match.group(0) + token = value.split(None, 1)[0] + if (b"KEY" in name or b"TOKEN" in name) and not re.search( + br"SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?", name) \ + and not endpoint.search(name) and len(token) >= 20: + replacement = b"" + else: + replacement = b"" + return match.group(1) + replacement + +data = var.sub(mask, data) +data = re.sub(br"sk-([A-Za-z0-9_-]{4})[A-Za-z0-9_-]{12,}([A-Za-z0-9_-]{4})", br"", data) +data = re.sub(br"sk-[A-Za-z0-9_-]{8,}", b"", data) +sys.stdout.buffer.write(data) +' > "$tmp" 2>/dev/null; then mv -f "$tmp" "$f" tmp="" else rm -f "$tmp" tmp="" + return 1 fi done ) @@ -670,10 +692,6 @@ _is_secret_name() { # same notion as shellm's trajectory masking || "$u" =~ (^|_)(URL|URI|ENDPOINT|DSN)(_|$) ]] } -_sed_escape() { # escape a literal for use in an ERE sed s/// pattern or replacement - printf '%s' "$1" | sed -e 's/[][\/.*^$&+?(){}|\\]/\\&/g' -} - usage() { cat < Date: Fri, 4 Sep 2026 18:07:51 +0800 Subject: [PATCH 8/8] fix: finish bug report secret scrubbing --- README.md | 2 +- docs/shellm.md | 13 +++++++------ install.sh | 2 +- tests/test_persona_bugreport.sh | 21 +++++++++++++++++++++ tools/persona | 8 +++++--- 5 files changed, 35 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 719c61aa..fedd7e99 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ life, and opens a dashboard where you can watch its mind run: curl -fsSL https://headlong.ai/install.sh | bash ``` -You'll need bash 3.2+, git, curl, jq, and an LLM API key (Anthropic, +You'll need bash 3.2+, git, curl, jq, Python 3, and an LLM API key (Anthropic, OpenAI, Gemini, or OpenRouter) — or a local model on any OpenAI-compatible server (llama.cpp, Ollama, vLLM, LM Studio; see [Local models](#local-models) below, no key needed); the dashboard also diff --git a/docs/shellm.md b/docs/shellm.md index d6d5f85e..0e6d5b18 100644 --- a/docs/shellm.md +++ b/docs/shellm.md @@ -292,12 +292,13 @@ Set `LLM_API_KEY` if the endpoint wants a bearer token. The policy for which providers live in core is in [design/providers.md](../design/providers.md). -Under shellm, set `SHELLM_API_URL` instead of `LLM_API_URL` — shellm -clears any inherited `LLM_API_URL`, and `llm` itself falls back to -`SHELLM_API_URL` for this provider, so thinkers and other tools that -call `llm` directly reach the endpoint too. Name the model as well, or -shellm falls back to its default Claude model and sends that to your -endpoint: +Under shellm, `SHELLM_API_URL` is the preferred name and `LLM_API_URL` is +accepted as an alias. A value already in the process wins over either name +loaded from `.env`. If both names are already in the process, +`SHELLM_API_URL` wins. Shellm exports the resolved value as `LLM_API_URL`, +so thinkers and other tools that call `llm` directly reach the same +endpoint. Name the model as well, or shellm falls back to its default Claude +model and sends that to your endpoint: ```bash LLM_PROVIDER=openai-compatible \ diff --git a/install.sh b/install.sh index eeed5e02..ff24b964 100755 --- a/install.sh +++ b/install.sh @@ -606,7 +606,7 @@ main() { fi unset HEADLONG_INSTALL_LOCATION_CHOSEN - _require_deps jq curl + _require_deps jq curl python3 mkdir -p "$PREFIX" _install_tools diff --git a/tests/test_persona_bugreport.sh b/tests/test_persona_bugreport.sh index 77765d6c..cf90cda4 100755 --- a/tests/test_persona_bugreport.sh +++ b/tests/test_persona_bugreport.sh @@ -51,11 +51,15 @@ export PATH="$REPO/bin:$REPO/tools:$PATH" KEY="sk-or-v1-0123456789abcdef0123456789abcdef0123456789abcdef" PW="hunter2hunter2-very-secret" DSN="postgresql://bugreport-user:bugreport-pass@example.invalid/private-db" +SERVICE_URL="https://overlap.example.invalid" +PRIVATE_DSN="$SERVICE_URL/private-secret-suffix" export BROKEN_SECRET=$'first-line\nsecond-line' cat > "$HEADLONG_HOME/.env" < "$HEADLONG_HOME/logs/web.log" @@ -64,6 +68,10 @@ printf 'init ok\n' > "$HEADLONG_HOME/logs/init.log" (cd "$HEADLONG_APP_DIR" && identity new alpha >/dev/null 2>&1) || { bad "identity new alpha"; exit 1; } ID="$HEADLONG_APP_DIR/.identities/alpha" ln -s alpha "$HEADLONG_APP_DIR/.identities/default" +cat > "$ID/.env" <> "$TJ/trajectory.jsonl" <> "$TJ/trajectory.jsonl" mkdir -p "$ID/run/logs" "$ID/memories" "$ID/workdir" "$TJ/blobs" printf 'env: OPENROUTER_API_KEY=%s\nmultiline: %s\n' "$KEY" "$BROKEN_SECRET" > "$ID/run/logs/monolith.log" printf -- '---\ntitle: db\n---\nthe db password is %s\nthe dsn is %s\n' "$PW" "$DSN" > "$ID/memories/db.md" +printf 'the private endpoint is %s\n' "$PRIVATE_DSN" >> "$ID/memories/db.md" printf 'scratch file\n' > "$ID/workdir/scratch.txt" head -c 2048 /dev/urandom > "$TJ/blobs/bin.dat" @@ -123,6 +134,9 @@ check "binary blob carried intact" cmp -s "$TJ/blobs/bin.dat" "$TOP/ident check_not "API key value nowhere in bundle" grep -rqF "$KEY" "$TOP" check_not "DB password nowhere in bundle" grep -rqF "$PW" "$TOP" check_not "DSN literal nowhere in bundle" grep -rqF "$DSN" "$TOP" +check_not "overlapping secret suffix nowhere in bundle" grep -rqF 'private-secret-suffix' "$TOP" +check_not "identity password nowhere in bundle" grep -rqF 'identity-password-canary' "$TOP" +check_not "identity config nowhere in bundle" grep -rqF 'identity-config-canary' "$TOP" check_not "no sk-... shaped string survives" grep -rqE 'sk-[A-Za-z0-9_-]{8,}' "$TOP" check_not "other token value nowhere in bundle" grep -rqF 'ghp_OTHERTOKEN0123456789abcdefghijkl' "$TOP" check_not "other password nowhere in bundle" grep -rqF 'correcthorsebatterystaple' "$TOP" @@ -130,6 +144,7 @@ check_not "compact API key values nowhere in bundle" grep -rqE '(svc|api)_compac check_not "compact access token nowhere in bundle" grep -rqF 'tok_compact_0123456789abcdef' "$TOP" check_not "multiline literal nowhere in bundle" grep -rqF "$BROKEN_SECRET" "$TOP" check_not "spaced password suffix nowhere in bundle" grep -rqF 'horse battery staple' "$TOP" +check_not "quoted password suffix nowhere in bundle" grep -rqF 'quote-secret-suffix' "$TOP" check "legacy --var KEY=value row masked, 4+4 hint kept" grep -q -- '--var OPENROUTER_API_KEY= think' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "token not in .env still masked by pattern (hint kept)" grep -q -- '--var GH_TOKEN= ' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "password on argv masked whole" grep -q -- '--var PG_PASSWORD= --var SHELLM_ENV=local' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" @@ -143,7 +158,11 @@ check "thinker log key scrubbed (hint kept)" grep -q 'OPENROUTER_API_KEY=' "$TOP/identity/memories/db.md" check "memory DSN literal scrubbed" grep -q 'dsn is ' "$TOP/identity/memories/db.md" check "plain thought text kept" grep -q 'all is well' "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" +check "scrubbed trajectory remains valid JSONL" bash -c \ + 'while IFS= read -r line; do printf "%s\n" "$line" | jq -e . >/dev/null || exit 1; done < "$1"' \ + _ "$TOP/identity/trajectories/$(basename "$TJ")/trajectory.jsonl" check "source trajectory untouched" grep -qF "$KEY" "$TJ/trajectory.jsonl" +check "installer requires Python 3" grep -q '_require_deps jq curl python3' "$REPO/install.sh" # 3. helper failure is fail-closed; literals stay out of argv and temp files REAL_PYTHON=$(command -v python3) @@ -181,6 +200,8 @@ check_not "implementation writes no secret helper script" grep -q 'headlong-reda check "--include-workdir adds workdir" bash -c ' persona alpha bugreport --out "$1" --include-workdir >/dev/null 2>&1 && tar -tzf "$1" | grep -q "identity/workdir/scratch.txt"' _ "$WORK/b2.tgz" +check_not "--include-workdir still leaves .env out" bash -c \ + 'tar -tzf "$1" | grep -q "/identity/.env$"' _ "$WORK/b2.tgz" check "default --out lands in HOME" bash -c ' out=$(persona alpha bugreport 2>/dev/null) && [[ "$out" == "$HOME/headlong-bugreport-alpha-"*.tgz && -s "$out" ]]' check "--help exits 0 and mentions Usage" bash -c 'persona alpha bugreport --help | grep -q "^Usage:"' diff --git a/tools/persona b/tools/persona index 614f4061..67778996 100755 --- a/tools/persona +++ b/tools/persona @@ -432,6 +432,7 @@ EOF *) die "bugreport: unknown option: $1" ;; esac done + command -v python3 >/dev/null 2>&1 || die "bugreport requires python3 for secret scrubbing" local stamp; stamp=$(date +%Y%m%d-%H%M%S) [[ -n "$out" ]] || out="$HOME/headlong-bugreport-$NAME-$stamp.tgz" local iddir="$IDENTITY_ROOT/$NAME" @@ -446,8 +447,8 @@ EOF # carry com.apple.provenance; we don't want xattrs in the bundle anyway. local -a tarq=() case "$(tar --version 2>/dev/null)" in *bsdtar*) tarq=(--no-xattrs --no-mac-metadata) ;; esac - local excl=(--exclude='./workdir') - [[ "$BUGREPORT_INCLUDE_WORKDIR" -eq 1 ]] && excl=() + local excl=(--exclude='./.env') + [[ "$BUGREPORT_INCLUDE_WORKDIR" -eq 1 ]] || excl+=(--exclude='./workdir') # Follow the default -> name symlink; copy files, not the symlink itself. # Skip single files over 50MB (noted in report.txt) so a runaway blob # doesn't turn the bundle into a gigabyte. @@ -629,6 +630,7 @@ def framed(): return value pairs = [(framed(), framed()) for _ in range(int(source.readline()))] +pairs.sort(key=lambda pair: len(pair[0]), reverse=True) data = source.read() for needle, replacement in pairs: if needle: @@ -636,7 +638,7 @@ for needle, replacement in pairs: credential = re.compile(br"(?:SECRET|PASSWORD|PASSWD|PASSPHRASE|CREDENTIALS?|KEY|TOKEN)") endpoint = re.compile(br"(?:^|_)(?:URL|URI|ENDPOINT|DSN)(?:_|$)") -var = re.compile(br"(--var ([A-Za-z_][A-Za-z0-9_]*)=)(.*?)(?= --var |\"|$)", re.M) +var = re.compile(br"(--var ([A-Za-z_][A-Za-z0-9_]*)=)((?:\\.|[^\"\\])*?)(?= --var |\"|$)", re.M) def mask(match): name = match.group(2).upper() value = match.group(3)