diff --git a/tests/policy/README.md b/tests/policy/README.md index f0c46bce..b80eae75 100644 --- a/tests/policy/README.md +++ b/tests/policy/README.md @@ -7,7 +7,7 @@ authoring contract: every rule below states the invariant it enforces first, the mechanism that enforces it. Read the whole thing before pinning anything. This README is invisible to the fixture loader: discovery is -`find tests/policy -maxdepth 1 -name 'safety-*.json'` (`tools/policy_check.sh:1473`), +`find tests/policy -maxdepth 1 -name 'safety-*.json'` (the `SAFETY_FIXTURES` discovery block in `tools/policy_check.sh`), so only `safety-*.json` files are ever evaluated. ## 1. Honest capability statement @@ -178,3 +178,23 @@ file (or a new section) fails to say X," no fixture in this directory can see it that requires a structural check in `tools/policy_check.sh` itself (compare the CHECK-numbered guards). Do not paper over the gap with a presence pin; name the uncovered half in the fixture `description` or add the structural check. + +## 8. Runtime dependencies + +**Invariant: a missing interpreter must fail the check loudly and name itself, +never silently degrade to a weaker check or a false pass.** + +`set_check` fixtures require `perl` to run them. There is no fallback: each +fixture's `extract_regex` is compiled and executed by `perl` (`qr//`). If `perl` +is not available on the runner, the check fails closed and names `perl` in the +failure message, rather than silently passing. + +Each fixture's `extract_regex` reaches `perl` as data, not as program source: it +is never interpolated into a shell command line or a Perl program string, so +characters such as `/`, `$`, and `@` need no extra escaping for the shell or for +Perl. + +The regex MUST contain a capture group; the first group is what `set_check` +captures as a set member. A regex that fails to compile, or whose extraction +fails at run time, FAILS the check -- a broken extraction is never silently +treated as a zero-match (empty set) result. diff --git a/tests/policy/fixtures/set-check-zero-match-canary.md b/tests/policy/fixtures/set-check-zero-match-canary.md index 6b8498f6..fa95f20d 100644 --- a/tests/policy/fixtures/set-check-zero-match-canary.md +++ b/tests/policy/fixtures/set-check-zero-match-canary.md @@ -5,7 +5,7 @@ self-test in `tools/policy_check.sh`. It is NOT a policy fixture (fixture discovery is `safety-*.json` at the `tests/policy/` top level) and nothing else in the repo reads it. -It exists to exercise two branches of `test_set_check`: +It exists to exercise six branches of `test_set_check`: 1. ZERO-MATCH branch. The self-test extracts with the regex `SETCHECK-ZERO-MATCH-CANARY :` (digit-suffixed form), which matches @@ -20,7 +20,34 @@ It exists to exercise two branches of `test_set_check`: must FAIL — proving the zero-match pass in (1) is a real assertion and not a predicate that accepts anything. +3. NON-COMPILING REGEX branch. The self-test also extracts with a regex that + does not compile under perl. A regex that fails to compile must FAIL the + check rather than being silently treated as a zero-match pass. + +4. BROKEN-EXTRACTION branch. The self-test also extracts with a regex that + compiles, HAS a capture group, and MATCHES the present-canary lines below, + but whose capture group is optional and never takes part in those matches. + Because the group exists, the independent capture-group count lets it + through, so it reaches the extractor. The extractor stops with an error + when a match leaves group 1 empty, and that error must FAIL the check. It + must never be read as a clean zero-match result. + +5. UNESCAPED-SLASH branch. The self-test also extracts with a regex containing + an unescaped `/` delimiter character, matched against the slash-canary + marker line below whose value is `a/b`. The regex reaches perl as data, not + as program source, so the unescaped `/` must not break the match: the check + must PASS and capture `a/b`. + +6. CAPTURE-FREE ZERO-MATCH branch. The self-test also extracts with the same + digit-suffixed `SETCHECK-ZERO-MATCH-CANARY :` pattern from item 1 (still + matching NOTHING in this file), but written with no capture group. Without + an independent capture-group count run before extraction, this exact + combination -- no match AND no capture group -- would never reach the + extraction loop's capture guard and would vacuously pass as the same clean + empty result as branch 1. That must FAIL the check. + Present-canary lines (one occurrence each, do not duplicate or remove): - SETCHECK-PRESENT-CANARY 1: first present marker - SETCHECK-PRESENT-CANARY 2: second present marker +- SETCHECK-SLASH-CANARY a/b: unescaped-slash delimiter marker diff --git a/tools/policy_check.sh b/tools/policy_check.sh index 40e61efc..b1975494 100644 --- a/tools/policy_check.sh +++ b/tools/policy_check.sh @@ -2229,6 +2229,13 @@ test_set_check() { local set_check_json="$2" local passed=true + if ! command -v perl > /dev/null 2>&1; then + add_finding 'SAFETY' '' 0 \ + "[$rule_name] set_check requires perl (REQUIRED dependency: it is the only regex engine set_check compiles and extracts with) and perl was not found on PATH" + TEST_SET_CHECK_RESULT="false" + return + fi + local regex_text regex_text="$(echo "$set_check_json" | jq -r '.extract_regex // empty')" if [[ -z "$regex_text" ]]; then @@ -2238,15 +2245,37 @@ test_set_check() { return fi - # Validate regex compiles - if ! echo "" | grep -P "$regex_text" > /dev/null 2>&1; then - # grep -P returns 1 for no match but 2 for bad regex; test specifically - if echo "" | grep -P "$regex_text" 2>&1 | grep -qi 'error\|invalid\|unknown'; then - add_finding 'SAFETY' '' 0 \ - "[$rule_name] set_check.extract_regex did not compile" - TEST_SET_CHECK_RESULT="false" - return - fi + # The compile oracle is the extraction engine itself: the regex reaches + # perl as DATA through the environment and is compiled with qr//, exactly + # as the extractor below compiles it. + if ! RE="$regex_text" perl -e 'qr/$ENV{RE}/' 2>/dev/null; then + add_finding 'SAFETY' '' 0 \ + "[$rule_name] set_check.extract_regex did not compile" + TEST_SET_CHECK_RESULT="false" + return + fi + + # Independent capture-group validation: the compile oracle above only + # proves the regex compiles -- it says nothing about whether the regex + # captures a group. A capture-free regex that matches ZERO times in the + # extraction loop below never runs the `defined($1) or die` guard (the + # while-loop body never executes), so it would otherwise vacuously pass as + # a legitimate empty-capture zero-match result. Count the regex's own + # capture groups independently of whether it matches anything: wrap it in + # a synthetic always-matching alternation `(?:$re)|(?:)` against the empty + # string. Perl populates @+ with one slot per capture group defined + # ANYWHERE in the pattern (participating or not), regardless of which + # alternation branch actually matched, so `scalar(@+) - 1` reports the + # true group count without requiring the regex to match real content. + if ! RE="$regex_text" perl -e ' + my $re = qr/$ENV{RE}/; + "" =~ /(?:$re)|(?:)/; + exit(scalar(@+) - 1 > 0 ? 0 : 1); + ' 2>/dev/null; then + add_finding 'SAFETY' '' 0 \ + "[$rule_name] set_check.extract_regex has no capture group -- a capture-free regex that matches nothing would vacuously pass as an empty result" + TEST_SET_CHECK_RESULT="false" + return fi # Build expected set @@ -2281,22 +2310,37 @@ test_set_check() { local content content="$(<"$abs_path")" - # Extract capture group 1 matches; Perl is primary (handles PCRE regexes correctly). - # The fallback runs ONLY when perl is ABSENT -- an empty capture ('{}') from a - # SUCCESSFUL perl run is a legitimate zero-match result, not a missing interpreter. - # Each branch normalizes its own failure to '' (never `|| echo '{}'` INSIDE the - # substitution: the pipeline's last stage has already printed one document, so the - # echo APPENDS a second and yields invalid JSON for the --argjson below). - local captured_json - captured_json='' - if command -v perl > /dev/null 2>&1; then - captured_json="$(echo "$content" | perl -ne "while (/$regex_text/g) { print \"\$1\n\" }" 2>/dev/null | jq -R . | jq -s 'group_by(.) | map({key: .[0], value: length}) | from_entries' 2>/dev/null)" || captured_json='' - else - # Fallback: grep -oP + sed -E for environments without Perl - captured_json="$(echo "$content" | grep -oP "$regex_text" 2>/dev/null | sed -E "s/$regex_text/\1/" | jq -R . | jq -s 'group_by(.) | map({key: .[0], value: length}) | from_entries' 2>/dev/null)" || captured_json='' + # Extract capture group 1 of every match, line by line. The regex is + # passed to perl as DATA (the RE environment variable) and compiled + # with qr//, never spliced into program source, so a `/` in the regex + # cannot break the program. A match with no group-1 capture dies: a + # regex that cannot capture is a broken extraction, not zero matches. + # perl prints a trailing \x1f record mark so the command substitution + # cannot strip a final empty capture; the mark is removed below. + # INVARIANT: the capture must never swallow the extractor's status. It + # is read into extract_rc and tested explicitly; the `|| extract_rc=$?` + # form also suppresses errexit for this one command only. Rewriting it + # as `|| extract_out=''` (or defaulting an empty result to '{}') makes + # a failed extraction indistinguishable from a real zero-match result + # and lets a zero-count subset fixture pass vacuously. + local extract_out extract_rc jq_rc captured_json + extract_rc=0 + extract_out="$(RE="$regex_text" perl -ne 'BEGIN { $re = qr/$ENV{RE}/ } while (/$re/g) { defined($1) or die "no capture group\n"; print "$1\n" } END { print "\x1f" }' <<< "$content" 2>/dev/null)" || extract_rc=$? + if [[ "$extract_rc" -ne 0 ]]; then + passed=false + add_finding 'SAFETY' "$rel_path" 0 \ + "[$rule_name] set_check extraction FAILED for ${rel_path} (perl rc=$extract_rc): the extract_regex did not compile or matched without a group-1 capture -- a broken extraction is not a zero-match result" + continue fi - if [[ -z "$captured_json" ]]; then - captured_json='{}' + extract_out="${extract_out%$'\x1f'}" + + jq_rc=0 + captured_json="$(printf '%s' "$extract_out" | jq -R . | jq -s 'group_by(.) | map({key: .[0], value: length}) | from_entries')" || jq_rc=$? + if [[ "$jq_rc" -ne 0 || -z "$captured_json" ]]; then + passed=false + add_finding 'SAFETY' "$rel_path" 0 \ + "[$rule_name] set_check capture aggregation FAILED for ${rel_path} (jq rc=$jq_rc): no capture object was produced" + continue fi local captured_set @@ -2541,17 +2585,32 @@ fi # ── SAFETY-CANARY: set_check zero-match self-test ────────────────────────── # A files entry whose extract_regex matches NOTHING must still produce a valid -# empty capture object. No standing green fixture witnesses that branch: real -# fixtures always capture something, so a regression in the capture plumbing -# (an interpreter-fallback misfire that appends a second JSON document and -# makes the downstream --argjson reject) aborts the whole run instead of -# failing one check. Assertion 1 is the regression witness; because the -# regression ABORTS rather than returns false, test_set_check is called in a -# SUBSHELL and its verdict read back over stdout -- an abort kills only the -# subshell, empty output means FAIL, and the run continues to a summary. -# Subshell isolation also keeps the control assertion's findings out of the -# real report. Assertion 2 is the non-vacuity control: a deliberately wrong -# occurrence count must fail, proving assertion 1 asserts something. +# empty capture object, and a BROKEN extraction must never be mistaken for +# that zero-match result. No standing green fixture witnesses these branches: +# real fixtures always capture something. Each assertion calls test_set_check +# in a SUBSHELL and reads its verdict back over stdout, so a regression that +# ABORTS the call (e.g. malformed capture JSON rejected by the downstream +# --argjson) kills only the subshell, empty output means FAIL, and the run +# continues to a summary. Subshell isolation also keeps the must-fail +# assertions' findings out of the real report. +# 1. zero-match: a regex matching nothing must pass (the zero-match witness). +# 2. non-vacuity control: a deliberately wrong occurrence count must fail, +# proving assertion 1 asserts something. +# 3. non-compiling regex: must fail through the perl compile oracle. +# 4. broken extraction: a regex that compiles, HAS a capture group (so it +# clears the independent group count), and matches, but whose group 1 +# does not participate in the match must fail -- the extractor's +# `defined($1) or die` fires at run time, and that nonzero status must +# never collapse into an empty capture object that reads as a vacuous +# zero-match pass. This is the only assertion that reaches the extractor +# exit-status path; branch 6 is stopped earlier by the group count. +# 5. unescaped slash: a regex containing `/` must extract normally, proving +# the regex reaches perl as data rather than as program source. +# 6. capture-free zero-match: a regex with NO capture group that ALSO +# matches nothing must fail -- without an independent capture-group +# count, this exact combination never reaches the `defined($1) or die` +# guard (the while-loop body never runs) and would vacuously pass as the +# same clean empty result as branch 1. # INVARIANT: the capture must NOT be written as `out="$( ... )" || out=''` -- # bash disables errexit inside a command substitution that is part of an # AND-OR list, so the regression would be swallowed INSIDE the subshell and @@ -2583,7 +2642,7 @@ else if [[ "$set_check_zero_result" != 'true' ]]; then set_check_zero_canary_ok=false add_finding 'SAFETY-CANARY' "$SET_CHECK_ZERO_CANARY_REL" 0 \ - "set_check over a zero-match file did not return a clean pass (empty result = the call aborted the run; the grep fallback appended a second '{}' to a pipeline that already printed one -- see test_set_check in tools/policy_check.sh)" + "set_check over a zero-match file did not return a clean pass (empty result = the call aborted the run; a false result = the zero-match capture was not a valid empty object -- see test_set_check in tools/policy_check.sh)" fi set_check_control_spec="$(jq -n --arg path "$SET_CHECK_ZERO_CANARY_REL" '{ @@ -2606,6 +2665,96 @@ else add_finding 'SAFETY-CANARY' "$SET_CHECK_ZERO_CANARY_REL" 0 \ 'set_check control did not fail -- capture/count assertion is vacuous' fi + + set_check_uncompiled_spec="$(jq -n --arg path "$SET_CHECK_ZERO_CANARY_REL" '{ + extract_regex: "(SETCHECK-UNCLOSED", + expected_set: [], + expected_counts: {}, + files: [{path: $path, mode: "subset"}] + }')" + set_check_uncompiled_result='' + set +e + set_check_uncompiled_result="$( + set -e + TEST_SET_CHECK_RESULT='' + test_set_check 'set-check-zero-match-canary-uncompiled' "$set_check_uncompiled_spec" > /dev/null 2>&1 + echo "$TEST_SET_CHECK_RESULT" + )" + set -e + if [[ "$set_check_uncompiled_result" != 'false' ]]; then + set_check_zero_canary_ok=false + add_finding 'SAFETY-CANARY' "$SET_CHECK_ZERO_CANARY_REL" 0 \ + 'set_check accepted a non-compiling extract_regex -- the perl compile oracle is not failing closed' + fi + + set_check_nocapture_spec="$(jq -n --arg path "$SET_CHECK_ZERO_CANARY_REL" '{ + extract_regex: "SETCHECK-PRESENT-CANARY (x)?[0-9]:", + expected_set: [], + expected_counts: {}, + files: [{path: $path, mode: "subset"}] + }')" + set_check_nocapture_result='' + set +e + set_check_nocapture_result="$( + set -e + TEST_SET_CHECK_RESULT='' + test_set_check 'set-check-zero-match-canary-nocapture' "$set_check_nocapture_spec" > /dev/null 2>&1 + echo "$TEST_SET_CHECK_RESULT" + )" + set -e + if [[ "$set_check_nocapture_result" != 'false' ]]; then + set_check_zero_canary_ok=false + add_finding 'SAFETY-CANARY' "$SET_CHECK_ZERO_CANARY_REL" 0 \ + 'set_check passed a broken extraction (the regex has a capture group, but group 1 did not participate in a match, so the extractor died with a nonzero status) -- a failed extraction is being read as a zero-match result' + fi + + set_check_slash_spec="$(jq -n --arg path "$SET_CHECK_ZERO_CANARY_REL" '{ + extract_regex: "SETCHECK-SLASH-CANARY (a/[a-z]+):", + expected_set: ["a/b"], + expected_counts: {"a/b": 1}, + files: [{path: $path, mode: "equal"}] + }')" + set_check_slash_result='' + set +e + set_check_slash_result="$( + set -e + TEST_SET_CHECK_RESULT='' + test_set_check 'set-check-zero-match-canary-slash' "$set_check_slash_spec" 1>&2 + echo "$TEST_SET_CHECK_RESULT" + )" + set -e + if [[ "$set_check_slash_result" != 'true' ]]; then + set_check_zero_canary_ok=false + add_finding 'SAFETY-CANARY' "$SET_CHECK_ZERO_CANARY_REL" 0 \ + 'set_check failed to extract with a regex containing an unescaped slash -- the regex is reaching perl as program source instead of data' + fi + + # Branch 6: a capture-free regex reusing the reserved zero-match token + # prefix (guaranteed to match nothing in this fixture per the file's own + # docstring) must still FAIL -- without the independent capture-group + # count above, this exact combination never runs the extraction loop's + # `defined($1) or die` guard and would vacuously pass as a clean + # zero-match result identical to branch 1. + set_check_nocapture_zero_match_spec="$(jq -n --arg path "$SET_CHECK_ZERO_CANARY_REL" '{ + extract_regex: "SETCHECK-ZERO-MATCH-CANARY [0-9]+:", + expected_set: [], + expected_counts: {}, + files: [{path: $path, mode: "subset"}] + }')" + set_check_nocapture_zero_match_result='' + set +e + set_check_nocapture_zero_match_result="$( + set -e + TEST_SET_CHECK_RESULT='' + test_set_check 'set-check-zero-match-canary-nocapture-zero-match' "$set_check_nocapture_zero_match_spec" > /dev/null 2>&1 + echo "$TEST_SET_CHECK_RESULT" + )" + set -e + if [[ "$set_check_nocapture_zero_match_result" != 'false' ]]; then + set_check_zero_canary_ok=false + add_finding 'SAFETY-CANARY' "$SET_CHECK_ZERO_CANARY_REL" 0 \ + 'set_check passed a capture-free extract_regex that matches nothing -- a capture-free regex is being vacuously accepted as a zero-match result' + fi fi if [[ "$set_check_zero_canary_ok" == true ]]; then echo '[PASS] SAFETY-CANARY: set_check returns a clean pass over a zero-match file'