diff --git a/CHANGELOG.md b/CHANGELOG.md index 03102ff..f9e4fe3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,8 @@ once it reaches a published 0.1.0 release. - [S-168] A colon-introduced list of bare-name choices sitting directly under a placeholder row — or a `+word`/`-word` pair sharing one — now becomes that placeholder's own choices instead of folding into the row's own description, so `mandible Xvfb`'s `+extension`/`-extension` pair shows its 19 run-time-toggleable extension names as choices on both halves. - [S-145] A single-dash-long table with no column padding at all now recovers a row's genuine bracketed value one space after its name, so `mandible Xvfb`'s `-render` and `-deferglyphs` keep their own choices instead of losing them outright. - [S-167] A `Usage:` line whose leading word carries a bracketed optional-abbreviation suffix (`v[ersion]`, `g[dbserver]`) now reaches the tree as its own subcommand, named and shown by its full word with the short prefix kept as an alias, and probed for its own flags, instead of folding into the usage block (`mandible lldb-server`). +- [S-169] A command table sitting directly under the root's own usage synopsis, rows never repeating the tool's own name, now recovers as subcommands carrying their own flags and operands instead of fabricated flag groups on the root, so `mandible dmsetup` and `mandible dmstats` show their full command lists. +- [S-170] A usage-synopsis bracket group whose own value spec is itself entirely option spellings no longer invents a value from it, so `mandible dmsetup`'s `-h, --help` and `-v, --verbose` show no value name instead of `-c|-C|--columns`/`-v|--verbose ...`. ## [0.7.0] - 2026-09-05 diff --git a/corpus/dmsetup/1.02.185/expected.snap b/corpus/dmsetup/1.02.185/expected.snap new file mode 100644 index 0000000..6f67946 --- /dev/null +++ b/corpus/dmsetup/1.02.185/expected.snap @@ -0,0 +1,789 @@ +name: dmsetup +usage: +- dmsetup [--version] [-h|--help [-c|-C|--columns]] [-v|--verbose [-v|--verbose ...]] [-f|--force] [--checks] [--manglename {none|hex|auto}] [-r|--readonly] [--noopencount] [--noflush] [--nolockfs] [--inactive] [--udevcookie ] [--noudevrules] [--noudevsync] [--verifyudev] [-y|--yes] [--readahead {[+]|auto|none}] [--retry] [-c|-C|--columns] [-o ] [-O|--sort ] [-S|--select ] [--nameprefixes] [--noheadings] [--separator ] +flags: +- spellings: + - --version + provenance: + sources: + - help-text-synopsis +- spellings: + - -h + - --help + provenance: + sources: + - help-text-synopsis +- spellings: + - -v + - --verbose + provenance: + sources: + - help-text-synopsis +- spellings: + - -f + - --force + provenance: + sources: + - help-text-synopsis +- spellings: + - --checks + provenance: + sources: + - help-text-synopsis +- spellings: + - --manglename + value_name: '{none|hex|auto}' + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - -r + - --readonly + provenance: + sources: + - help-text-synopsis +- spellings: + - --noopencount + provenance: + sources: + - help-text-synopsis +- spellings: + - --noflush + provenance: + sources: + - help-text-synopsis +- spellings: + - --nolockfs + provenance: + sources: + - help-text-synopsis +- spellings: + - --inactive + provenance: + sources: + - help-text-synopsis +- spellings: + - --udevcookie + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - --noudevrules + provenance: + sources: + - help-text-synopsis +- spellings: + - --noudevsync + provenance: + sources: + - help-text-synopsis +- spellings: + - --verifyudev + provenance: + sources: + - help-text-synopsis +- spellings: + - -y + - --yes + provenance: + sources: + - help-text-synopsis +- spellings: + - --readahead + value_name: '{[+]|auto|none}' + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - --retry + provenance: + sources: + - help-text-synopsis +- spellings: + - -c + provenance: + sources: + - help-text-synopsis +- spellings: + - -C + provenance: + sources: + - help-text-synopsis +- spellings: + - --columns + provenance: + sources: + - help-text-synopsis +- spellings: + - -o + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - -O + - --sort + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - -S + - --select + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - --nameprefixes + provenance: + sources: + - help-text-synopsis +- spellings: + - --noheadings + provenance: + sources: + - help-text-synopsis +- spellings: + - --separator + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +provenance: + sources: + - help-text + confidence: 0.27 +children_filled: true +subcommands: +- name: help + flags: + - spellings: + - -c + provenance: + sources: + - help-text-synopsis + - spellings: + - -C + provenance: + sources: + - help-text-synopsis + - spellings: + - --columns + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: create + positionals: + - name: dev_name + required: true + provenance: + sources: + - help-text + flags: + - spellings: + - -j + - --major + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - -U + - --uid + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - -G + - --gid + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - -M + - --mode + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - -u + - --uuid + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --addnodeonresume + provenance: + sources: + - help-text-synopsis + - spellings: + - --addnodeoncreate + provenance: + sources: + - help-text-synopsis + - spellings: + - --readahead + value_name: '{[+]|auto|none}' + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - -n + - --notable + - --table + value_name: '{|}' + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --concise + value_name: + value_kind: Optional + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: remove + flags: + - spellings: + - --deferred + provenance: + sources: + - help-text-synopsis + - spellings: + - -f + - --force + provenance: + sources: + - help-text-synopsis + - spellings: + - --retry + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: remove_all + flags: + - spellings: + - -f + - --force + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: suspend + flags: + - spellings: + - --noflush + provenance: + sources: + - help-text-synopsis + - spellings: + - --nolockfs + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: resume + flags: + - spellings: + - --noflush + provenance: + sources: + - help-text-synopsis + - spellings: + - --nolockfs + provenance: + sources: + - help-text-synopsis + - spellings: + - --addnodeonresume + - --addnodeoncreate + provenance: + sources: + - help-text-synopsis + - spellings: + - --readahead + value_name: '{[+]|auto|none}' + value_kind: Required + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: load + positionals: + - name: device + required: true + provenance: + sources: + - help-text + - name: table + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: clear + positionals: + - name: device + required: true + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: reload + positionals: + - name: device + required: true + provenance: + sources: + - help-text + - name: table + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: wipe_table + flags: + - spellings: + - -f + - --force + provenance: + sources: + - help-text-synopsis + - spellings: + - --noflush + provenance: + sources: + - help-text-synopsis + - spellings: + - --nolockfs + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: rename + positionals: + - name: device + required: true + provenance: + sources: + - help-text + flags: + - spellings: + - --setuuid + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: measure + positionals: + - name: device + variadic: true + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: message + positionals: + - name: device + required: true + provenance: + sources: + - help-text + - name: sector + required: true + provenance: + sources: + - help-text + - name: message + required: true + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: ls + flags: + - spellings: + - --target + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --exec + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - -o + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --tree + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: info + positionals: + - name: device + variadic: true + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: deps + positionals: + - name: device + variadic: true + provenance: + sources: + - help-text + flags: + - spellings: + - -o + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: stats + positionals: + - name: command + required: true + provenance: + sources: + - help-text + - name: device + variadic: true + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: status + positionals: + - name: device + variadic: true + provenance: + sources: + - help-text + flags: + - spellings: + - --noflush + provenance: + sources: + - help-text-synopsis + - spellings: + - --target + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: table + positionals: + - name: device + variadic: true + provenance: + sources: + - help-text + flags: + - spellings: + - --concise + provenance: + sources: + - help-text-synopsis + - spellings: + - --target + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --showkeys + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: wait + positionals: + - name: device + required: true + provenance: + sources: + - help-text + - name: event_nr + provenance: + sources: + - help-text + flags: + - spellings: + - --noflush + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: mknodes + positionals: + - name: device + variadic: true + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: mangle + positionals: + - name: device + variadic: true + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: udevcreatecookie + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: udevreleasecookie + positionals: + - name: cookie + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: udevflags + positionals: + - name: cookie + required: true + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: udevcomplete + positionals: + - name: cookie + required: true + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: udevcomplete_all + positionals: + - name: age_in_minutes + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: udevcookies + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: target-version + positionals: + - name: target + variadic: true + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: targets + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: version + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: setgeometry + positionals: + - name: device + required: true + provenance: + sources: + - help-text + - name: cyl + required: true + provenance: + sources: + - help-text + - name: head + required: true + provenance: + sources: + - help-text + - name: sect + required: true + provenance: + sources: + - help-text + - name: start + required: true + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: splitname + positionals: + - name: device + required: true + provenance: + sources: + - help-text + - name: subsystem + provenance: + sources: + - help-text + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true diff --git a/corpus/dmsetup/1.02.185/help.stderr.txt b/corpus/dmsetup/1.02.185/help.stderr.txt new file mode 100644 index 0000000..80633df --- /dev/null +++ b/corpus/dmsetup/1.02.185/help.stderr.txt @@ -0,0 +1,68 @@ +Usage: + +dmsetup + [--version] [-h|--help [-c|-C|--columns]] + [-v|--verbose [-v|--verbose ...]] [-f|--force] + [--checks] [--manglename {none|hex|auto}] + [-r|--readonly] [--noopencount] [--noflush] [--nolockfs] [--inactive] + [--udevcookie ] [--noudevrules] [--noudevsync] [--verifyudev] + [-y|--yes] [--readahead {[+]|auto|none}] [--retry] + [-c|-C|--columns] [-o ] [-O|--sort ] + [-S|--select ] [--nameprefixes] [--noheadings] + [--separator ] + + help [-c|-C|--columns] + create + [-j|--major -m|--minor ] + [-U|--uid ] [-G|--gid ] [-M|--mode ] + [-u|--uuid ] [--addnodeonresume|--addnodeoncreate] + [--readahead {[+]|auto|none}] + [-n|--notable|--table {
|}] + create --concise [] + remove [--deferred] [-f|--force] [--retry] ... + remove_all [-f|--force] + suspend [--noflush] [--nolockfs] ... + resume [--noflush] [--nolockfs] ... + [--addnodeonresume|--addnodeoncreate] + [--readahead {[+]|auto|none}] + load [
|] + clear + reload [
|] + wipe_table [-f|--force] [--noflush] [--nolockfs] ... + rename [--setuuid] + measure [...] + message + ls [--target ] [--exec ] [-o ] [--tree] + info [...] + deps [-o ] [...] + stats [] [...] + status [...] [--noflush] [--target ] + table [...] [--concise] [--target ] [--showkeys] + wait [] [--noflush] + mknodes [...] + mangle [...] + udevcreatecookie + udevreleasecookie [] + udevflags + udevcomplete + udevcomplete_all [] + udevcookies + target-version [...] + targets + version + setgeometry + splitname [] + + may be device name or (if only one) -u or -j -m + is one of 'none', 'auto' and 'hex'. + are comma-separated. Use 'help -c' for list. + has single-device entries separated by semi-colons: + ,,,,
+ where is 'ro' or 'rw' (the default) and any of , + and may be empty. Separate extra table lines with commas. + E.g.: dev1,,,,0 100 linear 253:1 0,100 100 error;dev2,,,ro,0 1 error +Table_file contents may be supplied on stdin. +Options are: devno, devname, blkdevname. +Tree specific options are: ascii, utf, vt100; compact, inverted, notrunc; + blkdevname, [no]device, active, open, rw and uuid. + diff --git a/corpus/dmsetup/1.02.185/help.txt b/corpus/dmsetup/1.02.185/help.txt new file mode 100644 index 0000000..e69de29 diff --git a/corpus/dmsetup/1.02.185/meta.toml b/corpus/dmsetup/1.02.185/meta.toml new file mode 100644 index 0000000..f9f56d3 --- /dev/null +++ b/corpus/dmsetup/1.02.185/meta.toml @@ -0,0 +1,66 @@ +# dmsetup's `Usage:` block is two shapes at once (docs/shapes.md S-169, +# S-170): a bare `Usage:` label, a blank line, then the program's own +# name alone on its line with its global flags on continuation lines +# (`[--version] [-h|--help [-c|-C|--columns]]`, ...), a blank line, then +# a tab-indented table of one line per command — `help`, `create`, +# `remove`, ... — each with its own flags/operands, never repeating +# `dmsetup`'s own name. +# +# Before the fix, `scan_usage_section` ended the usage block at the first +# blank line after the bare `Usage:` label, so the root's own flags and +# the whole command table fell through to the generic body scanner, +# which read the tab-indented command lines as fabricated group labels +# (`Create `) — S-169. Every command word here is now +# recovered as its own `invocation_attested` (never `heading_attested`) +# subcommand, carrying its own flags/operands, with only the root's own +# global-flags block kept as `root.usage`. +# +# `[-h|--help [-c|-C|--columns]]` and `[-v|--verbose [-v|--verbose +# ...]]` also fed the parser's ordinary `try_value` grammar a bracket +# group whose entire content is itself option spellings (S-170) — before +# the fix, `--help`/`--verbose` carried an invented value +# (`-c|-C|--columns`, `-v|--verbose ...`). Neither carries a value now. +# +# Residual, pre-existing and out of scope for S-169/S-170: `create`'s own +# row glues two flag-alternations with a bare space and no separating +# pipe or bracket (`-j|--major -m|--minor `) — a shape +# `extract_usage_flags` has never handled at the usage-line level. `-j`/ +# `--major` recovers correctly; `-m`/`--minor` does not yet. This was +# already true of the shared engine before this round; recovering +# `create` as a real subcommand at all is what newly exposes it here. +# Not asserted below. + +[bless] +provenance = "agent" + +[tool] +name = "dmsetup" +version = "1.02.185" +platform = "ubuntu-24.04" +captured_with = "mandible 0.7.0" + +[[capture]] +argv = ["dmsetup", "--help"] +stdout = "help.txt" +stderr = "help.stderr.txt" + +[contract] +expected_framework = "generic" +min_status = "ok" +min_subcommands = 30 +must_contain_flags = ["--version", "--force", "--readonly"] +verdict_scope = ["flags", "subcommands"] + +# S-170: neither carries the nested bracket group of flag spellings as +# an invented value. +[contract.must_not_value_name] +"--help" = "-c" +"--verbose" = "-v" + +[contract.must_contain_flags_by_path] +create = ["--major", "--uuid"] +remove = ["--deferred", "--force", "--retry"] + +# An ordinary root flag keeps its real value, unaffected by the S-170 fix. +[contract.must_value_name] +"--separator" = "separator" diff --git a/corpus/dmstats/1.02.185/expected.snap b/corpus/dmstats/1.02.185/expected.snap new file mode 100644 index 0000000..282daf2 --- /dev/null +++ b/corpus/dmstats/1.02.185/expected.snap @@ -0,0 +1,532 @@ +name: dmstats +usage: +- dmstats [-h|--help] [-v|--verbose [-v|--verbose ...]] [--areas ] [--areasize ] [--userdata ] [--clear] [--count ] [--interval ] [-o ] [-O|--sort ] [--programid ] [--start ] [--length ] [--segments] [--units ] +flags: +- spellings: + - -h + - --help + provenance: + sources: + - help-text-synopsis +- spellings: + - -v + - --verbose + provenance: + sources: + - help-text-synopsis +- spellings: + - --areas + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - --areasize + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - --userdata + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - --clear + provenance: + sources: + - help-text-synopsis +- spellings: + - --count + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - --interval + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - -o + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - -O + - --sort + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - --programid + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - --start + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - --length + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +- spellings: + - --segments + provenance: + sources: + - help-text-synopsis +- spellings: + - --units + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis +provenance: + sources: + - help-text + confidence: 0.5 +children_filled: true +subcommands: +- name: help + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: clear + flags: + - spellings: + - --allregions + provenance: + sources: + - help-text-synopsis + - spellings: + - --regionid + value_name: id + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --alldevices + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: create + flags: + - spellings: + - --length + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --areas + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --areasize + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --programid + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --userdata + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --bounds + value_name: histogram_boundaries + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --precise + provenance: + sources: + - help-text-synopsis + - spellings: + - --segments + provenance: + sources: + - help-text-synopsis + - spellings: + - --alldevices + provenance: + sources: + - help-text-synopsis + - spellings: + - --filemap + provenance: + sources: + - help-text-synopsis + - spellings: + - --nogroup + provenance: + sources: + - help-text-synopsis + - spellings: + - --nomonitor + provenance: + sources: + - help-text-synopsis + - spellings: + - --follow + value_name: mode + value_kind: Required + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: delete + flags: + - spellings: + - --allprograms + provenance: + sources: + - help-text-synopsis + - spellings: + - --programid + value_name: id + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --allregions + - --regionid + value_name: id + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --alldevices + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: group + flags: + - spellings: + - --alias + value_name: NAME + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --regions + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --allprograms + provenance: + sources: + - help-text-synopsis + - spellings: + - --programid + value_name: id + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --alldevices + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: list + flags: + - spellings: + - --allprograms + provenance: + sources: + - help-text-synopsis + - spellings: + - --programid + value_name: id + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --allregions + provenance: + sources: + - help-text-synopsis + - spellings: + - --regionid + value_name: id + value_kind: Required + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: print + flags: + - spellings: + - --clear + provenance: + sources: + - help-text-synopsis + - spellings: + - --allprograms + provenance: + sources: + - help-text-synopsis + - spellings: + - --programid + value_name: id + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --allregions + - --regionid + value_name: id + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --alldevices + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: report + positionals: + - name: device + variadic: true + provenance: + sources: + - help-text + flags: + - spellings: + - --interval + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --count + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --units + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --programid + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --regionid + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - -o + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - -O + - --sort + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - -S + - --select + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --nameprefixes + provenance: + sources: + - help-text-synopsis + - spellings: + - --noheadings + provenance: + sources: + - help-text-synopsis + - spellings: + - --separator + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --allprograms + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: ungroup + flags: + - spellings: + - --groupid + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --allprograms + provenance: + sources: + - help-text-synopsis + - spellings: + - --programid + value_name: id + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --alldevices + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: update_filemap + positionals: + - name: file_path + required: true + provenance: + sources: + - help-text + flags: + - spellings: + - --groupid + value_name: + value_kind: Required + provenance: + sources: + - help-text-synopsis + - spellings: + - --nomonitor + provenance: + sources: + - help-text-synopsis + - spellings: + - --follow + value_name: mode + value_kind: Required + provenance: + sources: + - help-text-synopsis + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true +- name: version + provenance: + sources: + - help-text + children_filled: true + invocation_attested: true diff --git a/corpus/dmstats/1.02.185/help.stderr.txt b/corpus/dmstats/1.02.185/help.stderr.txt new file mode 100644 index 0000000..cf065d2 --- /dev/null +++ b/corpus/dmstats/1.02.185/help.stderr.txt @@ -0,0 +1,47 @@ +Usage: + +dmstats + [-h|--help] + [-v|--verbose [-v|--verbose ...]] + [--areas ] [--areasize ] + [--userdata ] [--clear] + [--count ] [--interval ] + [-o ] [-O|--sort ] + [--programid ] + [--start ] [--length ] + [--segments] [--units ] + + help + clear [--allregions|--regionid id] [--alldevices|...] + create [--start [--length ] + [--areas ] [--areasize ] + [--programid ] [--userdata ] + [--bounds histogram_boundaries] [--precise] + [--segments] [--alldevices|...] + create --filemap [--nogroup] [--nomonitor] [--follow mode] + [--programid ] [--userdata ] + [--bounds histogram_boundaries] [--precise] + delete [--allprograms|--programid id] + [--allregions|--regionid id] + [--alldevices|...] + group [--alias NAME] --regions + [--allprograms|--programid id] [--alldevices|...] + list [--allprograms|--programid id] [--allregions|--regionid id] + print [--clear] [--allprograms|--programid id] + [--allregions|--regionid id] + [--alldevices|...] + report [--interval ] [--count ] + [--units ] [--programid ] [--regionid ] + [-o ] [-O|--sort ] + [-S|--select ] [--nameprefixes] + [--noheadings] [--separator ] + [--allprograms|--programid id] [...] + ungroup --groupid [--allprograms|--programid id] + [--alldevices|...] + update_filemap --groupid + [--nomonitor] [--follow mode] + version + + may be device name or (if only one) -u or -j -m + are comma-separated. Use 'help -c' for list. + diff --git a/corpus/dmstats/1.02.185/help.txt b/corpus/dmstats/1.02.185/help.txt new file mode 100644 index 0000000..e69de29 diff --git a/corpus/dmstats/1.02.185/meta.toml b/corpus/dmstats/1.02.185/meta.toml new file mode 100644 index 0000000..292bc59 --- /dev/null +++ b/corpus/dmstats/1.02.185/meta.toml @@ -0,0 +1,32 @@ +# dmstats shares dmsetup's own shape (docs/shapes.md S-169): a bare +# `Usage:` label, a blank line, the program's own name alone on its +# line with its global flags on continuation lines, a blank line, then +# a tab-indented table of one line per command (`help`, `clear`, +# `create`, `delete`, `group`, `list`, `print`, `report`, `ungroup`, ...), +# never repeating `dmstats`'s own name. See `corpus/dmsetup/1.02.185` for +# the shared history. + +[bless] +provenance = "agent" + +[tool] +name = "dmstats" +version = "1.02.185" +platform = "ubuntu-24.04" +captured_with = "mandible 0.7.0" + +[[capture]] +argv = ["dmstats", "--help"] +stdout = "help.txt" +stderr = "help.stderr.txt" + +[contract] +expected_framework = "generic" +min_status = "ok" +min_subcommands = 10 +must_contain_flags = ["--verbose", "--areas", "--interval"] +verdict_scope = ["flags", "subcommands"] + +[contract.must_contain_flags_by_path] +create = ["--areas", "--areasize", "--precise"] +delete = ["--allprograms", "--allregions"] diff --git a/docs/shapes.md b/docs/shapes.md index 6d63ddd..ff7a2fc 100644 --- a/docs/shapes.md +++ b/docs/shapes.md @@ -3595,6 +3595,101 @@ entry's `tools` field and nothing else. It does not get a new entry. nobody read. `Xvfb` is the one fixture fixed and corpus-pinned this round. 2026-09-13. +### S-169: a command table inside the usage block + +- id: S-169 +- looks like: | + Usage: + + dmsetup + [--version] [-h|--help [-c|-C|--columns]] + [-v|--verbose [-v|--verbose ...]] [-f|--force] + ... + + help [-c|-C|--columns] + create + [-j|--major -m|--minor ] + ... + remove [--deferred] [-f|--force] [--retry] ... + ls [--target ] [--exec ] [--tree] + ... +- tools: dmsetup, dmstats; a fleet-wide raw-shape grep over + `audit/queue-captures` also named claude, cryptsetup, + dpkg-maintscript-helper, ethtool, fail2ban-client, ffplay, fwupdmgr, + fwupdtool, integritysetup, kernel-install, modprobe, + update-xmlcatalog, veritysetup (15 tools, a loose upper bound, not a + calibrated count) +- handling: Fixed. A bare `Usage:` label (nothing else on its line) + followed by a blank line and the tool's own name alone on its line, + heading its root invocation form, is now recognized as continuing the + *same* labelled usage block (`scan_usage_section`'s own + `bare_usage_label_only` gate, reusing the unlabelled-synopsis + continuation machinery via the new + `looks_like_bare_name_then_usage_fragment` predicate — the existing + `looks_like_stanza_continuation_head` requires a *single* self-closed + docopt bracket row on the next line, which a line carrying several + bracket groups at once is not). Before this, the block ended at that + first blank line, and the root's own flags plus the whole command + table fell through to the generic body scanner, which read the + tab-indented command lines as fabricated group labels + (`Create `). Once the root stanza is recovered this way + (`UsageScan::recovered_bare_root_stanza`), the very next block is + tried against `scan_headingless_usage_command_table` + (`help_text/sections/usage_command_table.rs`): one row per command, + never repeating the tool's own name (S-016's own shape, a table whose + rows *do* repeat it, is distinct and takes priority), each row's own + flags/operands recovered from its own line and any deeper-indented + continuation via the existing `extract_usage_flags`/ + `extract_positionals`. A repeated command word (`create ` / + `create --concise ...`) folds into one node, its second invocation + form's flags merged in. Every emitted node is `invocation_attested: + true`, `heading_attested: false` — a usage block is not a heading, so + spec §6 rule 0's second gate keeps declining to probe these words, + proven by the corpus fixtures' snapshots carrying no + `heading_attested` field at all. Gated on `recovered_bare_root_stanza` + specifically, not merely "a labelled usage block existed somewhere": + an ordinary `Usage: prog [opts]` tool whose body happens to start at + the same indent as a real table (`ar`'s modifier tables) must never be + mistaken for this shape, caught by a regression this fix's own + break-it check reproduced and fixed before shipping. +- fleet: `usage-command-table` + (`xtask/src/detector/usage_command_table.rs`), self-checks held both + directions (fires on the pre-fix shape, silent once every row is a + real subcommand); no seed-labelled tool carries this shape, so + `Detector::family` returns `None`. A calibrated fleet-wide count + needs a full-`PATH` sweep, not run as part of this fix. + +### S-170: a bracket group of flags is not a value + +- id: S-170 +- looks like: | + [-h|--help [-c|-C|--columns]] + [-v|--verbose [-v|--verbose ...]] +- tools: dmsetup; a fleet-wide raw-shape grep over `audit/queue-captures` + named 186 tools, certainly over-matching (any bracketed alternation + glued to another bracket, most of them real values) — not a calibrated + count +- handling: Fixed. `help_text::grammar::try_value`'s bracketed-value arm + now refuses to fold a `[VALUE]` group into a value at all when + `VALUE`'s own `|`-separated members are entirely option spellings + (`-c`, `-C`, `--columns`) or a bare ellipsis word — `try_value` returns + `None` for that case, so the flag carries no value and nothing is + invented in its place, exactly as an unrecognized value spec already + does elsewhere in this function. Requires a genuine `|` alternation: + a lone dash-led word inside one bracket with no alternation at all + (nvim's own `--remote[-subcommand]`) is a real, if oddly spelled, + value placeholder and must keep it — caught by this fix's own + regression run against the existing corpus before shipping (`nvim` + briefly lost `--remote`'s value during development) and now asserted + by `bracket_group_is_pure_flag_alternation`'s own `content.contains('|')` + gate. One shared engine, so every framework gets the fix at once (spec + §7 Tier B rule 1). +- fleet: `nested-flag-group-as-value` + (`xtask/src/detector/nested_flag_group_as_value.rs`), self-checks held + both directions; no seed-labelled tool carries this shape, so + `Detector::family` returns `None`. A calibrated fleet-wide count needs + a full-`PATH` sweep, not run as part of this fix. + ### S-171: a numbered `X1 X2 ...` pair sits ahead of a later required operand - id: S-171 diff --git a/mandible-extract/src/help_text/grammar.rs b/mandible-extract/src/help_text/grammar.rs index 8fb6024..3ac6548 100644 --- a/mandible-extract/src/help_text/grammar.rs +++ b/mandible-extract/src/help_text/grammar.rs @@ -842,6 +842,56 @@ fn foldable_value(value: &str) -> bool { !value.contains('[') && value.chars().any(|c| c.is_ascii_alphanumeric()) } +/// True when `tok` is itself shaped like an option spelling (`-c`, +/// `--columns`) or a bare ellipsis (`...`) — the two things a nested +/// optional group inside a bracketed value spec is ever built from. See +/// docs/shapes.md S-170. +fn is_flag_spelling_or_ellipsis(tok: &str) -> bool { + let tok = tok.trim(); + if tok.is_empty() { + return false; + } + if tok.chars().all(|c| c == '.') { + return true; + } + let stripped = tok.trim_start_matches('-'); + stripped.len() != tok.len() + && !stripped.is_empty() + && stripped + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-') +} + +/// True when a bracket group's content is entirely option spellings +/// (optionally repeated via a bare ellipsis word), split on `|` — the +/// shape `-h|--help [-c|-C|--columns]` names for its own inner group, and +/// `-v|--verbose [-v|--verbose ...]` for its own. Neither is a value: it is +/// a nested optional group of flags, and folding it into `value_name` +/// would invent a value that names no real placeholder. See +/// docs/shapes.md S-170. +fn bracket_group_is_pure_flag_alternation(content: &str) -> bool { + // Requires a real `|` alternation, not merely a lone dash-led word: + // nvim's own `--remote[-subcommand]` names a real, if oddly spelled, + // value placeholder — one member, no alternation at all — and must + // keep its value. See docs/shapes.md S-170. + if !content.contains('|') { + return false; + } + let parts: Vec<&str> = content + .split('|') + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect(); + parts.len() > 1 + && parts.iter().all(|part| { + let mut words = part.split_whitespace(); + let Some(first) = words.next() else { + return false; + }; + is_flag_spelling_or_ellipsis(first) && words.all(is_flag_spelling_or_ellipsis) + }) +} + fn try_value(input: &str) -> Option<(String, ValueKind, &str)> { let mut s = input; @@ -871,6 +921,14 @@ fn try_value(input: &str) -> Option<(String, ValueKind, &str)> { } let name = value_inside_brackets(&mut s).ok()?; close_bracket(&mut s).ok()?; + // A bracket group whose every `|`-separated member is itself an + // option spelling (or a bare ellipsis) is a nested optional group + // of flags, never a value — `dmsetup --help`'s + // `[-h|--help [-c|-C|--columns]]` names no value for `--help`. See + // docs/shapes.md S-170. + if bracket_group_is_pure_flag_alternation(name) { + return None; + } let mut combined = name.to_string(); let mut current = name; let mut folded_any = false; diff --git a/mandible-extract/src/help_text/sections/mod.rs b/mandible-extract/src/help_text/sections/mod.rs index 9b40eb0..43fcbab 100644 --- a/mandible-extract/src/help_text/sections/mod.rs +++ b/mandible-extract/src/help_text/sections/mod.rs @@ -46,6 +46,7 @@ mod spelling; #[cfg(test)] mod test_support; mod usage; +mod usage_command_table; mod usage_optional_word; use backfill::*; @@ -63,6 +64,7 @@ use spelling::*; #[cfg(test)] use test_support::*; use usage::*; +use usage_command_table::*; pub use usage_optional_word::reconstruct_abbrev_word; use usage_optional_word::scan_usage_optional_word_table; @@ -369,6 +371,15 @@ struct UsageScan { next_index: usize, entries: Vec, positionals: Vec, + /// True when a bare `Usage:` label (nothing else on its own line) was + /// followed by the tool's own name-only line opening its root + /// invocation form (dmsetup's shape) — the positional evidence + /// [`scan_headingless_usage_command_table`] requires, so that scan is + /// only ever tried for documents that actually have this idiom, never + /// for an ordinary `Usage: prog [opts]` tool whose body happens to + /// start at the same indent as a table row (`ar`'s modifier tables). + /// See docs/shapes.md S-169. + recovered_bare_root_stanza: bool, } /// Walk the usage block starting at `start`, folding wrapped continuation @@ -486,6 +497,121 @@ fn record_usage_line( line_entry_index.push(usage_entries.len() - 1); } +/// True when `usage_lines` names nothing yet but the bare `Usage:` marker +/// itself — the precondition for [`stanza_continuation_head`]'s own +/// `bare_usage_label_only` gate. See docs/shapes.md S-169. +fn usage_label_names_nothing_yet( + labelled_usage_start: Option, + usage_lines: &[String], +) -> bool { + labelled_usage_start.is_some() + && usage_lines.len() == 1 + && usage_lines[0] + .trim_end_matches(':') + .eq_ignore_ascii_case("usage") +} + +/// `lines[j]` continues the open usage stanza: either the ordinary +/// multi-stanza shape ([`looks_like_stanza_continuation_head`]), or — only +/// when `bare_usage_label_only` — the dmsetup idiom +/// ([`looks_like_bare_name_then_usage_fragment`]). See docs/shapes.md +/// S-169. +fn stanza_continuation_head( + lines: &[&str], + j: usize, + name: &str, + bare_usage_label_only: bool, +) -> bool { + j < lines.len() + && (looks_like_stanza_continuation_head(lines, j, name) + || (bare_usage_label_only && looks_like_bare_name_then_usage_fragment(lines, j, name))) +} + +/// A blank line inside a usage block ends it, except for the one +/// multi-stanza shape below. Returns the index to resume at, plus whether +/// the resumed stanza was pulled in under a bare `Usage:` label. `None` +/// means the block really has ended. Split out of [`scan_usage_section`] +/// to keep that function under the size ceiling. +fn continue_usage_block_across_blank( + lines: &[&str], + i: usize, + labelled_usage_start: Option, + tool_name: Option<&str>, + usage_lines: &mut Vec, + usage_entries: &mut Vec, + line_entry_index: &mut Vec, +) -> Option<(usize, bool)> { + let bare_usage_label_only = usage_label_names_nothing_yet(labelled_usage_start, usage_lines); + if labelled_usage_start.is_none() || bare_usage_label_only { + if let Some(name) = tool_name { + let mut j = i + 1; + // Deliberately not `looks_like_unlabeled_synopsis_line` + // here: that test alone would also admit corepack's + // headingless invocation-table rows (`corepack + // enable [--install-directory #0] ...`), demoting a + // real subcommand into fabricated usage text. See + // S-016. + let is_head = |lines: &[&str], j: usize| { + stanza_continuation_head(lines, j, name, bare_usage_label_only) + }; + if !is_head(lines, j) { + if let Some(next) = lines.get(j) { + let t = next.trim_start(); + if !t.is_empty() && is_prose_sentence(t) { + j += 1; + } + } + } + if is_head(lines, j) { + let trimmed = lines[j].trim().to_string(); + usage_lines.push(trimmed.clone()); + usage_entries.push(trimmed); + line_entry_index.push(usage_entries.len() - 1); + return Some((j + 1, bare_usage_label_only)); + } + } + } + None +} + +/// Where the block resumes past a bare usage label's own blank gap, and +/// whether a stanza was pulled in there. The stanza recognizer gets first +/// refusal: under a bare `Usage:` the line past the gap may be the tool's +/// own root invocation form, which is a stanza to record rather than a gap +/// to step over (`dmsetup`, `dmstats`, S-169). Otherwise the gap is only +/// skipped, since a bare label contributed no content to lose +/// (`perlthanks`'s `Advanced usage:`, S-151). +fn resume_after_bare_label( + lines: &[&str], + mut i: usize, + labelled_usage_start: Option, + tool_name: Option<&str>, + usage_lines: &mut Vec, + usage_entries: &mut Vec, + line_entry_index: &mut Vec, +) -> (usize, bool) { + // Only a real gap is the recognizer's business: `fdisk` writes its + // forms directly under its bare label, and reading past the first one + // would drop it. + if lines.get(i).is_some_and(|l| l.trim().is_empty()) { + if let Some((next, recovered)) = continue_usage_block_across_blank( + lines, + i, + labelled_usage_start, + tool_name, + usage_lines, + usage_entries, + line_entry_index, + ) { + return (next, recovered); + } + } + while i < lines.len() && lines[i].trim().is_empty() { + i += 1; + } + (i, false) +} + fn scan_usage_section( lines: &[&str], start: usize, @@ -519,16 +645,21 @@ fn scan_usage_section( // start a fresh usage entry even when it carries no marker or own-name // evidence of its own. See `is_bare_or_form_separator`. let mut force_new_entry_after_separator = false; + // See [`UsageScan::recovered_bare_root_stanza`]. + let mut recovered_bare_root_stanza = false; i += 1; if seed_is_bare { - // A bare label may sit on its own physical line with its forms a - // blank line further down (`perlthanks`'s `Advanced usage:`, a - // blank line, then its two forms) — skip past the gap rather than - // ending the block on it, since a bare label already contributed - // no content to lose. See S-151, corpus/perlthanks. - while i < lines.len() && lines[i].trim().is_empty() { - i += 1; - } + let (next, recovered) = resume_after_bare_label( + lines, + i, + labelled_usage_start, + tool_name, + usage_lines, + &mut usage_entries, + &mut line_entry_index, + ); + recovered_bare_root_stanza = recovered; + i = next; } while i < lines.len() { let l = lines[i]; @@ -574,35 +705,28 @@ fn scan_usage_section( // as a full sentence — the stanza's own description, // consumed here so it lands in neither the synopsis nor // the tool's description. See S-005. - if labelled_usage_start.is_none() { - if let Some(name) = tool_name { - let mut j = i + 1; - // Deliberately not `looks_like_unlabeled_synopsis_line` - // here: that test alone would also admit corepack's - // headingless invocation-table rows (`corepack - // enable [--install-directory #0] ...`), demoting a - // real subcommand into fabricated usage text. See - // S-016. - let is_head = |lines: &[&str], j: usize| { - j < lines.len() && looks_like_stanza_continuation_head(lines, j, name) - }; - if !is_head(lines, j) { - if let Some(next) = lines.get(j) { - let t = next.trim_start(); - if !t.is_empty() && is_prose_sentence(t) { - j += 1; - } - } - } - if is_head(lines, j) { - let trimmed = lines[j].trim().to_string(); - usage_lines.push(trimmed.clone()); - usage_entries.push(trimmed); - line_entry_index.push(usage_entries.len() - 1); - i = j + 1; - continue; - } + // dmsetup's own shape: a bare `Usage:` label with nothing else + // on its line, a blank line, then the tool's own name heading + // its root invocation form — the identical stanza-continuation + // shape the unlabelled path already recognizes, just labelled. + // Scoped to the label having named nothing yet: once a real + // stanza is pulled in this way, further blank lines fall back + // to the ordinary labelled rule below. See docs/shapes.md + // S-169. + if let Some((next, recovered)) = continue_usage_block_across_blank( + lines, + i, + labelled_usage_start, + tool_name, + usage_lines, + &mut usage_entries, + &mut line_entry_index, + ) { + if recovered { + recovered_bare_root_stanza = true; } + i = next; + continue; } break; } @@ -769,6 +893,7 @@ fn scan_usage_section( next_index: i, positionals: extract_positionals(usage_lines, primary_lines), entries: usage_entries, + recovered_bare_root_stanza, } } @@ -1948,6 +2073,27 @@ fn parse_body( i = scan.next_index; result.positionals = scan.positionals; result.usage = scan.entries; + // A command table sitting directly under the root's own labelled + // usage block, one line per command, rows never repeating the + // tool's own name (`dmsetup`'s second block). Gated on + // `recovered_bare_root_stanza`, not merely a labelled block, so an + // ordinary `Usage: prog [opts]` tool whose body starts at the same + // indent as a real table (`ar`'s modifier tables) is never + // mistaken for this shape — the position right after the tool's + // own *recovered root stanza* is the evidence, not "some usage + // block existed somewhere". Every emitted node is + // invocation_attested, never heading_attested: a usage block is + // not a heading. See docs/shapes.md S-169. + if scan.recovered_bare_root_stanza { + if let Some((end, nodes)) = + scan_headingless_usage_command_table(&lines, i, tool_name, raw) + { + i = end; + for node in nodes { + result.try_push_subcommand(node); + } + } + } // A block right under the usage line naming each positional's own // description (`invoke-rc.d`'s `basename - Initscript ID...`). // Consumed only when every row matches a positional this usage diff --git a/mandible-extract/src/help_text/sections/usage.rs b/mandible-extract/src/help_text/sections/usage.rs index d976378..190f70b 100644 --- a/mandible-extract/src/help_text/sections/usage.rs +++ b/mandible-extract/src/help_text/sections/usage.rs @@ -20,6 +20,33 @@ pub(super) fn looks_like_bare_synopsis_head(lines: &[&str], idx: usize, name: &s }) } +/// True if `lines[idx]` is the tool's own bare name and *nothing else* — +/// dmsetup's root `Usage:` block writes the program name alone on one +/// line, then a multi-bracket-group synopsis on however many continuation +/// lines follow. [`looks_like_bare_synopsis_head`]'s own next-row test +/// ([`looks_like_bracket_flag_row`]/[`looks_like_paren_alternation_open`]) +/// requires a *single* self-closed docopt row, which a line carrying +/// several bracket groups at once (`[--version] [-h|--help [...]]`) is +/// not; this widens the next-row evidence to any [`looks_like_usage_fragment`] +/// opener (`[`, `<`, `{`) — still unambiguous synopsis notation, never +/// prose. See docs/shapes.md S-169. +pub(super) fn looks_like_bare_name_then_usage_fragment( + lines: &[&str], + idx: usize, + name: &str, +) -> bool { + let t = lines[idx].trim_start(); + let Some(rest) = t.strip_prefix(name) else { + return false; + }; + if !rest.trim().is_empty() { + return false; + } + lines + .get(idx + 1) + .is_some_and(|next| looks_like_usage_fragment(next.trim_start())) +} + /// True if `lines[idx]` continues an already-open unlabelled synopsis into /// a **later stanza**: a line opening with the tool's own name whose /// remainder either carries a bare flag token directly (`vgck diff --git a/mandible-extract/src/help_text/sections/usage_command_table.rs b/mandible-extract/src/help_text/sections/usage_command_table.rs new file mode 100644 index 0000000..684341e --- /dev/null +++ b/mandible-extract/src/help_text/sections/usage_command_table.rs @@ -0,0 +1,227 @@ +//! A command table sitting directly under the tool's own root `Usage:` +//! synopsis, one line per command, each carrying its own flags/operands — +//! `dmsetup --help`'s second block (`docs/shapes.md` S-169). Distinct from +//! [`super::scan::scan_headingless_invocation_table`] (S-016): that shape's +//! rows repeat the tool's own name (`btrfs balance start ...`); this one's +//! rows are the bare command word alone (`create `), so the +//! evidence that they name real commands is their position — directly +//! after the root's own usage block, never introduced by any heading — +//! rather than a repeated name. Split out of `usage.rs` to keep that file +//! under its own size ceiling (AGENTS.md §2). + +use super::*; + +/// Fewest command rows required before a run of lines is read as a real +/// per-command table rather than one stray line. Mirrors +/// [`super::scan::MIN_INVOCATION_TABLE_ROWS`]'s own reasoning. See +/// docs/shapes.md S-169. +const MIN_USAGE_COMMAND_TABLE_ROWS: usize = 2; + +/// Recognize the **headingless usage command table**: a run of rows +/// directly beneath the tool's own root `Usage:` block (already scanned +/// into `usage_lines`, `lines[start]` is the first line after it), each +/// opening with a bare, `is_command_name_shaped` word that is never the +/// tool's own name, and carrying its own flags/operands on that line plus +/// any deeper-indented continuation. A repeated command word +/// (`dmsetup`'s `create ` / `create --concise ...`) is a second +/// invocation form of the same command, folded into that command's own +/// entities rather than starting a new node. Every emitted node is +/// `invocation_attested: true`, `heading_attested: false` — a usage block +/// is not a heading, and spec §6 rule 0's second gate must keep declining +/// to probe these words. See docs/shapes.md S-169. +pub(super) fn scan_headingless_usage_command_table( + lines: &[&str], + start: usize, + tool_name: Option<&str>, + raw: &str, +) -> Option<(usize, Vec)> { + let mut i = start; + while i < lines.len() && lines[i].trim().is_empty() { + i += 1; + } + if i >= lines.len() { + return None; + } + let table_indent = leading_whitespace(lines[i]); + // A flush-left line here is prose or a heading, never this shape — + // the table always sits under the root's own indented flags block. + if table_indent == 0 { + return None; + } + + let mut nodes: Vec = Vec::new(); + let mut index_by_name: std::collections::HashMap = + std::collections::HashMap::new(); + let mut node_lines: Vec> = Vec::new(); + let mut current: Option = None; + let mut rows_seen = 0usize; + + while i < lines.len() { + let line = lines[i]; + if line.trim().is_empty() { + break; + } + let indent = leading_whitespace(line); + if indent < table_indent { + break; + } + let trimmed = line.trim(); + if indent == table_indent { + let word_end = trimmed.find(char::is_whitespace).unwrap_or(trimmed.len()); + let word = &trimmed[..word_end]; + if !is_command_name_shaped(word) || !token_occurs_literally(raw, word) { + break; + } + if tool_name.is_some_and(|name| word == name) { + // This is S-016's own shape instead; defer to it entirely. + return None; + } + if nodes.len() >= MAX_RECOVERED_ENTRIES { + break; + } + rows_seen += 1; + let idx = *index_by_name.entry(word.to_string()).or_insert_with(|| { + let mut node = CommandNode::new(word, Provenance::single(Source::HelpText)); + node.invocation_attested = true; + node.heading_attested = false; + node.children_filled = true; + nodes.push(node); + node_lines.push(Vec::new()); + nodes.len() - 1 + }); + current = Some(idx); + node_lines[idx].push(trimmed.to_string()); + } else { + // A continuation line, deeper than the table's own row indent: + // belongs to whichever row is currently open. + let Some(idx) = current else { break }; + node_lines[idx].push(trimmed.to_string()); + } + i += 1; + } + + if rows_seen < MIN_USAGE_COMMAND_TABLE_ROWS || nodes.is_empty() { + return None; + } + + for (idx, node) in nodes.iter_mut().enumerate() { + let own_lines = &node_lines[idx]; + let mut flags: Vec = Vec::new(); + for flag in extract_usage_flags(own_lines) { + if !flag_spelling_already_present(&flag, &flags) { + flags.push(flag); + } + } + let positionals = extract_positionals(own_lines, std::collections::HashSet::new()); + node.entities.extend(flags); + node.entities.extend(positionals); + } + + Some((i, nodes)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn find_subcommand<'a>(nodes: &'a [CommandNode], name: &str) -> &'a CommandNode { + nodes + .iter() + .find(|n| n.name == name) + .unwrap_or_else(|| panic!("{name} missing from {nodes:?}")) + } + + /// The dmsetup shape itself: a root usage block, a blank line, then a + /// tab-indented command table whose rows never repeat `dmsetup`'s own + /// name. See docs/shapes.md S-169. + #[test] + fn recovers_commands_never_naming_the_tool_with_their_own_flags() { + let raw = "Usage:\n\ + \n\ + dmsetup\n \ + [--version] [-h|--help [-c|-C|--columns]]\n \ + [-v|--verbose [-v|--verbose ...]] [-f|--force]\n\ + \n\ + \thelp [-c|-C|--columns]\n\ + \tcreate \n\ + \t [-j|--major -m|--minor ]\n\ + \tremove [--deferred] [-f|--force] [--retry] ...\n\ + \tremove_all [-f|--force]\n"; + let parsed = parse_named(raw, "dmsetup"); + + assert!( + parsed.flags.iter().all(|f| f.long() != Some("major")), + "create's own -j/--major must not leak onto the root: {:?}", + parsed.flags + ); + + let create = find_subcommand(&parsed.subcommands, "create"); + assert!(create.invocation_attested); + assert!(!create.heading_attested, "a usage block is not a heading"); + let major = create + .flags() + .find(|f| f.long() == Some("major")) + .expect("create's own -j/--major recovered"); + assert_eq!(major.short(), Some('j')); + + let remove = find_subcommand(&parsed.subcommands, "remove"); + assert!(remove.flags().any(|f| f.long() == Some("force"))); + assert!(remove.invocation_attested && !remove.heading_attested); + + assert!(find_subcommand(&parsed.subcommands, "remove_all").invocation_attested); + } + + /// `create ` / `create --concise ...` are two invocation + /// forms of the one command `create`, never two nodes. + #[test] + fn a_repeated_command_word_folds_into_one_node() { + let raw = "Usage:\n\ + \n\ + dmsetup\n \ + [--version]\n\ + \n\ + \tcreate \n\ + \t [-j|--major -m|--minor ]\n\ + \tcreate --concise []\n\ + \tremove [--deferred] [-f|--force] ...\n"; + let parsed = parse_named(raw, "dmsetup"); + let creates: Vec<_> = parsed + .subcommands + .iter() + .filter(|n| n.name == "create") + .collect(); + assert_eq!(creates.len(), 1, "must fold to one node: {:?}", creates); + assert!(creates[0].flags().any(|f| f.long() == Some("concise"))); + assert!(creates[0].flags().any(|f| f.long() == Some("major"))); + } + + /// A table whose rows repeat the tool's own name is S-016's shape, not + /// this one — refuse outright so the caller can try that recognizer. + #[test] + fn refuses_when_rows_repeat_the_tools_own_name() { + let raw = "Usage:\n\ + \n\ + btrfs\n \ + [--version]\n\ + \n\ + \tbtrfs balance start \n\ + \tbtrfs balance pause \n"; + let lines: Vec<&str> = raw.lines().collect(); + // Lines 0..=3 are the usage block; the table starts at the blank + // line, index 4. + assert!(scan_headingless_usage_command_table(&lines, 4, Some("btrfs"), raw).is_none()); + } + + /// One row alone, below the floor, must not be promoted. + #[test] + fn refuses_a_single_row() { + let raw = "Usage:\n\ + \n\ + dmsetup\n \ + [--version]\n\ + \n\ + \thelp [-c|-C|--columns]\n"; + let lines: Vec<&str> = raw.lines().collect(); + assert!(scan_headingless_usage_command_table(&lines, 4, Some("dmsetup"), raw).is_none()); + } +} diff --git a/xtask/src/detector/mod.rs b/xtask/src/detector/mod.rs index 13bc17b..4be7454 100644 --- a/xtask/src/detector/mod.rs +++ b/xtask/src/detector/mod.rs @@ -128,6 +128,10 @@ pub(crate) mod header_declared_env_column; // list under a `+word`/`-word` placeholder pair), same direct-`Detector`- // impl shape. pub(crate) mod choice_list_under_placeholder; +// Round-11 family detectors (queue item 2: a command table inside the +// usage block, atlas S-169 and S-170), same direct-`Detector`-impl shape. +pub(crate) mod nested_flag_group_as_value; +pub(crate) mod usage_command_table; pub(crate) use calibration::*; pub(crate) use commands::*; @@ -792,6 +796,8 @@ pub fn registry() -> Vec> { Box::new(header_declared_env_column::HeaderDeclaredEnvColumn), Box::new(choice_list_under_placeholder::ChoiceListUnderPlaceholder), Box::new(UsageOptionalWordTable), + Box::new(usage_command_table::UsageCommandTable), + Box::new(nested_flag_group_as_value::NestedFlagGroupAsValue), ] } diff --git a/xtask/src/detector/nested_flag_group_as_value.rs b/xtask/src/detector/nested_flag_group_as_value.rs new file mode 100644 index 0000000..d33d349 --- /dev/null +++ b/xtask/src/detector/nested_flag_group_as_value.rs @@ -0,0 +1,206 @@ +//! `nested-flag-group-as-value` (round 11, atlas S-170): a usage-synopsis +//! bracket group whose own value spec is itself entirely option spellings +//! (optionally an ellipsis) — `dmsetup --help`'s `[-h|--help +//! [-c|-C|--columns]]` and `[-v|--verbose [-v|--verbose ...]]` — is a +//! nested optional group of flags, never a value. Fires when a flag's own +//! `value_name` still reads that way (the defect itself); the fix +//! (`help_text::grammar::try_value`, shared by every framework) refuses to +//! fabricate that value at all, so a repaired tree carries no such +//! `value_name` and this goes silent. + +use crate::detector::{Detector, Expect, Scope, SelfCheck, ToolEvidence}; +use mandible_core::CommandNode; + +pub struct Finding { + pub spelling: String, + pub value: String, +} + +pub struct Report { + pub findings: Vec, +} + +/// Same shape test `mandible_extract::help_text::grammar` uses at the fix +/// site, re-derived locally (this crate cannot depend on +/// `mandible-extract`): an option spelling, dash-led with a name-shaped +/// tail, or a bare ellipsis word. +fn is_flag_spelling_or_ellipsis(tok: &str) -> bool { + let tok = tok.trim(); + if tok.is_empty() { + return false; + } + if tok.chars().all(|c| c == '.') { + return true; + } + let stripped = tok.trim_start_matches('-'); + stripped.len() != tok.len() + && !stripped.is_empty() + && stripped + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-') +} + +/// True when `value`, brackets stripped, is entirely `|`-separated option +/// spellings (each optionally followed by a bare ellipsis word) — never a +/// real value a reader could supply. +fn value_is_pure_flag_alternation(value: &str) -> bool { + let content = value.trim().trim_start_matches('[').trim_end_matches(']'); + // Requires a real `|` alternation — a lone dash-led word (nvim's own + // `--remote`'s value `-subcommand`) is an oddly spelled but real + // value, never this shape. + if !content.contains('|') { + return false; + } + let parts: Vec<&str> = content + .split('|') + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect(); + parts.len() > 1 + && parts.iter().all(|part| { + let mut words = part.split_whitespace(); + let Some(first) = words.next() else { + return false; + }; + is_flag_spelling_or_ellipsis(first) && words.all(is_flag_spelling_or_ellipsis) + }) +} + +pub fn detect(raw: &str, root: &CommandNode) -> Report { + let mut findings = Vec::new(); + for flag in root.flags() { + let Some(value) = flag.value_name.as_deref() else { + continue; + }; + if !value_is_pure_flag_alternation(value) { + continue; + } + // A light existence check: the suspicious value text should occur + // literally in the tool's own raw output, the same discipline the + // existence oracle applies elsewhere (spec §13.1). + let bare = value.trim_matches(|c| c == '[' || c == ']'); + if !raw.contains(bare) { + continue; + } + let spelling = flag + .long() + .map(str::to_string) + .or_else(|| flag.short().map(|c| c.to_string())) + .unwrap_or_default(); + findings.push(Finding { + spelling, + value: value.to_string(), + }); + } + Report { findings } +} + +pub struct NestedFlagGroupAsValue; + +impl Detector for NestedFlagGroupAsValue { + fn name(&self) -> &'static str { + "nested-flag-group-as-value" + } + + fn family(&self) -> Option<&'static str> { + None + } + + fn describes(&self) -> &'static str { + "a flag whose own value_name is entirely `|`-separated option spellings (optionally an \ + ellipsis) — a nested optional group of flags a usage synopsis wrote, never a real value" + } + + fn hits(&self, evidence: &ToolEvidence<'_>) -> Vec { + detect(evidence.raw, evidence.root) + .findings + .iter() + .map(|f| { + format!( + "{} carries value_name {:?}, itself only flag spellings", + f.spelling, f.value + ) + }) + .collect() + } + + fn scope(&self) -> Scope { + Scope::full() + } + + fn self_checks(&self) -> Vec { + self_checks() + } +} + +// ---------------------------------------------------------------------- +// Self-checks +// ---------------------------------------------------------------------- + +use mandible_core::{Entity, Provenance, Source}; + +pub(crate) const DMSETUP_HELP_ROW: &str = "[--version] [-h|--help [-c|-C|--columns]]\n"; +pub(crate) const DMSETUP_VERBOSE_ROW: &str = "[-v|--verbose [-v|--verbose ...]] [-f|--force]\n"; + +fn flag_with_value(long: &str, short: Option, value_name: Option<&str>) -> Entity { + let mut e = Entity::flag_long(long, Provenance::single(Source::HelpText)); + if let Some(c) = short { + e.spellings.insert(0, mandible_core::Spelling::short(c)); + } + e.value_name = value_name.map(str::to_string); + e +} + +fn node_with_flags(name: &str, flags: Vec) -> CommandNode { + let mut root = CommandNode::new(name, Provenance::single(Source::HelpText)); + root.set_entities_of(mandible_core::EntityKind::Flag, flags); + root +} + +pub(crate) fn self_checks() -> Vec { + vec![ + SelfCheck { + name: "dmsetup's own bytes, --help carrying its own nested group as a value", + why: "the defect itself: -c|-C|--columns is a nested optional group of flags, never \ + --help's own value", + expect: Expect::Fires(1), + raw: DMSETUP_HELP_ROW.to_string(), + root: node_with_flags( + "dmsetup", + vec![flag_with_value("help", Some('h'), Some("-c|-C|--columns"))], + ), + }, + SelfCheck { + name: "dmsetup's own bytes, --help fixed to carry no value", + why: "once the fix refuses to fabricate the value, the same raw row must go silent", + expect: Expect::Silent, + raw: DMSETUP_HELP_ROW.to_string(), + root: node_with_flags("dmsetup", vec![flag_with_value("help", Some('h'), None)]), + }, + SelfCheck { + name: "dmsetup's own bytes, --verbose's ellipsis-carrying nested group", + why: "the same shape with a bare ellipsis word inside the nested group", + expect: Expect::Fires(1), + raw: DMSETUP_VERBOSE_ROW.to_string(), + root: node_with_flags( + "dmsetup", + vec![flag_with_value( + "verbose", + Some('v'), + Some("-v|--verbose ..."), + )], + ), + }, + SelfCheck { + name: "a real choice value, never flag-shaped", + why: "`--manglename {none|hex|auto}` names real choice words, not option spellings — \ + this rule must stay silent", + expect: Expect::Silent, + raw: "[--manglename {none|hex|auto}]\n".to_string(), + root: node_with_flags( + "dmsetup", + vec![flag_with_value("manglename", None, Some("none|hex|auto"))], + ), + }, + ] +} diff --git a/xtask/src/detector/usage_command_table.rs b/xtask/src/detector/usage_command_table.rs new file mode 100644 index 0000000..efb7269 --- /dev/null +++ b/xtask/src/detector/usage_command_table.rs @@ -0,0 +1,231 @@ +//! `usage-command-table` (round 11, atlas S-169): a command table sitting +//! directly under the tool's own root `Usage:` synopsis — a bare `Usage:` +//! label, a blank line, the tool's own name alone on its line, its own +//! bracketed global flags on continuation lines, a blank line, then one +//! row per command, rows never repeating the tool's own name +//! (`dmsetup --help`'s second block; `docs/shapes.md` S-169). Local, +//! independent shape check (no shared code with `mandible-extract`), the +//! same reasoning `invocation_form_head_as_flag_group` (S-137) uses for +//! its own local re-derivation. + +use crate::detector::{Detector, Expect, Scope, SelfCheck, ToolEvidence}; +use mandible_core::CommandNode; + +pub struct Finding { + pub name: String, +} + +pub struct Report { + pub findings: Vec, +} + +/// A command-word-shaped token: `^[a-z][a-z0-9_.-]*$`, spec §7 Tier B +/// rule 7's own candidate test, re-derived locally rather than imported — +/// this crate cannot depend on `mandible-extract`. +fn is_command_word_shaped(tok: &str) -> bool { + let mut chars = tok.chars(); + match chars.next() { + Some(c) if c.is_ascii_lowercase() => {} + _ => return false, + } + chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '_' | '.' | '-')) +} + +fn leading_whitespace(line: &str) -> usize { + let mut col = 0usize; + for c in line.chars() { + if c == '\t' { + col = (col / 8 + 1) * 8; + } else if c.is_whitespace() { + col += 1; + } else { + break; + } + } + col +} + +/// The distinct command words the raw text's own layout names, under +/// `docs/shapes.md` S-169's exact shape — or `None` when the document +/// does not open with it at all. +fn raw_command_table_rows(raw: &str, tool_name: &str) -> Option> { + let lines: Vec<&str> = raw.lines().collect(); + let mut i = lines + .iter() + .position(|l| l.trim().trim_end_matches(':').eq_ignore_ascii_case("usage"))?; + i += 1; + if !lines.get(i).is_some_and(|l| l.trim().is_empty()) { + return None; + } + i += 1; + if lines.get(i).map(|l| l.trim()) != Some(tool_name) { + return None; + } + i += 1; + // The root's own bracketed continuation, until the blank line that + // separates it from the command table. + while lines.get(i).is_some_and(|l| !l.trim().is_empty()) { + i += 1; + } + if !lines.get(i).is_some_and(|l| l.trim().is_empty()) { + return None; + } + i += 1; + let first = lines.get(i)?; + if first.trim().is_empty() { + return None; + } + let table_indent = leading_whitespace(first); + if table_indent == 0 { + return None; + } + let mut names = Vec::new(); + while let Some(line) = lines.get(i) { + if line.trim().is_empty() { + break; + } + let indent = leading_whitespace(line); + if indent < table_indent { + break; + } + if indent == table_indent { + let trimmed = line.trim(); + let word_end = trimmed.find(char::is_whitespace).unwrap_or(trimmed.len()); + let word = &trimmed[..word_end]; + if !is_command_word_shaped(word) { + break; + } + if word == tool_name { + // S-016's own shape, not this one. + return None; + } + if !names.contains(&word.to_string()) { + names.push(word.to_string()); + } + } + i += 1; + } + (names.len() >= 2).then_some(names) +} + +pub fn detect(raw: &str, root: &CommandNode) -> Report { + let Some(names) = raw_command_table_rows(raw, &root.name) else { + return Report { + findings: Vec::new(), + }; + }; + let findings = names + .into_iter() + .filter(|name| !root.subcommands.iter().any(|c| &c.name == name)) + .map(|name| Finding { name }) + .collect(); + Report { findings } +} + +pub struct UsageCommandTable; + +impl Detector for UsageCommandTable { + fn name(&self) -> &'static str { + "usage-command-table" + } + + fn family(&self) -> Option<&'static str> { + None + } + + fn describes(&self) -> &'static str { + "a command table under the tool's own root `Usage:` synopsis (bare label, blank, own \ + name, bracketed continuation, blank, one row per command) whose rows never repeat the \ + tool's own name, and are missing from the parsed tree's own subcommands" + } + + fn hits(&self, evidence: &ToolEvidence<'_>) -> Vec { + detect(evidence.raw, evidence.root) + .findings + .iter() + .map(|f| { + format!( + "{} named by the usage command table but not a subcommand", + f.name + ) + }) + .collect() + } + + fn scope(&self) -> Scope { + Scope::full() + } + + fn self_checks(&self) -> Vec { + self_checks() + } +} + +// ---------------------------------------------------------------------- +// Self-checks +// ---------------------------------------------------------------------- + +use mandible_core::{Provenance, Source}; + +/// dmsetup's own shape, byte-exact enough to exercise the recognizer. +pub(crate) const DMSETUP_USAGE_SHAPE: &str = "Usage:\n\ + \n\ + dmsetup\n \ + [--version] [-h|--help [-c|-C|--columns]]\n\ + \n\ + \thelp [-c|-C|--columns]\n\ + \tcreate \n\ + \tremove [--deferred] [-f|--force] [--retry] ...\n"; + +fn empty_node(name: &str) -> CommandNode { + CommandNode::new(name, Provenance::single(Source::HelpText)) +} + +fn node_with_subcommands(name: &str, subs: &[&str]) -> CommandNode { + let mut root = empty_node(name); + root.subcommands = subs.iter().map(|s| empty_node(s)).collect(); + root +} + +pub(crate) fn self_checks() -> Vec { + vec![ + SelfCheck { + name: "dmsetup's own bytes, no commands recovered at all", + why: "the defect itself: the whole table is missing from the tree", + expect: Expect::Fires(3), + raw: DMSETUP_USAGE_SHAPE.to_string(), + root: empty_node("dmsetup"), + }, + SelfCheck { + name: "dmsetup's own bytes, every row recovered as its own subcommand", + why: "once every row is a real subcommand, the same raw text must go silent", + expect: Expect::Silent, + raw: DMSETUP_USAGE_SHAPE.to_string(), + root: node_with_subcommands("dmsetup", &["help", "create", "remove"]), + }, + SelfCheck { + name: "an ordinary `Usage: prog [opts]` tool, not this shape at all", + why: "a one-line labelled usage synopsis must never be mistaken for the bare-label \ + idiom this rule requires", + expect: Expect::Silent, + raw: "Usage: prog [opts]\n\n frob do the frobbing\n twist do the twisting\n" + .to_string(), + root: empty_node("prog"), + }, + SelfCheck { + name: "btrfs's own shape, rows repeating the tool's own name", + why: "S-016's own shape, a table whose rows repeat the tool's name, is a different \ + rule entirely and must never double-count here", + expect: Expect::Silent, + raw: "Usage:\n\ + \n\ + btrfs\n \ + [--version]\n\ + \n\ + \tbtrfs balance start \n\ + \tbtrfs balance pause \n" + .to_string(), + root: empty_node("btrfs"), + }, + ] +}