From 8727c513b5af661fcf1253af08087a340d43d98a Mon Sep 17 00:00:00 2001 From: Sadig Akhund Date: Sun, 13 Sep 2026 16:24:35 +0400 Subject: [PATCH] extract: join a usage line's bracket group only when it is flat S-154 tested no flatness, so a nested group fused too: `uniq`'s `[INPUT [OUTPUT]]` became one positional called `INPUT OUTPUT`. docs/shapes.md S-154. corpus/uniq/9.4, corpus/env/9.4. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 1 + corpus/env/9.4/expected.snap | 111 ++++++++ corpus/env/9.4/help.txt | 34 +++ corpus/env/9.4/meta.toml | 25 ++ corpus/uniq/9.4/expected.snap | 116 ++++++++ corpus/uniq/9.4/help.txt | 34 +++ corpus/uniq/9.4/meta.toml | 26 ++ docs/shapes.md | 32 ++- .../src/help_text/sections/multiword.rs | 66 ++++- .../src/help_text/sections/usage.rs | 103 +++++++- xtask/src/coverage/round10.rs | 11 +- xtask/src/detector/commands.rs | 4 +- xtask/src/detector/mod.rs | 6 + .../nested_bracket_group_fused_operand.rs | 250 ++++++++++++++++++ 14 files changed, 794 insertions(+), 25 deletions(-) create mode 100644 corpus/env/9.4/expected.snap create mode 100644 corpus/env/9.4/help.txt create mode 100644 corpus/env/9.4/meta.toml create mode 100644 corpus/uniq/9.4/expected.snap create mode 100644 corpus/uniq/9.4/help.txt create mode 100644 corpus/uniq/9.4/meta.toml create mode 100644 xtask/src/detector/nested_bracket_group_fused_operand.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e6f9cba..cd00b5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ once it reaches a published 0.1.0 release. ### Fixed +- [S-154] A nested bracket group in a usage line no longer fuses its words into one operand name, so `mandible uniq` shows `INPUT` and `OUTPUT` and `mandible env` shows `COMMAND` and `ARG` again; a flat group (`mandible mknod`'s `[MAJOR MINOR]`) still reads as one operand. - [S-155] A value spec written as one delimited alternation of literal values (`--compression=(xz|none|auto)`, `--crate-type `, argparse's own `{c,java,ruby,tcl}`) now attaches as that flag's choices, instead of rendering as one opaque placeholder (`mandible grub-mkimage`, `mandible rustc`, `mandible tclobjnew-bpfcc`). - `xtask sweep-diff` now reports subcommand-count gains and losses per tool, separately from flags, so an invented subcommand row no longer passes as a clean sweep. - `xtask detector calibrate` now prints `NOT EVALUABLE` instead of `DOES NOT PASS` for a detector whose family has zero labelled members in the audit seed, so a legitimate gap in the sample is no longer reported as a failing detector. diff --git a/corpus/env/9.4/expected.snap b/corpus/env/9.4/expected.snap new file mode 100644 index 0000000..2cbeb43 --- /dev/null +++ b/corpus/env/9.4/expected.snap @@ -0,0 +1,111 @@ +name: env +description: Set each NAME to VALUE in the environment and run COMMAND. +usage: +- 'Usage: env [OPTION]... [-] [NAME=VALUE]... [COMMAND [ARG]...]' +positionals: +- name: COMMAND + provenance: + sources: + - help-text +- name: ARG + variadic: true + provenance: + sources: + - help-text +flags: +- spellings: + - -i + - --ignore-environment + description: start with an empty environment + provenance: + sources: + - help-text +- spellings: + - '-0' + - --null + description: end each output line with NUL, not newline + provenance: + sources: + - help-text +- spellings: + - -u + - --unset + value_name: NAME + value_kind: Required + description: remove variable from the environment + provenance: + sources: + - help-text +- spellings: + - -C + - --chdir + value_name: DIR + value_kind: Required + description: change working directory to DIR + provenance: + sources: + - help-text +- spellings: + - -S + - --split-string + value_name: S + value_kind: Required + description: process and split S into separate arguments; used to pass multiple arguments on shebang lines + provenance: + sources: + - help-text +- spellings: + - --block-signal + value_name: SIG + value_kind: Optional + description: block delivery of SIG signal(s) to COMMAND + provenance: + sources: + - help-text +- spellings: + - --default-signal + value_name: SIG + value_kind: Optional + description: reset handling of SIG signal(s) to the default + provenance: + sources: + - help-text +- spellings: + - --ignore-signal + value_name: SIG + value_kind: Optional + description: set handling of SIG signal(s) to do nothing + provenance: + sources: + - help-text +- spellings: + - --list-signal-handling + description: list non default signal handling to stderr + provenance: + sources: + - help-text +- spellings: + - -v + - --debug + description: print verbose information for each processing step + provenance: + sources: + - help-text +- spellings: + - --help + description: display this help and exit + provenance: + sources: + - help-text +- spellings: + - --version + description: output version information and exit + provenance: + sources: + - help-text +detected_framework: GNU argp/getopt_long +provenance: + sources: + - help-text + confidence: 1.0 +children_filled: true diff --git a/corpus/env/9.4/help.txt b/corpus/env/9.4/help.txt new file mode 100644 index 0000000..601b93c --- /dev/null +++ b/corpus/env/9.4/help.txt @@ -0,0 +1,34 @@ +Usage: env [OPTION]... [-] [NAME=VALUE]... [COMMAND [ARG]...] +Set each NAME to VALUE in the environment and run COMMAND. + +Mandatory arguments to long options are mandatory for short options too. + -i, --ignore-environment start with an empty environment + -0, --null end each output line with NUL, not newline + -u, --unset=NAME remove variable from the environment + -C, --chdir=DIR change working directory to DIR + -S, --split-string=S process and split S into separate arguments; + used to pass multiple arguments on shebang lines + --block-signal[=SIG] block delivery of SIG signal(s) to COMMAND + --default-signal[=SIG] reset handling of SIG signal(s) to the default + --ignore-signal[=SIG] set handling of SIG signal(s) to do nothing + --list-signal-handling list non default signal handling to stderr + -v, --debug print verbose information for each processing step + --help display this help and exit + --version output version information and exit + +A mere - implies -i. If no COMMAND, print the resulting environment. + +SIG may be a signal name like 'PIPE', or a signal number like '13'. +Without SIG, all known signals are included. Multiple signals can be +comma-separated. An empty SIG argument is a no-op. + +Exit status: + 125 if the env command itself fails + 126 if COMMAND is found but cannot be invoked + 127 if COMMAND cannot be found + - the exit status of COMMAND otherwise + +GNU coreutils online help: +Report any translation bugs to +Full documentation +or available locally via: info '(coreutils) env invocation' diff --git a/corpus/env/9.4/meta.toml b/corpus/env/9.4/meta.toml new file mode 100644 index 0000000..4c4e405 --- /dev/null +++ b/corpus/env/9.4/meta.toml @@ -0,0 +1,25 @@ +# docs/shapes.md S-154: `[COMMAND [ARG]...]` names two optional operands, +# `COMMAND` and a repeatable `ARG` nested inside it. The flat multi-word +# group rule joined them into one positional spelled `COMMAND ARG` and +# carried the repetition marker onto that joined name, so the tree said +# `env` takes one operand where its own synopsis names two. + +[bless] +provenance = "agent" + +[tool] +name = "env" +version = "9.4" +platform = "ubuntu-24.04" +captured_with = "manual capture per corpus/README.md step 2 (TERM=dumb NO_COLOR=1 COLUMNS=100 LC_ALL=C.UTF-8); GNU coreutils 9.4" + +[[capture]] +argv = ["env", "--help"] +stdout = "help.txt" + +[contract] +verdict_scope = ["flags", "usage"] +expected_framework = "GNU argp/getopt_long" +must_contain_positionals = ["COMMAND", "ARG..."] +must_not_contain_positionals = ["COMMAND ARG"] +must_contain_flags = ["-S", "-C", "--block-signal", "--list-signal-handling"] diff --git a/corpus/uniq/9.4/expected.snap b/corpus/uniq/9.4/expected.snap new file mode 100644 index 0000000..c2ad0ec --- /dev/null +++ b/corpus/uniq/9.4/expected.snap @@ -0,0 +1,116 @@ +name: uniq +description: Filter adjacent matching lines from INPUT (or standard input), writing to OUTPUT (or standard output). +usage: +- 'Usage: uniq [OPTION]... [INPUT [OUTPUT]]' +positionals: +- name: INPUT + provenance: + sources: + - help-text +- name: OUTPUT + provenance: + sources: + - help-text +flags: +- spellings: + - -c + - --count + description: prefix lines by the number of occurrences + provenance: + sources: + - help-text +- spellings: + - -d + - --repeated + description: only print duplicate lines, one for each group + provenance: + sources: + - help-text +- spellings: + - -D + description: print all duplicate lines + provenance: + sources: + - help-text +- spellings: + - --all-repeated + value_name: METHOD + value_kind: Optional + description: like -D, but allow separating groups with an empty line; METHOD={none(default),prepend,separate} + provenance: + sources: + - help-text +- spellings: + - -f + - --skip-fields + value_name: N + value_kind: Required + description: avoid comparing the first N fields + provenance: + sources: + - help-text +- spellings: + - --group + value_name: METHOD + value_kind: Optional + description: show all items, separating groups with an empty line; METHOD={separate(default),prepend,append,both} + provenance: + sources: + - help-text +- spellings: + - -i + - --ignore-case + description: ignore differences in case when comparing + provenance: + sources: + - help-text +- spellings: + - -s + - --skip-chars + value_name: N + value_kind: Required + description: avoid comparing the first N characters + provenance: + sources: + - help-text +- spellings: + - -u + - --unique + description: only print unique lines + provenance: + sources: + - help-text +- spellings: + - -z + - --zero-terminated + description: line delimiter is NUL, not newline + provenance: + sources: + - help-text +- spellings: + - -w + - --check-chars + value_name: N + value_kind: Required + description: compare no more than N characters in lines + provenance: + sources: + - help-text +- spellings: + - --help + description: display this help and exit + provenance: + sources: + - help-text +- spellings: + - --version + description: output version information and exit + provenance: + sources: + - help-text +detected_framework: GNU argp/getopt_long +provenance: + sources: + - help-text + confidence: 1.0 +children_filled: true diff --git a/corpus/uniq/9.4/help.txt b/corpus/uniq/9.4/help.txt new file mode 100644 index 0000000..b76b8b7 --- /dev/null +++ b/corpus/uniq/9.4/help.txt @@ -0,0 +1,34 @@ +Usage: uniq [OPTION]... [INPUT [OUTPUT]] +Filter adjacent matching lines from INPUT (or standard input), +writing to OUTPUT (or standard output). + +With no options, matching lines are merged to the first occurrence. + +Mandatory arguments to long options are mandatory for short options too. + -c, --count prefix lines by the number of occurrences + -d, --repeated only print duplicate lines, one for each group + -D print all duplicate lines + --all-repeated[=METHOD] like -D, but allow separating groups + with an empty line; + METHOD={none(default),prepend,separate} + -f, --skip-fields=N avoid comparing the first N fields + --group[=METHOD] show all items, separating groups with an empty line; + METHOD={separate(default),prepend,append,both} + -i, --ignore-case ignore differences in case when comparing + -s, --skip-chars=N avoid comparing the first N characters + -u, --unique only print unique lines + -z, --zero-terminated line delimiter is NUL, not newline + -w, --check-chars=N compare no more than N characters in lines + --help display this help and exit + --version output version information and exit + +A field is a run of blanks (usually spaces and/or TABs), then non-blank +characters. Fields are skipped before chars. + +Note: 'uniq' does not detect repeated lines unless they are adjacent. +You may want to sort the input first, or use 'sort -u' without 'uniq'. + +GNU coreutils online help: +Report any translation bugs to +Full documentation +or available locally via: info '(coreutils) uniq invocation' diff --git a/corpus/uniq/9.4/meta.toml b/corpus/uniq/9.4/meta.toml new file mode 100644 index 0000000..a54d990 --- /dev/null +++ b/corpus/uniq/9.4/meta.toml @@ -0,0 +1,26 @@ +# docs/shapes.md S-154: `[INPUT [OUTPUT]]` is a nested bracket group, so it +# names two optional operands, `INPUT` and `OUTPUT`, the second nested +# inside the first's own group. The rule that collapses a flat multi-word +# group to one operand (`gdk-pixbuf-thumbnailer`'s `[INPUT FILE]`) also +# fired here and joined the two words into one positional spelled +# `INPUT OUTPUT`, a name the tool never documents. + +[bless] +provenance = "agent" + +[tool] +name = "uniq" +version = "9.4" +platform = "ubuntu-24.04" +captured_with = "manual capture per corpus/README.md step 2 (TERM=dumb NO_COLOR=1 COLUMNS=100 LC_ALL=C.UTF-8); GNU coreutils 9.4" + +[[capture]] +argv = ["uniq", "--help"] +stdout = "help.txt" + +[contract] +verdict_scope = ["flags", "usage"] +expected_framework = "GNU argp/getopt_long" +must_contain_positionals = ["INPUT", "OUTPUT"] +must_not_contain_positionals = ["INPUT OUTPUT"] +must_contain_flags = ["-c", "-w", "--all-repeated", "--group"] diff --git a/docs/shapes.md b/docs/shapes.md index 47ada3c..50a0439 100644 --- a/docs/shapes.md +++ b/docs/shapes.md @@ -3079,17 +3079,37 @@ entry's `tools` field and nothing else. It does not get a new entry. - id: S-154 - looks like: | gdk-pixbuf-thumbnailer [OPTION…] [INPUT FILE] [OUTPUT FILE] Thumbnail images -- tools: gdk-pixbuf-thumbnailer, mknod, sg_format, udevadm, fzf-tmux -- handling: Fixed. A bracket group holding several words is one operand whose - name is the whole run, not one operand per word, so `mandible +- tools: gdk-pixbuf-thumbnailer, mknod, accessdb, systemd-sysusers, + systemd-tmpfiles, sg_format, udevadm, fzf-tmux +- handling: Fixed, then corrected. A FLAT bracket group holding several words + is one operand whose name is the whole run, so `mandible gdk-pixbuf-thumbnailer` shows two positionals, `INPUT FILE` and `OUTPUT FILE`, where it showed three (`INPUT`, `FILE`, `OUTPUT`) before. The name keeps its source spelling, following S-097's ruling that a glued group is - quoted as written. + quoted as written. Flat means one bracket pair holding no `|` and no `<`, + with any repetition marker at the group's own end + (`systemd-sysusers`'s `[CONFIGURATION FILE...]` is one repeatable + operand). The rule as first shipped tested no flatness at all, so it also + fused a NESTED group, where each bracket pair names its own operand: + `uniq`'s `[INPUT [OUTPUT]]` became one positional called `INPUT OUTPUT` + and `env`'s `[COMMAND [ARG]...]` became `COMMAND ARG`, neither a name its + tool documents. Fifteen tools read that way, `chroot`, `env`, `gettext`, + `grub-menulst2cfg`, `inetutils-telnet`, `mkpasswd`, `nice`, `parted`, + `patch`, `perror`, `ptx`, `split`, `sqlite3`, `telnet` and `uniq`. A + nested group now reads exactly as it did before this rule existed, one + operand per bracket pair, the inner ones optional + (`bare_bracket_group_is_flat`, `mandible-extract/src/help_text/sections/ + multiword.rs`). The sweep that admitted the rule could not see this, + because sweep-diff compares flags and subcommands and never positionals. - fleet: `trailing-bracket-group-multiword-operand` reads 0 tools/0 findings post-fix on a full-`PATH` sweep of 2323 tools, 2026-09-13, and is ratcheted - there. Zero flag losses, zero subcommand movement, all nine named controls - byte-identical. + there. `nested-bracket-group-fused-operand` + (`xtask/src/detector/nested_bracket_group_fused_operand.rs`) counts the + regression instead of the original defect and reads 15 tools/15 findings + on the fused parser against 0 tools/0 findings on the corrected one, also + ratcheted at zero. Zero flag losses, zero subcommand movement, all nine + named controls byte-identical. `corpus/uniq/9.4` and `corpus/env/9.4` + assert both words separately and refuse the fused name. ### S-155: an alternation value name becomes choices diff --git a/mandible-extract/src/help_text/sections/multiword.rs b/mandible-extract/src/help_text/sections/multiword.rs index 2551718..3d30e65 100644 --- a/mandible-extract/src/help_text/sections/multiword.rs +++ b/mandible-extract/src/help_text/sections/multiword.rs @@ -301,19 +301,61 @@ pub(super) fn recover_leading_numbered_pair( None } -/// A bracket group that opened on a bare ALL-CAPS word (`extract_positionals`'s -/// own token loop), closed. `words` pairs each raw token with its own -/// cleaned (bracket/dot-trimmed) spelling, in source order. Every word -/// ALL-CAPS and none an option-list placeholder (`mknod`'s `[MAJOR -/// MINOR]`, `gdk-pixbuf-thumbnailer`'s `[INPUT FILE]`) collapses to one -/// multi-word operand; otherwise this is exactly the per-word reading a -/// lone-word ALL-CAPS bracket already gets (`udevadm`'s `[COMMAND -/// OPTIONS]` keeps `COMMAND`, declines `OPTIONS`) — a batch of the same -/// tokens behind one bracket, not a new rule, so this never removes a -/// reading the lone-word branch already gave. See docs/shapes.md S-154. -pub(super) fn finalize_bare_bracket_group(words: &[(String, String)], line: &str) -> Vec { +/// True when a bare bracket group's own source text names one operand +/// rather than several: exactly one bracket pair, and no `|` or `<` +/// inside it. A group's own trailing repetition marker is allowed +/// (`systemd-sysusers`'s `[CONFIGURATION FILE...]` is one repeatable +/// operand), an interior one is not. `[MAJOR MINOR]` is flat; `[INPUT +/// [OUTPUT]]` and `[COMMAND [ARG]...]` are two optional operands, the +/// inner one nested inside the outer, which is what the words mean and +/// what the parser read before S-154 shipped. `raw` is the group's own +/// whitespace-normalized source, from its opening bracket on. +/// See docs/shapes.md S-154. +pub(super) fn bare_bracket_group_is_flat(raw: &str) -> bool { + let Some(open) = raw.find('[') else { + return false; + }; + let mut depth = 0i32; + let mut inner = None; + for (offset, c) in raw[open..].char_indices() { + match c { + '[' => depth += 1, + ']' => { + depth -= 1; + if depth == 0 { + inner = Some(&raw[open + 1..open + offset]); + break; + } + } + _ => {} + } + } + let Some(inner) = inner else { + return false; + }; + let inner = inner.trim_end_matches('.'); + !inner.contains(['[', ']', '|', '<']) && !inner.contains("...") +} + +/// A bracket group that opened on a bare ALL-CAPS word, closed. `words` +/// pairs each raw token with its cleaned spelling in source order, and +/// `flat` is [`bare_bracket_group_is_flat`] for the same group. A flat +/// group whose every word is ALL-CAPS and no option-list placeholder +/// (`mknod`'s `[MAJOR MINOR]`) collapses to one multi-word operand. +/// Anything else gets exactly the per-word reading a lone-word ALL-CAPS +/// bracket already gets: `udevadm`'s `[COMMAND OPTIONS]` keeps +/// `COMMAND` and declines `OPTIONS`, `uniq`'s `[INPUT [OUTPUT]]` keeps +/// both words apart. That fallback is a batch of the tokens the +/// lone-word branch would have read one by one, so this never removes a +/// reading. See docs/shapes.md S-154. +pub(super) fn finalize_bare_bracket_group( + words: &[(String, String)], + line: &str, + flat: bool, +) -> Vec { let is_real_word = |w: &str| w.chars().all(|c| c.is_uppercase() || c == '_') && w.len() > 1; - if words.len() >= 2 + if flat + && words.len() >= 2 && words .iter() .all(|(_, w)| is_real_word(w) && !is_option_list_placeholder(w)) diff --git a/mandible-extract/src/help_text/sections/usage.rs b/mandible-extract/src/help_text/sections/usage.rs index 190f70b..fe62835 100644 --- a/mandible-extract/src/help_text/sections/usage.rs +++ b/mandible-extract/src/help_text/sections/usage.rs @@ -282,6 +282,10 @@ pub(super) fn extract_positionals( // repeatability exactly the way the lone-word ALL-CAPS branch // below does for the same token. let mut bare_group_words: Vec<(String, String)> = Vec::new(); + // The group's own source text, from its opening bracket on, for + // `bare_bracket_group_is_flat`. Only a flat group names one + // multi-word operand; a nested one names an operand per group. + let mut bare_group_raw = String::new(); for token in line.split_whitespace() { let cleaned = token.trim_matches(|c| c == '[' || c == ']' || c == '.'); let opens = token.matches('[').count() as i32; @@ -319,23 +323,28 @@ pub(super) fn extract_positionals( if cleaned.starts_with('-') || consumed_by_prior_flag { continue; } - // A bracket group opened by a bare ALL-CAPS word that does not - // close on the same token collapses to one multi-word operand - // (`finalize_bare_bracket_group`), rather than one positional - // per word inside it. See docs/shapes.md S-154. + // A flat bracket group opened by a bare ALL-CAPS word that + // does not close on the same token collapses to one + // multi-word operand (`finalize_bare_bracket_group`), rather + // than one positional per word inside it. A nested group + // keeps its words apart. See docs/shapes.md S-154. if bare_group_active { bare_group_depth += opens - closes; + bare_group_raw.push(' '); + bare_group_raw.push_str(token); if !cleaned.is_empty() { bare_group_words.push((token.to_string(), cleaned.to_string())); } if bare_group_depth <= 0 { - for p in finalize_bare_bracket_group(&bare_group_words, line) { + let flat = bare_bracket_group_is_flat(&bare_group_raw); + for p in finalize_bare_bracket_group(&bare_group_words, line, flat) { if seen.insert(p.primary_name().to_string()) { out.push(p); } } bare_group_active = false; bare_group_words.clear(); + bare_group_raw.clear(); } continue; } @@ -348,6 +357,7 @@ pub(super) fn extract_positionals( bare_group_active = true; bare_group_depth = opens - closes; bare_group_words = vec![(token.to_string(), cleaned.to_string())]; + bare_group_raw = token.to_string(); continue; } let (name, variadic) = if let Some(stripped) = cleaned.strip_prefix('<') { @@ -3225,4 +3235,87 @@ mod tests { .collect(); assert_eq!(names, vec!["FILE"], "{names:?}"); } + + /// `mknod`'s own bytes (docs/shapes.md S-154): one flat bracket pair + /// holding two ALL-CAPS words is one operand named by the whole run. + #[test] + fn a_flat_bracket_group_of_two_all_caps_words_is_one_operand() { + let parsed = parse("Usage: mknod [OPTION]... NAME TYPE [MAJOR MINOR]\n"); + let names: Vec<&str> = parsed + .positionals + .iter() + .map(|p| p.primary_name()) + .collect(); + assert_eq!(names, vec!["NAME", "TYPE", "MAJOR MINOR"], "{names:?}"); + } + + /// `uniq`'s own bytes (docs/shapes.md S-154): the group nests, so it + /// names two optional operands, never one joined across the inner + /// bracket. This is the regression S-154 shipped with. + #[test] + fn a_nested_bracket_group_names_one_operand_per_word() { + let parsed = parse("Usage: uniq [OPTION]... [INPUT [OUTPUT]]\n"); + let names: Vec<&str> = parsed + .positionals + .iter() + .map(|p| p.primary_name()) + .collect(); + assert_eq!(names, vec!["INPUT", "OUTPUT"], "{names:?}"); + assert!(!parsed.positionals[0].required); + assert!(!parsed.positionals[1].required); + } + + /// `env`'s own bytes: the inner group carries the repetition marker, + /// which marks that inner operand and never fuses the two words. + #[test] + fn a_nested_bracket_group_with_an_inner_repetition_marker_keeps_both_words() { + let parsed = parse("Usage: env [OPTION]... [-] [NAME=VALUE]... [COMMAND [ARG]...]\n"); + let names: Vec<&str> = parsed + .positionals + .iter() + .map(|p| p.primary_name()) + .collect(); + assert_eq!(names, vec!["COMMAND", "ARG"], "{names:?}"); + assert!(parsed.positionals[1].repeatable); + } + + /// `systemd-sysusers`'s own bytes: the group is flat and its dots sit + /// at the group's own end, so it stays one repeatable operand. + #[test] + fn a_flat_bracket_group_ending_in_dots_is_one_repeatable_operand() { + let parsed = parse("Usage: systemd-sysusers [OPTIONS...] [CONFIGURATION FILE...]\n"); + let names: Vec<&str> = parsed + .positionals + .iter() + .map(|p| p.primary_name()) + .collect(); + assert_eq!(names, vec!["CONFIGURATION FILE"], "{names:?}"); + assert!(parsed.positionals[0].repeatable); + } + + /// `gettext`'s own bytes: the outer group's first word is itself + /// bracketed, which is a nested group, not a two-word name. + #[test] + fn a_bracketed_first_word_inside_a_group_keeps_the_words_apart() { + let parsed = parse("Usage: gettext [OPTION] [[TEXTDOMAIN] MSGID]\n"); + let names: Vec<&str> = parsed + .positionals + .iter() + .map(|p| p.primary_name()) + .collect(); + assert_eq!(names, vec!["TEXTDOMAIN", "MSGID"], "{names:?}"); + } + + /// `parted`'s own bytes: three words behind two nested brackets are + /// three operands, one per group. + #[test] + fn a_doubly_nested_bracket_group_names_three_operands() { + let parsed = parse("Usage: parted [OPTION]... [DEVICE [COMMAND [PARAMETERS]...]...]\n"); + let names: Vec<&str> = parsed + .positionals + .iter() + .map(|p| p.primary_name()) + .collect(); + assert_eq!(names, vec!["DEVICE", "COMMAND", "PARAMETERS"], "{names:?}"); + } } diff --git a/xtask/src/coverage/round10.rs b/xtask/src/coverage/round10.rs index 4c89777..79c115c 100644 --- a/xtask/src/coverage/round10.rs +++ b/xtask/src/coverage/round10.rs @@ -1,6 +1,7 @@ //! The round-10 family detectors, atlas S-150 upward. Split into its own //! file for the same line-count reason `round7.rs`, `round8.rs` and -//! `round9.rs` are. +//! `round9.rs` are. `nested-bracket-group-fused-operand` belongs here +//! because the rule it fences, S-154, is one of this round's own. use super::score::FAMILY_DETECTOR_SAMPLES_PER_ROW; use crate::detector::{Detector, ToolEvidence}; @@ -32,6 +33,9 @@ pub(super) fn round10_family_counts( let description_reused_as_group = crate::detector::description_reused_as_group_label::DescriptionReusedAsGroupLabel .hits(&evidence); + let fused_operand = + crate::detector::nested_bracket_group_fused_operand::NestedBracketGroupFusedOperand + .hits(&evidence); let headingless_table_in_description = crate::detector::headingless_table_in_root_description::HeadinglessTableInRootDescription .hits(&evidence); @@ -137,5 +141,10 @@ pub(super) fn round10_family_counts( env_col.len(), env_col.into_iter().take(cap).collect(), ), + ( + "nested-bracket-group-fused-operand", + fused_operand.len(), + fused_operand.into_iter().take(cap).collect(), + ), ] } diff --git a/xtask/src/detector/commands.rs b/xtask/src/detector/commands.rs index 03401e8..ceca152 100644 --- a/xtask/src/detector/commands.rs +++ b/xtask/src/detector/commands.rs @@ -399,6 +399,7 @@ pub fn check_round9_family_ratchets( } /// [`check_vim_family_ratchet`] for round 10's repaired families, atlas +/// S-154 (a nested bracket group's words fused into one operand name), /// S-150 (a bare usage label seeding an empty form) and S-142's two /// halves (issue #143: the glued label and the open-bracket /// continuation, both now tree-aware — see @@ -413,6 +414,7 @@ pub fn check_round10_family_ratchets( previous: &crate::coverage::Aggregate, fresh: &crate::coverage::Aggregate, ) -> anyhow::Result { + let fused = check_vim_family_ratchet("nested-bracket-group-fused-operand", previous, fresh)?; let bare = check_vim_family_ratchet("bare-usage-label-form", previous, fresh)?; let glued = check_vim_family_ratchet("usage-label-glued-to-program-name", previous, fresh)?; let bracket = check_vim_family_ratchet( @@ -420,7 +422,7 @@ pub fn check_round10_family_ratchets( previous, fresh, )?; - Ok(bare && glued && bracket) + Ok(fused && bare && glued && bracket) } /// pnpm's two families (atlas S-103, S-104), fixed in diff --git a/xtask/src/detector/mod.rs b/xtask/src/detector/mod.rs index a697138..185d2d4 100644 --- a/xtask/src/detector/mod.rs +++ b/xtask/src/detector/mod.rs @@ -89,6 +89,11 @@ pub(crate) mod usage_spelling_duplicates_table_row; pub(crate) mod trailing_bracket_group_multiword_operand; pub(crate) mod usage_bracket_group_multiword_value; +// S-154's own regression fence, the inverse of the S-132 detector above: +// that one counts an operand the tree never got, this one counts a fused +// operand name the tree invented. Same direct-`Detector`-impl shape. +pub(crate) mod nested_bracket_group_fused_operand; + // Round-8 family detector (issue "lvm2 invocation forms read as section // headings", atlas S-137), same direct-`Detector`-impl shape. pub(crate) mod invocation_form_head_as_flag_group; @@ -767,6 +772,7 @@ pub fn registry() -> Vec> { Box::new(or_joined_alias_single_space_gap::OrJoinedAliasSingleSpaceGap), Box::new(usage_bracket_group_multiword_value::UsageBracketGroupMultiwordValue), Box::new(trailing_bracket_group_multiword_operand::TrailingBracketGroupMultiwordOperand), + Box::new(nested_bracket_group_fused_operand::NestedBracketGroupFusedOperand), Box::new(UsageTextContinuationFold), Box::new(NumberedVariadicUsageTail), Box::new(invocation_form_head_as_flag_group::InvocationFormHeadAsFlagGroup), diff --git a/xtask/src/detector/nested_bracket_group_fused_operand.rs b/xtask/src/detector/nested_bracket_group_fused_operand.rs new file mode 100644 index 0000000..3b3efde --- /dev/null +++ b/xtask/src/detector/nested_bracket_group_fused_operand.rs @@ -0,0 +1,250 @@ +//! `nested-bracket-group-fused-operand` (atlas S-154): a usage line's +//! nested bracket group of ALL-CAPS words reaches the tree as ONE +//! positional whose name is those words joined by a space (`uniq`'s +//! `[INPUT [OUTPUT]]` becoming `INPUT OUTPUT`). The sibling of +//! `trailing_bracket_group_multiword_operand` (S-132), inverted: that one +//! counts an operand the tree never got, this one counts an operand name +//! the tree invented. A FLAT group's joined name is the legitimate S-154 +//! read (`mknod`'s `[MAJOR MINOR]`) and is silent here. No labelled tool +//! carries this shape, so [`Detector::family`] returns `None`. Fixtures: +//! corpus/uniq/9.4 and corpus/env/9.4. + +use crate::detector::{Detector, Expect, Scope, SelfCheck, ToolEvidence}; +use mandible_core::CommandNode; + +pub struct Finding { + pub name: String, + pub group: String, + pub line: String, +} + +pub struct Report { + pub findings: Vec, +} + +/// One ALL-CAPS operand word: uppercase letters or underscores only, and +/// more than one character, so a stray `A` or a lowercase prose word is +/// never counted. The same test the parser's own token loop applies. +fn all_caps_word(w: &str) -> bool { + w.len() > 1 && w.chars().all(|c| c.is_uppercase() || c == '_') +} + +/// Every depth-zero bracket group in `line`, each as its own source span +/// including both brackets. An unmatched `[` yields nothing, the way an +/// unterminated group means nothing to a reader either. +fn bracket_groups(line: &str) -> Vec<&str> { + let mut out = Vec::new(); + let mut depth = 0i32; + let mut start = None; + for (i, c) in line.char_indices() { + match c { + '[' => { + if depth == 0 { + start = Some(i); + } + depth += 1; + } + ']' => { + depth -= 1; + if depth == 0 { + if let Some(s) = start.take() { + out.push(&line[s..=i]); + } + } + depth = depth.max(0); + } + _ => {} + } + } + out +} + +/// The space-joined name a nested group's own ALL-CAPS words would fuse +/// into, or `None` when the group is not that shape. Nested means the +/// group holds a further `[` inside it, which is what makes the words +/// separate operands rather than one name: `[INPUT [OUTPUT]]` is `INPUT` +/// and an optional `OUTPUT`, never an operand called `INPUT OUTPUT`. +/// Every whitespace-delimited word in the group must read ALL-CAPS once +/// brackets and dots are trimmed, and there must be two or more, so a +/// group carrying prose or a `name=value` pair claims nothing. +fn fused_name(group: &str) -> Option { + let inner = group.get(1..group.len() - 1)?; + if !inner.contains('[') { + return None; + } + let mut words = Vec::new(); + for token in inner.split_whitespace() { + let cleaned = token.trim_matches(|c| c == '[' || c == ']' || c == '.'); + if cleaned.is_empty() { + continue; + } + if !all_caps_word(cleaned) { + return None; + } + words.push(cleaned); + } + (words.len() >= 2).then(|| words.join(" ")) +} + +/// Every line of the raw capture is read, not only the primary synopsis: +/// `ptx` writes its nested group on its second invocation form (`or: +/// ptx -G [OPTION]... [INPUT [OUTPUT]]`), and the parser reads operands +/// off whichever form it treats as primary. Requiring the fused name to +/// be present in the tree keeps that breadth from costing precision — +/// nothing but the fusing rule itself produces a positional spelled with +/// a space that the group's own words spell out in order. +pub fn detect(raw: &str, root: &CommandNode) -> Report { + let mut findings = Vec::new(); + for line in raw.lines() { + for group in bracket_groups(line) { + let Some(name) = fused_name(group) else { + continue; + }; + if root.positionals().any(|p| p.primary_name() == name) { + findings.push(Finding { + name, + group: group.to_string(), + line: line.to_string(), + }); + } + } + } + Report { findings } +} + +pub struct NestedBracketGroupFusedOperand; + +impl Detector for NestedBracketGroupFusedOperand { + fn name(&self) -> &'static str { + "nested-bracket-group-fused-operand" + } + + fn family(&self) -> Option<&'static str> { + None + } + + fn describes(&self) -> &'static str { + "a usage line's nested bracket group of ALL-CAPS words reaches the tree as one positional \ + whose name is those words joined by a space" + } + + fn hits(&self, evidence: &ToolEvidence<'_>) -> Vec { + detect(evidence.raw, evidence.root) + .findings + .iter() + .map(|f| { + format!( + "positional {:?} fused the nested group {:?}, from {:?}", + f.name, f.group, f.line + ) + }) + .collect() + } + + fn scope(&self) -> Scope { + Scope::full() + } + + fn self_checks(&self) -> Vec { + self_checks() + } +} + +// ---------------------------------------------------------------------- +// Self-checks +// ---------------------------------------------------------------------- + +use mandible_core::{Entity, Provenance, Source}; + +/// `uniq`'s real usage line, byte-exact (corpus/uniq/9.4's `help.txt`). +pub(crate) const UNIQ_USAGE: &str = "Usage: uniq [OPTION]... [INPUT [OUTPUT]]\n"; + +/// `env`'s real usage line, byte-exact (corpus/env/9.4's `help.txt`). +pub(crate) const ENV_USAGE: &str = + "Usage: env [OPTION]... [-] [NAME=VALUE]... [COMMAND [ARG]...]\n"; + +fn node_with_positionals(name: &str, names: &[&str]) -> CommandNode { + let mut root = CommandNode::new(name, Provenance::single(Source::HelpText)); + let entities = names + .iter() + .map(|n| Entity::positional(*n, Provenance::single(Source::HelpText))) + .collect(); + root.set_entities_of(mandible_core::EntityKind::Positional, entities); + root +} + +pub(crate) fn self_checks() -> Vec { + vec![ + SelfCheck { + name: "uniq's own bytes, the two words fused into one operand", + why: "the defect itself: `[INPUT [OUTPUT]]` names two optional operands and the \ + tree carries one called `INPUT OUTPUT`, a name uniq never documents", + expect: Expect::Fires(1), + raw: UNIQ_USAGE.to_string(), + root: node_with_positionals("uniq", &["INPUT OUTPUT"]), + }, + SelfCheck { + name: "uniq's own bytes, the two operands read apart", + why: "once the tree carries `INPUT` and `OUTPUT` separately, the same usage line \ + must go silent", + expect: Expect::Silent, + raw: UNIQ_USAGE.to_string(), + root: node_with_positionals("uniq", &["INPUT", "OUTPUT"]), + }, + SelfCheck { + name: "env's own bytes, the inner repetition marker fused too", + why: "`[COMMAND [ARG]...]` marks the inner operand repeatable, not a joined name; \ + the `[NAME=VALUE]...` group ahead of it must claim nothing here, since its \ + own word is not ALL-CAPS once the `=` pair is read as one word", + expect: Expect::Fires(1), + raw: ENV_USAGE.to_string(), + root: node_with_positionals("env", &["COMMAND ARG"]), + }, + SelfCheck { + name: "mknod's own bytes, a flat group's joined name is correct", + why: "the legitimate S-154 read: `[MAJOR MINOR]` has no inner bracket, so its two \ + words are one operand name and this detector must not call it a defect", + expect: Expect::Silent, + raw: "Usage: mknod [OPTION]... NAME TYPE [MAJOR MINOR]\n".to_string(), + root: node_with_positionals("mknod", &["NAME", "TYPE", "MAJOR MINOR"]), + }, + SelfCheck { + name: "gdk-pixbuf-thumbnailer's own bytes, two flat groups side by side", + why: "two flat groups on one line are two operand names (issue #135's own fixture), \ + not a nesting — neither may be reported", + expect: Expect::Silent, + raw: " gdk-pixbuf-thumbnailer [OPTION\u{2026}] [INPUT FILE] [OUTPUT FILE] Thumbnail \ + images\n" + .to_string(), + root: node_with_positionals("gdk-pixbuf-thumbnailer", &["INPUT FILE", "OUTPUT FILE"]), + }, + SelfCheck { + name: "systemd-sysusers's own bytes, a flat group ending in its own dots", + why: "`[CONFIGURATION FILE...]` is one repeatable operand: the dots mark the \ + group's own repetition and add no nesting, so the joined name stands", + expect: Expect::Silent, + raw: "systemd-sysusers [OPTIONS...] [CONFIGURATION FILE...]\n".to_string(), + root: node_with_positionals("systemd-sysusers", &["CONFIGURATION FILE"]), + }, + SelfCheck { + name: "parted's own bytes, three words behind two nestings", + why: "the deepest fleet specimen: `[DEVICE [COMMAND [PARAMETERS]...]...]` fused all \ + three words into one name, so a detector that only looked one level deep \ + would miss it", + expect: Expect::Fires(1), + raw: "Usage: parted [OPTION]... [DEVICE [COMMAND [PARAMETERS]...]...]\n".to_string(), + root: node_with_positionals("parted", &["DEVICE COMMAND PARAMETERS"]), + }, + SelfCheck { + name: "ptx's own bytes, the nested group on a later invocation form", + why: "ptx writes `[INPUT [OUTPUT]]` on its second form, not its first, and the \ + parser still read operands off it — reading only the primary synopsis line \ + would report zero while the tree carries the fused name", + expect: Expect::Fires(1), + raw: "Usage: ptx [OPTION]... [INPUT]... (without -G)\n or: ptx -G [OPTION]... \ + [INPUT [OUTPUT]]\n" + .to_string(), + root: node_with_positionals("ptx", &["INPUT", "INPUT OUTPUT"]), + }, + ] +}