Skip to content

Fix --posix to disable the GNU regular expression escapes #550

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LoukasPap wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Fix --posix to disable the GNU regular expression escapes #550

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
99e302e
Fix --posix to disable the GNU regular expression escapes
LoukasPap Sep 5, 2026
d26d8d5
Split test case to separate POSIX and GNU operators and skip branch f…
LoukasPap Sep 24, 2026
a39e090
Fix backslash handling inside bracket expressions
LoukasPap Sep 24, 2026
4d29500
Address review comments on bracket expressions
LoukasPap Oct 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions src/sed/compiler.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -478,7 +478,13 @@ fn compile_address(
} else {
RegexMode::Basic
};
let re = parse_regex_for_mode(lines, line, regex_mode, context.character_mode)?;
let re = parse_regex_for_mode(
lines,
line,
regex_mode,
context.character_mode,
context.posix,
)?;
// Skip over delimiter
line.advance();

Expand Down Expand Up @@ -861,7 +867,13 @@ fn compile_subst_command(
} else {
RegexMode::Basic
};
let pattern = parse_regex_for_mode(lines, line, regex_mode, context.character_mode)?;
let pattern = parse_regex_for_mode(
lines,
line,
regex_mode,
context.character_mode,
context.posix,
)?;
let mut subst = Box::new(Substitution::default());

subst.replacement = compile_replacement(lines, line, context.character_mode)?;
Expand Down Expand Up @@ -3572,7 +3584,7 @@ mod tests {
compile_text_command(&mut lines, &mut chars, &mut cmd, &mut context).unwrap();
match &cmd.data {
CommandData::Text(text) => {
assert_eq!(text.as_ref(), b">helll\x08o\nto\nall\x07\n");
assert_eq!(text.as_ref(), b">helllbo\nto\nall\x07\n");
}
_ => panic!("Expected CommandData::Text"),
}
Expand Down
146 changes: 110 additions & 36 deletions src/sed/delimited_parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,6 @@ pub fn parse_char_escape(line: &mut ScriptCharProvider) -> Option<char> {
line.advance();
Some('\x07')
}
'b' => {
line.advance();
Some('\x08')
}
'f' => {
line.advance();
Some('\x0c')
Expand Down Expand Up @@ -221,10 +217,13 @@ pub fn parse_char_escape(line: &mut ScriptCharProvider) -> Option<char> {
/// This functionality is needed to avoid terminating delimited
/// sequences when a delimiter appears within a character class.
/// While at it, handle escaped characters for the sake of consistency.
/// Under --posix the GNU character escapes are disabled here, although
/// they remain available outside bracket expressions.
fn parse_character_class(
lines: &ScriptLineProvider,
line: &mut ScriptCharProvider,
character_mode: CharacterMode,
posix: bool,
) -> UResult<Vec<u8>> {
let mut result = Vec::new();

Expand Down Expand Up @@ -304,10 +303,8 @@ fn parse_character_class(

continue;
}
// Not a POSIX construct — treat as literal
result.push(b'[');
result.push(line.current_byte());
line.advance();
// Not a POSIX construct — an ordinary member of the set
result.extend_from_slice(br"\[");
continue;
}

Expand All @@ -317,12 +314,14 @@ fn parse_character_class(
if line.eol() {
break;
}
if let Some(decoded) = parse_char_escape(line) {
let decoded = if posix { None } else { parse_char_escape(line) };

@sylvestre sylvestre Oct 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

under --posix [\n] now holds \ and n, are you sure GNU sed doesn't still turn it into a newline there?
printf 'a\nb\n' | sed --posix 'N;s/[\n]/X/' please check and add a test :)

@LoukasPap LoukasPap Oct 11, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just verified it holds \ and n under --posix so the newline survives. I'll include a test.

if let Some(decoded) = decoded {
push_script_char(&mut result, decoded, character_mode);
} else {
result.push(b'\\');
result.push(line.current_byte());
line.advance();
// Outside the character escapes a \ is an ordinary member of the set,
// so the next character is left for the loop to handle. This is what
// makes `[a\]bc]` end at the first `]`, as GNU sed does.
result.extend_from_slice(br"\\");
Comment thread
Copilot marked this conversation as resolved.

@sylvestre sylvestre Oct 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this also changes [\/] (escaped delimiter in a class): it now holds \ and /, GNU keeps only /
echo 'a\b/c' | sed 's/[\/]/X/g' should give a\bXc
could you please handle the delimiter before this and add a test? common idiom for paths

@LoukasPap LoukasPap Oct 11, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

GNU keeps both members here. Inside a bracket expression the backslash doesn’t quote the delimiter, it’s an ordinary member of the set:

$ echo 'a\b/c' | sed 's/[\/]/X/g'
aXbXc

$ echo 'a\b/c' | ./target/debug/sed 's/[\/]/X/g'
aXbXc

}
} else {
result.push(line.current_byte());
Expand Down Expand Up @@ -359,22 +358,36 @@ pub fn parse_regex(
line: &mut ScriptCharProvider,
regex_mode: RegexMode,
) -> UResult<Vec<u8>> {
parse_regex_for_mode(lines, line, regex_mode, CharacterMode::Utf8)
parse_regex_for_mode(lines, line, regex_mode, CharacterMode::Utf8, false)
}

/// Return true if a backslash followed by `c` is a GNU regular expression
/// extension that must be emitted as the literal character `c` under --posix.
/// In Extended mode `? + |` are excluded, because `\?` is already a literal there.
fn is_gnu_regex_escape(c: char, regex_mode: RegexMode) -> bool {
match c {
'?' | '+' | '|' => matches!(regex_mode, RegexMode::Basic),
'w' | 'W' | 's' | 'S' | 'b' | 'B' | '<' | '>' | '`' | '\'' => true,
_ => false,
}
}

/// Parse a regular expression according to the current character mode.
/// Under `posix` the GNU regular expression extensions are disabled, with each
/// one standing for the literal character that follows its backslash.
pub fn parse_regex_for_mode(
lines: &ScriptLineProvider,
line: &mut ScriptCharProvider,
regex_mode: RegexMode,
character_mode: CharacterMode,
posix: bool,
) -> UResult<Vec<u8>> {
let delimiter = scan_delimiter(lines, line)?;
let mut result = Vec::new();
while !line.eol() {
match line.current() {
'[' if delimiter != '[' => {
let cc = parse_character_class(lines, line, character_mode)?;
let cc = parse_character_class(lines, line, character_mode, posix)?;
result.extend_from_slice(&cc);
continue;
}
Expand All @@ -389,6 +402,11 @@ pub fn parse_regex_for_mode(
line.advance();
continue;
}
if posix && is_gnu_regex_escape(line.current(), regex_mode) {
result.push(line.current_byte());
line.advance();
continue;
}
if line.current() == '{' && matches!(regex_mode, RegexMode::Basic) {
validate_quantifier_structure(lines, line, delimiter, RegexMode::Basic)?;
let quantifier = validate_quantifier_numbers(lines, line)?;
Expand Down Expand Up @@ -937,144 +955,146 @@ mod tests {
fn test_basic_character_class() {
let mut line = char_provider_from("[qr]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, b"[qr]");
}

#[test]
fn test_negated_class() {
let mut line = char_provider_from("[^abc]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, b"[^abc]");
}

#[test]
fn test_leading_close_bracket() {
let mut line = char_provider_from("[]abc]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, b"[]abc]");
}

#[test]
fn test_leading_negated_close_bracket() {
let mut line = char_provider_from("[^]abc]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, b"[^]abc]");
}

#[test]
fn test_escaped_character_begin() {
let mut line = char_provider_from("[\\nabc]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, b"[\nabc]");
}

#[test]
fn test_escaped_character_middle() {
let mut line = char_provider_from("[a\\nbc]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, b"[a\nbc]");
}

#[test]
fn test_escaped_character_end() {
let mut line = char_provider_from("[abc\\n]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, b"[abc\n]");
}

#[test]
fn test_escaped_delimiter() {
// A backslash does not quote the `]`, so the class ends at it and
// holds `a` and a literal backslash, as GNU sed does.
let mut line = char_provider_from("[a\\]bc]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
assert_eq!(result, br"[a\]bc]");
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, br"[a\\]");
}

#[test]
fn test_posix_class() {
let mut line = char_provider_from("[[:digit:]]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, b"[[:digit:]]");
}

#[test]
fn test_colon_literal_character_class() {
let mut line = char_provider_from("[:]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, b"[:]");
}

#[test]
fn test_equivalence_class() {
let mut line = char_provider_from("[[=a=]]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, b"[[=a=]]");
}

#[test]
fn test_collating_symbol() {
let mut line = char_provider_from("[[.ch.]]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, b"[[.ch.]]");
}

#[test]
fn test_unterminated_class_error() {
let mut line = char_provider_from("[abc"); // missing closing ]
let lines = test_lines();
let err = parse_character_class(&lines, &mut line, CharacterMode::Utf8);
let err = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false);
assert!(err.is_err());
}

#[test]
fn test_open_bracket_at_eol_errors() {
let mut line = char_provider_from("[");
let lines = test_lines();
let err = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap_err();
let err = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap_err();
assert!(err.to_string().contains("Unterminated bracket expression"));
}

#[test]
fn test_unterminated_posix_class_error() {
let mut line = char_provider_from("[[:digit:]");
let lines = test_lines();
let err = parse_character_class(&lines, &mut line, CharacterMode::Utf8);
let err = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false);
assert!(err.is_err());
}

#[test]
fn test_unterminated_escape_error() {
let mut line = char_provider_from("[abc\\"); // missing closing ]
let lines = test_lines();
let err = parse_character_class(&lines, &mut line, CharacterMode::Utf8);
let err = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false);
assert!(err.is_err());
}

#[test]
fn test_malformed_posix_like_pattern_treated_as_literal() {
let mut line = char_provider_from("[[x]yz]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
assert_eq!(result, b"[[x]");
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, br"[\[x]");
}

#[test]
fn test_literal_open_bracket_in_character_class() {
let mut line = char_provider_from("[a[b]");
let lines = test_lines();
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
assert_eq!(result, b"[a[b]");
let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8, false).unwrap();
assert_eq!(result, br"[a\[b]");
}

// parse_regex
Expand Down Expand Up @@ -1234,7 +1254,8 @@ mod tests {
fn test_regex_with_escaped_bracket_in_character_class() {
let (lines, mut line) = make_providers("/[a\\]z]/");
let parsed = parse_regex(&lines, &mut line, RegexMode::Basic).unwrap();
assert_eq!(parsed, br"[a\]z]");
// The class ends at the `]`, leaving `z]` as literal characters.
assert_eq!(parsed, br"[a\\]z]");
assert_eq!(line.current(), '/');
}

Expand All @@ -1254,6 +1275,59 @@ mod tests {
assert_eq!(line.current(), '/');
}

// parse_regex_for_mode: GNU extensions under --posix
#[test]
fn test_posix_bre_emits_bare_characters() {
// bre_to_ere() escapes the ERE metacharacters, so they are left bare.
let (lines, mut line) = make_providers(r"/0\?a\+b\|/");
let parsed = parse_regex_for_mode(
&lines,
&mut line,
RegexMode::Basic,
CharacterMode::Utf8,
true,
)
.unwrap();
assert_eq!(parsed, b"0?a+b|");

let (lines, mut line) = make_providers(r"/\w\s\b\<\`/");
let parsed = parse_regex_for_mode(
&lines,
&mut line,
RegexMode::Basic,
CharacterMode::Utf8,
true,
)
.unwrap();
assert_eq!(parsed, b"wsb<`");
}

#[test]
fn test_posix_ere_keeps_metacharacters_escaped() {
// In EREs \? \+ \| are already literals, so the backslash stays.
let (lines, mut line) = make_providers(r"/a\?b\+c\|/");
let parsed = parse_regex_for_mode(
&lines,
&mut line,
RegexMode::Extended,
CharacterMode::Utf8,
true,
)
.unwrap();
assert_eq!(parsed, br"a\?b\+c\|");

let (lines, mut line) = make_providers(r"/\w\s\b/");
let parsed = parse_regex_for_mode(
&lines,
&mut line,
RegexMode::Extended,
CharacterMode::Utf8,
true,
)
.unwrap();
assert_eq!(parsed, b"wsb");
}

// validate_quantifier_structure
//BRE tests
#[test]
Expand Down
Loading
Loading