Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 0 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ rand = { version = "0.10.0" }
regex = "1.10.4"
sha2 = { version = "0.11.0", default-features = false, features = ["alloc"] }
tempfile = "3.10.1"
terminal_size = "0.4.2"
textwrap = { version = "0.16.1", features = ["terminal_size"] }
unicode-width = "0.2.2"
uucore = { version = "0.12.0", features = ["diagnostics", "libc"] }
Expand All @@ -77,7 +76,6 @@ phf = { workspace = true }
predicates = { workspace = true }
regex = { workspace = true }
tempfile = { workspace = true }
terminal_size = { workspace = true }
textwrap = { workspace = true }
unicode-width = { workspace = true }
uucore = { workspace = true }
Expand Down
16 changes: 2 additions & 14 deletions src/sed/compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,8 @@ use std::mem;
use std::path::PathBuf;
use std::rc::Rc;

use terminal_size::{Width, terminal_size};
use uucore::error::{UResult, USimpleError};

const DEFAULT_OUTPUT_WIDTH: usize = 60;

const ERR_ADDRESS_0_USAGE: &str =
"address 0 can only be used with ~step, a second regular expression, or a read command";
const ERR_SANDBOX: &str = "command not allowed with --sandbox";
Expand Down Expand Up @@ -1218,22 +1215,13 @@ fn compile_label_command(
Ok(CommandHandling::Continue)
}

/// Return the width of the command's terminal or a default.
fn output_width() -> usize {
if let Some((Width(w), _)) = terminal_size() {
w as usize
} else {
DEFAULT_OUTPUT_WIDTH
}
}

/// Compile commands that take a number as an argument.
// Handles l q Q
fn compile_number_command(
lines: &mut ScriptLineProvider,
line: &mut ScriptCharProvider,
cmd: &mut Command,
_context: &mut ProcessingContext,
context: &mut ProcessingContext,
) -> UResult<CommandHandling> {
line.advance(); // Skip the command character
line.eat_spaces(); // Skip any leading whitespace
Expand All @@ -1247,7 +1235,7 @@ fn compile_number_command(
cmd.data = CommandData::Number(0);
}
'l' => {
cmd.data = CommandData::Number(output_width());
cmd.data = CommandData::Number(context.length);
}
_ => panic!("invalid number-expecting command"),
},
Expand Down
69 changes: 69 additions & 0 deletions src/sed/fast_io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -665,6 +665,29 @@ impl OutputBuffer {
)))
}

/// Schedule the specified string for output as part of an output line.
/// Unlike `write_str`, the string is not treated as a complete line, so
/// no newline is deferred after it and output that follows continues on
/// the same line.
pub(crate) fn write_partial_str(&mut self, s: &str) -> io::Result<()> {
if s.is_empty() {
return Ok(());
}

// End a preceding complete line whose newline was deferred.
self.flush_pending_newline()?;

// Mmapped output can be pending without a deferred newline, e.g. a
// newline-terminated input line printed from the mmapped input, so
// keep it ahead of this write. (After a deferred newline was
// written it has already been flushed and this does nothing.)
#[cfg(unix)]
{
self.flush_mmap(WriteRange::Complete)?;
}
self.out.write_all(s.as_bytes())
}

/// Schedule the specified bytes for eventual output.
pub fn write_bytes(&mut self, bytes: &[u8]) -> io::Result<()> {
let (content, has_newline) = if bytes.ends_with(b"\n") {
Expand Down Expand Up @@ -2151,4 +2174,50 @@ mod tests {
file.read_to_string(&mut out).unwrap();
assert_eq!(out, "baz\n");
}

// write_partial_str emits a deferred newline before its string
#[test]
fn write_partial_str_after_pending_newline() {
let (mut buf, mut file) = new_for_test();
buf.write_str("foo").unwrap();
assert!(buf.pending_newline);
buf.write_partial_str("bar").unwrap();
buf.out.flush().unwrap();
file.seek(SeekFrom::Start(0)).unwrap();
let mut out = String::new();
file.read_to_string(&mut out).unwrap();
assert_eq!(out, "foo\nbar");
}

// write_partial_str leaves no newline pending, so the output that
// follows continues its line
#[test]
fn write_partial_str_defers_no_newline() {
let (mut buf, mut file) = new_for_test();
buf.write_str("foo").unwrap();
buf.write_partial_str("bar").unwrap();
assert!(!buf.pending_newline);
buf.write_str("baz\n").unwrap();
buf.out.flush().unwrap();
file.seek(SeekFrom::Start(0)).unwrap();
let mut out = String::new();
file.read_to_string(&mut out).unwrap();
assert_eq!(out, "foo\nbarbaz\n");
}

// write_partial_str writes out pending mmapped output before its string
#[cfg(unix)]
#[test]
fn write_partial_str_after_mmap_chunk() {
let (mut buf, mut file) = new_for_test();
buf.write_chunk(&make_mmap_chunk(b"abc\n")).unwrap();
assert_eq!(buf.mmap_chunk.as_ref().unwrap().len, 4);
buf.write_partial_str("xyz").unwrap();
assert_eq!(buf.mmap_chunk.as_ref().unwrap().len, 0);
buf.flush().unwrap();
file.seek(SeekFrom::Start(0)).unwrap();
let mut out = String::new();
file.read_to_string(&mut out).unwrap();
assert_eq!(out, "abc\nxyz");
}
}
8 changes: 6 additions & 2 deletions src/sed/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,9 @@ pub fn uu_app() -> Command {
.require_equals(true)
.default_missing_value(""),
// Access with .get_one::<u32>("line-length")
arg!(-l --length <NUM> "Specify the 'l' command line-wrap length.")
arg!(-l --"line-length" <NUM> "Specify the 'l' command line-wrap length.")
// The long name used before GNU sed's --line-length was accepted.
.alias("length")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we need to keep the --length alias? it isn't a GNU option

.allow_hyphen_values(true)
.value_parser(clap::value_parser!(u32)),
arg!(-n --quiet "Suppress automatic printing of pattern space.").aliases(["silent"]),
Expand Down Expand Up @@ -364,7 +366,9 @@ fn build_context(matches: &ArgMatches) -> UResult<ProcessingContext> {
in_place_suffix: matches
.get_one::<String>("in-place")
.and_then(|s| if s.is_empty() { None } else { Some(s.clone()) }),
length: matches.get_one::<u32>("length").map_or(70, |v| *v as usize),
length: matches
.get_one::<u32>("line-length")
.map_or(70, |v| *v as usize),
quiet: matches.get_flag("quiet"),
posix: matches.get_flag("posix"),
separate: matches.get_flag("separate") || matches.contains_id("in-place"),
Expand Down
88 changes: 82 additions & 6 deletions src/sed/processor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -506,29 +506,56 @@ fn readable_char(ch: char) -> Cow<'static, str> {
}
}

/// Bound on the rendered list output buffered when never wrapping.
// A wrap width of zero never folds, so without periodic draining the whole
// rendered line would be held in memory. A few KB amortize the write calls
// while keeping the buffer bounded.
const LIST_FLUSH_THRESHOLD: usize = 4 * 1024;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we really need the draining? the pattern space is already whole in memory, the rendered line is at most a few times that.
without it we could drop write_partial_str, never_wrap and half of the tests, no?


/// Buffered state for rendering one list command output line.
struct ListLine {
buffer: String,
// Rendered bytes buffered since the last fold or drain; when wrapping,
// also the current column.
width: usize,
max_width: usize,
// Width at which write_item() acts: the wrap width, or
// LIST_FLUSH_THRESHOLD when never wrapping.
fold_at: usize,
// The wrap width; zero means never wrap.
wrap_width: usize,
}

impl ListLine {
/// Create an empty list output line with the specified maximum width.
fn new(max_width: usize) -> Self {
// Width zero never wraps, per the GNU sed manual; draining every few
// KB bounds the buffer and keeps one width test in write_item().
Self {
buffer: String::new(),
width: 0,
max_width,
fold_at: if max_width == 0 {
LIST_FLUSH_THRESHOLD
} else {
max_width
},
wrap_width: max_width,
}
}

/// Write a rendered list item, folding before the item if needed.
/// Write a rendered list item, folding the line before it when wrapping
/// and draining the buffer when not.
fn write_item(&mut self, output: &mut OutputBuffer, out_str: &str) -> UResult<()> {
let out_len = out_str.len();
if self.width + out_len + 1 > self.max_width {
self.buffer.push_str("\\\n");
output.write_str(std::mem::take(&mut self.buffer))?;
if self.width + out_len + 1 > self.fold_at {
if self.wrap_width == 0 {
// Drain before out_str: finish() terminates only a non-empty
// buffer, so draining after the last item would lose the "$".
output.write_partial_str(&self.buffer)?;
self.buffer.clear();
} else {
self.buffer.push_str("\\\n");
output.write_str(std::mem::take(&mut self.buffer))?;
}
self.width = 0;
}
self.buffer.push_str(out_str);
Expand Down Expand Up @@ -1103,6 +1130,55 @@ mod tests {
assert_eq!(written, "abcd\\\n");
}

#[test]
fn test_write_list_item_zero_width_never_folds() {
let mut file = tempfile().unwrap();
let mut output = OutputBuffer::new(Box::new(file.try_clone().unwrap()));
let mut line = ListLine::new(0);
line.write_item(&mut output, "abcd").unwrap();

line.write_item(&mut output, "e").unwrap();
output.flush().unwrap();

assert_eq!(line.buffer, "abcde");
assert_eq!(line.width, 5);
file.seek(SeekFrom::Start(0)).unwrap();
let mut written = String::new();
file.read_to_string(&mut written).unwrap();
assert_eq!(written, "");
}

#[test]
fn test_write_list_item_zero_width_drains_buffer() {
let mut file = tempfile().unwrap();
let mut output = OutputBuffer::new(Box::new(file.try_clone().unwrap()));
let mut line = ListLine::new(0);

// Exactly as many items as the drain threshold.
let items = LIST_FLUSH_THRESHOLD;
for _ in 0..items {
line.write_item(&mut output, "a").unwrap();
}

// The drain took place before the item that reached the threshold
// was appended, so that item is still buffered.
assert_eq!(line.buffer, "a");
assert_eq!(line.width, 1);
output.flush().unwrap();
file.seek(SeekFrom::Start(0)).unwrap();
let mut drained = String::new();
file.read_to_string(&mut drained).unwrap();
assert_eq!(drained, "a".repeat(items - 1));

// The complete output is still the unfolded line and its terminator.
line.finish(&mut output).unwrap();
output.flush().unwrap();
file.seek(SeekFrom::Start(0)).unwrap();
let mut written = String::new();
file.read_to_string(&mut written).unwrap();
assert_eq!(written, format!("{}$\n", "a".repeat(items)));
}

#[test]
fn test_list_line_finish_writes_terminator() {
let mut file = tempfile().unwrap();
Expand Down
Loading
Loading