Restore bracketed paste mode instead of forcing it off

CHANGELOG.md

@@ -12,6 +12,8 @@ CHANGELOG
     - ASCII input is unaffected
 - Fixed an image from a preview command being torn apart when its rows are separated by IND instead of newlines, as `chafa` does under tmux (#4885)
 - Fixed `replace-query` corrupting the item text when the query is edited afterwards
+- fzf no longer turns bracketed paste mode off on exit when the terminal already had it on, which broke pasting in shells that run fzf from a line editor widget (#4887)
+- Fixed startup blocking on terminals that never answer escape sequences, such as FreeBSD virtual terminals. fzf waited for a reply until a key was pressed, then dropped that keystroke (#2860, #976)
 
 0.74.2
 ------

src/algo/algo.go

@@ -304,7 +304,7 @@ func bonusAt(input *util.Chars, idx int) int16 {
 
 func normalizeRune(r rune) rune {
 	// Every key of the map folds to ASCII, so a rune the bitmap rejects cannot
-	// be in it. TestNormalizedKeysAreFlagged pins that.
+	// be in it. TestNormalizedKeysAreFlagged verifies that.
 	if !util.MayFoldToAscii(r) {
 		return r
 	}

src/algo/fastpath_equiv_test.go

@@ -1,7 +1,7 @@
 package algo
 
 // Equivalence tests for the single- and two-character fast paths against the
-// general FuzzyMatchV2 algorithm, which serves as the oracle.
+// general FuzzyMatchV2 algorithm, which serves as the reference.
 //
 // Two complementary strategies:
 //   - Exhaustive: every string up to a fixed length over an alphabet that

src/algo/runeprefilter_test.go

@@ -29,9 +29,9 @@ func foldForTest(r rune, normalize bool) rune {
 }
 
 // The prefilter is only safe on items whose runes cannot become ASCII. This
-// pins util.MayFoldToAscii as a superset of the runes that actually can, over
-// the whole Unicode range and both normalization modes. If normalize.go or the
-// Go unicode tables change, this fails.
+// verifies util.MayFoldToAscii as a superset of the runes that actually can,
+// over the whole Unicode range and both normalization modes. If normalize.go
+// or the Go unicode tables change, this fails.
 func TestMayFoldToAsciiIsSuperset(t *testing.T) {
 	missed := 0
 	for r := rune(utf8.RuneSelf); r <= unicode.MaxRune; r++ {
@@ -51,8 +51,8 @@ func TestMayFoldToAsciiIsSuperset(t *testing.T) {
 	}
 }
 
-// Scripts that must stay unflagged, otherwise the prefilter never engages for
-// them and Step C buys nothing.
+// Scripts that must stay unflagged, otherwise the prefilter never runs for
+// them and Step C has no effect.
 func TestMayFoldToAsciiExcludesMajorScripts(t *testing.T) {
 	for _, s := range []struct {
 		name   string
@@ -62,9 +62,9 @@ func TestMayFoldToAsciiExcludesMajorScripts(t *testing.T) {
 		{"Arabic", 0x0600, 0x06FF}, {"Thai", 0x0E00, 0x0E7F}, {"Devanagari", 0x0900, 0x097F},
 		{"CJK", 0x4E00, 0x9FFF}, {"Hangul", 0xAC00, 0xD7A3}, {"kana", 0x3040, 0x30FF},
 		{"box drawing", 0x2500, 0x257F}, {"emoji", 0x1F300, 0x1FAFF},
-		// These sit between the Latin blocks and were swallowed by an earlier,
-		// wider grouping of foldableRanges. General Punctuation is the costly
-		// one: curly quotes, en and em dashes and the ellipsis live there.
+		// These sit between the Latin blocks and were included in an earlier,
+		// wider grouping of foldableRanges. General Punctuation matters most:
+		// curly quotes, en and em dashes and the ellipsis are in it.
 		{"Greek Extended", 0x1F00, 0x1FFF}, {"General Punctuation", 0x2000, 0x206F},
 		{"Currency Symbols", 0x20A0, 0x20CF}, {"CJK Symbols", 0x3000, 0x303F},
 	} {
@@ -257,7 +257,7 @@ func TestNormalizeRuneUnchangedByGuard(t *testing.T) {
 
 // Step G lets non-ASCII pattern runes use the scan, but only when no other
 // rune can transform into them. Being uncased is not sufficient: U+00DF has no
-// simple uppercase yet U+1E9E lowercases onto it. This pins the guard against
+// simple uppercase yet U+1E9E lowercases onto it. This checks the guard against
 // the full preimage relation over all of Unicode.
 func TestRunePrefilterableGuardIsSound(t *testing.T) {
 	preimage := map[rune][]rune{}
@@ -306,7 +306,7 @@ func TestRunePrefilterableGuardIsSound(t *testing.T) {
 	}
 }
 
-// The Step G path must actually engage and reject, otherwise the equivalence
+// The Step G path must actually run and reject, otherwise the equivalence
 // test above proves nothing about non-ASCII patterns.
 func TestNonAsciiPatternPrefilterEngages(t *testing.T) {
 	rng := rand.New(rand.NewSource(6))
@@ -413,7 +413,7 @@ func FuzzRunePrefilter(f *testing.F) {
 		}
 		chars := util.ToChars([]byte(input))
 		if chars.IsBytes() {
-			return // byte mode is the existing fuzzers' territory
+			return // byte mode is covered by the existing fuzzers
 		}
 		for _, cs := range []bool{false, true} {
 			for _, norm := range []bool{false, true} {

src/tui/light.go

@@ -24,15 +24,38 @@ const (
 	defaultEscDelay = 100
 	escPollInterval = 5
 	offsetPollTries = 10
+	queryTimeout    = 500 * time.Millisecond
 	maxInputBuffer  = 1024 * 1024
 	maxSelectTries  = 100
 )
 
 const DefaultTtyDevice string = "/dev/tty"
 
-var offsetRegexp = regexp.MustCompile("(.*?)\x00?\x1b\\[([0-9]+);([0-9]+)R")
+var offsetRegexp = regexp.MustCompile("\x00?\x1b\\[([0-9]+);([0-9]+)R")
 var offsetRegexpBegin = regexp.MustCompile("^\x1b\\[[0-9]+;[0-9]+R")
 
+// DECRPM reply to the DECRQM query for bracketed paste mode. Ps is 1 or 3 when
+// the mode was already set, 2 or 4 when reset, 0 when the terminal does not
+// recognize the mode.
+var pasteModeRegexp = regexp.MustCompile("\x00?\x1b\\[\\?2004;([0-4])\\$y")
+var pasteModeRegexpBegin = regexp.MustCompile("^\x1b\\[\\?2004;[0-4]\\$y")
+
+// A report to ask the terminal for, and the reply to recognize it by.
+type termQuery struct {
+	seq   string
+	reply *regexp.Regexp
+}
+
+// What we ask the terminal at startup, in the order the queries go out. A
+// terminal answers them in that order, so the cursor position query is last and
+// also ends the wait: every terminal fzf supports answers it, so once its reply
+// arrives, a query still unanswered is one the terminal does not know rather
+// than one we stopped waiting for too early.
+var startupQueries = []termQuery{
+	{"?2004$p", pasteModeRegexp},
+	{"6n", offsetRegexp},
+}
+
 func (r *LightRenderer) Bell() {
 	r.flushRaw("\a")
 }
@@ -158,6 +181,10 @@ type LightRenderer struct {
 	showCursor    bool
 	mutex         sync.Mutex
 
+	// Whether bracketed paste was already on before we enabled it. Nil when
+	// the terminal did not answer the query.
+	pasteWasSet *bool
+
 	// Windows only
 	ttyinChannel    chan byte
 	inHandle        uintptr
@@ -230,8 +257,13 @@ func (r *LightRenderer) Init() error {
 
 	if r.fullscreen {
 		r.smcup()
-	} else {
-		y, x := r.findOffset()
+	}
+
+	// Ask everything in one round trip, before the offset is needed.
+	y, x, pasteWasSet := r.queryStartup()
+	r.pasteWasSet = pasteWasSet
+
+	if !r.fullscreen {
 		r.mouse = r.mouse && y >= 0
 		// When --no-clear is used for repetitive relaunching, there is a small
 		// time frame between fzf processes where the user keystrokes are not
@@ -318,7 +350,11 @@ func (r *LightRenderer) getBytesInternal(cancellable bool, buffer []byte, nonblo
 	if c == Esc.Int() || nonblock {
 		retries = r.escDelay / escPollInterval
 	}
-	buffer = append(buffer, byte(c))
+	// A non-blocking read that found nothing has no byte to record. Recording
+	// one would put a NUL in the middle of a reply still being assembled.
+	if result.ok() {
+		buffer = append(buffer, byte(c))
+	}
 
 	pc := c
 	for {
@@ -446,6 +482,12 @@ func (r *LightRenderer) escSequence(sz *int) Event {
 		return Event{Invalid, 0, nil}
 	}
 
+	loc = pasteModeRegexpBegin.FindIndex(r.buffer)
+	if loc != nil && loc[0] == 0 {
+		*sz = loc[1]
+		return Event{Invalid, 0, nil}
+	}
+
 	*sz = 2
 	if r.buffer[1] == 8 {
 		return Event{CtrlAltBackspace, 0, nil}
@@ -1019,7 +1061,16 @@ func (r *LightRenderer) disableMouse() {
 
 func (r *LightRenderer) disableModes() {
 	r.disableMouse()
-	r.csi("?2004l")
+	// Put bracketed paste back the way we found it. A shell that runs fzf from
+	// a line editor widget re-enables the mode only when the editor starts, so
+	// forcing it off here would leave it off for the rest of the session.
+	// Terminals that did not answer the query fall back to disabling, which is
+	// what fzf has always done.
+	if r.pasteWasSet != nil && *r.pasteWasSet {
+		r.csi("?2004h")
+	} else {
+		r.csi("?2004l")
+	}
 }
 
 func (r *LightRenderer) Resume(clear bool, sigcont bool) {

src/tui/light_unix.go

@@ -8,6 +8,7 @@ import (
 	"os/exec"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/junegunn/fzf/src/util"
 	"golang.org/x/sys/unix"
@@ -93,25 +94,113 @@ func (r *LightRenderer) updateTerminalSize() {
 	}
 }
 
-func (r *LightRenderer) findOffset() (row int, col int) {
-	r.csi("6n")
+// waitReadable reports whether the tty has something to read before the
+// deadline. A terminal that does not recognize a query answers nothing at all,
+// so the read that follows must be able to stop waiting, or fzf would wait for
+// the user to press a key instead of drawing itself. The timeout is generous
+// because a terminal that does answer exceeds it only when the link is slow
+// enough to be unusable anyway.
+func (r *LightRenderer) waitReadable(timeout time.Duration) bool {
+	fd := r.fd()
+	deadline := time.Now().Add(timeout)
+	for {
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			return false
+		}
+		var rfds unix.FdSet
+		if fd >= len(rfds.Bits)*unix.NFDBITS {
+			return false
+		}
+		rfds.Set(fd)
+		// Recomputed each time round: Linux select rewrites the timeout with
+		// the time left, other systems leave it alone
+		tv := unix.NsecToTimeval(int64(remaining))
+		n, err := unix.Select(fd+1, &rfds, nil, nil, &tv)
+		if err == syscall.EINTR {
+			continue
+		}
+		if err != nil {
+			// Nothing was confirmed readable, and the read that follows
+			// reports the failure if the fd is really broken
+			return false
+		}
+		return n > 0
+	}
+}
+
+// queryTerminal sends every query in a single write and reads until the last
+// one is answered. Returns the submatches of each reply, nil for a query the
+// terminal ignored. Replies are cut out of what we read as they are recognized,
+// so whatever is left over is input the user typed during the round trip.
+func (r *LightRenderer) queryTerminal(queries []termQuery) [][][]byte {
+	for _, query := range queries {
+		r.csi(query.seq)
+	}
 	r.flush()
-	var err error
-	bytes := []byte{}
+
+	replies := make([][][]byte, len(queries))
+	buffer := []byte{}
 	for tries := range offsetPollTries {
-		bytes, _, err = r.getBytesInternal(false, bytes, tries > 0)
+		// Only the first read blocks, so that is the one to put a bound on
+		if tries == 0 && !r.waitReadable(queryTimeout) {
+			return replies
+		}
+
+		var err error
+		buffer, _, err = r.getBytesInternal(false, buffer, tries > 0)
 		if err != nil {
-			return -1, -1
+			return replies
 		}
 
-		offsets := offsetRegexp.FindSubmatch(bytes)
-		if len(offsets) > 3 {
-			// Add anything we skipped over to the input buffer
-			r.buffer = append(r.buffer, offsets[1]...)
-			return atoi(string(offsets[2]), 0) - 1, atoi(string(offsets[3]), 0) - 1
+		for idx, query := range queries {
+			if replies[idx] != nil {
+				continue
+			}
+			loc := query.reply.FindSubmatchIndex(buffer)
+			if loc == nil {
+				continue
+			}
+			groups := make([][]byte, len(loc)/2)
+			for group := range groups {
+				if loc[group*2] >= 0 {
+					groups[group] = buffer[loc[group*2]:loc[group*2+1]]
+				}
+			}
+			replies[idx] = groups
+			// Capping the prefix makes append copy, leaving groups valid
+			buffer = append(buffer[:loc[0]:loc[0]], buffer[loc[1]:]...)
+		}
+
+		if replies[len(queries)-1] != nil {
+			break
 		}
 	}
-	return -1, -1
+
+	r.buffer = append(r.buffer, buffer...)
+	return replies
+}
+
+func (r *LightRenderer) queryStartup() (row int, col int, pasteWasSet *bool) {
+	replies := r.queryTerminal(startupQueries)
+	if paste := replies[0]; paste != nil && paste[1][0] != '0' {
+		// 1 = set, 3 = permanently set
+		set := paste[1][0] == '1' || paste[1][0] == '3'
+		pasteWasSet = &set
+	}
+	row, col = parseOffset(replies[1])
+	return
+}
+
+func parseOffset(reply [][]byte) (row int, col int) {
+	if reply == nil {
+		return -1, -1
+	}
+	return atoi(string(reply[1]), 0) - 1, atoi(string(reply[2]), 0) - 1
+}
+
+func (r *LightRenderer) findOffset() (row int, col int) {
+	return parseOffset(r.queryTerminal(startupQueries[1:])[0])
 }
 
 func (r *LightRenderer) getch(cancellable bool, nonblock bool) (int, getCharResult) {

src/tui/light_windows.go

@@ -151,6 +151,13 @@ func (r *LightRenderer) findOffset() (row int, col int) {
 	return int(bufferInfo.CursorPosition.Y), int(bufferInfo.CursorPosition.X)
 }
 
+// The console API answers for the cursor, and there is no reply to parse for
+// bracketed paste, so fzf keeps disabling the mode on exit here.
+func (r *LightRenderer) queryStartup() (row int, col int, pasteWasSet *bool) {
+	row, col = r.findOffset()
+	return
+}
+
 func (r *LightRenderer) getch(cancellable bool, nonblock bool) (int, getCharResult) {
 	if !nonblock && !cancellable {
 		bc := <-r.ttyinChannel

src/util/chars.go

@@ -36,9 +36,9 @@ type Chars struct {
 // Rune ranges that case folding or normalization can turn into ASCII, derived
 // from algo's normalization table and unicode.ToLower, then merged. They are a
 // superset of the exact set, which TestMayFoldToAsciiIsSuperset in the algo
-// package pins. Grouped tightly on purpose: a wider merge would swallow Greek
-// Extended, General Punctuation and the currency and letterlike blocks, and
-// every line holding a curly quote or an em dash would then lose the
+// package verifies. Grouped tightly on purpose: a wider merge would include
+// Greek Extended, General Punctuation and the currency and letterlike blocks,
+// and every line holding a curly quote or an em dash would then lose the
 // prefilter. Cyrillic, Greek, Hebrew, Arabic, Thai, Devanagari, CJK, Hangul,
 // kana, emoji, punctuation and box drawing are all outside.
 const (