Performance optimizations for non-ASCII input

CHANGELOG.md

@@ -3,7 +3,15 @@ CHANGELOG
 
 0.74.3
 ------
+- Performance optimizations for non-ASCII input
+    - A line holding any non-ASCII character is kept as a rune array, and the prefilter did not run on those lines, so every item went through the full score matrix
+    - Queries are up to 16x faster, the gain growing with how much of the line is non-ASCII
+    - Non-ASCII queries are up to 12x faster
+    - Reading non-ASCII input is up to 37% faster and uses up to 29% less memory, the gain depending on how early the first non-ASCII character appears in the line
+    - Accented Latin and fullwidth forms get the faster reading but not the faster queries, because those characters can still match their ASCII counterparts
+    - ASCII input is unaffected
 - Fixed an image from a preview command being torn apart when its rows are separated by IND instead of newlines, as `chafa` does under tmux (#4885)
+- Fixed `replace-query` corrupting the item text when the query is edited afterwards
 
 0.74.2
 ------

Makefile

@@ -102,7 +102,7 @@ itest:
 # FUZZTIME (e.g. make fuzz FUZZTIME=5m).
 FUZZTIME ?= 30s
 fuzz:
-	@for t in FuzzFuzzyMatchV2Single FuzzFuzzyMatchV2Two; do \
+	@for t in FuzzFuzzyMatchV2Single FuzzFuzzyMatchV2Two FuzzRunePrefilter; do \
 		echo "== $$t =="; \
 		$(GO) test -run '^$$' -fuzz "^$$t$$" -fuzztime $(FUZZTIME) ./src/algo || exit 1; \
 	done

src/algo/algo.go

@@ -303,7 +303,9 @@ func bonusAt(input *util.Chars, idx int) int16 {
 }
 
 func normalizeRune(r rune) rune {
-	if r < 0x00C0 || r > 0xFF61 {
+	// Every key of the map folds to ASCII, so a rune the bitmap rejects cannot
+	// be in it. TestNormalizedKeysAreFlagged pins that.
+	if !util.MayFoldToAscii(r) {
 		return r
 	}
 
@@ -345,10 +347,90 @@ func isAscii(runes []rune) bool {
 	return true
 }
 
+// runePrefilterable reports whether scanning the rune array can decide this
+// pattern against this item without missing a match. Phase 2 lowercases an
+// uppercase text rune and then normalizes it, and the scan sees neither
+// transform, so every pattern rune must be unreachable by them.
+func runePrefilterable(input *util.Chars, pattern []rune, caseSensitive bool) bool {
+	if input.MayFoldToAscii() {
+		// A non-ASCII rune of this item could fold onto an ASCII pattern char
+		for _, r := range pattern {
+			if r < utf8.RuneSelf {
+				return false
+			}
+		}
+	}
+	if caseSensitive {
+		// No case transform is applied, and normalization only ever produces
+		// ASCII, so nothing can reach a non-ASCII pattern rune
+		return true
+	}
+	for _, r := range pattern {
+		// Another rune must not lowercase onto this one. Being uncased is not
+		// enough by itself: U+00DF has no simple uppercase yet U+1E9E
+		// lowercases to it. Excluding the foldable set covers that.
+		if r >= utf8.RuneSelf &&
+			(unicode.ToUpper(r) != r || unicode.ToLower(r) != r || util.MayFoldToAscii(r)) {
+			return false
+		}
+	}
+	return true
+}
+
+// runeFuzzyIndex is asciiFuzzyIndex for rune-mode input. Only valid when
+// runePrefilterable says so.
+func runeFuzzyIndex(input *util.Chars, pattern []rune, caseSensitive bool) (int, int) {
+	runes := input.Runes()
+	firstIdx, idx, lastIdx := 0, 0, 0
+	var last rune
+	for pidx := range pattern {
+		last = pattern[pidx]
+		if last < utf8.RuneSelf {
+			idx = indexAsciiRune(runes, caseSensitive, byte(last), idx)
+		} else {
+			idx = indexRune(runes, last, idx)
+		}
+		if idx < 0 {
+			return -1, -1
+		}
+		if pidx == 0 && idx > 0 {
+			// Step back to find the right bonus point
+			firstIdx = idx - 1
+		}
+		lastIdx = idx
+		idx++
+	}
+
+	// Find the last appearance of the last character of the pattern to limit
+	// the search scope
+	if lastIdx+1 < len(runes) {
+		var end int
+		if last < utf8.RuneSelf {
+			end = lastIndexAsciiRune(runes, caseSensitive, byte(last), lastIdx+1)
+		} else {
+			end = lastIndexRune(runes, last, lastIdx+1)
+		}
+		if end >= 0 {
+			return firstIdx, end + 1
+		}
+	}
+	return firstIdx, lastIdx + 1
+}
+
 func asciiFuzzyIndex(input *util.Chars, pattern []rune, caseSensitive bool) (int, int) {
-	// Can't determine
 	if !input.IsBytes() {
-		return 0, input.Length()
+		if disableRunePrefilter {
+			return 0, input.Length()
+		}
+		// runePrefilterable does not inline, so keep the common case out of
+		// it: an ASCII pattern against an item that cannot fold to ASCII is
+		// always scannable, and both of these checks do inline.
+		if input.MayFoldToAscii() || !isAscii(pattern) {
+			if !runePrefilterable(input, pattern, caseSensitive) {
+				return 0, input.Length()
+			}
+		}
+		return runeFuzzyIndex(input, pattern, caseSensitive)
 	}
 
 	// Not possible
@@ -466,8 +548,9 @@ func fuzzyMatchV2Single(caseSensitive bool, forward bool, input *util.Chars, b b
 // Test hooks: force the general path instead of a fast path, so the two can
 // be compared for equivalence.
 var (
-	disableSingle bool
-	disableTwo    bool
+	disableSingle        bool
+	disableTwo           bool
+	disableRunePrefilter bool
 )
 
 // fuzzyMatchV2Two is a fused fast path for a two-character ASCII pattern on

src/algo/runeindex_others.go

@@ -0,0 +1,23 @@
+//go:build !386 && !amd64 && !arm64
+
+package algo
+
+// The byte-view scanners in runeindex_x86.go reinterpret a []rune as
+// little-endian 4-byte lanes, which is not valid everywhere. Elsewhere the
+// reference scanners are the implementation.
+
+func indexAsciiRune(runes []rune, caseSensitive bool, b byte, from int) int {
+	return indexAsciiRuneRef(runes, caseSensitive, b, from)
+}
+
+func lastIndexAsciiRune(runes []rune, caseSensitive bool, b byte, from int) int {
+	return lastIndexAsciiRuneRef(runes, caseSensitive, b, from)
+}
+
+func indexRune(runes []rune, r rune, from int) int {
+	return indexRuneRef(runes, r, from)
+}
+
+func lastIndexRune(runes []rune, r rune, from int) int {
+	return lastIndexRuneRef(runes, r, from)
+}

src/algo/runeindex_ref.go

@@ -0,0 +1,55 @@
+package algo
+
+// Reference scanners over a []rune, with no representation tricks.
+//
+// They have two roles. Where reinterpreting a []rune as little-endian bytes is
+// not valid, they are the shipped implementation, via runeindex_others.go.
+// Everywhere else, the tests feed the same inputs to these and to the byte-view
+// scanners in runeindex_x86.go and require identical answers.
+//
+// They carry no build tag so that both roles hold on every platform. Otherwise
+// the portable build would be code that nothing here ever runs.
+
+func indexAsciiRuneRef(runes []rune, caseSensitive bool, b byte, from int) int {
+	lower, upper := rune(b), rune(-1)
+	if !caseSensitive && b >= 'a' && b <= 'z' {
+		upper = rune(b - 32)
+	}
+	for i := from; i < len(runes); i++ {
+		if runes[i] == lower || runes[i] == upper {
+			return i
+		}
+	}
+	return -1
+}
+
+func lastIndexAsciiRuneRef(runes []rune, caseSensitive bool, b byte, from int) int {
+	lower, upper := rune(b), rune(-1)
+	if !caseSensitive && b >= 'a' && b <= 'z' {
+		upper = rune(b - 32)
+	}
+	for i := len(runes) - 1; i >= from; i-- {
+		if runes[i] == lower || runes[i] == upper {
+			return i
+		}
+	}
+	return -1
+}
+
+func indexRuneRef(runes []rune, r rune, from int) int {
+	for i := from; i < len(runes); i++ {
+		if runes[i] == r {
+			return i
+		}
+	}
+	return -1
+}
+
+func lastIndexRuneRef(runes []rune, r rune, from int) int {
+	for i := len(runes) - 1; i >= from; i-- {
+		if runes[i] == r {
+			return i
+		}
+	}
+	return -1
+}

src/algo/runeindex_x86.go

@@ -0,0 +1,128 @@
+//go:build 386 || amd64 || arm64
+
+package algo
+
+import (
+	"bytes"
+	"unsafe"
+)
+
+// On these architectures a []rune is a little-endian array of 4-byte lanes, so
+// an ASCII rune is the byte itself followed by three zero bytes at a 4-byte
+// aligned offset. That lets the SIMD byte scanners run over the rune array
+// directly: find the low byte, then confirm alignment and the three zeroes.
+// A byte equal to the needle can also appear as the low byte of a multi-byte
+// rune (0x0165 has low byte 'e'), which those two checks reject.
+
+func runeBytes(runes []rune) []byte {
+	return unsafe.Slice((*byte)(unsafe.Pointer(unsafe.SliceData(runes))), len(runes)*4)
+}
+
+// indexAsciiRune returns the index of the first rune equal to b, or to its
+// uppercase form when ignoring case, at or after rune index from.
+func indexAsciiRune(runes []rune, caseSensitive bool, b byte, from int) int {
+	view := runeBytes(runes)
+	both := !caseSensitive && b >= 'a' && b <= 'z'
+	for off := from * 4; off < len(view); {
+		var idx int
+		if both {
+			idx = IndexByteTwo(view[off:], b, b-32)
+		} else {
+			idx = bytes.IndexByte(view[off:], b)
+		}
+		if idx < 0 {
+			return -1
+		}
+		pos := off + idx
+		if pos&3 == 0 && view[pos+1]|view[pos+2]|view[pos+3] == 0 {
+			return pos >> 2
+		}
+		off = pos + 1
+	}
+	return -1
+}
+
+// runeNeedle picks which of the rune's four bytes to scan for, and returns its
+// lane index and value. A zero byte is a useless needle because every ASCII
+// rune contributes three of them, so U+AE00 scanned by its low byte would hit
+// on almost every character of an ASCII-heavy line. Prefer a byte that cannot
+// occur in an ASCII rune at all, then any non-zero byte.
+func runeNeedle(r rune) (int, byte) {
+	var b [4]byte
+	b[0], b[1], b[2], b[3] = byte(r), byte(r>>8), byte(r>>16), byte(r>>24)
+	for i, v := range b {
+		if v >= 0x80 {
+			return i, v
+		}
+	}
+	for i, v := range b {
+		if v != 0 {
+			return i, v
+		}
+	}
+	return 0, 0
+}
+
+func runeAt(view []byte, start int) rune {
+	return rune(view[start]) | rune(view[start+1])<<8 | rune(view[start+2])<<16 | rune(view[start+3])<<24
+}
+
+// indexRune returns the index of the first rune equal to r at or after rune
+// index from. Case is not folded, so the caller must have established that no
+// other rune can transform into r.
+func indexRune(runes []rune, r rune, from int) int {
+	view := runeBytes(runes)
+	lane, needle := runeNeedle(r)
+	for off := from*4 + lane; off < len(view); {
+		idx := bytes.IndexByte(view[off:], needle)
+		if idx < 0 {
+			return -1
+		}
+		pos := off + idx
+		if start := pos - lane; start&3 == 0 && runeAt(view, start) == r {
+			return start >> 2
+		}
+		off = pos + 1
+	}
+	return -1
+}
+
+// lastIndexRune is indexRune scanning backwards from the end.
+func lastIndexRune(runes []rune, r rune, from int) int {
+	view := runeBytes(runes)
+	lane, needle := runeNeedle(r)
+	for end := len(view); end > from*4+lane; {
+		idx := bytes.LastIndexByte(view[from*4+lane:end], needle)
+		if idx < 0 {
+			return -1
+		}
+		pos := from*4 + lane + idx
+		if start := pos - lane; start&3 == 0 && runeAt(view, start) == r {
+			return start >> 2
+		}
+		end = pos
+	}
+	return -1
+}
+
+// lastIndexAsciiRune is indexAsciiRune scanning backwards from the end.
+func lastIndexAsciiRune(runes []rune, caseSensitive bool, b byte, from int) int {
+	view := runeBytes(runes)[from*4:]
+	both := !caseSensitive && b >= 'a' && b <= 'z'
+	for end := len(view); end > 0; {
+		var idx int
+		if both {
+			idx = lastIndexByteTwo(view[:end], b, b-32)
+		} else {
+			idx = bytes.LastIndexByte(view[:end], b)
+		}
+		if idx < 0 {
+			return -1
+		}
+		if idx&3 == 0 && view[idx+1]|view[idx+2]|view[idx+3] == 0 {
+			return from + idx>>2
+		}
+		end = idx
+	}
+	return -1
+}