From b627c352ac326b52d1b0695b13aeb053495b7af4 Mon Sep 17 00:00:00 2001 From: rajeshsub <4209324+rajeshsub@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:42:24 +1000 Subject: [PATCH 1/3] Phase 1 of 3 - performance improvements --- docs/adr/0006-pool-based-index-layout.md | 64 ++++++++ src/core/CMakeLists.txt | 1 + src/core/search/ISearchEngine.h | 29 ++-- src/core/search/SearchEngine.cpp | 128 ++++++++-------- src/core/search/SearchEngine.h | 26 ++-- src/core/storage/IIndexStore.h | 44 ------ src/core/storage/IndexPool.cpp | 71 +++++++++ src/core/storage/IndexPool.h | 51 +++++++ src/core/storage/IndexSerializer.cpp | 187 ++++++++++++++++------- src/core/storage/IndexSerializer.h | 44 +++--- src/core/storage/IndexStore.cpp | 138 ++++++++++------- src/core/storage/IndexStore.h | 20 ++- src/ui/MainWindow.cpp | 121 ++++++++------- src/ui/MainWindow.h | 21 ++- tests/CMakeLists.txt | 1 + tests/mocks/MockIndexStore.h | 1 - tests/test_IndexPool.cpp | 122 +++++++++++++++ tests/test_IndexSerializer.cpp | 128 +++++++++++----- tests/test_SearchEngine.cpp | 142 +++++++++-------- 19 files changed, 903 insertions(+), 436 deletions(-) create mode 100644 docs/adr/0006-pool-based-index-layout.md create mode 100644 src/core/storage/IndexPool.cpp create mode 100644 src/core/storage/IndexPool.h create mode 100644 tests/test_IndexPool.cpp diff --git a/docs/adr/0006-pool-based-index-layout.md b/docs/adr/0006-pool-based-index-layout.md new file mode 100644 index 0000000..18072fb --- /dev/null +++ b/docs/adr/0006-pool-based-index-layout.md @@ -0,0 +1,64 @@ +Status: Accepted + +## Context + +Profiling on a 624 K-entry index showed two root causes of slow search: + +1. **Cache thrashing from heap-scattered strings.** Each `FileEntry` owns three + heap-allocated `std::wstring` objects (`name`, `nameLower`, `path`). Searching + 624 K entries requires chasing ~1.9 M scattered heap pointers, producing near-zero + L2/L3 cache reuse and dominating search latency. + +2. **No early exit across parallel search threads.** Threads continue scanning after + `maxResults` hits are found collectively, wasting CPU time on every keystroke. + +A third issue — `GetEntries()` returning a raw pointer with no lock while ChangeWatcher +threads can concurrently call `ApplyAdd()` and reallocate the vector — is an active data +race. See also ADR-0003 (binary format) which this supersedes for the in-memory layout. + +## Options + +| Option | Fits when | Cost now | Extension path | Trade-off | +|--------|-----------|----------|----------------|-----------| +| a. Keep `vector` with per-entry `wstring` | Index is small (<100 K entries) | None | None | Cache-hostile at 600 K+ entries; scales poorly | +| b. Flat pool: single `vector` with offsets | Sequential scan is the primary access pattern | Medium refactor | Add sorted/trigram index later | O(n) scan but L3-resident; correct for 600 K scale | +| c. Sorted array + binary search | Prefix-match queries dominate | High refactor | Replace with trie later | Only fast for prefix queries; substring still O(n) | +| d. Trigram inverted index | Arbitrary substring at >10 M entries | High complexity | Standard IR approach | Overkill at 600 K; large memory overhead | + +## Decision + +Adopt **option b** — separate flat string pools: + +- **`nameLower` pool** (`vector`): all lowercased filenames concatenated, + ~18 MB for 624 K entries. Fits in L3 cache. This is the default search target. +- **`path` pool** (`vector`): all full paths original-case concatenated, + ~75 MB. Used only when `matchPath = true` (opt-in). +- **Metadata array** (`vector`): fixed-size structs (32 bytes each) holding + offsets and lengths into both pools, plus `size`, `lastModified`, `attributes`. + +`SearchResult::entry` changes from `const FileEntry*` to `uint32_t entryIndex`. The UI +looks up display fields via a thin `IndexPool::GetEntry(index)` call. + +Concurrency is managed by `std::shared_mutex` (backed by Windows SRWLOCK — no heavier +than a CRITICAL_SECTION). Search acquires a shared read lock for the duration of the +scan (~5-10 ms). ChangeWatcher acquires an exclusive write lock to append entries. + +A `pathLower` pool is deferred: path search (`matchPath = true`) is opt-in and not the +latency-sensitive default. It can be added as a fourth pool if path-search performance +becomes a concern. + +On-disk format is bumped to **version 2** (see ADR-0003). Version mismatch triggers a +one-time silent re-index on first launch after the upgrade. + +## Consequences + +- Name-only search scans ~18 MB of contiguous memory instead of chasing ~1.9 M heap + pointers — expected 3-5x throughput improvement. +- Data race on `GetEntries()` is eliminated by `shared_mutex`. +- `SearchResult` is a breaking API change: callers must use `entryIndex` + pool lookup + rather than a `FileEntry*`. +- `nameLower` is not persisted on disk; it is recomputed from `name` at load time + (one-time cost at startup, O(n) `towlower` pass). +- `pathLower` is not stored; path search uses a per-thread lowercase buffer as before. +- On-disk format version 2 is incompatible with version 1; existing `.idx` files are + discarded and rebuilt on first launch. diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index f52c6ed..5f5b1b0 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -8,6 +8,7 @@ add_library(winindex_core STATIC search/SearchEngine.cpp search/SimdSearch.cpp search/TokenMatcher.cpp + storage/IndexPool.cpp storage/IndexStore.cpp storage/IndexSerializer.cpp settings/Settings.cpp diff --git a/src/core/search/ISearchEngine.h b/src/core/search/ISearchEngine.h index b7652e8..ef7ff2f 100644 --- a/src/core/search/ISearchEngine.h +++ b/src/core/search/ISearchEngine.h @@ -1,7 +1,8 @@ #pragma once #include "../indexer/IFileSystemScanner.h" +#include "../storage/IndexPool.h" +#include #include -#include #include namespace winindex { @@ -17,30 +18,34 @@ struct SearchOptions { /// @brief A single match returned by a search query. struct SearchResult { - const FileEntry* - entry; ///< Pointer into the live index — valid for the lifetime of IIndexStore. + uint32_t entryIndex; ///< Index into the IndexPool that produced this result. uint32_t matchStart; ///< Character offset within the matched string (for highlight rendering). - uint32_t matchLen; ///< Length of the matched substring in characters. + uint32_t + matchLen; ///< Length of the matched substring in characters (0 for regex/token matches). }; /// @brief Interface for searching the in-memory file index. -/// -/// Implementations: ScalarSearchEngine (simple substring), SimdSearchEngine -/// (AVX2/SSE4.2 accelerated), and RegexSearchEngine (RE2-based). class ISearchEngine { public: virtual ~ISearchEngine() = default; - /// @brief Searches @p entries for files matching @p query. + /// @brief Searches the pool for files matching @p query. + /// + /// Callers must hold a shared read lock on IndexPool for the duration of + /// this call (IndexStore::GetSearchMutex()). + /// /// @param query Search string or regular expression. - /// @param entries Pointer to the flat array of FileEntry objects in the index. + /// @param meta Pointer to the flat EntryMeta array. /// @param entryCount Number of entries in the array. + /// @param nameLowerPool Flat pool of lowercased filenames (searched by default). + /// @param pathPool Flat pool of full paths (searched when matchPath is true). /// @param options Match mode flags. /// @param maxResults Maximum number of results to return. /// @param cancelToken Set to true externally to abort a long-running search. - /// @return Matched entries with highlight offsets, capped at @p maxResults. - virtual std::vector Search(const std::wstring& query, const FileEntry* entries, - uint64_t entryCount, const SearchOptions& options, + /// @return Matched entry indices with highlight offsets, capped at @p maxResults. + virtual std::vector Search(const std::wstring& query, const EntryMeta* meta, + uint64_t entryCount, const wchar_t* nameLowerPool, + const wchar_t* pathPool, const SearchOptions& options, uint32_t maxResults, const std::atomic& cancelToken) = 0; }; diff --git a/src/core/search/SearchEngine.cpp b/src/core/search/SearchEngine.cpp index ccbc4fb..82f525e 100644 --- a/src/core/search/SearchEngine.cpp +++ b/src/core/search/SearchEngine.cpp @@ -8,90 +8,94 @@ #include #include +#include #include -#include #include namespace winindex { -// Convert wide string to UTF-8 for RE2 -std::string SearchEngine::WideToUtf8(const std::wstring& s) { - if (s.empty()) +std::string SearchEngine::WideToUtf8(const wchar_t* s, size_t len) { + if (len == 0) return {}; - int sz = WideCharToMultiByte(CP_UTF8, 0, s.c_str(), -1, nullptr, 0, nullptr, nullptr); - std::string r(sz - 1, '\0'); - WideCharToMultiByte(CP_UTF8, 0, s.c_str(), -1, r.data(), sz, nullptr, nullptr); + int sz = + WideCharToMultiByte(CP_UTF8, 0, s, static_cast(len), nullptr, 0, nullptr, nullptr); + std::string r(static_cast(sz), '\0'); + WideCharToMultiByte(CP_UTF8, 0, s, static_cast(len), r.data(), sz, nullptr, nullptr); return r; } -bool SearchEngine::MatchesWholeWord(const std::wstring& text, size_t pos, size_t len) { +bool SearchEngine::MatchesWholeWord(const wchar_t* text, size_t textLen, size_t pos, size_t len) { auto isWordChar = [](wchar_t c) { return iswalnum(c) || c == L'_'; }; if (pos > 0 && isWordChar(text[pos - 1])) return false; - if (pos + len < text.size() && isWordChar(text[pos + len])) + if (pos + len < textLen && isWordChar(text[pos + len])) return false; return true; } -std::vector SearchEngine::Search(const std::wstring& query, const FileEntry* entries, - uint64_t entryCount, const SearchOptions& options, - uint32_t maxResults, +std::vector SearchEngine::Search(const std::wstring& query, const EntryMeta* meta, + uint64_t entryCount, const wchar_t* nameLowerPool, + const wchar_t* pathPool, + const SearchOptions& options, uint32_t maxResults, const std::atomic& cancelToken) { if (query.size() < 2) return {}; - if (options.useRegex) - return SearchRegex(query, entries, entryCount, options, maxResults, cancelToken); - else - return SearchSubstring(query, entries, entryCount, options, maxResults, cancelToken); + return SearchRegex(query, meta, entryCount, nameLowerPool, pathPool, options, maxResults, + cancelToken); + return SearchSubstring(query, meta, entryCount, nameLowerPool, pathPool, options, maxResults, + cancelToken); } -std::vector SearchEngine::SearchRegex(const std::wstring& query, - const FileEntry* entries, uint64_t entryCount, - const SearchOptions& options, - uint32_t maxResults, - const std::atomic& cancelToken) { +std::vector SearchEngine::SearchRegex( + const std::wstring& query, const EntryMeta* meta, uint64_t entryCount, + [[maybe_unused]] const wchar_t* nameLowerPool, const wchar_t* pathPool, + const SearchOptions& options, uint32_t maxResults, const std::atomic& cancelToken) { RE2::Options re2opts; re2opts.set_case_sensitive(options.caseSensitive); re2opts.set_encoding(RE2::Options::EncodingUTF8); - std::string utf8Query = WideToUtf8(query); + std::string utf8Query = WideToUtf8(query.c_str(), query.size()); RE2 re(utf8Query, re2opts); if (!re.ok()) - return {}; // Invalid regex — return empty + return {}; - // Capture-group version used only for whole-word boundary checks. - // RE2::PartialMatch with a StringPiece* arg returns false if the pattern - // has no capture groups, so we keep the plain boolean call as the primary path. RE2 reCapture("(" + utf8Query + ")", re2opts); std::vector results; results.reserve(maxResults); for (uint64_t i = 0; i < entryCount && !cancelToken.load(std::memory_order_relaxed); ++i) { - const FileEntry& e = entries[i]; - const std::wstring& target = options.matchPath ? e.path : e.name; + const EntryMeta& m = meta[i]; + if (m.deleted) + continue; + + const wchar_t* targetData; + size_t targetLen; + if (options.matchPath) { + targetData = pathPool + m.pathOffset; + targetLen = m.pathLen; + } else { + // Use name (original case) from path tail for regex matching + targetData = pathPool + m.pathOffset + m.nameStart; + targetLen = static_cast(m.pathLen - m.nameStart); + } - std::string utf8Target = WideToUtf8(target); + std::string utf8Target = WideToUtf8(targetData, targetLen); if (options.wholeWord) { re2::StringPiece match; if (!reCapture.ok() || !RE2::PartialMatch(utf8Target, reCapture, &match)) continue; size_t matchPos = static_cast(match.data() - utf8Target.data()); - if (!MatchesWholeWord(target, matchPos, match.size())) + if (!MatchesWholeWord(targetData, targetLen, matchPos, match.size())) continue; } else { if (!RE2::PartialMatch(utf8Target, re)) continue; } - SearchResult sr; - sr.entry = &e; - sr.matchStart = 0; - sr.matchLen = 0; - results.push_back(sr); - + results.push_back({static_cast(i), 0u, 0u}); if (results.size() >= maxResults) break; } @@ -99,23 +103,20 @@ std::vector SearchEngine::SearchRegex(const std::wstring& query, } std::vector SearchEngine::SearchSubstring( - const std::wstring& query, const FileEntry* entries, uint64_t entryCount, - const SearchOptions& options, uint32_t maxResults, const std::atomic& cancelToken) { + const std::wstring& query, const EntryMeta* meta, uint64_t entryCount, + const wchar_t* nameLowerPool, const wchar_t* pathPool, const SearchOptions& options, + uint32_t maxResults, const std::atomic& cancelToken) { std::wstring needle = options.caseSensitive ? query : [&] { std::wstring q = query; std::transform(q.begin(), q.end(), q.begin(), ::towlower); return q; }(); - // Token-set matching: pre-compute once, shared read-only across threads. - // Only activated when the query contains separator chars (space/_/-/.) - // so single-word queries take the unmodified SIMD-only path. const bool doTokenMatch = !options.caseSensitive && TokenMatcher::QueryHasSeparators(query); - // lowercaseQuery owns the storage that sortedQueryTokens views reference. std::wstring lowercaseQuery; std::vector sortedQueryTokens; if (doTokenMatch) { - lowercaseQuery = needle; // needle is already lowercased at this point + lowercaseQuery = needle; sortedQueryTokens = TokenMatcher::TokenizeView(lowercaseQuery); std::sort(sortedQueryTokens.begin(), sortedQueryTokens.end()); } @@ -123,6 +124,9 @@ std::vector SearchEngine::SearchSubstring( unsigned int numThreads = std::max(1u, std::thread::hardware_concurrency()); uint64_t chunkSize = (entryCount + numThreads - 1) / numThreads; + // Shared early-exit counter: stops all threads once maxResults hits are collected. + std::atomic collected{0}; + std::vector>> futures; futures.reserve(numThreads); @@ -134,34 +138,41 @@ std::vector SearchEngine::SearchSubstring( futures.push_back( std::async(std::launch::async, [&, begin, end]() -> std::vector { - // Thread-local buffer for matchPath case-insensitive (avoids per-entry alloc) thread_local std::wstring tlsPathBuf; std::vector local; for (uint64_t i = begin; i < end; ++i) { if (cancelToken.load(std::memory_order_relaxed)) break; + if (collected.load(std::memory_order_relaxed) >= maxResults) + break; - const FileEntry& e = entries[i]; + const EntryMeta& m = meta[i]; + if (m.deleted) + continue; - // Hot path: name search uses pre-lowercased nameLower — zero allocation const wchar_t* haystackData; size_t haystackLen; + if (options.matchPath) { if (options.caseSensitive) { - haystackData = e.path.c_str(); - haystackLen = e.path.size(); + haystackData = pathPool + m.pathOffset; + haystackLen = m.pathLen; } else { - tlsPathBuf.assign(e.path.begin(), e.path.end()); + tlsPathBuf.assign(pathPool + m.pathOffset, m.pathLen); std::transform(tlsPathBuf.begin(), tlsPathBuf.end(), tlsPathBuf.begin(), ::towlower); haystackData = tlsPathBuf.c_str(); haystackLen = tlsPathBuf.size(); } + } else if (options.caseSensitive) { + // Case-sensitive name: read original-case name from path tail + haystackData = pathPool + m.pathOffset + m.nameStart; + haystackLen = static_cast(m.pathLen - m.nameStart); } else { - const std::wstring& hay = options.caseSensitive ? e.name : e.nameLower; - haystackData = hay.c_str(); - haystackLen = hay.size(); + // Hot path: pre-lowercased nameLower — zero allocation, pool-direct + haystackData = nameLowerPool + m.nameLowerOffset; + haystackLen = m.nameLowerLen; } size_t pos = @@ -169,8 +180,6 @@ std::vector SearchEngine::SearchSubstring( bool tokenMatch = false; if (pos == std::wstring::npos) { - // Token-set fallback: fires only when SIMD missed and the query - // had separators (e.g. "just rosy", "rosy guitar flac"). if (!doTokenMatch) continue; std::wstring haystackStr(haystackData, haystackLen); @@ -183,16 +192,13 @@ std::vector SearchEngine::SearchSubstring( } if (options.wholeWord && !tokenMatch) { - std::wstring_view hayView(haystackData, haystackLen); - if (!MatchesWholeWord(std::wstring(hayView), pos, needle.size())) + if (!MatchesWholeWord(haystackData, haystackLen, pos, needle.size())) continue; } - SearchResult sr; - sr.entry = &e; - sr.matchStart = static_cast(pos); - sr.matchLen = tokenMatch ? 0u : static_cast(needle.size()); - local.push_back(sr); + local.push_back({static_cast(i), static_cast(pos), + tokenMatch ? 0u : static_cast(needle.size())}); + collected.fetch_add(1, std::memory_order_relaxed); } return local; })); diff --git a/src/core/search/SearchEngine.h b/src/core/search/SearchEngine.h index 22f58d3..af8ad41 100644 --- a/src/core/search/SearchEngine.h +++ b/src/core/search/SearchEngine.h @@ -8,27 +8,27 @@ namespace winindex { class SearchEngine : public ISearchEngine { public: - std::vector Search(const std::wstring& query, const FileEntry* entries, - uint64_t entryCount, const SearchOptions& options, + std::vector Search(const std::wstring& query, const EntryMeta* meta, + uint64_t entryCount, const wchar_t* nameLowerPool, + const wchar_t* pathPool, const SearchOptions& options, uint32_t maxResults, const std::atomic& cancelToken) override; private: - // Regex search path - static std::vector SearchRegex(const std::wstring& query, - const FileEntry* entries, uint64_t entryCount, + static std::vector SearchRegex(const std::wstring& query, const EntryMeta* meta, + uint64_t entryCount, const wchar_t* nameLowerPool, + const wchar_t* pathPool, const SearchOptions& options, uint32_t maxResults, const std::atomic& cancelToken); - // SIMD substring search path (parallelized) - static std::vector SearchSubstring(const std::wstring& query, - const FileEntry* entries, uint64_t entryCount, - const SearchOptions& options, - uint32_t maxResults, - const std::atomic& cancelToken); + static std::vector SearchSubstring( + const std::wstring& query, const EntryMeta* meta, uint64_t entryCount, + const wchar_t* nameLowerPool, const wchar_t* pathPool, const SearchOptions& options, + uint32_t maxResults, const std::atomic& cancelToken); - static bool MatchesWholeWord(const std::wstring& text, size_t matchPos, size_t matchLen); - static std::string WideToUtf8(const std::wstring& s); + static bool MatchesWholeWord(const wchar_t* text, size_t textLen, size_t matchPos, + size_t matchLen); + static std::string WideToUtf8(const wchar_t* s, size_t len); }; } // namespace winindex diff --git a/src/core/storage/IIndexStore.h b/src/core/storage/IIndexStore.h index 605c995..cd511a1 100644 --- a/src/core/storage/IIndexStore.h +++ b/src/core/storage/IIndexStore.h @@ -1,75 +1,31 @@ #pragma once #include "../indexer/IFileSystemScanner.h" #include -#include namespace winindex { /// @brief Interface for managing the in-memory file index and its on-disk persistence. -/// -/// The store owns the flat array of FileEntry objects and exposes it read-only for -/// search. Write access follows a Begin/Add/End transaction pattern to allow bulk -/// loading. Incremental updates (from the USN journal or ChangeWatcher) use the -/// Apply* methods. class IIndexStore { public: virtual ~IIndexStore() = default; - /// @brief Returns true if a valid, non-expired index exists on disk. virtual bool IsIndexValid() const = 0; - - /// @brief Loads the index from disk into memory. virtual void Load() = 0; - - /// @brief Persists the current in-memory index to disk (CRC-32 validated binary format). virtual void Save() = 0; - /// @brief Begins a bulk-write transaction. Must be paired with EndWrite(). virtual void BeginWrite() = 0; - - /// @brief Appends @p entry to the store during a bulk-write transaction. - /// @param entry File entry to add. virtual void AddEntry(const FileEntry& entry) = 0; - - /// @brief Commits the bulk-write transaction and makes entries visible to readers. virtual void EndWrite() = 0; - /// @brief Incrementally adds a new file entry (e.g. from a USN Added event). - /// @param entry The newly created file. virtual void ApplyAdd(const FileEntry& entry) = 0; - - /// @brief Incrementally removes an entry by path (e.g. from a USN Removed event). - /// @param path Full path of the deleted file. virtual void ApplyRemove(const std::wstring& path) = 0; - - /// @brief Removes all entries whose path starts with @p prefix (case-insensitive). - /// @param prefix Drive root or folder path, e.g. L"C:\\" or L"D:\\Music\\". virtual void RemoveEntriesUnderPath(const std::wstring& prefix) = 0; - - /// @brief Incrementally renames an entry (e.g. from a USN Renamed event). - /// @param oldPath Previous full path. - /// @param newPath New full path. virtual void ApplyRename(const std::wstring& oldPath, const std::wstring& newPath) = 0; - /// @brief Returns the number of entries currently held in the store. virtual uint64_t GetEntryCount() const = 0; - /// @brief Returns a read-only pointer to the flat entry array. - /// - /// The pointer is valid until the next write operation. - virtual const FileEntry* GetEntries() const = 0; - - /// @brief Returns the persisted USN cursor for @p root from the last index save. - /// @param root Drive root, e.g. L"C:\\". virtual uint64_t GetSavedUsn(const std::wstring& root) const = 0; - - /// @brief Stores the USN cursor for @p root so it can be used for delta replay on restart. - /// @param root Drive root, e.g. L"C:\\". - /// @param usn New USN cursor value returned by IUsnJournalMonitor::ReplaySince. virtual void SetSavedUsn(const std::wstring& root, uint64_t usn) = 0; - - /// @brief Returns the age of the on-disk index file in seconds. - /// Returns UINT64_MAX if the file does not exist. virtual uint64_t GetIndexAgeSeconds() const = 0; }; diff --git a/src/core/storage/IndexPool.cpp b/src/core/storage/IndexPool.cpp new file mode 100644 index 0000000..1f87f35 --- /dev/null +++ b/src/core/storage/IndexPool.cpp @@ -0,0 +1,71 @@ +#include "IndexPool.h" + +#include + +namespace winindex { + +void IndexPool::Clear() { + meta.clear(); + nameLowerPool.clear(); + pathPool.clear(); +} + +void IndexPool::Reserve(size_t capacity) { + meta.reserve(capacity); + nameLowerPool.reserve(capacity * 15); // avg filename ~15 chars + pathPool.reserve(capacity * 60); // avg full path ~60 chars +} + +void IndexPool::AddEntry(const FileEntry& e) { + EntryMeta m{}; + m.size = e.size; + m.lastModified = e.lastModified; + m.attributes = e.attributes; + m.deleted = 0; + + // Append full path to pathPool + m.pathOffset = static_cast(pathPool.size()); + m.pathLen = static_cast(e.path.size()); + pathPool.insert(pathPool.end(), e.path.begin(), e.path.end()); + + // nameStart: offset within path to the last path component + size_t slash = e.path.rfind(L'\\'); + m.nameStart = static_cast(slash != std::wstring::npos ? slash + 1 : 0); + + // Append lowercased filename to nameLowerPool + m.nameLowerOffset = static_cast(nameLowerPool.size()); + if (!e.nameLower.empty()) { + m.nameLowerLen = static_cast(e.nameLower.size()); + nameLowerPool.insert(nameLowerPool.end(), e.nameLower.begin(), e.nameLower.end()); + } else { + // Compute from name (or from path tail if name is empty) + const wchar_t* srcData = e.name.empty() ? e.path.data() + m.nameStart : e.name.data(); + size_t srcLen = + e.name.empty() ? static_cast(m.pathLen - m.nameStart) : e.name.size(); + m.nameLowerLen = static_cast(srcLen); + size_t base = nameLowerPool.size(); + nameLowerPool.insert(nameLowerPool.end(), srcData, srcData + srcLen); + std::transform(nameLowerPool.begin() + static_cast(base), nameLowerPool.end(), + nameLowerPool.begin() + static_cast(base), ::towlower); + } + + meta.push_back(m); +} + +std::wstring_view IndexPool::GetNameLower(uint32_t idx) const noexcept { + const auto& m = meta[idx]; + return {nameLowerPool.data() + m.nameLowerOffset, m.nameLowerLen}; +} + +std::wstring_view IndexPool::GetPath(uint32_t idx) const noexcept { + const auto& m = meta[idx]; + return {pathPool.data() + m.pathOffset, m.pathLen}; +} + +std::wstring_view IndexPool::GetName(uint32_t idx) const noexcept { + const auto& m = meta[idx]; + return {pathPool.data() + m.pathOffset + m.nameStart, + static_cast(m.pathLen - m.nameStart)}; +} + +} // namespace winindex diff --git a/src/core/storage/IndexPool.h b/src/core/storage/IndexPool.h new file mode 100644 index 0000000..e9a91a1 --- /dev/null +++ b/src/core/storage/IndexPool.h @@ -0,0 +1,51 @@ +#pragma once +#include "../indexer/IFileSystemScanner.h" +#include +#include +#include + +namespace winindex { + +// Fixed-size metadata record for one file entry. +// Offsets address into the flat wchar_t pools in IndexPool. +// sizeof = 40 bytes (36 + 4 trailing alignment pad from uint64_t members). +struct EntryMeta { + uint64_t size; + uint64_t lastModified; + uint32_t pathOffset; // char offset into IndexPool::pathPool + uint32_t nameLowerOffset; // char offset into IndexPool::nameLowerPool + uint32_t attributes; + uint16_t pathLen; // char count of full path + uint16_t nameLowerLen; // char count of lowercased filename + uint16_t nameStart; // chars from pathOffset where filename begins + uint8_t deleted; // non-zero = tombstoned by ApplyRemove / ApplyRename + uint8_t _pad{}; +}; + +// Flat contiguous string pool for the entire file index. +// +// Separate pools for nameLower and path allow name-only search (the default) +// to scan a compact ~18 MB working set without touching path data, keeping +// the hot scan in L3 cache on modern hardware. +class IndexPool { +public: + std::vector meta; + std::vector nameLowerPool; + std::vector pathPool; + + void Clear(); + void Reserve(size_t capacity); + + // Appends one entry. name/nameLower in e must be set by caller; + // nameLower may be empty, in which case it is computed from name. + void AddEntry(const FileEntry& e); + + uint64_t Size() const noexcept { return meta.size(); } + + // Zero-copy views into pool memory. Valid until the next mutation. + std::wstring_view GetNameLower(uint32_t idx) const noexcept; + std::wstring_view GetPath(uint32_t idx) const noexcept; + std::wstring_view GetName(uint32_t idx) const noexcept; // tail of path after last backslash +}; + +} // namespace winindex diff --git a/src/core/storage/IndexSerializer.cpp b/src/core/storage/IndexSerializer.cpp index 12fadc5..f643b7e 100644 --- a/src/core/storage/IndexSerializer.cpp +++ b/src/core/storage/IndexSerializer.cpp @@ -5,12 +5,23 @@ #include #include #include -#include -#include namespace winindex { -// CRC-32 (ISO 3309 polynomial) +static constexpr uint32_t kMagic = 0x58444957u; // "WIDX" +static constexpr uint16_t kVersion = 2; + +// Per-entry record as stored on disk (24 bytes, naturally aligned). +#pragma pack(push, 1) +struct DiskEntry { + uint64_t size; + uint64_t lastModified; + uint32_t attributes; + uint16_t pathLen; + uint16_t nameStart; +}; +#pragma pack(pop) + uint32_t IndexSerializer::Crc32(const void* data, size_t len) { static uint32_t table[256] = {}; static bool initialized = false; @@ -28,44 +39,51 @@ uint32_t IndexSerializer::Crc32(const void* data, size_t len) { return crc ^ 0xFFFFFFFFu; } -bool IndexSerializer::Serialize(const std::wstring& filePath, const std::vector& entries, +bool IndexSerializer::Serialize(const std::wstring& filePath, const IndexPool& pool, const std::unordered_map& usnMap) { - // Build payload (everything after header) in memory first so we can CRC it + // Build payload in memory so we can CRC it before writing. std::vector payload; - payload.reserve(entries.size() * 128); - auto writeU16 = [&](uint16_t v) { - payload.insert(payload.end(), reinterpret_cast(&v), - reinterpret_cast(&v) + sizeof(v)); - }; - auto writeU32 = [&](uint32_t v) { - payload.insert(payload.end(), reinterpret_cast(&v), - reinterpret_cast(&v) + sizeof(v)); - }; - auto writeU64 = [&](uint64_t v) { - payload.insert(payload.end(), reinterpret_cast(&v), - reinterpret_cast(&v) + sizeof(v)); - }; - auto writeWStr = [&](const std::wstring& s) { - const uint8_t* b = reinterpret_cast(s.data()); - payload.insert(payload.end(), b, b + s.size() * sizeof(wchar_t)); + auto writeRaw = [&](const void* src, size_t bytes) { + const auto* b = static_cast(src); + payload.insert(payload.end(), b, b + bytes); }; - - for (const auto& e : entries) { - writeU16(static_cast(e.name.size())); - writeU16(static_cast(e.path.size())); - writeU64(e.size); - writeU64(e.lastModified); - writeU32(e.attributes); - writeWStr(e.name); - writeWStr(e.path); + auto writeU16 = [&](uint16_t v) { writeRaw(&v, sizeof(v)); }; + auto writeU32 = [&](uint32_t v) { writeRaw(&v, sizeof(v)); }; + auto writeU64 = [&](uint64_t v) { writeRaw(&v, sizeof(v)); }; + + // Path pool + uint64_t pathPoolSize = pool.pathPool.size(); + writeU64(pathPoolSize); + if (pathPoolSize > 0) + writeRaw(pool.pathPool.data(), pathPoolSize * sizeof(wchar_t)); + + // Per-entry disk records (only non-deleted entries) + // We'll need to recount, so build in two passes: count first, then fill. + // Actually we write all (deleted bit is not persisted; deleted entries are dropped on Save). + uint64_t liveCount = 0; + for (size_t i = 0; i < pool.meta.size(); ++i) + if (!pool.meta[i].deleted) + ++liveCount; + + for (size_t i = 0; i < pool.meta.size(); ++i) { + const EntryMeta& m = pool.meta[i]; + if (m.deleted) + continue; + DiskEntry de{}; + de.size = m.size; + de.lastModified = m.lastModified; + de.attributes = m.attributes; + de.pathLen = m.pathLen; + de.nameStart = m.nameStart; + writeRaw(&de, sizeof(de)); } // USN map writeU32(static_cast(usnMap.size())); for (const auto& [root, usn] : usnMap) { writeU16(static_cast(root.size())); - writeWStr(root); + writeRaw(root.data(), root.size() * sizeof(wchar_t)); writeU64(usn); } @@ -74,10 +92,10 @@ bool IndexSerializer::Serialize(const std::wstring& filePath, const std::vector< uint64_t ts = (static_cast(now.dwHighDateTime) << 32) | now.dwLowDateTime; IndexFileHeader hdr{}; - hdr.magic = 0x58444957u; - hdr.version = 1; + hdr.magic = kMagic; + hdr.version = kVersion; hdr.timestamp = ts; - hdr.entryCount = static_cast(entries.size()); + hdr.entryCount = liveCount; hdr.crc32 = Crc32(payload.data(), payload.size()); std::ofstream f(filePath, std::ios::binary | std::ios::trunc); @@ -89,7 +107,7 @@ bool IndexSerializer::Serialize(const std::wstring& filePath, const std::vector< return f.good(); } -bool IndexSerializer::Deserialize(const std::wstring& filePath, std::vector& entries, +bool IndexSerializer::Deserialize(const std::wstring& filePath, IndexPool& pool, std::unordered_map& usnMap, uint64_t& outTimestamp) { std::ifstream f(filePath, std::ios::binary | std::ios::ate); @@ -104,10 +122,10 @@ bool IndexSerializer::Deserialize(const std::wstring& filePath, std::vector(&hdr), sizeof(hdr)); - if (hdr.magic != 0x58444957u) - return false; - if (hdr.version != 1) + if (hdr.magic != kMagic) return false; + if (hdr.version != kVersion) + return false; // Version 1 file: caller triggers re-index size_t payloadSize = fileSize - sizeof(IndexFileHeader); std::vector payload(payloadSize); @@ -119,7 +137,7 @@ bool IndexSerializer::Deserialize(const std::wstring& filePath, std::vector(hdr.entryCount)); + pool.Clear(); const uint8_t* p = payload.data(); const uint8_t* end = payload.data() + payloadSize; @@ -142,27 +160,79 @@ bool IndexSerializer::Deserialize(const std::wstring& filePath, std::vector std::wstring { - std::wstring s(chars, L'\0'); - memcpy(s.data(), p, chars * sizeof(wchar_t)); - p += chars * sizeof(wchar_t); - return s; - }; - for (auto& e : entries) { - if (p + 4 > end) - return false; - uint16_t nameLen = readU16(); - uint16_t pathLen = readU16(); - e.size = readU64(); - e.lastModified = readU64(); - e.attributes = readU32(); - e.name = readWStr(nameLen); - e.nameLower = e.name; - std::transform(e.nameLower.begin(), e.nameLower.end(), e.nameLower.begin(), ::towlower); - e.path = readWStr(pathLen); + // Path pool + if (p + sizeof(uint64_t) > end) + return false; + uint64_t pathPoolSize = readU64(); + size_t pathPoolBytes = static_cast(pathPoolSize) * sizeof(wchar_t); + if (p + pathPoolBytes > end) + return false; + + pool.pathPool.resize(static_cast(pathPoolSize)); + if (pathPoolSize > 0) + memcpy(pool.pathPool.data(), p, pathPoolBytes); + p += pathPoolBytes; + + // Per-entry records: reconstruct metadata + nameLower pool + uint64_t entryCount = hdr.entryCount; + pool.meta.reserve(static_cast(entryCount)); + pool.nameLowerPool.reserve(static_cast(entryCount) * 15); + + size_t diskEntrySize = sizeof(DiskEntry); + if (p + diskEntrySize * entryCount > end) + return false; + + for (uint64_t i = 0; i < entryCount; ++i) { + DiskEntry de{}; + memcpy(&de, p, diskEntrySize); + p += diskEntrySize; + + EntryMeta m{}; + m.size = de.size; + m.lastModified = de.lastModified; + m.attributes = de.attributes; + m.pathLen = de.pathLen; + m.nameStart = de.nameStart; + m.deleted = 0; + + // pathOffset: we need to figure out where this entry's path sits in the pool. + // Since we read pathPool as a whole flat buffer and DiskEntry doesn't store pathOffset + // (we need to recompute it), we track the running offset using pathLen values. + // This is done below after the loop. + // For now, store a sentinel — we'll fix it up. + m.pathOffset = 0; // fixed up below + + // Build nameLower from the path tail + m.nameLowerOffset = static_cast(pool.nameLowerPool.size()); + uint16_t nameLen = static_cast(de.pathLen - de.nameStart); + m.nameLowerLen = nameLen; + // pathOffset not yet known; use running sum from previous entries + // We'll fix pathOffset in a second pass below. + pool.meta.push_back(m); + pool.nameLowerPool.resize(pool.nameLowerPool.size() + nameLen); + } + + // Second pass: assign pathOffsets and fill nameLowerPool. + uint32_t runningPathOffset = 0; + size_t runningNlOffset = 0; + for (uint64_t i = 0; i < entryCount; ++i) { + EntryMeta& m = pool.meta[i]; + m.pathOffset = runningPathOffset; + m.nameLowerOffset = static_cast(runningNlOffset); + + // Fill nameLower: lowercase the filename portion of path + const wchar_t* nameSrc = pool.pathPool.data() + runningPathOffset + m.nameStart; + wchar_t* nlDst = pool.nameLowerPool.data() + runningNlOffset; + uint16_t nameLen = m.nameLowerLen; + for (uint16_t c = 0; c < nameLen; ++c) + nlDst[c] = static_cast(::towlower(nameSrc[c])); + + runningPathOffset += m.pathLen; + runningNlOffset += nameLen; } + // USN map if (p + sizeof(uint32_t) <= end) { uint32_t usnCount = readU32(); for (uint32_t i = 0; i < usnCount; ++i) { @@ -171,7 +241,8 @@ bool IndexSerializer::Deserialize(const std::wstring& filePath, std::vector end) break; - std::wstring root = readWStr(rootLen); + std::wstring root(reinterpret_cast(p), rootLen); + p += rootLen * sizeof(wchar_t); uint64_t usn = readU64(); usnMap[root] = usn; } diff --git a/src/core/storage/IndexSerializer.h b/src/core/storage/IndexSerializer.h index 9665e1d..0d8b707 100644 --- a/src/core/storage/IndexSerializer.h +++ b/src/core/storage/IndexSerializer.h @@ -1,35 +1,37 @@ #pragma once -#include "../indexer/IFileSystemScanner.h" +#include "IndexPool.h" #include #include -#include +#include namespace winindex { -// Binary index file layout: +// Binary index file layout (version 2): // -// Header (fixed size): +// Header (26 bytes, #pragma pack(1)): // u32 magic = 0x58444957 ("WIDX") -// u16 version +// u16 version = 2 // u64 timestamp (FILETIME of index build) // u64 entryCount // u32 crc32 (of everything after the header) // -// Per entry (variable length, packed): -// u16 nameLen (chars, not bytes) -// u16 pathLen -// u64 size -// u64 lastModified -// u32 attributes -// wchar_t name[nameLen] -// wchar_t path[pathLen] +// Payload: +// u64 pathPoolSize (char count) +// wchar_t pathPool[pathPoolSize] (UTF-16 LE, no null terminators) // -// USN map (after entries): -// u32 usnEntryCount -// per usn entry: -// u16 rootLen -// wchar_t root[rootLen] -// u64 usn +// Per-entry disk record (entryCount records, 24 bytes each): +// u64 size +// u64 lastModified +// u32 attributes +// u16 pathLen +// u16 nameStart +// +// USN map: +// u32 usnEntryCount +// per entry: u16 rootLen + wchar_t root[rootLen] + u64 usn +// +// nameLower is NOT stored on disk; it is rebuilt from path+nameStart at load time. +// Version 1 files are detected and trigger a silent re-index. #pragma pack(push, 1) struct IndexFileHeader { @@ -43,10 +45,10 @@ struct IndexFileHeader { class IndexSerializer { public: - static bool Serialize(const std::wstring& filePath, const std::vector& entries, + static bool Serialize(const std::wstring& filePath, const IndexPool& pool, const std::unordered_map& usnMap); - static bool Deserialize(const std::wstring& filePath, std::vector& entries, + static bool Deserialize(const std::wstring& filePath, IndexPool& pool, std::unordered_map& usnMap, uint64_t& outTimestamp); diff --git a/src/core/storage/IndexStore.cpp b/src/core/storage/IndexStore.cpp index c9f9e71..fb6d540 100644 --- a/src/core/storage/IndexStore.cpp +++ b/src/core/storage/IndexStore.cpp @@ -21,10 +21,9 @@ bool IndexStore::IsIndexValid() const { if (!GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &fad)) return false; - // Check age against configured reindex interval uint64_t reindexIntervalHours = m_settings->GetReindexIntervalHours(); if (reindexIntervalHours == 0) - return true; // Manual only — always treat as valid + return true; FILETIME now{}; GetSystemTimeAsFileTime(&now); @@ -32,111 +31,138 @@ bool IndexStore::IsIndexValid() const { uint64_t fileVal = (static_cast(fad.ftLastWriteTime.dwHighDateTime) << 32) | fad.ftLastWriteTime.dwLowDateTime; - // FILETIME is in 100-nanosecond intervals constexpr uint64_t hundredNsPerHour = 36000000000ULL; uint64_t ageHours = (nowVal - fileVal) / hundredNsPerHour; return ageHours < reindexIntervalHours; } void IndexStore::Load() { - std::lock_guard lock(m_mutex); - m_entries.clear(); - m_usnMap.clear(); + IndexPool tmp; + std::unordered_map usnTmp; uint64_t ts = 0; - if (!IndexSerializer::Deserialize(IndexFilePath(), m_entries, m_usnMap, ts)) { - m_entries.clear(); + bool ok = IndexSerializer::Deserialize(IndexFilePath(), tmp, usnTmp, ts); + + std::unique_lock lock(m_mutex); + if (ok) { + m_pool = std::move(tmp); + m_usnMap = std::move(usnTmp); + } else { + m_pool.Clear(); m_usnMap.clear(); } } void IndexStore::Save() { - std::lock_guard lock(m_mutex); - IndexSerializer::Serialize(IndexFilePath(), m_entries, m_usnMap); + std::shared_lock lock(m_mutex); + IndexSerializer::Serialize(IndexFilePath(), m_pool, m_usnMap); } void IndexStore::BeginWrite() { - std::lock_guard lock(m_mutex); - m_entries.clear(); + m_stagingBuf.clear(); } void IndexStore::AddEntry(const FileEntry& e) { - std::lock_guard lock(m_mutex); - m_entries.push_back(e); + // Called from single indexing thread — no lock needed on staging buffer. + m_stagingBuf.push_back(e); } void IndexStore::EndWrite() { - // Nothing to flush — entries are in memory until Save() + IndexPool fresh; + fresh.Reserve(m_stagingBuf.size()); + for (const auto& e : m_stagingBuf) fresh.AddEntry(e); + m_stagingBuf.clear(); + + std::unique_lock lock(m_mutex); + m_pool = std::move(fresh); } void IndexStore::ApplyAdd(const FileEntry& entry) { - std::lock_guard lock(m_mutex); - m_entries.push_back(entry); + std::unique_lock lock(m_mutex); + m_pool.AddEntry(entry); } void IndexStore::ApplyRemove(const std::wstring& path) { - std::lock_guard lock(m_mutex); std::wstring lower = path; std::transform(lower.begin(), lower.end(), lower.begin(), ::towlower); - m_entries.erase(std::remove_if(m_entries.begin(), m_entries.end(), - [&](const FileEntry& e) { - std::wstring p = e.path; - std::transform(p.begin(), p.end(), p.begin(), ::towlower); - return p == lower; - }), - m_entries.end()); -} -void IndexStore::RemoveEntriesUnderPath(const std::wstring& prefix) { - std::wstring lp = prefix; - std::transform(lp.begin(), lp.end(), lp.begin(), ::towlower); - if (!lp.empty() && lp.back() != L'\\') - lp += L'\\'; - - std::lock_guard lock(m_mutex); - m_entries.erase(std::remove_if(m_entries.begin(), m_entries.end(), - [&](const FileEntry& e) { - std::wstring ep = e.path; - std::transform(ep.begin(), ep.end(), ep.begin(), ::towlower); - return ep.compare(0, lp.size(), lp) == 0; - }), - m_entries.end()); + std::unique_lock lock(m_mutex); + for (size_t i = 0; i < m_pool.meta.size(); ++i) { + if (m_pool.meta[i].deleted) + continue; + auto pv = m_pool.GetPath(static_cast(i)); + std::wstring pl(pv.begin(), pv.end()); + std::transform(pl.begin(), pl.end(), pl.begin(), ::towlower); + if (pl == lower) { + m_pool.meta[i].deleted = 1; + break; + } + } } void IndexStore::ApplyRename(const std::wstring& oldPath, const std::wstring& newPath) { - std::lock_guard lock(m_mutex); std::wstring oldLower = oldPath; std::transform(oldLower.begin(), oldLower.end(), oldLower.begin(), ::towlower); - for (auto& e : m_entries) { - std::wstring p = e.path; - std::transform(p.begin(), p.end(), p.begin(), ::towlower); - if (p == oldLower) { - e.path = newPath; + + std::unique_lock lock(m_mutex); + for (size_t i = 0; i < m_pool.meta.size(); ++i) { + if (m_pool.meta[i].deleted) + continue; + auto pv = m_pool.GetPath(static_cast(i)); + std::wstring pl(pv.begin(), pv.end()); + std::transform(pl.begin(), pl.end(), pl.begin(), ::towlower); + if (pl == oldLower) { + m_pool.meta[i].deleted = 1; + + // Construct FileEntry for the new path preserving metadata + const EntryMeta& om = m_pool.meta[i]; + FileEntry fe; + fe.path = newPath; size_t slash = newPath.rfind(L'\\'); - e.name = (slash != std::wstring::npos) ? newPath.substr(slash + 1) : newPath; - e.nameLower = e.name; - std::transform(e.nameLower.begin(), e.nameLower.end(), e.nameLower.begin(), ::towlower); + fe.name = (slash != std::wstring::npos) ? newPath.substr(slash + 1) : newPath; + fe.nameLower = fe.name; + std::transform(fe.nameLower.begin(), fe.nameLower.end(), fe.nameLower.begin(), + ::towlower); + fe.size = om.size; + fe.lastModified = om.lastModified; + fe.attributes = om.attributes; + m_pool.AddEntry(fe); break; } } } -uint64_t IndexStore::GetEntryCount() const { - std::lock_guard lock(m_mutex); - return m_entries.size(); +void IndexStore::RemoveEntriesUnderPath(const std::wstring& prefix) { + std::wstring lp = prefix; + std::transform(lp.begin(), lp.end(), lp.begin(), ::towlower); + if (!lp.empty() && lp.back() != L'\\') + lp += L'\\'; + + std::unique_lock lock(m_mutex); + for (size_t i = 0; i < m_pool.meta.size(); ++i) { + if (m_pool.meta[i].deleted) + continue; + auto pv = m_pool.GetPath(static_cast(i)); + std::wstring pl(pv.begin(), pv.end()); + std::transform(pl.begin(), pl.end(), pl.begin(), ::towlower); + if (pl.compare(0, lp.size(), lp) == 0) + m_pool.meta[i].deleted = 1; + } } -const FileEntry* IndexStore::GetEntries() const { - return m_entries.data(); +uint64_t IndexStore::GetEntryCount() const { + std::shared_lock lock(m_mutex); + return static_cast(std::count_if(m_pool.meta.begin(), m_pool.meta.end(), + [](const EntryMeta& m) { return !m.deleted; })); } uint64_t IndexStore::GetSavedUsn(const std::wstring& root) const { - std::lock_guard lock(m_mutex); + std::shared_lock lock(m_mutex); auto it = m_usnMap.find(root); return (it != m_usnMap.end()) ? it->second : 0; } void IndexStore::SetSavedUsn(const std::wstring& root, uint64_t usn) { - std::lock_guard lock(m_mutex); + std::unique_lock lock(m_mutex); m_usnMap[root] = usn; } diff --git a/src/core/storage/IndexStore.h b/src/core/storage/IndexStore.h index 86e0b61..1ec04ea 100644 --- a/src/core/storage/IndexStore.h +++ b/src/core/storage/IndexStore.h @@ -1,8 +1,9 @@ #pragma once #include "../settings/Settings.h" #include "IIndexStore.h" +#include "IndexPool.h" #include -#include +#include #include #include #include @@ -27,23 +28,26 @@ class IndexStore : public IIndexStore { void RemoveEntriesUnderPath(const std::wstring& prefix) override; uint64_t GetEntryCount() const override; - const FileEntry* GetEntries() const override; uint64_t GetSavedUsn(const std::wstring& root) const override; void SetSavedUsn(const std::wstring& root, uint64_t usn) override; uint64_t GetIndexAgeSeconds() const override; + // Pool access for search — callers must hold GetSearchMutex() shared lock. + const IndexPool& GetPool() const noexcept { return m_pool; } + std::shared_mutex& GetSearchMutex() noexcept { return m_mutex; } + private: std::shared_ptr m_settings; - std::vector m_entries; - mutable std::mutex m_mutex; - std::unordered_map m_usnMap; // root -> last USN + IndexPool m_pool; + mutable std::shared_mutex m_mutex; + std::unordered_map m_usnMap; - static constexpr uint32_t MAGIC = 0x58444957; // "WIDX" - static constexpr uint16_t VERSION = 1; + // Staging buffer for BeginWrite / AddEntry / EndWrite bulk transactions. + // Filled without holding m_mutex; swapped in under exclusive lock in EndWrite. + std::vector m_stagingBuf; std::wstring IndexFilePath() const; - uint32_t ComputeCrc32(const void* data, size_t len) const; }; } // namespace winindex diff --git a/src/ui/MainWindow.cpp b/src/ui/MainWindow.cpp index 0d7dc8e..5831bbc 100644 --- a/src/ui/MainWindow.cpp +++ b/src/ui/MainWindow.cpp @@ -434,22 +434,46 @@ void MainWindow::ExecuteSearch(const std::wstring& query) { m_searchThread.join(); m_searchCancel.store(false); - auto* results = new std::vector(); - auto entries = m_indexStore->GetEntries(); - auto count = m_indexStore->GetEntryCount(); + // Acquire shared lock so pool pointers remain stable for the search thread. + std::shared_lock lock(m_indexStore->GetSearchMutex()); + const IndexPool& pool = m_indexStore->GetPool(); + const EntryMeta* meta = pool.meta.data(); + uint64_t entryCount = static_cast(pool.meta.size()); + const wchar_t* nlPool = pool.nameLowerPool.data(); + const wchar_t* pathPool = pool.pathPool.data(); + auto opts = m_settings->GetSearchOptions(); auto engine = m_searchEngine; auto& cancelRef = m_searchCancel; HWND hwnd = m_hwnd; - m_searchThread = std::thread([=, &cancelRef]() mutable { - *results = engine->Search(query, entries, count, opts, 10000, cancelRef); - PostMessageW(hwnd, WM_SEARCH_RESULTS, 0, reinterpret_cast(results)); + // Move lock into thread — prevents exclusive writes (EndWrite/ApplyAdd) until search completes. + m_searchThread = std::thread([=, lk = std::move(lock), &cancelRef]() mutable { + auto raw = + engine->Search(query, meta, entryCount, nlPool, pathPool, opts, 10000, cancelRef); + + auto* display = new std::vector(); + display->reserve(raw.size()); + for (const auto& r : raw) { + const EntryMeta& m = meta[r.entryIndex]; + uint16_t nameLen = static_cast(m.pathLen - m.nameStart); + DisplayEntry de; + de.name = std::wstring(pathPool + m.pathOffset + m.nameStart, nameLen); + de.path = std::wstring(pathPool + m.pathOffset, m.pathLen); + de.size = m.size; + de.lastModified = m.lastModified; + de.attributes = m.attributes; + de.matchStart = r.matchStart; + de.matchLen = r.matchLen; + display->push_back(std::move(de)); + } + // lk released here — exclusive writes may now proceed. + PostMessageW(hwnd, WM_SEARCH_RESULTS, 0, reinterpret_cast(display)); }); } -void MainWindow::OnSearchResults(std::vector* results) { - m_totalMatches = results->size(); // simplified; actual total would need separate count pass +void MainWindow::OnSearchResults(std::vector* results) { + m_totalMatches = results->size(); m_currentResults = std::move(*results); delete results; @@ -494,10 +518,10 @@ void MainWindow::OnListDblClick() { int sel = ListView_GetNextItem(m_hListView, -1, LVNI_SELECTED); if (sel < 0 || sel >= static_cast(m_currentResults.size())) return; - const FileEntry* entry = m_currentResults[sel].entry; - if (!PreCheckFileExists(entry)) + const std::wstring& path = m_currentResults[sel].path; + if (!PreCheckFileExists(path)) return; - ShellExecuteW(m_hwnd, L"open", entry->path.c_str(), nullptr, nullptr, SW_SHOWNORMAL); + ShellExecuteW(m_hwnd, L"open", path.c_str(), nullptr, nullptr, SW_SHOWNORMAL); } void MainWindow::OnListKeyDown(const NMLVKEYDOWN* kd) { @@ -547,12 +571,12 @@ void MainWindow::OnContextMenu(HWND hwndFrom, int x, int y) { DestroyMenu(hBase); } -bool MainWindow::PreCheckFileExists(const FileEntry* entry) { - if (GetFileAttributesW(entry->path.c_str()) != INVALID_FILE_ATTRIBUTES) +bool MainWindow::PreCheckFileExists(const std::wstring& path) { + if (GetFileAttributesW(path.c_str()) != INVALID_FILE_ATTRIBUTES) return true; std::wstring msg = - L"The file no longer exists:\n\n" + entry->path + + L"The file no longer exists:\n\n" + path + L"\n\nThe index may be out of date. Would you like to rebuild the index now?"; int ret = MessageBoxW(m_hwnd, msg.c_str(), L"File Not Found", MB_YESNO | MB_ICONWARNING); if (ret == IDYES) @@ -564,20 +588,19 @@ void MainWindow::OpenSelectedFile() { int sel = ListView_GetNextItem(m_hListView, -1, LVNI_SELECTED); if (sel < 0 || sel >= static_cast(m_currentResults.size())) return; - const FileEntry* entry = m_currentResults[sel].entry; - if (!PreCheckFileExists(entry)) + const std::wstring& path = m_currentResults[sel].path; + if (!PreCheckFileExists(path)) return; - ShellExecuteW(m_hwnd, L"open", entry->path.c_str(), nullptr, nullptr, SW_SHOWNORMAL); + ShellExecuteW(m_hwnd, L"open", path.c_str(), nullptr, nullptr, SW_SHOWNORMAL); } void MainWindow::OpenContainingFolder() { int sel = ListView_GetNextItem(m_hListView, -1, LVNI_SELECTED); if (sel < 0 || sel >= static_cast(m_currentResults.size())) return; - const FileEntry* entry = m_currentResults[sel].entry; + const std::wstring& path = m_currentResults[sel].path; - // Open Explorer with file selected - PIDLIST_ABSOLUTE pidl = ILCreateFromPathW(entry->path.c_str()); + PIDLIST_ABSOLUTE pidl = ILCreateFromPathW(path.c_str()); if (pidl) { SHOpenFolderAndSelectItems(pidl, 0, nullptr, 0); ILFree(pidl); @@ -590,8 +613,8 @@ void MainWindow::CopySelectedPaths(bool filenameOnly) { while ((i = ListView_GetNextItem(m_hListView, i, LVNI_SELECTED)) != -1) { if (i >= static_cast(m_currentResults.size())) break; - const FileEntry* e = m_currentResults[i].entry; - text += (filenameOnly ? e->name : e->path) + L"\r\n"; + const DisplayEntry& e = m_currentResults[i]; + text += (filenameOnly ? e.name : e.path) + L"\r\n"; } if (text.empty()) return; @@ -617,7 +640,7 @@ void MainWindow::CutSelectedFiles() { while ((i = ListView_GetNextItem(m_hListView, i, LVNI_SELECTED)) != -1) { if (i >= static_cast(m_currentResults.size())) break; - paths += m_currentResults[i].entry->path + L'\0'; + paths += m_currentResults[i].path + L'\0'; } if (paths.empty()) return; @@ -658,7 +681,7 @@ void MainWindow::OnBeginDrag() { while ((i = ListView_GetNextItem(m_hListView, i, LVNI_SELECTED)) != -1) { if (i >= static_cast(m_currentResults.size())) break; - paths += m_currentResults[i].entry->path + L'\0'; + paths += m_currentResults[i].path + L'\0'; } if (paths.empty()) return; @@ -693,13 +716,12 @@ void MainWindow::DeleteSelectedFiles() { if (MessageBoxW(m_hwnd, msg.c_str(), L"Confirm Delete", MB_YESNO | MB_ICONWARNING) != IDYES) return; - // Build double-null-terminated path list for SHFileOperation std::wstring paths; int i = -1; while ((i = ListView_GetNextItem(m_hListView, i, LVNI_SELECTED)) != -1) { if (i >= static_cast(m_currentResults.size())) break; - paths += m_currentResults[i].entry->path + L'\0'; + paths += m_currentResults[i].path + L'\0'; } if (paths.empty()) return; @@ -740,24 +762,22 @@ void MainWindow::ApplyCurrentSort() { int col = m_sortColumn; bool desc = m_sortDescending; std::sort(m_currentResults.begin(), m_currentResults.end(), - [col, desc](const SearchResult& a, const SearchResult& b) { + [col, desc](const DisplayEntry& a, const DisplayEntry& b) { int cmp = 0; switch (col) { case 0: - cmp = a.entry->name.compare(b.entry->name); + cmp = a.name.compare(b.name); break; case 1: - cmp = a.entry->path.compare(b.entry->path); + cmp = a.path.compare(b.path); break; case 2: - cmp = (a.entry->size < b.entry->size) ? -1 - : (a.entry->size > b.entry->size) ? 1 - : 0; + cmp = (a.size < b.size) ? -1 : (a.size > b.size) ? 1 : 0; break; case 3: - cmp = (a.entry->lastModified < b.entry->lastModified) ? -1 - : (a.entry->lastModified > b.entry->lastModified) ? 1 - : 0; + cmp = (a.lastModified < b.lastModified) ? -1 + : (a.lastModified > b.lastModified) ? 1 + : 0; break; default: break; @@ -899,34 +919,33 @@ LRESULT CALLBACK MainWindow::WndProc(HWND hwnd, UINT msg, WPARAM wp, LPARAM lp) int idx = di->item.iItem; if (idx < 0 || idx >= static_cast(self->m_currentResults.size())) break; - const FileEntry* e = self->m_currentResults[idx].entry; + const DisplayEntry& e = self->m_currentResults[idx]; if (di->item.mask & LVIF_TEXT) { switch (di->item.iSubItem) { case 0: - di->item.pszText = const_cast(e->name.c_str()); + di->item.pszText = const_cast(e.name.c_str()); break; case 1: { - // Show path without filename static thread_local std::wstring pathBuf; - size_t slash = e->path.rfind(L'\\'); + size_t slash = e.path.rfind(L'\\'); pathBuf = (slash != std::wstring::npos) - ? e->path.substr(0, slash) - : e->path; + ? e.path.substr(0, slash) + : e.path; di->item.pszText = const_cast(pathBuf.c_str()); break; } case 2: { static thread_local std::wstring sizeBuf; wchar_t szBuf[32]{}; - if (e->size < 1024ULL) - swprintf_s(szBuf, L"%llu B", e->size); - else if (e->size < 1024ULL * 1024) - swprintf_s(szBuf, L"%.2f KB", e->size / 1024.0); - else if (e->size < 1024ULL * 1024 * 1024) - swprintf_s(szBuf, L"%.2f MB", e->size / (1024.0 * 1024)); + if (e.size < 1024ULL) + swprintf_s(szBuf, L"%llu B", e.size); + else if (e.size < 1024ULL * 1024) + swprintf_s(szBuf, L"%.2f KB", e.size / 1024.0); + else if (e.size < 1024ULL * 1024 * 1024) + swprintf_s(szBuf, L"%.2f MB", e.size / (1024.0 * 1024)); else swprintf_s(szBuf, L"%.2f GB", - e->size / (1024.0 * 1024 * 1024)); + e.size / (1024.0 * 1024 * 1024)); sizeBuf = szBuf; di->item.pszText = const_cast(sizeBuf.c_str()); break; @@ -934,8 +953,8 @@ LRESULT CALLBACK MainWindow::WndProc(HWND hwnd, UINT msg, WPARAM wp, LPARAM lp) case 3: { static thread_local std::wstring dateBuf; FILETIME ft; - ft.dwLowDateTime = static_cast(e->lastModified); - ft.dwHighDateTime = static_cast(e->lastModified >> 32); + ft.dwLowDateTime = static_cast(e.lastModified); + ft.dwHighDateTime = static_cast(e.lastModified >> 32); SYSTEMTIME st{}; FileTimeToLocalFileTime(&ft, &ft); FileTimeToSystemTime(&ft, &st); @@ -969,7 +988,7 @@ LRESULT CALLBACK MainWindow::WndProc(HWND hwnd, UINT msg, WPARAM wp, LPARAM lp) } case WM_SEARCH_RESULTS: { - auto* r = reinterpret_cast*>(lp); + auto* r = reinterpret_cast*>(lp); if (self && r) self->OnSearchResults(r); return 0; diff --git a/src/ui/MainWindow.h b/src/ui/MainWindow.h index 3bf1a4b..220ac45 100644 --- a/src/ui/MainWindow.h +++ b/src/ui/MainWindow.h @@ -17,6 +17,18 @@ namespace winindex { +// Snapshot of display fields copied from the pool under shared lock. +// Owned by the UI thread; never references into the pool after creation. +struct DisplayEntry { + std::wstring name; + std::wstring path; + uint64_t size; + uint64_t lastModified; + uint32_t attributes; + uint32_t matchStart; + uint32_t matchLen; +}; + class MainWindow { public: static bool Register(HINSTANCE hInst); @@ -29,11 +41,11 @@ class MainWindow { void OnSize(int cx, int cy); void OnCommand(WORD id); void OnContextMenu(HWND hwndFrom, int x, int y); - void OnSearchChanged(); // called from SearchBar on text change + void OnSearchChanged(); void OnListDblClick(); void OnListKeyDown(const NMLVKEYDOWN* kd); void OnIndexerStatus(const IndexerStatus& status); - void OnSearchResults(std::vector* results); + void OnSearchResults(std::vector* results); void OnDeviceChange(WPARAM event, LPARAM lp); static LRESULT CALLBACK WndProc(HWND hwnd, UINT msg, WPARAM wp, LPARAM lp); @@ -52,12 +64,11 @@ class MainWindow { std::shared_ptr m_indexer; std::shared_ptr m_searchEngine; - std::vector m_currentResults; + std::vector m_currentResults; uint64_t m_totalMatches = 0; int m_sortColumn = -1; bool m_sortDescending = false; - // Debounce timer static constexpr UINT_PTR kSearchTimerId = 1; static constexpr UINT kDebounceMs = 150; std::atomic m_searchCancel{false}; @@ -74,7 +85,7 @@ class MainWindow { void CutSelectedFiles(); void DeleteSelectedFiles(); void OnBeginDrag(); - bool PreCheckFileExists(const FileEntry* entry); + bool PreCheckFileExists(const std::wstring& path); void ShowAbout(); void SetStatusText(const std::wstring& text); void OnColumnClick(int col); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6dec665..efa2ce4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,5 +1,6 @@ add_executable(winindex_tests test_main.cpp + test_IndexPool.cpp test_IndexSerializer.cpp test_PathUtils.cpp test_SearchEngine.cpp diff --git a/tests/mocks/MockIndexStore.h b/tests/mocks/MockIndexStore.h index beeb3fc..6c393ba 100644 --- a/tests/mocks/MockIndexStore.h +++ b/tests/mocks/MockIndexStore.h @@ -18,7 +18,6 @@ class MockIndexStore : public IIndexStore { MOCK_METHOD(void, RemoveEntriesUnderPath, (const std::wstring& prefix), (override)); MOCK_METHOD(void, ApplyRename, (const std::wstring& o, const std::wstring& n), (override)); MOCK_METHOD(uint64_t, GetEntryCount, (), (const, override)); - MOCK_METHOD(const FileEntry*, GetEntries, (), (const, override)); MOCK_METHOD(uint64_t, GetSavedUsn, (const std::wstring& root), (const, override)); MOCK_METHOD(void, SetSavedUsn, (const std::wstring& root, uint64_t usn), (override)); MOCK_METHOD(uint64_t, GetIndexAgeSeconds, (), (const, override)); diff --git a/tests/test_IndexPool.cpp b/tests/test_IndexPool.cpp new file mode 100644 index 0000000..907615f --- /dev/null +++ b/tests/test_IndexPool.cpp @@ -0,0 +1,122 @@ +#include +#define WIN32_LEAN_AND_MEAN +#include + +#include "storage/IndexPool.h" +#include + +using namespace winindex; + +static FileEntry MakeEntry(const wchar_t* name, const wchar_t* path, uint64_t size = 0, + uint64_t modified = 0, uint32_t attrs = FILE_ATTRIBUTE_NORMAL) { + FileEntry e; + e.name = name; + e.nameLower = name; + std::transform(e.nameLower.begin(), e.nameLower.end(), e.nameLower.begin(), ::towlower); + e.path = path; + e.size = size; + e.lastModified = modified; + e.attributes = attrs; + return e; +} + +TEST(IndexPoolTest, EmptyPool) { + IndexPool pool; + EXPECT_EQ(pool.Size(), 0u); +} + +TEST(IndexPoolTest, AddOneEntry) { + IndexPool pool; + pool.AddEntry(MakeEntry(L"report.txt", L"C:\\docs\\report.txt", 1024, 999)); + + ASSERT_EQ(pool.Size(), 1u); + EXPECT_EQ(pool.GetName(0), L"report.txt"); + EXPECT_EQ(pool.GetPath(0), L"C:\\docs\\report.txt"); + EXPECT_EQ(pool.GetNameLower(0), L"report.txt"); + EXPECT_EQ(pool.meta[0].size, 1024u); + EXPECT_EQ(pool.meta[0].lastModified, 999u); +} + +TEST(IndexPoolTest, NameLowerIsCaseInsensitive) { + IndexPool pool; + pool.AddEntry(MakeEntry(L"REPORT.TXT", L"C:\\REPORT.TXT")); + + EXPECT_EQ(pool.GetNameLower(0), L"report.txt"); + EXPECT_EQ(pool.GetName(0), L"REPORT.TXT"); +} + +TEST(IndexPoolTest, NameLowerComputedWhenMissing) { + IndexPool pool; + FileEntry e; + e.name = L"MixedCase.Pdf"; + e.nameLower = L""; // intentionally empty — pool must compute it + e.path = L"C:\\MixedCase.Pdf"; + pool.AddEntry(e); + + EXPECT_EQ(pool.GetNameLower(0), L"mixedcase.pdf"); + EXPECT_EQ(pool.GetName(0), L"MixedCase.Pdf"); +} + +TEST(IndexPoolTest, GetNameExtractsLastComponent) { + IndexPool pool; + pool.AddEntry(MakeEntry(L"deep.log", L"C:\\a\\b\\c\\deep.log")); + + EXPECT_EQ(pool.GetName(0), L"deep.log"); + EXPECT_EQ(pool.GetPath(0), L"C:\\a\\b\\c\\deep.log"); +} + +TEST(IndexPoolTest, MultipleEntriesIndependent) { + IndexPool pool; + pool.AddEntry(MakeEntry(L"alpha.txt", L"C:\\alpha.txt", 10)); + pool.AddEntry(MakeEntry(L"BETA.PDF", L"D:\\docs\\BETA.PDF", 20)); + pool.AddEntry(MakeEntry(L"gamma.mp3", L"E:\\music\\gamma.mp3", 30)); + + ASSERT_EQ(pool.Size(), 3u); + EXPECT_EQ(pool.GetName(0), L"alpha.txt"); + EXPECT_EQ(pool.GetNameLower(1), L"beta.pdf"); + EXPECT_EQ(pool.GetPath(2), L"E:\\music\\gamma.mp3"); + EXPECT_EQ(pool.meta[0].size, 10u); + EXPECT_EQ(pool.meta[1].size, 20u); + EXPECT_EQ(pool.meta[2].size, 30u); +} + +TEST(IndexPoolTest, ClearResetsAll) { + IndexPool pool; + pool.AddEntry(MakeEntry(L"file.txt", L"C:\\file.txt")); + pool.Clear(); + + EXPECT_EQ(pool.Size(), 0u); + EXPECT_TRUE(pool.nameLowerPool.empty()); + EXPECT_TRUE(pool.pathPool.empty()); +} + +TEST(IndexPoolTest, TombstoneFlag) { + IndexPool pool; + pool.AddEntry(MakeEntry(L"alive.txt", L"C:\\alive.txt")); + pool.AddEntry(MakeEntry(L"dead.txt", L"C:\\dead.txt")); + + EXPECT_EQ(pool.meta[0].deleted, 0u); + pool.meta[1].deleted = 1; + EXPECT_EQ(pool.meta[1].deleted, 1u); + EXPECT_EQ(pool.meta[0].deleted, 0u); // first entry unaffected +} + +TEST(IndexPoolTest, ReserveDoesNotChangeSize) { + IndexPool pool; + pool.Reserve(100000); + EXPECT_EQ(pool.Size(), 0u); + EXPECT_GE(pool.meta.capacity(), 100000u); +} + +TEST(IndexPoolTest, PathAtRootLevel) { + // No backslash in name (root of drive) + IndexPool pool; + FileEntry e; + e.name = L"autorun.inf"; + e.nameLower = L"autorun.inf"; + e.path = L"C:\\autorun.inf"; + pool.AddEntry(e); + + EXPECT_EQ(pool.GetName(0), L"autorun.inf"); + EXPECT_EQ(pool.GetPath(0), L"C:\\autorun.inf"); +} diff --git a/tests/test_IndexSerializer.cpp b/tests/test_IndexSerializer.cpp index 6abf0b9..829f658 100644 --- a/tests/test_IndexSerializer.cpp +++ b/tests/test_IndexSerializer.cpp @@ -2,6 +2,7 @@ #define WIN32_LEAN_AND_MEAN #include +#include "storage/IndexPool.h" #include "storage/IndexSerializer.h" #include #include @@ -20,88 +21,147 @@ class IndexSerializerTest : public ::testing::Test { } void TearDown() override { _wremove(tmpPath.c_str()); } + + static FileEntry MakeEntry(const wchar_t* path, uint64_t size = 0, uint64_t lastModified = 0, + uint32_t attributes = 0) { + FileEntry e; + e.path = path; + size_t slash = e.path.rfind(L'\\'); + e.name = (slash != std::wstring::npos) ? e.path.substr(slash + 1) : e.path; + e.nameLower = e.name; + std::transform(e.nameLower.begin(), e.nameLower.end(), e.nameLower.begin(), ::towlower); + e.size = size; + e.lastModified = lastModified; + e.attributes = attributes; + return e; + } }; TEST_F(IndexSerializerTest, RoundtripEmptyIndex) { - std::vector entries; + IndexPool pool; std::unordered_map usnMap; - ASSERT_TRUE(IndexSerializer::Serialize(tmpPath, entries, usnMap)); + ASSERT_TRUE(IndexSerializer::Serialize(tmpPath, pool, usnMap)); - std::vector loaded; + IndexPool loaded; std::unordered_map loadedUsn; uint64_t ts = 0; ASSERT_TRUE(IndexSerializer::Deserialize(tmpPath, loaded, loadedUsn, ts)); - EXPECT_EQ(loaded.size(), 0u); + EXPECT_EQ(loaded.meta.size(), 0u); EXPECT_GT(ts, 0u); } TEST_F(IndexSerializerTest, RoundtripSingleEntry) { - FileEntry e; - e.name = L"hello.txt"; - e.path = L"C:\\Users\\test\\hello.txt"; - e.size = 12345; - e.lastModified = 999888777; - e.attributes = FILE_ATTRIBUTE_NORMAL; - - std::vector entries = {e}; + IndexPool pool; + pool.AddEntry( + MakeEntry(L"C:\\Users\\test\\hello.txt", 12345, 999888777, FILE_ATTRIBUTE_NORMAL)); + std::unordered_map usnMap; usnMap[L"C:\\"] = 42; - ASSERT_TRUE(IndexSerializer::Serialize(tmpPath, entries, usnMap)); + ASSERT_TRUE(IndexSerializer::Serialize(tmpPath, pool, usnMap)); - std::vector loaded; + IndexPool loaded; std::unordered_map loadedUsn; uint64_t ts = 0; ASSERT_TRUE(IndexSerializer::Deserialize(tmpPath, loaded, loadedUsn, ts)); - ASSERT_EQ(loaded.size(), 1u); - EXPECT_EQ(loaded[0].name, e.name); - EXPECT_EQ(loaded[0].path, e.path); - EXPECT_EQ(loaded[0].size, e.size); - EXPECT_EQ(loaded[0].lastModified, e.lastModified); - EXPECT_EQ(loaded[0].attributes, e.attributes); + ASSERT_EQ(loaded.meta.size(), 1u); + EXPECT_EQ(loaded.GetName(0), L"hello.txt"); + EXPECT_EQ(loaded.GetPath(0), L"C:\\Users\\test\\hello.txt"); + EXPECT_EQ(loaded.meta[0].size, 12345u); + EXPECT_EQ(loaded.meta[0].lastModified, 999888777u); + EXPECT_EQ(loaded.meta[0].attributes, static_cast(FILE_ATTRIBUTE_NORMAL)); EXPECT_EQ(loadedUsn[L"C:\\"], 42u); + EXPECT_GT(ts, 0u); } TEST_F(IndexSerializerTest, RoundtripManyEntries) { - std::vector entries; + IndexPool pool; for (int i = 0; i < 10000; ++i) { - FileEntry e; - e.name = L"file_" + std::to_wstring(i) + L".dat"; - e.path = L"C:\\data\\file_" + std::to_wstring(i) + L".dat"; - e.size = static_cast(i) * 1024; - e.lastModified = static_cast(i); - e.attributes = FILE_ATTRIBUTE_NORMAL; - entries.push_back(e); + pool.AddEntry(MakeEntry((L"C:\\data\\file_" + std::to_wstring(i) + L".dat").c_str(), + static_cast(i) * 1024, static_cast(i), + FILE_ATTRIBUTE_NORMAL)); } std::unordered_map usnMap; - ASSERT_TRUE(IndexSerializer::Serialize(tmpPath, entries, usnMap)); + ASSERT_TRUE(IndexSerializer::Serialize(tmpPath, pool, usnMap)); - std::vector loaded; + IndexPool loaded; std::unordered_map loadedUsn; uint64_t ts = 0; ASSERT_TRUE(IndexSerializer::Deserialize(tmpPath, loaded, loadedUsn, ts)); - EXPECT_EQ(loaded.size(), 10000u); + EXPECT_EQ(loaded.meta.size(), 10000u); +} + +TEST_F(IndexSerializerTest, NameLowerRebuiltCorrectly) { + IndexPool pool; + pool.AddEntry(MakeEntry(L"C:\\Docs\\Report_2024.xlsx")); + + std::unordered_map usnMap; + ASSERT_TRUE(IndexSerializer::Serialize(tmpPath, pool, usnMap)); + + IndexPool loaded; + std::unordered_map loadedUsn; + uint64_t ts = 0; + ASSERT_TRUE(IndexSerializer::Deserialize(tmpPath, loaded, loadedUsn, ts)); + + ASSERT_EQ(loaded.meta.size(), 1u); + EXPECT_EQ(loaded.GetNameLower(0), L"report_2024.xlsx"); + EXPECT_EQ(loaded.GetName(0), L"Report_2024.xlsx"); +} + +TEST_F(IndexSerializerTest, TombstonedEntriesNotPersisted) { + IndexPool pool; + pool.AddEntry(MakeEntry(L"C:\\keep.txt", 100)); + pool.AddEntry(MakeEntry(L"C:\\delete.txt", 200)); + pool.meta[1].deleted = 1; + + std::unordered_map usnMap; + ASSERT_TRUE(IndexSerializer::Serialize(tmpPath, pool, usnMap)); + + IndexPool loaded; + std::unordered_map loadedUsn; + uint64_t ts = 0; + ASSERT_TRUE(IndexSerializer::Deserialize(tmpPath, loaded, loadedUsn, ts)); + + ASSERT_EQ(loaded.meta.size(), 1u); + EXPECT_EQ(loaded.GetName(0), L"keep.txt"); + EXPECT_EQ(loaded.meta[0].size, 100u); +} + +TEST_F(IndexSerializerTest, UsnMapRoundtrip) { + IndexPool pool; + std::unordered_map usnMap; + usnMap[L"C:\\"] = 111; + usnMap[L"D:\\"] = 222; + + ASSERT_TRUE(IndexSerializer::Serialize(tmpPath, pool, usnMap)); + + IndexPool loaded; + std::unordered_map loadedUsn; + uint64_t ts = 0; + ASSERT_TRUE(IndexSerializer::Deserialize(tmpPath, loaded, loadedUsn, ts)); + + EXPECT_EQ(loadedUsn[L"C:\\"], 111u); + EXPECT_EQ(loadedUsn[L"D:\\"], 222u); } TEST_F(IndexSerializerTest, CorruptFileFails) { - // Write garbage FILE* f = _wfopen(tmpPath.c_str(), L"wb"); ASSERT_NE(f, nullptr); const char garbage[] = "this is not a valid index file at all!!!"; fwrite(garbage, 1, sizeof(garbage), f); fclose(f); - std::vector loaded; + IndexPool loaded; std::unordered_map usnMap; uint64_t ts = 0; EXPECT_FALSE(IndexSerializer::Deserialize(tmpPath, loaded, usnMap, ts)); } TEST_F(IndexSerializerTest, MissingFileFails) { - std::vector loaded; + IndexPool loaded; std::unordered_map usnMap; uint64_t ts = 0; EXPECT_FALSE( diff --git a/tests/test_SearchEngine.cpp b/tests/test_SearchEngine.cpp index 8ec38df..2a1a1d7 100644 --- a/tests/test_SearchEngine.cpp +++ b/tests/test_SearchEngine.cpp @@ -3,14 +3,14 @@ #include "search/SearchEngine.h" #include "search/SimdSearch.h" #include "search/TokenMatcher.h" +#include "storage/IndexPool.h" #include #include using namespace winindex; -static std::vector MakeEntries( - std::initializer_list> items) { - std::vector v; +static IndexPool MakePool(std::initializer_list> items) { + IndexPool pool; for (auto& [name, path] : items) { FileEntry e; e.name = name; @@ -20,9 +20,16 @@ static std::vector MakeEntries( e.size = 0; e.lastModified = 0; e.attributes = 0; - v.push_back(e); + pool.AddEntry(e); } - return v; + return pool; +} + +static std::vector DoSearch(SearchEngine& engine, const IndexPool& pool, + const std::wstring& query, const SearchOptions& opts, + uint32_t maxResults, const std::atomic& cancel) { + return engine.Search(query, pool.meta.data(), static_cast(pool.meta.size()), + pool.nameLowerPool.data(), pool.pathPool.data(), opts, maxResults, cancel); } class SearchEngineTest : public ::testing::Test { @@ -32,70 +39,70 @@ class SearchEngineTest : public ::testing::Test { }; TEST_F(SearchEngineTest, BasicSubstringMatch) { - auto entries = MakeEntries({ + auto pool = MakePool({ {L"report_2024.xlsx", L"C:\\docs\\report_2024.xlsx"}, {L"summary.pdf", L"C:\\docs\\summary.pdf"}, {L"report_q1.docx", L"C:\\docs\\report_q1.docx"}, }); SearchOptions opts{}; - auto results = engine.Search(L"report", entries.data(), entries.size(), opts, 100, cancel); + auto results = DoSearch(engine, pool, L"report", opts, 100, cancel); EXPECT_EQ(results.size(), 2u); } TEST_F(SearchEngineTest, CaseSensitiveMatch) { - auto entries = MakeEntries({ + auto pool = MakePool({ {L"Report.txt", L"C:\\Report.txt"}, {L"report.txt", L"C:\\report.txt"}, }); SearchOptions opts{}; opts.caseSensitive = true; - auto results = engine.Search(L"Report", entries.data(), entries.size(), opts, 100, cancel); + auto results = DoSearch(engine, pool, L"Report", opts, 100, cancel); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].entry->name, L"Report.txt"); + EXPECT_EQ(pool.GetName(results[0].entryIndex), L"Report.txt"); } TEST_F(SearchEngineTest, CaseInsensitiveMatch) { - auto entries = MakeEntries({ + auto pool = MakePool({ {L"REPORT.txt", L"C:\\REPORT.txt"}, {L"report.txt", L"C:\\report.txt"}, }); SearchOptions opts{}; opts.caseSensitive = false; - auto results = engine.Search(L"report", entries.data(), entries.size(), opts, 100, cancel); + auto results = DoSearch(engine, pool, L"report", opts, 100, cancel); EXPECT_EQ(results.size(), 2u); } TEST_F(SearchEngineTest, WholeWordMatch) { - auto entries = MakeEntries({ + auto pool = MakePool({ {L"report.txt", L"C:\\report.txt"}, {L"reports_final.txt", L"C:\\reports_final.txt"}, }); SearchOptions opts{}; opts.wholeWord = true; - auto results = engine.Search(L"report", entries.data(), entries.size(), opts, 100, cancel); + auto results = DoSearch(engine, pool, L"report", opts, 100, cancel); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].entry->name, L"report.txt"); + EXPECT_EQ(pool.GetName(results[0].entryIndex), L"report.txt"); } TEST_F(SearchEngineTest, MatchPathOption) { - auto entries = MakeEntries({ + auto pool = MakePool({ {L"file.txt", L"C:\\Projects\\report\\file.txt"}, {L"other.txt", L"C:\\Documents\\other.txt"}, }); SearchOptions opts{}; opts.matchPath = true; - auto results = engine.Search(L"report", entries.data(), entries.size(), opts, 100, cancel); + auto results = DoSearch(engine, pool, L"report", opts, 100, cancel); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].entry->name, L"file.txt"); + EXPECT_EQ(pool.GetName(results[0].entryIndex), L"file.txt"); } TEST_F(SearchEngineTest, RegexMatch) { - auto entries = MakeEntries({ + auto pool = MakePool({ {L"invoice_001.pdf", L"C:\\invoice_001.pdf"}, {L"invoice_abc.pdf", L"C:\\invoice_abc.pdf"}, {L"summary.pdf", L"C:\\summary.pdf"}, @@ -103,59 +110,57 @@ TEST_F(SearchEngineTest, RegexMatch) { SearchOptions opts{}; opts.useRegex = true; - auto results = - engine.Search(L"invoice_\\d+", entries.data(), entries.size(), opts, 100, cancel); + auto results = DoSearch(engine, pool, L"invoice_\\d+", opts, 100, cancel); ASSERT_EQ(results.size(), 1u); - EXPECT_EQ(results[0].entry->name, L"invoice_001.pdf"); + EXPECT_EQ(pool.GetName(results[0].entryIndex), L"invoice_001.pdf"); } TEST_F(SearchEngineTest, InvalidRegexReturnsEmpty) { - auto entries = MakeEntries({ - {L"file.txt", L"C:\\file.txt"}, - }); + auto pool = MakePool({{L"file.txt", L"C:\\file.txt"}}); SearchOptions opts{}; opts.useRegex = true; - auto results = - engine.Search(L"[invalid(regex", entries.data(), entries.size(), opts, 100, cancel); + auto results = DoSearch(engine, pool, L"[invalid(regex", opts, 100, cancel); EXPECT_TRUE(results.empty()); } TEST_F(SearchEngineTest, MaxResultsCap) { - std::vector entries; + IndexPool pool; for (int i = 0; i < 200; ++i) { FileEntry e; e.name = L"file_" + std::to_wstring(i) + L".txt"; e.path = L"C:\\" + e.name; - entries.push_back(e); + e.nameLower = e.name; + std::transform(e.nameLower.begin(), e.nameLower.end(), e.nameLower.begin(), ::towlower); + pool.AddEntry(e); } SearchOptions opts{}; - auto results = engine.Search(L"fi", entries.data(), entries.size(), opts, 10, cancel); + auto results = DoSearch(engine, pool, L"fi", opts, 10, cancel); EXPECT_LE(results.size(), 10u); } TEST_F(SearchEngineTest, QueryTooShortReturnsEmpty) { - auto entries = MakeEntries({{L"file.txt", L"C:\\file.txt"}}); + auto pool = MakePool({{L"file.txt", L"C:\\file.txt"}}); SearchOptions opts{}; - auto results = engine.Search(L"f", entries.data(), entries.size(), opts, 100, cancel); + auto results = DoSearch(engine, pool, L"f", opts, 100, cancel); EXPECT_TRUE(results.empty()); } TEST_F(SearchEngineTest, CancelTokenAbortsSearch) { - std::vector entries; + IndexPool pool; for (int i = 0; i < 100000; ++i) { FileEntry e; e.name = L"somefile_" + std::to_wstring(i) + L".txt"; e.path = L"C:\\" + e.name; - entries.push_back(e); + e.nameLower = e.name; + std::transform(e.nameLower.begin(), e.nameLower.end(), e.nameLower.begin(), ::towlower); + pool.AddEntry(e); } std::atomic cancelNow{true}; // already cancelled SearchOptions opts{}; - auto results = - engine.Search(L"somefile", entries.data(), entries.size(), opts, 10000, cancelNow); - // With immediate cancel the result set should be small or empty + auto results = DoSearch(engine, pool, L"somefile", opts, 10000, cancelNow); EXPECT_LT(results.size(), 10000u); } @@ -170,7 +175,7 @@ class TokenSetMatchTest : public ::testing::Test { protected: SearchEngine engine; std::atomic cancel{false}; - std::vector entries = MakeEntries({ + IndexPool pool = MakePool({ {kLedZepName, kLedZepPath}, {L"unrelated_song.mp3", L"C:\\music\\unrelated_song.mp3"}, }); @@ -178,96 +183,91 @@ class TokenSetMatchTest : public ::testing::Test { }; TEST_F(TokenSetMatchTest, SpaceSeparatorMatchesHyphen) { - auto r = engine.Search(L"just rosy", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"just rosy", opts, 100, cancel); ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(r[0].entry->name, kLedZepName); + EXPECT_EQ(pool.GetName(r[0].entryIndex), kLedZepName); } TEST_F(TokenSetMatchTest, UpperCaseQuerySpaceSep) { - auto r = engine.Search(L"Just rosy", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"Just rosy", opts, 100, cancel); ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(r[0].entry->name, kLedZepName); + EXPECT_EQ(pool.GetName(r[0].entryIndex), kLedZepName); } TEST_F(TokenSetMatchTest, UnderscoreSeparatorMatchesHyphen) { - auto r = engine.Search(L"just_rosy", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"just_rosy", opts, 100, cancel); ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(r[0].entry->name, kLedZepName); + EXPECT_EQ(pool.GetName(r[0].entryIndex), kLedZepName); } TEST_F(TokenSetMatchTest, MixedSeparatorsInQuery) { - auto r = engine.Search(L"just rosy june", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"just rosy june", opts, 100, cancel); ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(r[0].entry->name, kLedZepName); + EXPECT_EQ(pool.GetName(r[0].entryIndex), kLedZepName); } TEST_F(TokenSetMatchTest, NonAdjacentTokens) { - auto r = engine.Search(L"just rosy guitar", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"just rosy guitar", opts, 100, cancel); ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(r[0].entry->name, kLedZepName); + EXPECT_EQ(pool.GetName(r[0].entryIndex), kLedZepName); } TEST_F(TokenSetMatchTest, TokensFromDifferentParts) { - auto r = engine.Search(L"rosy guitar flac", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"rosy guitar flac", opts, 100, cancel); ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(r[0].entry->name, kLedZepName); + EXPECT_EQ(pool.GetName(r[0].entryIndex), kLedZepName); } TEST_F(TokenSetMatchTest, LedZepPlusRosy) { - auto r = engine.Search(L"LedZep rosy", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"LedZep rosy", opts, 100, cancel); ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(r[0].entry->name, kLedZepName); + EXPECT_EQ(pool.GetName(r[0].entryIndex), kLedZepName); } TEST_F(TokenSetMatchTest, LedZepPlusFlac) { - auto r = engine.Search(L"ledzep flac", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"ledzep flac", opts, 100, cancel); ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(r[0].entry->name, kLedZepName); + EXPECT_EQ(pool.GetName(r[0].entryIndex), kLedZepName); } TEST_F(TokenSetMatchTest, TokenOrderIrrelevant_GuitarFirst) { - auto r = engine.Search(L"just guitar rosy", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"just guitar rosy", opts, 100, cancel); ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(r[0].entry->name, kLedZepName); + EXPECT_EQ(pool.GetName(r[0].entryIndex), kLedZepName); } TEST_F(TokenSetMatchTest, TokenOrderIrrelevant_RosyFirst) { - auto r = engine.Search(L"rosy just guitar", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"rosy just guitar", opts, 100, cancel); ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(r[0].entry->name, kLedZepName); + EXPECT_EQ(pool.GetName(r[0].entryIndex), kLedZepName); } TEST_F(TokenSetMatchTest, AllTokensMustMatch_NegativeCase) { - // "piano" is not in the filename — should not match - auto r = engine.Search(L"just rosy piano", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"just rosy piano", opts, 100, cancel); EXPECT_EQ(r.size(), 0u); } TEST_F(TokenSetMatchTest, SingleWordQueryUsesSimdPath) { - // "ledzep" is a direct substring — still found via SIMD path - auto r = engine.Search(L"ledzep", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"ledzep", opts, 100, cancel); ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(r[0].entry->name, kLedZepName); + EXPECT_EQ(pool.GetName(r[0].entryIndex), kLedZepName); } TEST_F(TokenSetMatchTest, ExactHyphenSubstringStillWorks) { - // "just-rosy" is a literal substring — SIMD finds it without token path - auto r = engine.Search(L"just-rosy", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"just-rosy", opts, 100, cancel); ASSERT_EQ(r.size(), 1u); - EXPECT_EQ(r[0].entry->name, kLedZepName); + EXPECT_EQ(pool.GetName(r[0].entryIndex), kLedZepName); } TEST_F(TokenSetMatchTest, CaseSensitiveModeSkipsTokenPath) { SearchOptions caseSens{}; caseSens.caseSensitive = true; - // lowercase "just rosy" won't match mixed-case filename in case-sensitive mode - auto r = engine.Search(L"just rosy", entries.data(), entries.size(), caseSens, 100, cancel); + auto r = DoSearch(engine, pool, L"just rosy", caseSens, 100, cancel); EXPECT_EQ(r.size(), 0u); } TEST_F(TokenSetMatchTest, PartialTokenDoesNotMatch) { - // "led" and "zep" are not independent tokens in the filename ("ledzep" is one token) - auto r = engine.Search(L"led zep", entries.data(), entries.size(), opts, 100, cancel); + auto r = DoSearch(engine, pool, L"led zep", opts, 100, cancel); EXPECT_EQ(r.size(), 0u); } @@ -349,8 +349,6 @@ TEST(TokenMatcherTest, AllQueryTokensPresent_EmptyQueryReturnsFalse) { // SIMD detection test TEST(SimdTest, DetectCaps) { auto caps = winindex::DetectSimdCaps(); - // We can't assert specific caps since it depends on host CPU, - // but the call must not crash. (void)caps; } From 2623d0b5b513a18316f3e916892013da16e956cd Mon Sep 17 00:00:00 2001 From: rajeshsub <4209324+rajeshsub@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:57:05 +1000 Subject: [PATCH 2/3] Phase 2 of 3 - performance improvements --- src/core/CMakeLists.txt | 9 +++ src/core/search/SearchEngine.cpp | 99 ++++++++++++++++++---------- src/core/search/SimdSearch.cpp | 100 ++++++----------------------- src/core/search/SimdSearchAvx2.cpp | 49 ++++++++++++++ 4 files changed, 144 insertions(+), 113 deletions(-) create mode 100644 src/core/search/SimdSearchAvx2.cpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 5f5b1b0..cea9166 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -7,6 +7,7 @@ add_library(winindex_core STATIC indexer/Indexer.cpp search/SearchEngine.cpp search/SimdSearch.cpp + search/SimdSearchAvx2.cpp search/TokenMatcher.cpp storage/IndexPool.cpp storage/IndexStore.cpp @@ -26,6 +27,14 @@ target_link_libraries(winindex_core PRIVATE ntdll ) +# SimdSearchAvx2.cpp must be compiled with AVX2 enabled so intrinsics are available. +# Runtime detection in SimdFindSubstring() guards actual execution on non-AVX2 CPUs. +if(MSVC) + set_source_files_properties(search/SimdSearchAvx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2") +else() + set_source_files_properties(search/SimdSearchAvx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2") +endif() + # Expose interfaces for mocking in tests target_sources(winindex_core PUBLIC indexer/IFileSystemScanner.h diff --git a/src/core/search/SearchEngine.cpp b/src/core/search/SearchEngine.cpp index 82f525e..f667e35 100644 --- a/src/core/search/SearchEngine.cpp +++ b/src/core/search/SearchEngine.cpp @@ -56,49 +56,80 @@ std::vector SearchEngine::SearchRegex( re2opts.set_encoding(RE2::Options::EncodingUTF8); std::string utf8Query = WideToUtf8(query.c_str(), query.size()); + // Compile RE2 objects once; RE2::PartialMatch is thread-safe for concurrent reads. RE2 re(utf8Query, re2opts); if (!re.ok()) return {}; - RE2 reCapture("(" + utf8Query + ")", re2opts); - std::vector results; - results.reserve(maxResults); - - for (uint64_t i = 0; i < entryCount && !cancelToken.load(std::memory_order_relaxed); ++i) { - const EntryMeta& m = meta[i]; - if (m.deleted) - continue; - - const wchar_t* targetData; - size_t targetLen; - if (options.matchPath) { - targetData = pathPool + m.pathOffset; - targetLen = m.pathLen; - } else { - // Use name (original case) from path tail for regex matching - targetData = pathPool + m.pathOffset + m.nameStart; - targetLen = static_cast(m.pathLen - m.nameStart); - } + unsigned int numThreads = std::max(1u, std::thread::hardware_concurrency()); + uint64_t chunkSize = (entryCount + numThreads - 1) / numThreads; + std::atomic collected{0}; - std::string utf8Target = WideToUtf8(targetData, targetLen); - - if (options.wholeWord) { - re2::StringPiece match; - if (!reCapture.ok() || !RE2::PartialMatch(utf8Target, reCapture, &match)) - continue; - size_t matchPos = static_cast(match.data() - utf8Target.data()); - if (!MatchesWholeWord(targetData, targetLen, matchPos, match.size())) - continue; - } else { - if (!RE2::PartialMatch(utf8Target, re)) - continue; - } + std::vector>> futures; + futures.reserve(numThreads); - results.push_back({static_cast(i), 0u, 0u}); - if (results.size() >= maxResults) + for (unsigned int t = 0; t < numThreads; ++t) { + uint64_t begin = t * chunkSize; + uint64_t end = std::min(begin + chunkSize, entryCount); + if (begin >= entryCount) break; + + futures.push_back( + std::async(std::launch::async, [&, begin, end]() -> std::vector { + std::vector local; + for (uint64_t i = begin; i < end; ++i) { + if (cancelToken.load(std::memory_order_relaxed)) + break; + if (collected.load(std::memory_order_relaxed) >= maxResults) + break; + + const EntryMeta& m = meta[i]; + if (m.deleted) + continue; + + const wchar_t* targetData; + size_t targetLen; + if (options.matchPath) { + targetData = pathPool + m.pathOffset; + targetLen = m.pathLen; + } else { + targetData = pathPool + m.pathOffset + m.nameStart; + targetLen = static_cast(m.pathLen - m.nameStart); + } + + std::string utf8Target = WideToUtf8(targetData, targetLen); + + if (options.wholeWord) { + re2::StringPiece match; + if (!reCapture.ok() || !RE2::PartialMatch(utf8Target, reCapture, &match)) + continue; + size_t matchPos = static_cast(match.data() - utf8Target.data()); + if (!MatchesWholeWord(targetData, targetLen, matchPos, match.size())) + continue; + } else { + if (!RE2::PartialMatch(utf8Target, re)) + continue; + } + + local.push_back({static_cast(i), 0u, 0u}); + collected.fetch_add(1, std::memory_order_relaxed); + } + return local; + })); + } + + std::vector results; + results.reserve(maxResults); + for (auto& f : futures) { + auto chunk = f.get(); + for (const auto& sr : chunk) { + if (results.size() >= maxResults) + goto done; + results.push_back(sr); + } } +done: return results; } diff --git a/src/core/search/SimdSearch.cpp b/src/core/search/SimdSearch.cpp index ce9c053..fb308ad 100644 --- a/src/core/search/SimdSearch.cpp +++ b/src/core/search/SimdSearch.cpp @@ -1,12 +1,17 @@ #include "SimdSearch.h" +#include // SSE2 — baseline on x64 Windows, no /arch flag required #include #include #include +#include namespace winindex { +// Defined in SimdSearchAvx2.cpp, compiled with /arch:AVX2. +size_t Avx2Find(const wchar_t* hay, size_t hayLen, const wchar_t* needle, size_t needleLen); + SimdCaps DetectSimdCaps() { SimdCaps caps{}; int cpuInfo[4] = {}; @@ -20,22 +25,6 @@ SimdCaps DetectSimdCaps() { static const SimdCaps g_simdCaps = DetectSimdCaps(); -// --------------------------------------------------------------------------- -// Scalar fallback -// --------------------------------------------------------------------------- -static size_t ScalarFind(const wchar_t* hay, size_t hayLen, const wchar_t* needle, - size_t needleLen) { - if (needleLen == 0) - return 0; - if (needleLen > hayLen) - return std::wstring::npos; - for (size_t i = 0; i <= hayLen - needleLen; ++i) { - if (wmemcmp(hay + i, needle, needleLen) == 0) - return i; - } - return std::wstring::npos; -} - static size_t ScalarFindInsensitive(const wchar_t* hay, size_t hayLen, const wchar_t* needle, size_t needleLen) { if (needleLen == 0) @@ -57,103 +46,56 @@ static size_t ScalarFindInsensitive(const wchar_t* hay, size_t hayLen, const wch } // --------------------------------------------------------------------------- -// SSE4.2 accelerated search (operates on UTF-16 as pairs of bytes) -// Uses _mm_cmpistrm for byte-level first-char scan, then verifies. -// --------------------------------------------------------------------------- -#ifdef __SSE4_2__ -#include -static size_t Sse42Find(const wchar_t* hay, size_t hayLen, const wchar_t* needle, - size_t needleLen) { - if (needleLen == 0) - return 0; - if (needleLen > hayLen) - return std::wstring::npos; - - wchar_t firstChar = needle[0]; - const wchar_t* p = hay; - const wchar_t* end = hay + hayLen - needleLen + 1; - - while (p < end) { - // Scan for first character - if (*p == firstChar) { - if (wmemcmp(p, needle, needleLen) == 0) - return static_cast(p - hay); - } - ++p; - } - return std::wstring::npos; -} -#endif - -// --------------------------------------------------------------------------- -// AVX2 accelerated search — scan 16 wchar_t at a time for first character +// SSE2 — 8 wchar_t at a time; always compiled on x64, no /arch flag needed. // --------------------------------------------------------------------------- -#ifdef __AVX2__ -#include -static size_t Avx2Find(const wchar_t* hay, size_t hayLen, const wchar_t* needle, size_t needleLen) { +static size_t Sse2Find(const wchar_t* hay, size_t hayLen, const wchar_t* needle, size_t needleLen) { if (needleLen == 0) return 0; if (needleLen > hayLen) return std::wstring::npos; - const wchar_t firstChar = needle[0]; - const __m256i vFirst = _mm256_set1_epi16(static_cast(firstChar)); - + const __m128i vFirst = _mm_set1_epi16(static_cast(needle[0])); const wchar_t* p = hay; const wchar_t* end = hay + hayLen - needleLen + 1; - // Process 16 wchar_t at a time - while (p + 16 <= end) { - __m256i chunk = _mm256_loadu_si256(reinterpret_cast(p)); - __m256i cmp = _mm256_cmpeq_epi16(chunk, vFirst); - uint32_t mask = static_cast(_mm256_movemask_epi8(cmp)); + while (p + 8 <= end) { + __m128i chunk = _mm_loadu_si128(reinterpret_cast(p)); + __m128i cmp = _mm_cmpeq_epi16(chunk, vFirst); + unsigned int mask = static_cast(_mm_movemask_epi8(cmp)); while (mask) { - // Each bit pair corresponds to one wchar_t unsigned long bit; _BitScanForward(&bit, mask); - size_t offset = bit / 2; - if (p + offset + needleLen <= hay + hayLen) { - if (wmemcmp(p + offset, needle, needleLen) == 0) - return static_cast(p + offset - hay); - } + size_t off = bit / 2; // byte index → wchar_t index + if (wmemcmp(p + off, needle, needleLen) == 0) + return static_cast(p + off - hay); + // Clear both bytes of this wchar_t's match bits + mask &= mask - 1; mask &= mask - 1; - mask &= mask - 1; // clear both bits of the pair } - p += 16; + p += 8; } - // Tail: scalar cleanup while (p < end) { - if (*p == firstChar && wmemcmp(p, needle, needleLen) == 0) + if (*p == needle[0] && wmemcmp(p, needle, needleLen) == 0) return static_cast(p - hay); ++p; } return std::wstring::npos; } -#endif // --------------------------------------------------------------------------- -// Public API — dispatch at runtime +// Public API — runtime dispatch // --------------------------------------------------------------------------- size_t SimdFindSubstring(const wchar_t* haystack, size_t haystackLen, const wchar_t* needle, size_t needleLen) { -#ifdef __AVX2__ if (g_simdCaps.avx2) return Avx2Find(haystack, haystackLen, needle, needleLen); -#endif -#ifdef __SSE4_2__ - if (g_simdCaps.sse42) - return Sse42Find(haystack, haystackLen, needle, needleLen); -#endif - return ScalarFind(haystack, haystackLen, needle, needleLen); + return Sse2Find(haystack, haystackLen, needle, needleLen); } size_t SimdFindSubstringInsensitive(const wchar_t* haystack, size_t haystackLen, const wchar_t* needle, size_t needleLen) { - // Case-insensitive: lowercase the needle once, scan with lowercased comparison. - // SIMD for case-insensitive wide strings is complex; scalar is used here. - // The needle is short; the haystack length drives cost. return ScalarFindInsensitive(haystack, haystackLen, needle, needleLen); } diff --git a/src/core/search/SimdSearchAvx2.cpp b/src/core/search/SimdSearchAvx2.cpp new file mode 100644 index 0000000..f756853 --- /dev/null +++ b/src/core/search/SimdSearchAvx2.cpp @@ -0,0 +1,49 @@ +#include +#include + +#include "SimdSearch.h" +#include + +// This file is compiled with /arch:AVX2 (MSVC) or -mavx2 (GCC/Clang). +// All code here may emit AVX2 instructions; only call Avx2Find after +// confirming AVX2 support via DetectSimdCaps(). + +namespace winindex { + +size_t Avx2Find(const wchar_t* hay, size_t hayLen, const wchar_t* needle, size_t needleLen) { + if (needleLen == 0) + return 0; + if (needleLen > hayLen) + return std::wstring::npos; + + const __m256i vFirst = _mm256_set1_epi16(static_cast(needle[0])); + const wchar_t* p = hay; + const wchar_t* end = hay + hayLen - needleLen + 1; + + while (p + 16 <= end) { + __m256i chunk = _mm256_loadu_si256(reinterpret_cast(p)); + __m256i cmp = _mm256_cmpeq_epi16(chunk, vFirst); + uint32_t mask = static_cast(_mm256_movemask_epi8(cmp)); + + while (mask) { + unsigned long bit; + _BitScanForward(&bit, mask); + size_t off = bit / 2; // byte index → wchar_t index + if (wmemcmp(p + off, needle, needleLen) == 0) + return static_cast(p + off - hay); + // Clear both bytes of this wchar_t's match bits + mask &= mask - 1; + mask &= mask - 1; + } + p += 16; + } + + while (p < end) { + if (*p == needle[0] && wmemcmp(p, needle, needleLen) == 0) + return static_cast(p - hay); + ++p; + } + return std::wstring::npos; +} + +} // namespace winindex From e4eb3736fd2e88f41baf835fd640fa8a473a3ed7 Mon Sep 17 00:00:00 2001 From: rajeshsub <4209324+rajeshsub@users.noreply.github.com> Date: Fri, 3 Jul 2026 00:32:42 +1000 Subject: [PATCH 3/3] Phase 3 of 3 - performance improvements --- src/core/indexer/Indexer.cpp | 123 ++++++++++++++++--------- src/core/indexer/Indexer.h | 4 +- src/core/indexer/UsnJournalMonitor.cpp | 8 +- 3 files changed, 87 insertions(+), 48 deletions(-) diff --git a/src/core/indexer/Indexer.cpp b/src/core/indexer/Indexer.cpp index a8c235b..e6d0628 100644 --- a/src/core/indexer/Indexer.cpp +++ b/src/core/indexer/Indexer.cpp @@ -8,6 +8,7 @@ #include #include +#include #include namespace winindex { @@ -102,10 +103,16 @@ void Indexer::IndexPaths(std::vector paths) { void Indexer::IndexPathsThread(const std::vector& paths) { EmitStatus(IndexerState::Scanning, L"Indexing new paths..."); + std::vector excludedPaths = m_settings->GetExcludedPaths(); for (const auto& root : paths) { if (m_cancel.load(std::memory_order_relaxed)) break; - ScanDrive(root); + std::vector localEntries; + ScanDriveInto(root, excludedPaths, localEntries); + for (const auto& fe : localEntries) { + m_indexStore->ApplyAdd(fe); + ++m_filesIndexed; + } } if (!m_cancel.load(std::memory_order_relaxed)) { m_indexStore->Save(); @@ -206,7 +213,6 @@ void Indexer::EmitStatusDone(uint64_t filesIndexed, std::vector lo void Indexer::IndexingThread() { EmitStatus(IndexerState::Scanning, L"Starting index build..."); - m_indexStore->BeginWrite(); auto selectedDrives = m_settings->GetSelectedDrives(); if (selectedDrives.empty()) { @@ -216,45 +222,75 @@ void Indexer::IndexingThread() { std::transform(fixed.begin(), fixed.end(), std::back_inserter(selectedDrives), [](const DriveInfo& d) { return d.root; }); } - for (const auto& root : selectedDrives) { - if (m_cancel.load(std::memory_order_relaxed)) - break; - ScanDrive(root); + + // Snapshot settings before spawning threads; Settings isn't thread-safe. + std::vector excludedPaths = m_settings->GetExcludedPaths(); + + // Scan each drive on its own thread into a per-drive local buffer. + const size_t n = selectedDrives.size(); + std::vector> perDrive(n); + std::vector scanThreads; + scanThreads.reserve(n); + + for (size_t i = 0; i < n; ++i) { + scanThreads.emplace_back([this, &selectedDrives, &excludedPaths, &perDrive, i]() { + ScanDriveInto(selectedDrives[i], excludedPaths, perDrive[i]); + }); } + for (auto& t : scanThreads) t.join(); - if (!m_cancel.load(std::memory_order_relaxed)) { - m_indexStore->EndWrite(); - m_indexStore->Save(); - EmitStatusDone(m_filesIndexed, m_settings->GetSelectedDrives()); + if (m_cancel.load(std::memory_order_relaxed)) { + SetEvent(m_completionEvent); + return; } - if (!m_cancel.load(std::memory_order_relaxed)) - StartLiveMonitoring(); + m_indexStore->BeginWrite(); + for (const auto& entries : perDrive) { + for (const auto& fe : entries) { + m_indexStore->AddEntry(fe); + ++m_filesIndexed; + } + } + m_indexStore->EndWrite(); + m_indexStore->Save(); + EmitStatusDone(m_filesIndexed, m_settings->GetSelectedDrives()); + StartLiveMonitoring(); SetEvent(m_completionEvent); } +bool Indexer::BuildAndApplyAdd(const std::wstring& path) { + WIN32_FILE_ATTRIBUTE_DATA fad{}; + if (!GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &fad)) + return false; + if (fad.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + return false; + FileEntry fe; + size_t slash = path.rfind(L'\\'); + fe.name = (slash != std::wstring::npos) ? path.substr(slash + 1) : path; + fe.nameLower = fe.name; + std::transform(fe.nameLower.begin(), fe.nameLower.end(), fe.nameLower.begin(), ::towlower); + fe.path = path; + fe.size = (static_cast(fad.nFileSizeHigh) << 32) | fad.nFileSizeLow; + fe.lastModified = (static_cast(fad.ftLastWriteTime.dwHighDateTime) << 32) | + fad.ftLastWriteTime.dwLowDateTime; + fe.attributes = fad.dwFileAttributes; + m_indexStore->ApplyAdd(fe); + return true; +} + void Indexer::ApplyChange(const FileChangeEvent& evt) { - if (evt.type == FileChangeType::Added || evt.type == FileChangeType::Modified) { - WIN32_FILE_ATTRIBUTE_DATA fad{}; - if (!GetFileAttributesExW(evt.path.c_str(), GetFileExInfoStandard, &fad)) - return; - if (fad.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) - return; - if (evt.type == FileChangeType::Modified) - m_indexStore->ApplyRemove(evt.path); - FileEntry fe; - size_t slash = evt.path.rfind(L'\\'); - fe.name = (slash != std::wstring::npos) ? evt.path.substr(slash + 1) : evt.path; - fe.nameLower = fe.name; - std::transform(fe.nameLower.begin(), fe.nameLower.end(), fe.nameLower.begin(), ::towlower); - fe.path = evt.path; - fe.size = (static_cast(fad.nFileSizeHigh) << 32) | fad.nFileSizeLow; - fe.lastModified = (static_cast(fad.ftLastWriteTime.dwHighDateTime) << 32) | - fad.ftLastWriteTime.dwLowDateTime; - fe.attributes = fad.dwFileAttributes; - m_indexStore->ApplyAdd(fe); + if (evt.type == FileChangeType::Added) { + BuildAndApplyAdd(evt.path); + } else if (evt.type == FileChangeType::Modified) { + m_indexStore->ApplyRemove(evt.path); + BuildAndApplyAdd(evt.path); } else if (evt.type == FileChangeType::Removed) { m_indexStore->ApplyRemove(evt.path); + } else if (evt.type == FileChangeType::Renamed) { + if (!evt.oldPath.empty()) + m_indexStore->ApplyRename(evt.oldPath, evt.path); + else + BuildAndApplyAdd(evt.path); } } @@ -284,34 +320,31 @@ void Indexer::StartWatchersForRoots(const std::vector& roots) { } } -void Indexer::ScanDrive(const std::wstring& root) { +void Indexer::ScanDriveInto(const std::wstring& root, + const std::vector& excludedPaths, + std::vector& out) { DriveFilesystem fs = GetFilesystem(root); - // Select scanner: MFT for NTFS if admin available, FindFile otherwise + // Select scanner: MFT for NTFS if admin available, FindFile otherwise. IFileSystemScanner* scanner = m_findScanner.get(); if (fs == DriveFilesystem::NTFS && m_mftScanner->IsMftAvailable(root)) { scanner = m_mftScanner.get(); - EmitStatus(IndexerState::Scanning, L"Indexing " + root + L" (MFT mode)...", m_filesIndexed); + EmitStatus(IndexerState::Scanning, L"Indexing " + root + L" (MFT mode)..."); } else if (fs == DriveFilesystem::NTFS) { - EmitStatus( - IndexerState::Scanning, - L"Indexing " + root + L" (standard mode - run as administrator for faster MFT scan)...", - m_filesIndexed); + EmitStatus(IndexerState::Scanning, + L"Indexing " + root + + L" (standard mode - run as administrator for faster MFT scan)..."); } else { - EmitStatus(IndexerState::Scanning, L"Indexing " + root + L" (FAT32)...", m_filesIndexed); + EmitStatus(IndexerState::Scanning, L"Indexing " + root + L" (FAT32)..."); } ScanOptions opts; opts.rootPaths = {root}; - opts.excludedPaths = m_settings->GetExcludedPaths(); + opts.excludedPaths = excludedPaths; scanner->Scan( - opts, - [this](const FileEntry& fe) { - m_indexStore->AddEntry(fe); - ++m_filesIndexed; - }, + opts, [&out](const FileEntry& fe) { out.push_back(fe); }, [this, &root](uint64_t count, const std::wstring& /*dir*/) { EmitStatus(IndexerState::Scanning, L"Indexing " + root + L"... " + std::to_wstring(count) + L" files", count); diff --git a/src/core/indexer/Indexer.h b/src/core/indexer/Indexer.h index d487827..9a7cfe8 100644 --- a/src/core/indexer/Indexer.h +++ b/src/core/indexer/Indexer.h @@ -64,7 +64,9 @@ class Indexer { void IndexingThread(); void IndexPathsThread(const std::vector& paths); void RemovePathsThread(const std::vector& paths); - void ScanDrive(const std::wstring& root); + void ScanDriveInto(const std::wstring& root, const std::vector& excludedPaths, + std::vector& out); + bool BuildAndApplyAdd(const std::wstring& path); void ApplyChange(const FileChangeEvent& evt); void StartLiveMonitoring(); void StartWatchersForRoots(const std::vector& roots); diff --git a/src/core/indexer/UsnJournalMonitor.cpp b/src/core/indexer/UsnJournalMonitor.cpp index 1f1a52c..1fb4f8e 100644 --- a/src/core/indexer/UsnJournalMonitor.cpp +++ b/src/core/indexer/UsnJournalMonitor.cpp @@ -76,8 +76,10 @@ uint64_t UsnJournalMonitor::ReplaySince(const std::wstring& root, uint64_t saved evt.type = FileChangeType::Added; else if (record->Reason & USN_REASON_FILE_DELETE) evt.type = FileChangeType::Removed; + else if (record->Reason & USN_REASON_RENAME_OLD_NAME) + evt.type = FileChangeType::Removed; else if (record->Reason & USN_REASON_RENAME_NEW_NAME) - evt.type = FileChangeType::Renamed; + evt.type = FileChangeType::Added; else evt.type = FileChangeType::Modified; @@ -143,8 +145,10 @@ void UsnJournalMonitor::StartMonitoring(const std::wstring& root, uint64_t start evt.type = FileChangeType::Added; else if (record->Reason & USN_REASON_FILE_DELETE) evt.type = FileChangeType::Removed; + else if (record->Reason & USN_REASON_RENAME_OLD_NAME) + evt.type = FileChangeType::Removed; else if (record->Reason & USN_REASON_RENAME_NEW_NAME) - evt.type = FileChangeType::Renamed; + evt.type = FileChangeType::Added; else evt.type = FileChangeType::Modified;