diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff89cfe..d75f00b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,12 +42,8 @@ jobs: # actually exercised without a full 3x3 runner-minute blow-up. - os: macos-latest python-version: '3.12' - # windows-latest is intentionally omitted from the gating matrix: - # the suite has real Windows issues (chromadb holds open file - # handles so tempdir teardown hits WinError 32, event-log rotation - # relies on POSIX rename-over-open, and a concurrent-append test - # loses writes). Tracked separately before Windows can be claimed - # as supported — see the Windows-support tracking issue. + - os: windows-latest + python-version: '3.12' steps: - uses: actions/checkout@v6 diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 8f5518f..fbd7f42 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -23,7 +23,7 @@ |----|--------|-----------------|-------| | Linux | ✅ Full | Ubuntu 20.04+ | CI-verified on every PR (Python 3.10–3.12) | | macOS | ✅ Full | 11.0+ | CI-verified on every PR; x86 and Apple Silicon | -| Windows | ⚠️ Experimental | 10, 11 | Installs and runs; **not yet CI-green** — the test suite has known Windows issues (ChromaDB holds open file handles so temp-dir teardown hits `WinError 32`, event-log rotation relies on POSIX rename-over-open, and a concurrent-append path loses writes). Task Scheduler walkthrough works; full support is tracked before Windows is gated in CI. | +| Windows | ✅ Full | 10, 11 | CI-verified on every PR (`windows-latest`, Python 3.12); Task Scheduler walkthrough included | | Docker | ✅ Full | Docker 20.10+ | Included in releases | --- diff --git a/docs/index.html b/docs/index.html index 2b2b42a..75275cd 100644 --- a/docs/index.html +++ b/docs/index.html @@ -22,11 +22,11 @@ "@type": "SoftwareApplication", "name": "NeuralMind", "applicationCategory": "DeveloperApplication", - "operatingSystem": "Linux, macOS", + "operatingSystem": "Linux, macOS, Windows", "description": "Persistent memory and semantic code intelligence for AI coding agents. Local-first code indexing with 40–70× per-query token reduction, a brain-like synapse layer that learns associations from how you use the codebase, an Obsidian-style graph view, an MCP server, and a built-in install doctor. Works with Claude Code, Cursor, Cline, Continue, and any MCP-compatible agent.", "url": "https://dfrostar.github.io/neuralmind/", "downloadUrl": "https://pypi.org/project/neuralmind/", - "softwareVersion": "0.23.0", + "softwareVersion": "0.24.0", "license": "https://opensource.org/licenses/MIT", "programmingLanguage": "Python", "operatingSystemRequirements": "Python 3.10+", @@ -355,7 +355,7 @@
-

v0.23.0 — latest release · release notes

+

v0.24.0 — latest release · release notes

Persistent memory for AI coding agents

Your agent learns your codebase the way a senior engineer would — what files go together, what you usually edit next, what patterns matter. The memory persists across sessions and surfaces automatically.

100% local. Your code never leaves your machine. Side effect: 40–70× cheaper code questions, measured in CI on every commit.

@@ -584,11 +584,11 @@

What's new

Shipped in small, verifiable increments — every release gated by the CI benchmark.

-
v0.24.0In development
+
v0.24.0Latest release

Memory namespaces & branch isolation. The synapse layer becomes namespace-aware: branch:<name> / personal / shared / ephemeral memory live separately in the same store, so a feature-branch spike can't pollute what the agent learned about main. Recall reads a transparent merged view (active branch 1.0× > personal 0.8× > shared team baseline 0.5×, attributed per-namespace in query --trace), and the new neuralmind memory {inspect,reset,export,import} moves memory as versioned JSON bundles — the team-memory on-ramp. Existing learned memory migrates losslessly into personal. Release notes →

-
v0.23.0Latest release
+
v0.23.0

Four future-proofing foundations: a schema-versioned index contract (IR) checked by the new neuralmind validate, a retrieval-quality harness (benchmark --quality: precision@k / recall@k / MRR over 30 golden queries, failing CI on regression), debug traces (query --trace shows why a result came back), and an experimental local daemon that keeps project state warm. Release notes →

diff --git a/neuralmind/core.py b/neuralmind/core.py index 5078f06..b2a7ed9 100644 --- a/neuralmind/core.py +++ b/neuralmind/core.py @@ -23,6 +23,7 @@ """ import json +import os import threading from datetime import datetime, timezone from pathlib import Path @@ -37,6 +38,51 @@ DEFAULT_HYBRID_HIGHLIGHT_COUNT = 3 +# Serializes recent-queries appends within this process. POSIX O_APPEND +# already makes single-line appends atomic, but Windows' CRT implements +# append mode as a separate seek-to-end + write, so two handles writing +# concurrently can interleave and lose lines. +_RECENT_QUERIES_APPEND_LOCK = threading.Lock() + +try: + import msvcrt # Windows-only: cross-process advisory lock below. +except ImportError: # pragma: no cover - POSIX + msvcrt = None # type: ignore[assignment] + + +def _lock_byte0(fd: int) -> bool: + """Best-effort cross-process mutex on byte 0 of *fd* (Windows only). + + POSIX writers don't need it (O_APPEND is atomic) and use flock for + compaction instead; on Windows both the appender and the compactor + take this same region so a compaction's read-truncate-rewrite can't + drop a concurrent process's append. Non-blocking with a short retry + so a stuck holder can never stall a query. + """ + if msvcrt is None: + return False + import time as _time + + for _ in range(50): # ~50ms worst case + try: + os.lseek(fd, 0, os.SEEK_SET) + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + return True + except OSError: + _time.sleep(0.001) + return False + + +def _unlock_byte0(fd: int) -> None: + if msvcrt is None: + return + try: + os.lseek(fd, 0, os.SEEK_SET) + msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) + except OSError: + pass + + # Canonical IR artifacts (PRD 1), under /.neuralmind/. IR_FILENAME = "index_ir.json" IR_META_FILENAME = "ir_meta.json" @@ -176,6 +222,22 @@ def __init__( def backend_name(self) -> str: return self.backend_manager.backend_name + def close(self) -> None: + """Release backend resources (vector-store file handles). + + Windows can't delete files a process still holds open, so + anything that removes the project directory afterwards — test + teardown, ``neuralmind reset`` — needs this. Safe to call more + than once; the synapse store opens its sqlite database per + operation and holds nothing between calls. + """ + embedder = getattr(self, "embedder", None) + if embedder is not None and hasattr(embedder, "close"): + try: + embedder.close() + except Exception: + pass + @property def memory_namespace(self) -> str: """The active synapse-memory namespace for this project (PRD 4). @@ -763,12 +825,14 @@ def _record_recent_query(self, question: str, result: ContextResult) -> None: UI can highlight on the canvas. Always on (local-only data, readable only through the auth-gated server). - The hot path is an atomic single-line append (POSIX O_APPEND - guarantees writes < PIPE_BUF don't tear across processes), so - CLI and MCP-server processes can safely write to the same file - without losing entries. Trimming back to RECENT_QUERIES_MAX is - a lazy compaction step gated by file size and protected by an - advisory lock — see ``_compact_recent_queries``. + The hot path is a single-line O_APPEND write. On POSIX that is + atomic across processes by itself (writes < PIPE_BUF don't + tear); Windows' CRT implements append as seek-to-end + write, + so writes are additionally serialized by a process-local lock + and a best-effort cross-process byte-range lock. Trimming back + to RECENT_QUERIES_MAX is a lazy compaction step gated by file + size and protected by the same locks — see + ``_compact_recent_queries``. Gated on the same consent flag as the learning log (`NEURALMIND_MEMORY`): if the user opted out of persisting @@ -799,9 +863,17 @@ def _record_recent_query(self, question: str, result: ContextResult) -> None: } log_path = self._recent_queries_path() log_path.parent.mkdir(parents=True, exist_ok=True) - line = json.dumps(record, sort_keys=True) + "\n" - with log_path.open("a", encoding="utf-8") as f: - f.write(line) + encoded = (json.dumps(record, sort_keys=True) + "\n").encode("utf-8") + with _RECENT_QUERIES_APPEND_LOCK: + fd = os.open(str(log_path), os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644) + locked = False + try: + locked = _lock_byte0(fd) + os.write(fd, encoded) + finally: + if locked: + _unlock_byte0(fd) + os.close(fd) except Exception: # Recording must never block the actual query. return @@ -814,10 +886,11 @@ def _compact_recent_queries(self, log_path: Path) -> None: """Trim the recent-queries log back to RECENT_QUERIES_MAX entries. Only runs when the file has grown past the size threshold, so the - hot path stays append-only. Uses an advisory file lock on POSIX - so the read-truncate-rewrite isn't interleaved with another - process's append; on Windows the lock is best-effort and worst - case is a slightly oversized log file (no data loss). + hot path stays append-only. The read-truncate-rewrite is guarded + against another process's append by flock on POSIX and by the + byte-0 region lock shared with ``_record_recent_query`` on + Windows; both are best-effort — worst case is a slightly + oversized log file (no data loss). """ try: size = log_path.stat().st_size @@ -829,12 +902,13 @@ def _compact_recent_queries(self, log_path: Path) -> None: import fcntl except ImportError: fcntl = None # type: ignore[assignment] - with log_path.open("r+", encoding="utf-8") as f: + with _RECENT_QUERIES_APPEND_LOCK, log_path.open("r+", encoding="utf-8") as f: if fcntl is not None: try: fcntl.flock(f.fileno(), fcntl.LOCK_EX) except OSError: pass + locked = _lock_byte0(f.fileno()) try: lines = [ln for ln in f if ln.strip()] if len(lines) <= self.RECENT_QUERIES_MAX: @@ -842,7 +916,10 @@ def _compact_recent_queries(self, log_path: Path) -> None: f.seek(0) f.truncate() f.writelines(lines[-self.RECENT_QUERIES_MAX :]) + f.flush() finally: + if locked: + _unlock_byte0(f.fileno()) if fcntl is not None: try: fcntl.flock(f.fileno(), fcntl.LOCK_UN) diff --git a/neuralmind/daemon.py b/neuralmind/daemon.py index 3e36200..28f5605 100644 --- a/neuralmind/daemon.py +++ b/neuralmind/daemon.py @@ -104,6 +104,28 @@ def clear_discovery(path: Path | None = None) -> None: def _pid_alive(pid: int) -> bool: + if os.name == "nt": + # os.kill(pid, 0) is NOT a liveness probe on Windows: + # signal.CTRL_C_EVENT == 0, so it sends a real Ctrl-C to the + # console process group — interrupting the daemon's (or test + # runner's) own console. Query the process handle instead. + import ctypes + + PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 # noqa: N806 - WinAPI name + STILL_ACTIVE = 259 # noqa: N806 + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + handle = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid) + if not handle: + # ERROR_ACCESS_DENIED (5): the process exists but is owned + # by someone else — alive, same as the PermissionError arm. + return ctypes.get_last_error() == 5 + try: + code = ctypes.c_ulong() + if not kernel32.GetExitCodeProcess(handle, ctypes.byref(code)): + return False + return code.value == STILL_ACTIVE + finally: + kernel32.CloseHandle(handle) try: os.kill(pid, 0) except ProcessLookupError: diff --git a/neuralmind/embedder.py b/neuralmind/embedder.py index 8e91e73..0981184 100644 --- a/neuralmind/embedder.py +++ b/neuralmind/embedder.py @@ -501,9 +501,29 @@ def clear(self) -> None: pass def close(self) -> None: - """Close ChromaDB client and cleanup.""" - if hasattr(self, "client"): - try: - self.client.delete_collection(name=self.COLLECTION_NAME) - except Exception: - pass + """Release the ChromaDB client's file handles. + + Chroma has no public ``close()``: it caches one ``System`` per + storage path (holding the sqlite connection pool and HNSW + segment files) for the life of the process. Windows refuses to + delete open files, so anything that removes the store afterwards + — temp-dir teardown in tests, a user deleting ``.neuralmind/`` — + hits ``WinError 32`` unless the System is actually stopped and + evicted from that cache. Deleting the collection (the previous + behavior) released nothing and destroyed data a later open + expected to find. + """ + client = getattr(self, "client", None) + if client is None: + return + self._collection = None + self.client = None + try: + from chromadb.api.shared_system_client import SharedSystemClient + + client._system.stop() + SharedSystemClient._identifier_to_system.pop(getattr(client, "_identifier", ""), None) + except Exception: + # Best-effort across chromadb versions: worst case is the + # pre-close behavior (handles live until process exit). + pass diff --git a/neuralmind/event_log.py b/neuralmind/event_log.py index a706455..a7cfb38 100644 --- a/neuralmind/event_log.py +++ b/neuralmind/event_log.py @@ -37,6 +37,61 @@ _MAX_LINE_BYTES = 64 * 1024 # Skip pathologically long lines. +def _open_shared_rb(path: Path): + """Open *path* for binary read without blocking a concurrent rename. + + The tailer holds its read handle across poll intervals, and rotation + is a logrotate-style rename of the live file. POSIX allows renaming + an open file; Windows' ``open()`` omits FILE_SHARE_DELETE from the + sharing mode, so the rotating process gets a PermissionError instead. + Recreate the POSIX semantics with CreateFileW + share-delete so + rotation never depends on catching the tailer between polls. + """ + if os.name != "nt": + return open(path, "rb") + + import ctypes + import msvcrt + + GENERIC_READ = 0x80000000 # noqa: N806 - canonical WinAPI names + FILE_SHARE_READ = 0x00000001 # noqa: N806 + FILE_SHARE_WRITE = 0x00000002 # noqa: N806 + FILE_SHARE_DELETE = 0x00000004 # noqa: N806 + OPEN_EXISTING = 3 # noqa: N806 + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.CreateFileW.restype = ctypes.c_void_p + kernel32.CreateFileW.argtypes = [ + ctypes.c_wchar_p, + ctypes.c_uint32, + ctypes.c_uint32, + ctypes.c_void_p, + ctypes.c_uint32, + ctypes.c_uint32, + ctypes.c_void_p, + ] + handle = kernel32.CreateFileW( + str(path), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + None, + OPEN_EXISTING, + 0, + None, + ) + if handle is None or handle == ctypes.c_void_p(-1).value: + # WinError maps ERROR_FILE_NOT_FOUND to FileNotFoundError, which + # _open() relies on to distinguish "not yet created" from real + # failures. + raise ctypes.WinError(ctypes.get_last_error()) + try: + fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) + except OSError: + kernel32.CloseHandle(ctypes.c_void_p(handle)) + raise + return os.fdopen(fd, "rb") + + class EventLogWriter: """Append JSON events to a line-delimited file. Thread-safe.""" @@ -115,7 +170,7 @@ def _open_at_start(self): def _open(self, *, seek_to_end: bool): try: - fh = open(self.path, "rb") + fh = _open_shared_rb(self.path) except FileNotFoundError: return None, None except OSError: diff --git a/tests/conftest.py b/tests/conftest.py index d205fbb..20ef5ad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ """Pytest fixtures for NeuralMind tests.""" +import gc import json +import os import tempfile from collections.abc import Generator from pathlib import Path @@ -111,8 +113,14 @@ def sample_graph() -> dict[str, Any]: @pytest.fixture def temp_project(sample_graph: dict[str, Any]) -> Generator[Path, None, None]: - """Create a temporary project directory with graph.json.""" - with tempfile.TemporaryDirectory() as tmpdir: + """Create a temporary project directory with graph.json. + + ``ignore_cleanup_errors``: chromadb-backed tests can leave file + handles open at teardown (released afterwards by + ``_release_chroma_file_handles``), and Windows refuses to delete + open files — without the flag every such test errors in teardown. + """ + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir: project_path = Path(tmpdir) # Create graphify-out directory @@ -170,7 +178,7 @@ def temp_project_with_claude_md(temp_project: Path) -> Path: @pytest.fixture def empty_project() -> Generator[Path, None, None]: """Create an empty temporary project directory.""" - with tempfile.TemporaryDirectory() as tmpdir: + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir: yield Path(tmpdir) @@ -343,3 +351,34 @@ def isolate_tests(tmp_path, monkeypatch): """Ensure tests don't affect each other.""" # Use temp directory for any file operations monkeypatch.chdir(tmp_path) + + +@pytest.fixture(autouse=True) +def _release_chroma_file_handles(): + """Stop chromadb's cached Systems after each test. + + Chroma caches one System per storage path for the life of the + process, holding sqlite + HNSW file handles. Tests rarely close + their embedders, which is invisible on POSIX but fatal on Windows: + temp-dir teardown can't delete open files (WinError 32). Stopping + and evicting every cached System after each test releases the + handles regardless of whether the test cleaned up. + """ + yield + try: + from chromadb.api.shared_system_client import SharedSystemClient + except Exception: + return + for system in list(SharedSystemClient._identifier_to_system.values()): + try: + system.stop() + except Exception: + pass + try: + SharedSystemClient._identifier_to_system.clear() + except Exception: + pass + if os.name == "nt": + # Segment objects may hold the last reference to an open index + # file; make their finalizers run before directory teardown. + gc.collect() diff --git a/tests/test_cli.py b/tests/test_cli.py index 069f5f1..58450c3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ """Tests for NeuralMind CLI functionality with real assertions.""" import json +import sys from unittest.mock import MagicMock, patch import pytest @@ -792,6 +793,7 @@ def test_cmd_init_hook_no_git_dir(self, tmp_path, capsys): cmd_init_hook(args) assert exc_info.value.code == 1 + @pytest.mark.skipif(sys.platform.startswith("win"), reason="Windows has no executable bit") def test_cmd_init_hook_makes_executable(self, tmp_path): """init-hook makes the hook file executable.""" import os