diff --git a/lib/rubydex/server.rb b/lib/rubydex/server.rb index da69948e..ee656aa1 100644 --- a/lib/rubydex/server.rb +++ b/lib/rubydex/server.rb @@ -23,13 +23,25 @@ def supported? !State::NOFOLLOW.nil? end - #: (workspace_path: String, ?progress_io: IO?) -> Rubydex::Graph + # Builds a fully indexed + resolved graph for the workspace, and returns it with the errors the + # indexer reported. `progress_io`, when given, receives human-readable progress messages. + # + # A caller that discards the errors records a file the indexer never read as successfully + # indexed, which is why they come back rather than vanishing here. + #: (workspace_path: String, ?progress_io: IO?) -> [Rubydex::Graph, Array[String]] def build_graph(workspace_path:, progress_io: nil) # The server boot must build the same graph as the inline CLI path. graph = Rubydex::Graph.configure_for_workspace(workspace_path) - Progress.with_timer(progress_io, "Indexing workspace...") { graph.index_workspace } + + # `workspace_paths` lists every root to index, and it names gem directories that this install + # may not have. Each absent root costs one error, and those phantom errors would drown the + # ones that concern real files, so they never reach the indexer. + roots = graph.workspace_paths.select { |path| File.exist?(path) } + + errors = [] #: Array[String] + Progress.with_timer(progress_io, "Indexing workspace...") { errors = graph.index_all(roots) } Progress.with_timer(progress_io, "Resolving graph...") { graph.resolve } - graph + [graph, errors] end end end diff --git a/lib/rubydex/server/core.rb b/lib/rubydex/server/core.rb index bc7def65..9921d987 100644 --- a/lib/rubydex/server/core.rb +++ b/lib/rubydex/server/core.rb @@ -5,15 +5,32 @@ module Rubydex module Server # The resident server process. It answers one client at a time, in process, over a UNIX socket. - # - # The graph is a snapshot from boot. A file edited after that answers with its boot content. class Core + # Errors that describe one path, and not the health of this process. The walk skips the entry + # they name and carries on. + # + # A resource failure such as `EMFILE`, `ENFILE` or `EIO` is deliberately absent. Swallowing one + # of those would make the walk skip every entry, and a refresh would then read the empty result + # as "every file was deleted" and erase the graph. Those errors travel on instead, and the + # request fails without the manifest moving. + PATH_ERROR_NAMES = [:ENOENT, :EACCES, :ELOOP, :ENAMETOOLONG, :ENOTDIR].freeze #: Array[Symbol] + + # Resolved by lookup rather than named directly, because the `Errno` constants a build defines + # are platform-dependent and this file still loads on Windows, where server mode cannot run. A + # missing constant would otherwise break the load itself. A test pins that every name resolves + # here, so a typo cannot hide behind the lookup. + PATH_ERRORS = PATH_ERROR_NAMES.filter_map do |name| + Errno.const_get(name, false) if Errno.const_defined?(name, false) + end.freeze #: Array[Class] + #: (State state, ?lock: File?) -> void def initialize(state, lock: nil) @state = state @lock = lock + @mutex = Mutex.new @running = true @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @manifest = {} #: Hash[String, Float] end # Blocks for the lifetime of the server. @@ -25,7 +42,8 @@ def run @state.record! require "rubydex" - @graph = Server.build_graph(workspace_path: @state.workspace_path) + @graph, boot_errors = Server.build_graph(workspace_path: @state.workspace_path) + @manifest = initial_manifest(boot_errors) server = open_socket log("rdx server ready (pid=#{Process.pid}, workspace=#{@state.workspace_path})") @@ -142,14 +160,200 @@ def handle_query(request) return response(stderr: "rdx server: the request carried no query format string\n", status: 1) end - # Only the parse and the render answer for user input. - begin - response(stdout: Rubydex::Query.parse(query).render(@graph, format)) - rescue ArgumentError => e - response(stderr: "#{e.message}\n", status: 1) + @mutex.synchronize do + refresh_if_stale + + # Only the parse and the render answer for user input. An `ArgumentError` from the refresh + # above is a server fault, and reporting it as a bad query would blame the caller. + begin + response(stdout: Rubydex::Query.parse(query).render(@graph, format)) + rescue ArgumentError => e + response(stderr: "#{e.message}\n", status: 1) + end end end + # Detects workspace files that changed since the graph was built and applies incremental + # updates before answering. Always correct, occasionally slow (Phase 1 freshness model). + #: -> void + def refresh_if_stale + current, unreadable = workspace_manifest + previous = @manifest + + # A path the walk could not read this time contributes no entries, and whatever lives under + # it is still there. Without this they would all look deleted, and one `chmod`, or one moment + # during a checkout, would erase a whole subtree from the graph. + hidden = unreadable.empty? ? [] : previous.keys.select { |path| under_any?(path, unreadable) } + + changed = current.select { |path, mtime| previous[path] != mtime }.keys + deleted = previous.keys - current.keys - hidden + return if changed.empty? && deleted.empty? + + failed = changed - index(changed) + deleted.each { |path| @graph.delete_document(uri_for(path)) } + @graph.resolve + + # Only a file that indexed cleanly becomes fresh. One that failed keeps its previous mtime, + # so the next walk sees it as changed and tries it again. Recording the new mtime would call + # a file the server never read "fresh" for the rest of its life. A failed file that is new + # has no previous mtime, and stays out of the manifest for the same reason. + @manifest = current.reject { |path, _| failed.include?(path) } + failed.each { |path| @manifest[path] = previous[path] if previous.key?(path) } + hidden.each { |path| @manifest[path] = previous[path] } + end + + # Whether `path` is one of `prefixes` or sits beneath one. Equality matters because a single + # file can be the thing that could not be read, and not only a directory above it. + #: (String path, Array[String] prefixes) -> bool + def under_any?(path, prefixes) + prefixes.any? { |prefix| path == prefix || path.start_with?("#{prefix}/") } + end + + # Indexes `paths` and returns the ones that indexed cleanly. + # + # `index_all` reports opaque messages for a whole batch, so a failure cannot be attributed to a + # file. The graph cannot answer it either: a failed update leaves the previous document in + # place, and mapping a path to the URI a document is stored under is exactly the parity + # question that Group D still owns. + # + # So a failing batch is halved until each failure sits alone. A batch indexes far faster per + # file than single calls do, which makes this much cheaper than asking file by file. Measured + # on 1081 files with one unreadable among them: 21 calls in 75ms, against 1081 calls in 311ms. + #: (Array[String] paths) -> Array[String] + def index(paths) + return paths if paths.empty? + + errors = @graph.index_all(paths) + return paths if errors.empty? + + errors.each { |message| log("rdx server: index error: #{message}") } + isolate(paths) + end + + # Halves a batch that failed until every failure is isolated, and returns what indexed cleanly. + #: (Array[String] paths) -> Array[String] + def isolate(paths) + return [] if paths.size <= 1 + + middle = paths.size / 2 + [paths[0...middle], paths[middle..] || []].flat_map do |half| + @graph.index_all(half).empty? ? half : isolate(half) + end + end + + # The manifest a fresh server starts from. + # + # With no errors, every file the walk found is fresh. With errors, the indexer cannot say which + # file failed, so the walk's files go through `index` and only those that come back clean are + # recorded. The rest stay out, and the first request retries them. Recording them from the walk + # alone would call a file the server never read "fresh" for the rest of its life. + #: (Array[String] errors) -> Hash[String, Float] + def initial_manifest(errors) + files, = workspace_manifest + return files if errors.empty? + + # Logged before anything else. An unreadable workspace root leaves no files to attribute, and + # a server that started on an empty graph must still say why. + errors.each { |message| log("rdx server: boot index error: #{message}") } + return files if files.empty? + + indexed = index(files.keys).to_h { |path| [path, true] } + + # `build_graph` resolved before this ran, and `index` has replaced documents since, so the + # graph is resolved again before it serves anything. + @graph.resolve + files.select { |path, _| indexed.key?(path) } + end + + # The indexable files under the workspace, and the directories the walk could not read. + #: -> [Hash[String, Float], Array[String]] + def workspace_manifest + manifest = {} #: Hash[String, Float] + unreadable = [] #: Array[String] + # Mirror the indexer's discovery: recurse everything except paths matching the configured + # exclude globs. The Rust indexer has no ignore-by-name list; it applies these glob patterns + # to every entry (pruning directories and skipping files alike). + patterns = @graph.excluded_patterns + collect_files(@state.workspace_path, manifest, patterns, unreadable, top_level: true) + [manifest, unreadable] + end + + # The rescues sit at three levels on purpose: + # + # - The inner one isolates a single entry, so a file that vanished mid-walk cannot hide every + # entry after it in the same directory. + # - `EACCES` on the directory itself records it as unreadable. Its files are still there, and + # the caller keeps them rather than treating the subtree as deleted. + # - `ENOENT` on the directory means it really is gone, and so are its files. + # + # `each_child` streams. `Dir.children` would materialise every name in the directory, which + # this project cannot afford on a hyper-scale workspace. + #: (String dir, Hash[String, Float] manifest, Array[String] patterns, Array[String] unreadable, ?top_level: bool) -> void + def collect_files(dir, manifest, patterns, unreadable, top_level: false) + Dir.each_child(dir) do |entry| + full = File.join(dir, entry) + next if excluded_by_patterns?(full, patterns) + + begin + if directory_to_walk?(full, top_level) + collect_files(full, manifest, patterns, unreadable) + elsif Rubydex::Graph::INDEXABLE_EXTENSIONS.include?(File.extname(entry)) + manifest[full] = File.mtime(full).to_f + end + rescue *PATH_ERRORS => error + # One entry the platform will not answer for: it vanished mid-walk, it cannot be read, or + # it is a symlink loop, where `lstat` succeeds but `mtime` follows the link and raises + # `ELOOP`. Skipping it must not hide the rest of the directory. + # + # A permission error is different from the others, because the entry is still there and + # may be a whole subtree. A directory that is readable but not searchable, mode `0400`, + # lands here for every child: `each_child` lists the names and each `lstat` is refused. + # The path is recorded so the caller keeps what it already knew, rather than reading the + # gap as a deletion and erasing the subtree. + unreadable << full if error.is_a?(Errno::EACCES) + next + end + end + rescue Errno::ENOENT + # The directory is gone, and so are its files. + nil + rescue *PATH_ERRORS + # Present but unreadable right now. Its files are not gone, so the caller keeps their entries + # instead of erasing the subtree. + unreadable << dir + end + + # Whether the walk descends into `path`. The answer differs by depth, because the two sides of + # discovery do: + # + # - `Graph#workspace_paths` asks `File.directory?` about the workspace's own children, so a + # symlinked directory there becomes an explicit root, and the Rust walker does traverse an + # explicit root (`collect_files_indexes_symlinked_directory_roots`). + # - Below that the walker asks a `DirEntry` for its type, which never follows a symlink + # (`collect_files_does_not_follow_symlinked_directories`). + # + # Following at every depth is what let `ln -s .. sub/loop` record one file 32 times under ever + # longer paths, until the platform refused. Following at neither depth would hide a whole + # top-level symlinked directory that the indexer does read. + #: (String path, bool top_level) -> bool + def directory_to_walk?(path, top_level) + top_level ? File.directory?(path) : File.lstat(path).directory? + end + + # Mirrors the Rust indexer's `is_excluded`: an entry is skipped when any exclude glob matches + # its path. `FNM_PATHNAME` keeps `*` from crossing `/` and enables `**` recursion, matching the + # `glob` crate's `Pattern::matches_path` semantics. + #: (String path, Array[String] patterns) -> bool + def excluded_by_patterns?(path, patterns) + patterns.any? { |pattern| File.fnmatch?(pattern, path, File::FNM_PATHNAME) } + end + + #: (String path) -> String + def uri_for(path) + path = "/#{path}" if Gem.win_platform? + URI::File.build(path: path).to_s + end + #: (?stdout: String, ?stderr: String, ?status: Integer) -> Hash[String, untyped] def response(stdout: "", stderr: "", status: 0) { "stdout" => stdout, "stderr" => stderr, "status" => status } diff --git a/test/server/core_test.rb b/test/server/core_test.rb index d082d66e..d8b77be1 100644 --- a/test/server/core_test.rb +++ b/test/server/core_test.rb @@ -72,6 +72,7 @@ def test_a_request_with_a_malformed_query_format_is_a_client_error # A real graph ensures the request reaches the extension and raises the `TypeError` this # test prevents, not a missing-graph error. core = with_graph + core.instance_variable_set(:@manifest, {}) # `false` is in the list because `|| "table"` would turn it into the default. [12345, ["json"], { "format" => "json" }, false].each do |format| @@ -87,16 +88,182 @@ def test_a_request_with_a_malformed_query_format_is_a_client_error def test_a_request_without_a_query_format_uses_the_default core = with_graph + core.instance_variable_set(:@manifest, {}) response = core.send(:handle_query, { "command" => "query", "query" => "MATCH (c:Class) RETURN c.name" }) assert_equal(0, response["status"]) end + # One entry that cannot be read must cost that entry alone. The rescue used to sit on the whole + # method, so a broken entry ended the walk of its directory and hid every file after it. + def test_the_manifest_walk_survives_an_unreadable_entry + skip("symlinks are unavailable on this platform") if Gem.win_platform? + + # `File.mtime` raises `ENOENT` on a symlink that points nowhere. + File.symlink(File.join(@workspace, "missing.rb"), File.join(@workspace, "broken.rb")) + ["a.rb", "b.rb", "c.rb"].each { |name| File.write(File.join(@workspace, name), "class X; end\n") } + + manifest = {} #: Hash[String, Float] + Core.new(@state).send(:collect_files, @workspace, manifest, [], []) + + assert_equal(["a.rb", "b.rb", "c.rb"], manifest.keys.map { |path| File.basename(path) }.sort) + end + + def test_the_manifest_walk_survives_an_unreadable_directory + skip("POSIX permissions are not enforced on this platform") if Gem.win_platform? + skip("root reads every directory") if Process.uid.zero? + + File.write(File.join(@workspace, "visible.rb"), "class X; end\n") + locked = File.join(@workspace, "locked") + FileUtils.mkdir_p(locked) + File.write(File.join(locked, "hidden.rb"), "class Y; end\n") + File.chmod(0o000, locked) + + manifest = {} #: Hash[String, Float] + Core.new(@state).send(:collect_files, @workspace, manifest, [], []) + + assert_equal(["visible.rb"], manifest.keys.map { |path| File.basename(path) }) + ensure + File.chmod(0o700, locked) if locked && File.directory?(locked) + end + + # A file the indexer could not read must stay stale, so the next request tries it again. + # Recording its new mtime would call it fresh for the rest of the server's life. + def test_a_file_that_fails_to_index_is_retried + core = Core.new(@state) + graph = Rubydex::Graph.configure_for_workspace(@workspace) + core.instance_variable_set(:@graph, graph) + + # Invalid UTF-8 is the shape of failure the indexer reports as a `FileError`. + unreadable = File.join(@workspace, "invalid_utf8.rb") + File.binwrite(unreadable, "\xff\xfe not utf8 \xff\n") + readable = File.join(@workspace, "fine.rb") + File.write(readable, "class Fine; end\n") + + core.instance_variable_set(:@manifest, {}) + log, _err = capture_io { core.send(:refresh_if_stale) } + + assert_match(/index error: .*invalid_utf8\.rb/, log) + + manifest = core.instance_variable_get(:@manifest) + assert(manifest.key?(readable), "a file that indexed must be recorded as fresh") + refute(manifest.key?(unreadable), "a file that failed must stay stale, so it is retried") + + # And the retry really happens: a second walk still sees it, so it looks changed again. + files, = core.send(:workspace_manifest) + assert_includes(files.keys, unreadable) + end + + # A directory that cannot be read right now contributes nothing to the walk. Its files are + # still there, so treating them as deleted would erase a whole subtree from the graph over one + # `chmod`, or over one moment during a checkout. + def test_an_unreadable_directory_does_not_erase_its_subtree + skip("POSIX permissions are not enforced on this platform") if Gem.win_platform? + skip("root reads every directory") if Process.uid.zero? + + sub = File.join(@workspace, "sub") + FileUtils.mkdir_p(sub) + kept = File.join(sub, "kept.rb") + File.write(kept, "class Kept; end\n") + + core = with_graph + core.instance_variable_set(:@manifest, {}) + core.send(:refresh_if_stale) + + assert(core.instance_variable_get(:@manifest).key?(kept)) + refute_nil(core.instance_variable_get(:@graph)["Kept"], "the class must be indexed to start") + + File.chmod(0o000, sub) + core.send(:refresh_if_stale) + + assert( + core.instance_variable_get(:@manifest).key?(kept), + "a file under an unreadable directory must keep its entry", + ) + refute_nil( + core.instance_variable_get(:@graph)["Kept"], + "an unreadable directory must not delete the documents beneath it", + ) + ensure + File.chmod(0o700, sub) if sub && File.directory?(sub) + end + + # A directory that is really gone must take its files with it, or the graph would answer with + # classes that no longer exist. + def test_a_deleted_directory_removes_its_subtree + sub = File.join(@workspace, "gone") + FileUtils.mkdir_p(sub) + File.write(File.join(sub, "doomed.rb"), "class Doomed; end\n") + + core = with_graph + core.instance_variable_set(:@manifest, {}) + core.send(:refresh_if_stale) + refute_nil(core.instance_variable_get(:@graph)["Doomed"]) + + FileUtils.rm_rf(sub) + core.send(:refresh_if_stale) + + assert_empty(core.instance_variable_get(:@manifest).select { |path, _| path.start_with?(sub) }) + assert_nil(core.instance_variable_get(:@graph)["Doomed"], "a deleted file must leave the graph") + end + + # The boot half of the same bug the refresh path fixes: a file the initial index could not read + # must not start out fresh, or nothing would ever try it again. + def test_a_file_that_fails_the_initial_index_is_not_fresh + File.binwrite(File.join(@workspace, "invalid_utf8.rb"), "\xff\xfe not utf8 \xff\n") + File.write(File.join(@workspace, "fine.rb"), "class Fine; end\n") + + graph, errors = Server.build_graph(workspace_path: @state.workspace_path) + refute_empty(errors, "the unreadable file must be reported rather than discarded") + + core = Core.new(@state) + core.instance_variable_set(:@graph, graph) + manifest = nil #: Hash[String, Float]? + capture_io { manifest = core.send(:initial_manifest, errors) } + + assert(manifest.key?(File.join(@state.workspace_path, "fine.rb"))) + refute( + manifest.key?(File.join(@state.workspace_path, "invalid_utf8.rb")), + "a file that failed the initial index must stay stale", + ) + end + + # A file can fail the initial index and succeed on the retry, which puts a new document into a + # graph that was already resolved. Serving that graph would answer without it, so the + # reconciliation has to resolve again. + def test_the_boot_reconciliation_resolves_what_it_indexes + skip("POSIX permissions are not enforced on this platform") if Gem.win_platform? + skip("root reads every file") if Process.uid.zero? + + File.write(File.join(@workspace, "parent.rb"), "class Parent; end\n") + late = File.join(@workspace, "late.rb") + File.write(late, "class Late < Parent; end\n") + File.chmod(0o000, late) # unreadable while the initial index runs + + graph, errors = Server.build_graph(workspace_path: @state.workspace_path) + refute_empty(errors, "the unreadable file must be reported") + assert_nil(graph["Late"], "the file must be missing from the graph to begin with") + + File.chmod(0o600, late) # readable again by the time the reconciliation retries it + + core = Core.new(@state) + core.instance_variable_set(:@graph, graph) + manifest = nil #: Hash[String, Float]? + capture_io { manifest = core.send(:initial_manifest, errors) } + + assert(manifest.key?(late), "the retry succeeded, so the file counts as fresh") + refute_nil(graph["Late"], "a document the reconciliation added must be resolved") + assert_includes(graph["Late"].ancestors.map(&:name), "Parent") + ensure + File.chmod(0o600, late) if late && File.exist?(late) + end + # Invalid Cypher is the caller's mistake, so it must return the parser's message, not an # internal error. def test_invalid_cypher_is_reported_as_a_user_error core = with_graph + core.instance_variable_set(:@manifest, {}) response = exchange_with(core, { "command" => "query", "query" => "NOT A QUERY" }) @@ -105,9 +272,244 @@ def test_invalid_cypher_is_reported_as_a_user_error refute_match(/internal error/, response["stderr"]) end + # The refresh runs inside the query path, but it answers for the server and not for the caller. + # The rescue used to wrap it, so an `ArgumentError` from a refresh came back as a bad query and + # blamed the user for a server fault. + def test_a_failing_refresh_is_an_internal_error_and_the_server_survives + core = with_graph + core.instance_variable_set(:@manifest, {}) + core.define_singleton_method(:refresh_if_stale) { raise ArgumentError, "refresh exploded" } + + response = exchange_with(core, { "command" => "query", "query" => "MATCH (c:Class) RETURN c.name" }) + + assert_equal(1, response["status"]) + assert_match(/internal error/, response["stderr"]) + assert_match(/refresh exploded/, response["stderr"]) + assert_match(/internal error: ArgumentError: refresh exploded/, @log) + + # The connection failed, the server did not. Once the fault clears, queries work again. + core.singleton_class.send(:remove_method, :refresh_if_stale) + again = exchange_with(core, { "command" => "query", "query" => "MATCH (c:Class) RETURN c.name" }) + + assert_equal(0, again["status"]) + end + + # Attribution halves a failing batch until each bad file sits alone, so several good files + # around one bad one must all survive, and only the bad one must stay stale. + def test_index_isolates_the_bad_files_in_a_batch + good = 8.times.map do |i| + path = File.join(@workspace, "good#{i}.rb") + File.write(path, "class Good#{i}; end\n") + path + end + bad = 2.times.map do |i| + path = File.join(@workspace, "bad#{i}.rb") + File.binwrite(path, "\xff\xfe not utf8 \xff\n") + path + end + + core = with_graph + indexed = nil #: Array[String]? + capture_io { indexed = core.send(:index, (good + bad).sort) } + + assert_equal(good.sort, indexed.sort) + end + + # An unreadable workspace root leaves nothing to attribute. The server still starts, and it + # must still say why its graph is empty. + def test_a_boot_error_is_logged_even_when_the_walk_found_nothing + core = with_graph + core.define_singleton_method(:workspace_manifest) { [{}, []] } + + manifest = nil #: Hash[String, Float]? + log, _err = capture_io { manifest = core.send(:initial_manifest, ["FileError: Path `/nope` does not exist"]) } + + assert_empty(manifest) + assert_match(/boot index error: FileError/, log) + end + + # `ln -s .. sub/loop` used to walk the workspace into itself over and over. It recorded one + # file 32 times under ever longer paths before the platform refused, and every one of those + # phantom entries would look changed on every request for the life of the server. + def test_the_walk_does_not_follow_a_symlink_cycle + skip("symlinks are unavailable on this platform") if Gem.win_platform? + + FileUtils.mkdir_p(File.join(@workspace, "sub")) + File.write(File.join(@workspace, "real.rb"), "class Real; end\n") + File.symlink("..", File.join(@workspace, "sub", "loop")) + + files, = with_graph.send(:workspace_manifest) + + assert_equal([File.join(@state.workspace_path, "real.rb")], files.keys) + end + + # Below the workspace root the Rust walker asks a `DirEntry` for its type, which never follows a + # symlink. See `collect_files_does_not_follow_symlinked_directories` in `rust/rubydex/src`. + def test_the_walk_does_not_follow_a_nested_symlinked_directory + skip("symlinks are unavailable on this platform") if Gem.win_platform? + + outside = File.join(@workspace, "outside") + FileUtils.mkdir_p(outside) + File.write(File.join(outside, "hidden.rb"), "class Hidden; end\n") + + nested = File.join(@workspace, "nested") + FileUtils.mkdir_p(nested) + File.write(File.join(nested, "kept.rb"), "class Kept; end\n") + File.symlink(outside, File.join(nested, "link")) + + files, = with_graph.send(:workspace_manifest) + names = files.keys.map { |path| path.delete_prefix("#{@state.workspace_path}/") }.sort + + assert_equal(["nested/kept.rb", "outside/hidden.rb"], names) + refute_includes(names, "nested/link/hidden.rb", "a nested symlinked directory must not be followed") + end + + # A symlinked directory directly under the workspace is different: `Graph#workspace_paths` adds + # it as an explicit root, and the Rust walker traverses an explicit root. See + # `collect_files_indexes_symlinked_directory_roots`. The walk has to agree, or those files would + # sit in the graph and never be refreshed. + def test_the_walk_follows_a_top_level_symlinked_directory + skip("symlinks are unavailable on this platform") if Gem.win_platform? + + target = Dir.mktmpdir("rdx-core-linked") + File.write(File.join(target, "linked.rb"), "class Linked; end\n") + File.symlink(target, File.join(@workspace, "vendor")) + + files, = with_graph.send(:workspace_manifest) + names = files.keys.map { |path| path.delete_prefix("#{@state.workspace_path}/") } + + assert_equal(["vendor/linked.rb"], names) + # Compared by suffix: the graph canonicalises the workspace path and the walk does not, which + # is the same Ruby/Rust path parity question that Group D still owns. + roots = Rubydex::Graph.configure_for_workspace(@state.workspace_path).workspace_paths + assert( + roots.any? { |path| path.end_with?("/vendor") }, + "the indexer treats it as an explicit root, which is why the walk follows it", + ) + ensure + FileUtils.rm_rf(target) if target + end + + # A symlink to a file is indexed at its own path, not at the target's. See + # `collect_files_indexes_symlinked_files_at_their_own_path`. + def test_the_walk_records_a_symlinked_file_at_its_own_path + skip("symlinks are unavailable on this platform") if Gem.win_platform? + + outside = Dir.mktmpdir("rdx-core-target") + target = File.join(outside, "real.rb") + File.write(target, "class Real; end\n") + File.symlink(target, File.join(@workspace, "alias.rb")) + + files, = with_graph.send(:workspace_manifest) + + assert_equal([File.join(@state.workspace_path, "alias.rb")], files.keys) + ensure + FileUtils.rm_rf(outside) if outside + end + + # `lstat` classifies a self-referential symlink as a plain entry, and then `File.mtime` follows + # it and raises `ELOOP`. Nothing rescued that, so one such file made every query fail. + def test_the_walk_survives_a_symlink_loop_file + skip("symlinks are unavailable on this platform") if Gem.win_platform? + + loop_path = File.join(@workspace, "loop.rb") + File.symlink("loop.rb", loop_path) + File.write(File.join(@workspace, "good.rb"), "class Good; end\n") + + # The loop really does raise, which is what makes the rescue load-bearing. + assert_raises(Errno::ELOOP) { File.mtime(loop_path) } + + files, = with_graph.send(:workspace_manifest) + + assert_equal([File.join(@state.workspace_path, "good.rb")], files.keys) + end + + # A resource failure says nothing about one path. Skipping every entry would leave an empty + # walk, and the refresh would read that as "every file was deleted" and erase the graph. The + # error has to travel instead, and the manifest must not move. + def test_a_resource_failure_does_not_empty_the_manifest + File.write(File.join(@workspace, "kept.rb"), "class Kept; end\n") + + core = with_graph + core.instance_variable_set(:@manifest, {}) + core.send(:refresh_if_stale) + + before = core.instance_variable_get(:@manifest) + assert_equal(1, before.size) + refute_nil(core.instance_variable_get(:@graph)["Kept"]) + + core.define_singleton_method(:directory_to_walk?) { |_path, _top| raise Errno::EMFILE } + + assert_raises(Errno::EMFILE) { core.send(:refresh_if_stale) } + + assert_equal(before, core.instance_variable_get(:@manifest), "the manifest must not move") + refute_nil(core.instance_variable_get(:@graph)["Kept"], "the graph must keep its documents") + end + + # These two exist wherever Ruby runs, so the walk can always count on them. A resource failure + # must never be in the list: swallowing one would empty the walk and erase the graph. + def test_the_path_errors_hold_the_universal_names_and_no_resource_error + assert_includes(Core::PATH_ERRORS, Errno::ENOENT) + assert_includes(Core::PATH_ERRORS, Errno::EACCES) + refute_includes(Core::PATH_ERRORS, Errno::EMFILE) + end + + # The list is resolved by lookup so a platform missing an optional errno still loads this file. + # The lookup would also hide a misspelled name, so the full resolution is pinned here, where + # every one of these constants is guaranteed to exist. + def test_every_path_error_name_resolves_on_a_posix_platform + skip("the set of errno constants differs on this platform") if Gem.win_platform? + + assert_equal( + Core::PATH_ERROR_NAMES.size, + Core::PATH_ERRORS.size, + "a name in PATH_ERROR_NAMES did not resolve, which usually means a typo", + ) + assert_includes(Core::PATH_ERRORS, Errno::ELOOP) + end + + # A directory can be readable and still not searchable, mode `0400`. `each_child` then lists its + # names while every `lstat` is refused, so the walk sees nothing under it and the outer rescue + # never fires. The files are still there, and treating the gap as a deletion erased them. + def test_a_directory_that_cannot_be_searched_does_not_erase_its_subtree + skip("POSIX permissions are not enforced on this platform") if Gem.win_platform? + skip("root searches every directory") if Process.uid.zero? + + sub = File.join(@workspace, "sub") + FileUtils.mkdir_p(sub) + kept = File.join(sub, "kept.rb") + File.write(kept, "class Kept; end\n") + + core = with_graph + core.instance_variable_set(:@manifest, {}) + core.send(:refresh_if_stale) + + assert(core.instance_variable_get(:@manifest).key?(kept)) + refute_nil(core.instance_variable_get(:@graph)["Kept"]) + + File.chmod(0o400, sub) + + # The shape this test exists for: listing works, and classifying each entry does not. + assert_equal(["kept.rb"], Dir.each_child(sub).to_a) + assert_raises(Errno::EACCES) { File.lstat(kept) } + + core.send(:refresh_if_stale) + + assert( + core.instance_variable_get(:@manifest).key?(kept), + "a file under an unsearchable directory must keep its entry", + ) + refute_nil( + core.instance_variable_get(:@graph)["Kept"], + "an unsearchable directory must not delete the documents beneath it", + ) + ensure + File.chmod(0o700, sub) if sub && File.directory?(sub) + end + private - # The graph is empty because these tests check request handling, not what the graph holds. + # A core wired to a real graph for this workspace, which is what the refresh path needs. #: -> Core def with_graph core = Core.new(@state) diff --git a/test/server/integration_test.rb b/test/server/integration_test.rb index c8aa0f12..3d396684 100644 --- a/test/server/integration_test.rb +++ b/test/server/integration_test.rb @@ -77,6 +77,36 @@ def test_boots_and_indexes_files_in_subdirectories end end + def test_server_picks_up_file_changes + with_context do |context| + track(context) + context.write!("zoo.rb", "class Animal; end\nclass Dog < Animal; end\n") + + query = "MATCH (c:Class)-[:HAS_PARENT]->(p:Class) WHERE p.name = 'Animal' RETURN c.name ORDER BY c.name" + refute_match(/Fox/, query!(context, query)) + + sleep(0.01) # ensure a distinct mtime + context.write!("zoo.rb", "class Animal; end\nclass Dog < Animal; end\nclass Fox < Animal; end\n") + + assert_match(/Fox/, query!(context, query)) + end + end + + def test_server_drops_deleted_files + with_context do |context| + track(context) + context.write!("animal.rb", "class Animal; end") + context.write!("dog.rb", "class Dog < Animal; end") + + query = "MATCH (c:Class {name: 'Dog'}) RETURN c.name" + assert_match(/Dog/, query!(context, query)) + + File.delete(context.absolute_path_to("dog.rb")) + + refute_match(/Dog/, query!(context, query)) + end + end + def test_query_output_matches_inline with_context do |context| track(context)