From f4b973df05ce8566ef6c83d6369740f97da496ce Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 6 Jul 2026 02:15:49 +0300 Subject: [PATCH 01/41] fix(ui): stable DJB2 directory hues (no abs-trap) + remove silent 2000-child truncation --- Sources/Layout/TreemapLayout.swift | 8 ++++++-- Sources/Scanner/FSNode.swift | 2 +- Tests/FileScannerTests.swift | 9 +++++++++ Tests/TreemapLayoutTests.swift | 10 ++++++++++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/Sources/Layout/TreemapLayout.swift b/Sources/Layout/TreemapLayout.swift index 85afbb5..a4df1d6 100644 --- a/Sources/Layout/TreemapLayout.swift +++ b/Sources/Layout/TreemapLayout.swift @@ -95,11 +95,15 @@ public struct TreemapLayout { // ── Color assignment ───────────────────────────────────────────────────── + static func directoryHue(for name: String) -> Double { + let hash = name.utf8.reduce(UInt32(5381)) { ($0 &<< 5) &+ $0 &+ UInt32($1) } + return Double(hash % 360) / 360.0 + } + private static func color(for node: FSNode, depth: Int, colorMap: ExtensionColorMap) -> Color { if node.isDirectory { // Directories: muted tinted containers — hue from name hash, low saturation - let hash = abs(node.name.hashValue) % 360 - let hue = Double(hash) / 360.0 + let hue = directoryHue(for: node.name) let fade = Double(depth) * 0.05 return Color(hue: hue, saturation: max(0.12, 0.28 - fade), diff --git a/Sources/Scanner/FSNode.swift b/Sources/Scanner/FSNode.swift index da4699e..d33de37 100644 --- a/Sources/Scanner/FSNode.swift +++ b/Sources/Scanner/FSNode.swift @@ -26,6 +26,6 @@ public final class FSNode: Identifiable, @unchecked Sendable { public extension FSNode { var optionalChildren: [FSNode]? { guard isDirectory && !children.isEmpty else { return nil } - return children.count > 2_000 ? Array(children.prefix(2_000)) : children + return children } } diff --git a/Tests/FileScannerTests.swift b/Tests/FileScannerTests.swift index c4ebd6a..f3b0469 100644 --- a/Tests/FileScannerTests.swift +++ b/Tests/FileScannerTests.swift @@ -31,6 +31,15 @@ final class FileScannerTests: XCTestCase { XCTAssertNil(child.parent) } + func test_optional_children_returns_all_children() { + let dir = FSNode(url: URL(fileURLWithPath: "/tmp"), name: "tmp", isDirectory: true, size: 0, fileExtension: "", parent: nil) + for i in 0..<2001 { + let child = FSNode(url: URL(fileURLWithPath: "/tmp/\(i)"), name: "\(i)", isDirectory: false, size: 1, fileExtension: "txt", parent: dir) + dir.children.append(child) + } + XCTAssertEqual(dir.optionalChildren?.count, 2001) + } + func test_scanner_builds_tree_from_temp_directory() async throws { let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) diff --git a/Tests/TreemapLayoutTests.swift b/Tests/TreemapLayoutTests.swift index 51c77b2..8c4925c 100644 --- a/Tests/TreemapLayoutTests.swift +++ b/Tests/TreemapLayoutTests.swift @@ -26,6 +26,16 @@ final class TreemapLayoutTests: XCTestCase { XCTAssertNotEqual(map.color(for: "pdf"), map.color(for: "mp4")) } + func test_directory_hue_is_deterministic_and_bounded() { + let name = "Documents" + let expected = name.utf8.reduce(UInt32(5381)) { ($0 &<< 5) &+ $0 &+ UInt32($1) } + let hue = TreemapLayout.directoryHue(for: name) + XCTAssertEqual(hue, Double(expected % 360) / 360.0) + XCTAssertGreaterThanOrEqual(hue, 0) + XCTAssertLessThan(hue, 1) + XCTAssertNotEqual(TreemapLayout.directoryHue(for: "aaa"), TreemapLayout.directoryHue(for: "zzz")) + } + func test_byte_formatter_kb() { XCTAssertEqual(ByteFormatter.string(from: 1_000), "1.0 KB") } From 8c3c1e42386b86da1e960f13ce5ea13b9b5cc364 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 6 Jul 2026 02:16:23 +0300 Subject: [PATCH 02/41] fix(refresh): allocated sizes, recursive new-dir scan, symlink/hidden/excluded policy on FSEvents refresh --- Sources/ViewModels/ScanViewModel.swift | 115 +++++++++++++++++++--- Tests/ScanRefreshTests.swift | 130 +++++++++++++++++++++++++ 2 files changed, 230 insertions(+), 15 deletions(-) create mode 100644 Tests/ScanRefreshTests.swift diff --git a/Sources/ViewModels/ScanViewModel.swift b/Sources/ViewModels/ScanViewModel.swift index 904acce..1cf0a61 100644 --- a/Sources/ViewModels/ScanViewModel.swift +++ b/Sources/ViewModels/ScanViewModel.swift @@ -249,45 +249,78 @@ public final class ScanViewModel: ObservableObject { return current } + // Returns the allocated disk size (st_blocks * 512) for a path via lstat, or nil if + // the path can't be stat'd or is a symlink. + private nonisolated static func lstatInfo(path: String) -> (isDir: Bool, isSymlink: Bool, allocatedSize: Int64)? { + var st = stat() + guard lstat(path, &st) == 0 else { return nil } + let mode = st.st_mode & S_IFMT + let isSymlink = mode == S_IFLNK + let isDir = mode == S_IFDIR + let allocatedSize = Int64(st.st_blocks) * 512 + return (isDir, isSymlink, allocatedSize) + } + + // Parses the excludedFolderNames default the same way FileScanner does. + private nonisolated static func parseExcludedNames() -> Set { + let raw = UserDefaults.standard.string(forKey: "excludedFolderNames") + ?? ".git,node_modules,DerivedData,.Trash" + return Set(raw.components(separatedBy: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty }) + } + // Re-stat the directory on disk and update children to match. // Returns true if anything changed (additions/removals/size changes). @discardableResult - private nonisolated static func refreshDirectory(node: FSNode) -> Bool { + nonisolated static func refreshDirectory(node: FSNode) -> Bool { guard node.isDirectory else { return false } let fm = FileManager.default + let showHiddenFiles = UserDefaults.standard.bool(forKey: "showHiddenFiles") + let excludedNames = parseExcludedNames() guard let entries = try? fm.contentsOfDirectory( at: node.url, - includingPropertiesForKeys: [.fileSizeKey, .isDirectoryKey, .isSymbolicLinkKey], - options: [.skipsHiddenFiles] + includingPropertiesForKeys: nil, + options: showHiddenFiles ? [] : [.skipsHiddenFiles] ) else { return false } - let onDisk = Dictionary(uniqueKeysWithValues: entries.map { ($0.lastPathComponent, $0) }) + // Filter out excluded folder names and symlinks up front, so both the + // removal pass and the add/update pass agree on what's "on disk". + var onDisk: [String: (url: URL, info: (isDir: Bool, isSymlink: Bool, allocatedSize: Int64))] = [:] + for url in entries { + let name = url.lastPathComponent + if excludedNames.contains(name) { continue } + guard let info = lstatInfo(path: url.path) else { continue } + if info.isSymlink { continue } + onDisk[name] = (url, info) + } + var changed = false - // Remove children that no longer exist + // Remove children that no longer exist (or are now excluded/symlinks) let before = node.children.count node.children.removeAll { !onDisk.keys.contains($0.name) } if node.children.count != before { changed = true } // Add or update children - for (name, url) in onDisk { + for (name, entry) in onDisk { + let (url, info) = entry if let existing = node.children.first(where: { $0.name == name }) { // Update size for files (directories update via recursive bubble) - if !existing.isDirectory, - let attrs = try? url.resourceValues(forKeys: [.fileSizeKey]), - let newSize = attrs.fileSize.map(Int64.init) { + if !existing.isDirectory { + let newSize = info.allocatedSize if existing.size != newSize { existing.size = newSize changed = true } } + } else if info.isDir { + // New directory — scan its whole subtree so it isn't left as a 0-byte leaf. + let child = scanSubtree(url: url, parent: node, showHiddenFiles: showHiddenFiles, excludedNames: excludedNames) + node.children.append(child) + changed = true } else { - // New entry — create a minimal FSNode - let attrs = try? url.resourceValues(forKeys: [.fileSizeKey, .isDirectoryKey]) - let isDir = attrs?.isDirectory ?? false - let size = (attrs?.fileSize).map(Int64.init) ?? 0 - let ext = isDir ? "" : url.pathExtension.lowercased() - let child = FSNode(url: url, name: name, isDirectory: isDir, size: size, fileExtension: ext, parent: node) + // New file + let ext = url.pathExtension.lowercased() + let child = FSNode(url: url, name: name, isDirectory: false, size: info.allocatedSize, fileExtension: ext, parent: node) child.safetyLevel = SafetyAnalyzer.level(for: child) node.children.append(child) changed = true @@ -300,6 +333,58 @@ public final class ScanViewModel: ObservableObject { return changed } + // Synchronously walks a newly-discovered directory subtree, applying the same rules + // as the initial scan: skip symlinks, skip hidden files unless showHiddenFiles, skip + // excludedNames, allocated sizes via lstat, directory size = sum of children. + private nonisolated static func scanSubtree( + url: URL, + parent: FSNode?, + showHiddenFiles: Bool, + excludedNames: Set + ) -> FSNode { + let name = url.lastPathComponent + guard let info = lstatInfo(path: url.path), info.isDir else { + // Not actually a directory (or vanished) — return an empty leaf; caller only + // invokes this when it already believes the entry is a directory. + return FSNode(url: url, name: name, isDirectory: false, size: 0, fileExtension: url.pathExtension.lowercased(), parent: parent) + } + + let node = FSNode(url: url, name: name, isDirectory: true, size: 0, fileExtension: "", parent: parent) + node.safetyLevel = SafetyAnalyzer.level(for: node) + + let fm = FileManager.default + guard let entries = try? fm.contentsOfDirectory( + at: url, + includingPropertiesForKeys: nil, + options: showHiddenFiles ? [] : [.skipsHiddenFiles] + ) else { return node } + + var children: [FSNode] = [] + var totalSize: Int64 = 0 + for childURL in entries { + let childName = childURL.lastPathComponent + if excludedNames.contains(childName) { continue } + guard let childInfo = lstatInfo(path: childURL.path) else { continue } + if childInfo.isSymlink { continue } + + let child: FSNode + if childInfo.isDir { + child = scanSubtree(url: childURL, parent: node, showHiddenFiles: showHiddenFiles, excludedNames: excludedNames) + } else { + let ext = childURL.pathExtension.lowercased() + child = FSNode(url: childURL, name: childName, isDirectory: false, size: childInfo.allocatedSize, fileExtension: ext, parent: node) + child.safetyLevel = SafetyAnalyzer.level(for: child) + } + children.append(child) + totalSize += child.size + } + + children.sort { $0.size > $1.size } + node.children = children + node.size = totalSize + return node + } + // Walk up the parent chain recalculating folder sizes from their children. private nonisolated static func bubbleUpSizes(from node: FSNode) { var current: FSNode? = node diff --git a/Tests/ScanRefreshTests.swift b/Tests/ScanRefreshTests.swift new file mode 100644 index 0000000..d460314 --- /dev/null +++ b/Tests/ScanRefreshTests.swift @@ -0,0 +1,130 @@ +import XCTest +@testable import MacDirStat + +final class ScanRefreshTests: XCTestCase { + + // Returns allocated disk size (st_blocks * 512) for a path, matching the scanner's metric. + private func allocatedSize(at path: String) -> Int64 { + var st = stat() + guard lstat(path, &st) == 0 else { return -1 } + return Int64(st.st_blocks) * 512 + } + + func test_refresh_uses_allocated_size() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let fileURL = tmp.appendingPathComponent("a.bin") + try Data([0x42]).write(to: fileURL) + + let expectedSize = allocatedSize(at: fileURL.path) + XCTAssertGreaterThan(expectedSize, 0) + + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 999999, fileExtension: "", parent: nil) + let staleChild = FSNode(url: fileURL, name: "a.bin", isDirectory: false, size: 999999, fileExtension: "bin", parent: dirNode) + dirNode.children = [staleChild] + + ScanViewModel.refreshDirectory(node: dirNode) + + let updated = dirNode.children.first { $0.name == "a.bin" } + XCTAssertNotNil(updated) + XCTAssertEqual(updated?.size, expectedSize, "refresh must use allocated size (st_blocks * 512), not logical size") + } + + func test_refresh_scans_new_directory_recursively() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let newDir = tmp.appendingPathComponent("newdir") + let subDir = newDir.appendingPathComponent("sub") + try FileManager.default.createDirectory(at: subDir, withIntermediateDirectories: true) + let fileURL = subDir.appendingPathComponent("file.bin") + try Data(repeating: 7, count: 8192).write(to: fileURL) + + let expectedFileSize = allocatedSize(at: fileURL.path) + XCTAssertGreaterThan(expectedFileSize, 0) + + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + + ScanViewModel.refreshDirectory(node: dirNode) + + let newDirNode = dirNode.children.first { $0.name == "newdir" } + XCTAssertNotNil(newDirNode, "new directory should appear as a child") + XCTAssertEqual(newDirNode?.isDirectory, true) + XCTAssertEqual(newDirNode?.size, expectedFileSize, "new directory size should reflect the recursively-scanned subtree") + + let subDirNode = newDirNode?.children.first { $0.name == "sub" } + XCTAssertNotNil(subDirNode, "nested subdirectory should be scanned") + let fileNode = subDirNode?.children.first { $0.name == "file.bin" } + XCTAssertNotNil(fileNode, "nested file should be present") + XCTAssertEqual(fileNode?.size, expectedFileSize) + } + + func test_refresh_skips_symlinks() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let realURL = tmp.appendingPathComponent("real.bin") + try Data(repeating: 1, count: 4096).write(to: realURL) + let linkURL = tmp.appendingPathComponent("link") + try FileManager.default.createSymbolicLink(at: linkURL, withDestinationURL: realURL) + + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + + ScanViewModel.refreshDirectory(node: dirNode) + + XCTAssertNil(dirNode.children.first { $0.name == "link" }, "symlinks must not be added by refresh") + XCTAssertNotNil(dirNode.children.first { $0.name == "real.bin" }) + } + + func test_refresh_honors_hidden_files_setting() throws { + let defaults = UserDefaults.standard + let priorValue = defaults.object(forKey: "showHiddenFiles") + defaults.set(true, forKey: "showHiddenFiles") + defer { + if let priorValue { defaults.set(priorValue, forKey: "showHiddenFiles") } else { defaults.removeObject(forKey: "showHiddenFiles") } + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let hiddenURL = tmp.appendingPathComponent(".dotfile") + try Data(repeating: 3, count: 4096).write(to: hiddenURL) + let expectedSize = allocatedSize(at: hiddenURL.path) + + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + + ScanViewModel.refreshDirectory(node: dirNode) + + let hiddenChild = dirNode.children.first { $0.name == ".dotfile" } + XCTAssertNotNil(hiddenChild, "when showHiddenFiles is true, refresh should include dotfiles") + XCTAssertEqual(hiddenChild?.size, expectedSize) + } + + func test_refresh_skips_excluded_folders() throws { + let defaults = UserDefaults.standard + let priorValue = defaults.object(forKey: "excludedFolderNames") + defaults.set(".git,node_modules,DerivedData,.Trash", forKey: "excludedFolderNames") + defer { + if let priorValue { defaults.set(priorValue, forKey: "excludedFolderNames") } else { defaults.removeObject(forKey: "excludedFolderNames") } + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let excludedDir = tmp.appendingPathComponent("node_modules") + try FileManager.default.createDirectory(at: excludedDir, withIntermediateDirectories: true) + try Data(repeating: 5, count: 4096).write(to: excludedDir.appendingPathComponent("junk.bin")) + + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + + ScanViewModel.refreshDirectory(node: dirNode) + + XCTAssertNil(dirNode.children.first { $0.name == "node_modules" }, "excluded folder names must not be added by refresh") + } +} From 18a16cc38127fad795610f96bb5b876f6c082b2d Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 6 Jul 2026 02:16:41 +0300 Subject: [PATCH 03/41] perf(duplicates): parallel bounded hashing, single-read small files, cancellation in assignment --- Sources/Duplicates/DuplicateDetector.swift | 109 ++++++++++++++++---- Tests/DuplicateDetectorTests.swift | 113 +++++++++++++++++++++ 2 files changed, 203 insertions(+), 19 deletions(-) diff --git a/Sources/Duplicates/DuplicateDetector.swift b/Sources/Duplicates/DuplicateDetector.swift index 5f9ce3c..106b888 100644 --- a/Sources/Duplicates/DuplicateDetector.swift +++ b/Sources/Duplicates/DuplicateDetector.swift @@ -6,6 +6,11 @@ public actor DuplicateDetector { private let maxSize: Int64 = 512 * 1024 * 1024 // skip files > 512 MB (VM images, sparsebundles, etc.) private let quickHashBytes = 65_536 // 64 KB quick pre-filter + // Bounded concurrency for I/O-bound hashing work. + private static var maxConcurrency: Int { + min(8, ProcessInfo.processInfo.activeProcessorCount) + } + public init() {} public func detect(in root: FSNode) async { @@ -16,32 +21,51 @@ public actor DuplicateDetector { let bySize = Dictionary(grouping: candidates) { $0.size } .filter { $0.value.count > 1 } - // Phase 1: quick hash (first 64 KB) to rule out non-duplicates without reading entire files + let quickHashBytes = self.quickHashBytes + + // Phase 1: quick hash (first 64 KB) to rule out non-duplicates without reading entire files. + // Hashing is I/O-bound and embarrassingly parallel, so run it with bounded concurrency. var byQuickHash: [String: [FSNode]] = [:] - for (_, nodes) in bySize { - for node in nodes { - guard !Task.isCancelled else { return } - if let qh = partialHash(url: node.url, maxBytes: quickHashBytes) { - let key = "\(node.size)-\(qh)" - byQuickHash[key, default: []].append(node) - } - } + let quickHashResults: [(FSNode, String)]? = await Self.hashInParallel( + nodes: bySize.values.flatMap { $0 } + ) { node in + Self.partialHash(url: node.url, maxBytes: quickHashBytes) + } + guard let quickHashResults else { return } + for (node, qh) in quickHashResults { + let key = "\(node.size)-\(qh)" + byQuickHash[key, default: []].append(node) } - // Phase 2: full hash only for groups that survived the quick-hash filter + // Small-file shortcut: if a file's whole content fits within the quick-hash window, + // the quick hash already IS a full-content hash, so those groups are final as-is. + // Only groups whose files exceed the quick-hash window need a full-file hash pass. var hashGroups: [String: [FSNode]] = [:] - for (_, nodes) in byQuickHash where nodes.count > 1 { - for node in nodes { - guard !Task.isCancelled else { return } - if let hash = fullHash(url: node.url) { - let key = "\(node.size)-\(hash)" - hashGroups[key, default: []].append(node) - } + var toFullHash: [FSNode] = [] + for (key, nodes) in byQuickHash where nodes.count > 1 { + if nodes[0].size <= Int64(quickHashBytes) { + hashGroups[key] = nodes + } else { + toFullHash.append(contentsOf: nodes) + } + } + + // Phase 2: full hash only for groups that survived the quick-hash filter and are + // larger than the quick-hash window (their quick hash alone is not conclusive). + if !toFullHash.isEmpty { + let fullHashResults: [(FSNode, String)]? = await Self.hashInParallel(nodes: toFullHash) { node in + Self.fullHash(url: node.url) + } + guard let fullHashResults else { return } + for (node, hash) in fullHashResults { + let key = "\(node.size)-\(hash)" + hashGroups[key, default: []].append(node) } } // Assign group IDs to genuine duplicates for (_, nodes) in hashGroups where nodes.count > 1 { + guard !Task.isCancelled else { return } let groupID = UUID() for node in nodes { node.duplicateGroupID = groupID } } @@ -57,9 +81,56 @@ public actor DuplicateDetector { } } + // Runs `hash` over `nodes` with bounded sliding-window concurrency, returning nil if the + // task was cancelled before completion. Nodes for which `hash` returns nil are dropped. + private static func hashInParallel( + nodes: [FSNode], + hash: @escaping @Sendable (FSNode) -> String? + ) async -> [(FSNode, String)]? { + guard !nodes.isEmpty else { return [] } + guard !Task.isCancelled else { return nil } + + var results: [(FSNode, String)] = [] + results.reserveCapacity(nodes.count) + + await withTaskGroup(of: (FSNode, String?).self) { group in + var index = 0 + let limit = maxConcurrency + + func launchNext() { + guard index < nodes.count else { return } + let node = nodes[index] + index += 1 + group.addTask { + guard !Task.isCancelled else { return (node, nil) } + return (node, hash(node)) + } + } + + // Prime the sliding window. + while index < limit && index < nodes.count { + launchNext() + } + + while let (node, key) = await group.next() { + if let key { + results.append((node, key)) + } + if Task.isCancelled { + // Drain remaining in-flight tasks without launching more. + continue + } + launchNext() + } + } + + guard !Task.isCancelled else { return nil } + return results + } + // Reads up to `maxBytes` from the file and returns a SHA256 hex string. // Each chunk is drained from the autorelease pool immediately to prevent accumulation. - private func partialHash(url: URL, maxBytes: Int) -> String? { + private static func partialHash(url: URL, maxBytes: Int) -> String? { guard let handle = try? FileHandle(forReadingFrom: url) else { return nil } defer { try? handle.close() } var hasher = SHA256() @@ -77,7 +148,7 @@ public actor DuplicateDetector { // Full-file SHA256. Each 1 MB chunk is released immediately via autoreleasepool, // capping peak memory at ~1 MB regardless of file size. - private func fullHash(url: URL) -> String? { + private static func fullHash(url: URL) -> String? { guard let handle = try? FileHandle(forReadingFrom: url) else { return nil } defer { try? handle.close() } var hasher = SHA256() diff --git a/Tests/DuplicateDetectorTests.swift b/Tests/DuplicateDetectorTests.swift index 2d66b0f..171f7f4 100644 --- a/Tests/DuplicateDetectorTests.swift +++ b/Tests/DuplicateDetectorTests.swift @@ -61,4 +61,117 @@ final class DuplicateDetectorTests: XCTestCase { XCTAssertNil(root.children[0].duplicateGroupID, "files below threshold should not be grouped") XCTAssertNil(root.children[1].duplicateGroupID) } + + func test_same_prefix_different_tail_not_duplicates() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + // 128 KB files: identical first 64 KB (quick-hash prefix), different last bytes. + let sharedPrefix = Data(repeating: 7, count: 65_536) + let tail1 = Data(repeating: 1, count: 65_536) + let tail2 = Data(repeating: 2, count: 65_536) + let data1 = sharedPrefix + tail1 + let data2 = sharedPrefix + tail2 + + let f1 = tmp.appendingPathComponent("a.bin") + let f2 = tmp.appendingPathComponent("b.bin") + try data1.write(to: f1) + try data2.write(to: f2) + + let root = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + for url in [f1, f2] { + let child = FSNode(url: url, name: url.lastPathComponent, isDirectory: false, size: Int64(data1.count), fileExtension: "bin", parent: root) + root.children.append(child) + } + + let detector = DuplicateDetector() + await detector.detect(in: root) + + XCTAssertNil(root.children[0].duplicateGroupID, "files sharing only a quick-hash prefix must not be grouped") + XCTAssertNil(root.children[1].duplicateGroupID) + } + + func test_small_file_duplicates_detected() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + // 8 KB files: above minSize (4 KB), below quickHashBytes (64 KB) -> quick hash IS full hash. + let data = Data(repeating: 55, count: 8192) + let other = Data(repeating: 66, count: 8192) + let f1 = tmp.appendingPathComponent("s1.bin") + let f2 = tmp.appendingPathComponent("s2.bin") + let f3 = tmp.appendingPathComponent("s3.bin") + try data.write(to: f1) + try data.write(to: f2) + try other.write(to: f3) + + let root = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + for url in [f1, f2, f3] { + let child = FSNode(url: url, name: url.lastPathComponent, isDirectory: false, size: Int64(data.count), fileExtension: "bin", parent: root) + root.children.append(child) + } + + let detector = DuplicateDetector() + await detector.detect(in: root) + + let s1 = root.children.first { $0.name == "s1.bin" }! + let s2 = root.children.first { $0.name == "s2.bin" }! + let s3 = root.children.first { $0.name == "s3.bin" }! + + XCTAssertNotNil(s1.duplicateGroupID) + XCTAssertNotNil(s2.duplicateGroupID) + XCTAssertEqual(s1.duplicateGroupID, s2.duplicateGroupID) + XCTAssertNil(s3.duplicateGroupID) + } + + func test_many_duplicate_pairs_all_detected() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let root = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + + let pairCount = 20 + var pairFiles: [[FSNode]] = [] + + for i in 0..() + for pair in pairFiles { + let a = pair[0] + let b = pair[1] + XCTAssertNotNil(a.duplicateGroupID, "\(a.name) should be grouped") + XCTAssertNotNil(b.duplicateGroupID, "\(b.name) should be grouped") + XCTAssertEqual(a.duplicateGroupID, b.duplicateGroupID, "\(a.name) and \(b.name) should share a group") + if let gid = a.duplicateGroupID { + XCTAssertFalse(seenGroupIDs.contains(gid), "group ID \(gid) reused across pairs") + seenGroupIDs.insert(gid) + } + } + XCTAssertEqual(seenGroupIDs.count, pairCount) + } } From f2efab033d9a15ff698ba28d9778d5a139af5f9b Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 6 Jul 2026 02:17:12 +0300 Subject: [PATCH 04/41] perf(scanner): hoist UserDefaults out of hot path + bound task fan-out --- Sources/Scanner/FileScanner.swift | 87 +++++++++++++++--- Tests/FileScannerTests.swift | 148 ++++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+), 15 deletions(-) diff --git a/Sources/Scanner/FileScanner.swift b/Sources/Scanner/FileScanner.swift index a9ad5db..9025ba8 100644 --- a/Sources/Scanner/FileScanner.swift +++ b/Sources/Scanner/FileScanner.swift @@ -42,6 +42,48 @@ private final class ProgressCounter: @unchecked Sendable { } } +// Thread-safe counter bounding the number of concurrently-spawned subtree tasks. +// Prevents unbounded task-group fan-out (and the associated flood of open dirfds) +// on very deep/wide directory trees. +private final class TaskBudget: @unchecked Sendable { + private var lock = os_unfair_lock() + private var remaining: Int + + init(limit: Int) { + self.remaining = limit + } + + // Returns true if a slot was reserved (caller must call release() when done). + func tryAcquire() -> Bool { + os_unfair_lock_lock(&lock) + defer { os_unfair_lock_unlock(&lock) } + guard remaining > 0 else { return false } + remaining -= 1 + return true + } + + func release() { + os_unfair_lock_lock(&lock) + remaining += 1 + os_unfair_lock_unlock(&lock) + } +} + +// Snapshot of scan-time settings, read once per scan (not per directory) to avoid +// UserDefaults overhead on the hot path. +struct ScanConfig: Sendable { + let excludedNames: Set + let showHiddenFiles: Bool + + static func loadFromUserDefaults() -> ScanConfig { + let rawExcluded = UserDefaults.standard.string(forKey: "excludedFolderNames") + ?? ".git,node_modules,DerivedData,.Trash" + let excludedNames = Set(rawExcluded.components(separatedBy: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty }) + let showHiddenFiles = UserDefaults.standard.bool(forKey: "showHiddenFiles") + return ScanConfig(excludedNames: excludedNames, showHiddenFiles: showHiddenFiles) + } +} + public actor FileScanner { private var activeTask: Task? @@ -58,6 +100,8 @@ public actor FileScanner { activeTask = Task { let counter = ProgressCounter() let visited = VisitedSet() + let config = ScanConfig.loadFromUserDefaults() + let taskBudget = TaskBudget(limit: min(max(4, ProcessInfo.processInfo.activeProcessorCount), 16)) // Get the root device ID to detect mount points var rootStat = stat() let rootDev: dev_t? = (stat(url.path, &rootStat) == 0) ? rootStat.st_dev : nil @@ -72,7 +116,7 @@ public actor FileScanner { } do { - let root = try await _buildTree(path: url.path, url: url, parent: nil, rootDev: rootDev, counter: counter, visited: visited) + let root = try await _buildTree(path: url.path, url: url, parent: nil, rootDev: rootDev, counter: counter, visited: visited, config: config, taskBudget: taskBudget) progressTask.cancel() continuation.yield(.completed(root: root)) } catch is CancellationError { @@ -96,7 +140,9 @@ private func _buildTree( parent: FSNode?, rootDev: dev_t?, counter: ProgressCounter, - visited: VisitedSet + visited: VisitedSet, + config: ScanConfig, + taskBudget: TaskBudget ) async throws -> FSNode { try Task.checkCancellation() @@ -120,24 +166,39 @@ private func _buildTree( let node = FSNode(url: url, name: name, isDirectory: isDir, size: 0, fileExtension: ext, parent: parent) if isDir { - let listing = _listDirectory(path: path, url: url, rootDev: rootDev, node: node, counter: counter, visited: visited) + let listing = _listDirectory(path: path, url: url, rootDev: rootDev, node: node, counter: counter, visited: visited, config: config) // Accumulate direct file sizes immediately node.size = listing.totalSize node.children = listing.children - // Recurse into subdirectories in parallel + // Recurse into subdirectories, in parallel up to the task budget; beyond + // that, recurse inline in the current task to bound total concurrency. if !listing.subdirPaths.isEmpty { var subdirSize: Int64 = 0 try await withThrowingTaskGroup(of: FSNode?.self) { group in for (subPath, subURL) in listing.subdirPaths { - group.addTask { - // Propagate cancellation; silently skip symlinks / unreadable entries. + if taskBudget.tryAcquire() { + group.addTask { + defer { taskBudget.release() } + // Propagate cancellation; silently skip symlinks / unreadable entries. + try Task.checkCancellation() + do { + return try await _buildTree(path: subPath, url: subURL, parent: node, rootDev: rootDev, counter: counter, visited: visited, config: config, taskBudget: taskBudget) + } catch is SkipError { + return nil + } + } + } else { + // Budget exhausted: recurse inline (no new task spawned) to + // bound concurrency without blocking on any lock or semaphore. try Task.checkCancellation() do { - return try await _buildTree(path: subPath, url: subURL, parent: node, rootDev: rootDev, counter: counter, visited: visited) + let child = try await _buildTree(path: subPath, url: subURL, parent: node, rootDev: rootDev, counter: counter, visited: visited, config: config, taskBudget: taskBudget) + node.children.append(child) + subdirSize += child.size } catch is SkipError { - return nil + // skip } } } @@ -173,17 +234,13 @@ private struct DirectoryContents { var itemCount: Int // count of items processed here } -private func _listDirectory(path: String, url: URL, rootDev: dev_t?, node: FSNode, counter: ProgressCounter, visited: VisitedSet) -> DirectoryContents { +private func _listDirectory(path: String, url: URL, rootDev: dev_t?, node: FSNode, counter: ProgressCounter, visited: VisitedSet, config: ScanConfig) -> DirectoryContents { var result = DirectoryContents(children: [], subdirPaths: [], totalSize: 0, itemCount: 0) guard let dir = opendir(path) else { return result } defer { closedir(dir) } let directoryFD = dirfd(dir) - let rawExcluded = UserDefaults.standard.string(forKey: "excludedFolderNames") - ?? ".git,node_modules,DerivedData,.Trash" - let excludedNames = Set(rawExcluded.components(separatedBy: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty }) - while let entry = readdir(dir) { let nameBytes = entry.pointee.d_name let name: String = withUnsafeBytes(of: nameBytes) { ptr in @@ -191,8 +248,8 @@ private func _listDirectory(path: String, url: URL, rootDev: dev_t?, node: FSNod return String(cString: bytes.baseAddress!) } guard name != "." && name != ".." else { continue } - if name.hasPrefix("."), !UserDefaults.standard.bool(forKey: "showHiddenFiles") { continue } - if excludedNames.contains(name) { continue } + if name.hasPrefix("."), !config.showHiddenFiles { continue } + if config.excludedNames.contains(name) { continue } let childURL = url.appendingPathComponent(name, isDirectory: entry.pointee.d_type == DT_DIR) let dtype = entry.pointee.d_type diff --git a/Tests/FileScannerTests.swift b/Tests/FileScannerTests.swift index c4ebd6a..2f83f34 100644 --- a/Tests/FileScannerTests.swift +++ b/Tests/FileScannerTests.swift @@ -92,4 +92,152 @@ final class FileScannerTests: XCTestCase { XCTAssertEqual(root?.children.count, 1, "symlink should be skipped") XCTAssertEqual(root?.children.first?.name, "real.txt") } + + // MARK: - Pinning tests for perf refactor (ScanConfig hoisting + TaskBudget fan-out cap) + + func test_scanner_respects_excluded_folder_names() async throws { + let priorValue = UserDefaults.standard.string(forKey: "excludedFolderNames") + UserDefaults.standard.set("skipme", forKey: "excludedFolderNames") + defer { + if let priorValue { + UserDefaults.standard.set(priorValue, forKey: "excludedFolderNames") + } else { + UserDefaults.standard.removeObject(forKey: "excludedFolderNames") + } + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let skipDir = tmp.appendingPathComponent("skipme") + let keepDir = tmp.appendingPathComponent("keep") + try FileManager.default.createDirectory(at: skipDir, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: keepDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + try Data(repeating: 0, count: 100 * 1024).write(to: skipDir.appendingPathComponent("big.bin")) + try Data(repeating: 0, count: 4096).write(to: keepDir.appendingPathComponent("file.bin")) + + let scanner = FileScanner() + var root: FSNode? + for await progress in await scanner.scan(url: tmp) { + if case .completed(let node) = progress { root = node } + } + + XCTAssertNotNil(root) + let names = Set(root?.children.map { $0.name } ?? []) + XCTAssertFalse(names.contains("skipme"), "excluded folder should not appear as a child") + XCTAssertTrue(names.contains("keep")) + + guard let keepNode = root?.children.first(where: { $0.name == "keep" }) else { + return XCTFail("keep dir missing from scan results") + } + XCTAssertEqual(root?.size, keepNode.size, "root size should exclude the skipped folder's contents") + } + + func test_scanner_honors_show_hidden_files_setting() async throws { + let priorValue = UserDefaults.standard.object(forKey: "showHiddenFiles") as? Bool + defer { + if let priorValue { + UserDefaults.standard.set(priorValue, forKey: "showHiddenFiles") + } else { + UserDefaults.standard.removeObject(forKey: "showHiddenFiles") + } + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + try Data(repeating: 0, count: 10).write(to: tmp.appendingPathComponent(".hidden")) + try Data(repeating: 0, count: 10).write(to: tmp.appendingPathComponent("visible.txt")) + + UserDefaults.standard.set(false, forKey: "showHiddenFiles") + do { + let scanner = FileScanner() + var root: FSNode? + for await progress in await scanner.scan(url: tmp) { + if case .completed(let node) = progress { root = node } + } + let names = Set(root?.children.map { $0.name } ?? []) + XCTAssertFalse(names.contains(".hidden"), "hidden file should be excluded when showHiddenFiles is false") + XCTAssertTrue(names.contains("visible.txt")) + } + + UserDefaults.standard.set(true, forKey: "showHiddenFiles") + do { + let scanner = FileScanner() + var root: FSNode? + for await progress in await scanner.scan(url: tmp) { + if case .completed(let node) = progress { root = node } + } + let names = Set(root?.children.map { $0.name } ?? []) + XCTAssertTrue(names.contains(".hidden"), "hidden file should be included when showHiddenFiles is true") + XCTAssertTrue(names.contains("visible.txt")) + } + } + + func test_scanner_handles_deep_wide_tree_correctly() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + var allDirs: [URL] = [] + var allFiles: [URL] = [] + + // Builds a tree nested 5 levels deep with ~40 directories total, each + // holding exactly one 4 KB file. This stresses the bounded task-group + // fan-out (Issue 2) to make sure totals stay correct regardless of + // whether a subtree is scanned via the task group or inline recursion. + let branchingPerLevel = [2, 2, 2, 2, 1] + func buildLevel(base: URL, levelIndex: Int) throws { + guard levelIndex < branchingPerLevel.count else { return } + let branches = branchingPerLevel[levelIndex] + for i in 0.. (dirs: Int, files: Int) { + if node.isDirectory { + var dirs = 1 + var files = 0 + for child in node.children { + let c = countNodes(child) + dirs += c.dirs + files += c.files + } + return (dirs, files) + } else { + return (0, 1) + } + } + + let counts = countNodes(root!) + XCTAssertEqual(counts.dirs, allDirs.count + 1, "every directory (plus root) must be reachable in the tree") + XCTAssertEqual(counts.files, allFiles.count, "every file must be reachable in the tree") + } } From 69933ecf850dfea5475678ee6bf5ea32dda0cd4e Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 6 Jul 2026 02:24:00 +0300 Subject: [PATCH 05/41] fix(refresh): don't resurrect deduplicated hardlinks to full size on FSEvents refresh --- Sources/ViewModels/ScanViewModel.swift | 9 +++++--- Tests/ScanRefreshTests.swift | 29 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/Sources/ViewModels/ScanViewModel.swift b/Sources/ViewModels/ScanViewModel.swift index 1cf0a61..1e369aa 100644 --- a/Sources/ViewModels/ScanViewModel.swift +++ b/Sources/ViewModels/ScanViewModel.swift @@ -251,14 +251,14 @@ public final class ScanViewModel: ObservableObject { // Returns the allocated disk size (st_blocks * 512) for a path via lstat, or nil if // the path can't be stat'd or is a symlink. - private nonisolated static func lstatInfo(path: String) -> (isDir: Bool, isSymlink: Bool, allocatedSize: Int64)? { + private nonisolated static func lstatInfo(path: String) -> (isDir: Bool, isSymlink: Bool, allocatedSize: Int64, linkCount: Int)? { var st = stat() guard lstat(path, &st) == 0 else { return nil } let mode = st.st_mode & S_IFMT let isSymlink = mode == S_IFLNK let isDir = mode == S_IFDIR let allocatedSize = Int64(st.st_blocks) * 512 - return (isDir, isSymlink, allocatedSize) + return (isDir, isSymlink, allocatedSize, Int(st.st_nlink)) } // Parses the excludedFolderNames default the same way FileScanner does. @@ -284,7 +284,7 @@ public final class ScanViewModel: ObservableObject { // Filter out excluded folder names and symlinks up front, so both the // removal pass and the add/update pass agree on what's "on disk". - var onDisk: [String: (url: URL, info: (isDir: Bool, isSymlink: Bool, allocatedSize: Int64))] = [:] + var onDisk: [String: (url: URL, info: (isDir: Bool, isSymlink: Bool, allocatedSize: Int64, linkCount: Int))] = [:] for url in entries { let name = url.lastPathComponent if excludedNames.contains(name) { continue } @@ -306,6 +306,9 @@ public final class ScanViewModel: ObservableObject { if let existing = node.children.first(where: { $0.name == name }) { // Update size for files (directories update via recursive bubble) if !existing.isDirectory { + // A 0-byte node for a multi-link inode is a hardlink the initial scan + // already counted elsewhere — re-statting it would double-count. + if info.linkCount > 1 && existing.size == 0 { continue } let newSize = info.allocatedSize if existing.size != newSize { existing.size = newSize diff --git a/Tests/ScanRefreshTests.swift b/Tests/ScanRefreshTests.swift index d460314..aecf11d 100644 --- a/Tests/ScanRefreshTests.swift +++ b/Tests/ScanRefreshTests.swift @@ -32,6 +32,35 @@ final class ScanRefreshTests: XCTestCase { XCTAssertEqual(updated?.size, expectedSize, "refresh must use allocated size (st_blocks * 512), not logical size") } + func test_refresh_preserves_hardlink_dedup() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let hard1 = tmp.appendingPathComponent("hard1.bin") + let hard2 = tmp.appendingPathComponent("hard2.bin") + try Data(repeating: 1, count: 262_144).write(to: hard1) + try FileManager.default.linkItem(at: hard1, to: hard2) + let fullSize = allocatedSize(at: hard1.path) + XCTAssertGreaterThan(fullSize, 0) + + // Initial scan counts a hardlinked inode once: hard1 keeps the size, hard2 is 0. + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: fullSize, fileExtension: "", parent: nil) + let winner = FSNode(url: hard1, name: "hard1.bin", isDirectory: false, size: fullSize, fileExtension: "bin", parent: dirNode) + let loser = FSNode(url: hard2, name: "hard2.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirNode) + dirNode.children = [winner, loser] + + // Unrelated change in the same directory triggers a refresh. + try Data(repeating: 9, count: 4096).write(to: tmp.appendingPathComponent("other.bin")) + + ScanViewModel.refreshDirectory(node: dirNode) + + let refreshedLoser = dirNode.children.first { $0.name == "hard2.bin" } + let refreshedWinner = dirNode.children.first { $0.name == "hard1.bin" } + XCTAssertEqual(refreshedLoser?.size, 0, "refresh must not resurrect a deduplicated hardlink to full size") + XCTAssertEqual(refreshedWinner?.size, fullSize) + } + func test_refresh_scans_new_directory_recursively() throws { let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) From e95ee04e4dde6401403fa4bb9b2ca7bc0057e523 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 6 Jul 2026 03:00:59 +0300 Subject: [PATCH 06/41] chore(diag): env-gated scan diagnostics (MDS_DEBUG_SKIPS, MDS_DEBUG_TREE, MDS_DIAG_PATH harness) --- Sources/Scanner/FileScanner.swift | 7 ++++- Sources/ViewModels/ScanViewModel.swift | 12 ++++++++ Tests/DiagScanTests.swift | 41 ++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 Tests/DiagScanTests.swift diff --git a/Sources/Scanner/FileScanner.swift b/Sources/Scanner/FileScanner.swift index 9025ba8..f6bb9ba 100644 --- a/Sources/Scanner/FileScanner.swift +++ b/Sources/Scanner/FileScanner.swift @@ -157,7 +157,12 @@ private func _buildTree( // Deduplicate directories by (dev, ino) — prevents double-counting paths like // /Applications and /System/Volumes/Data/Applications (same inode, different paths). if isDir { - guard visited.visit(dev: st.st_dev, ino: st.st_ino) else { throw SkipError() } + guard visited.visit(dev: st.st_dev, ino: st.st_ino) else { + if ProcessInfo.processInfo.environment["MDS_DEBUG_SKIPS"] != nil { + FileHandle.standardError.write("DEDUP_SKIP dev=\(st.st_dev) ino=\(st.st_ino) \(path)\n".data(using: .utf8)!) + } + throw SkipError() + } } let name = url.lastPathComponent diff --git a/Sources/ViewModels/ScanViewModel.swift b/Sources/ViewModels/ScanViewModel.swift index 1e369aa..e1cf1cb 100644 --- a/Sources/ViewModels/ScanViewModel.swift +++ b/Sources/ViewModels/ScanViewModel.swift @@ -117,6 +117,13 @@ public final class ScanViewModel: ObservableObject { self.itemsScanned = items self.bytesFound = bytes case .completed(let node): + if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { + var dump = "TREE_COMPLETED total=\(node.size)\n" + for c in node.children.sorted(by: { $0.size > $1.size }).prefix(15) { + dump += "TREE_CHILD \(c.size) \(c.name)\n" + } + FileHandle.standardError.write(dump.data(using: .utf8)!) + } self.isScanning = false self.isComputingLayout = true // keep spinner until treemap is ready // Sort + safety-tag the entire tree off-thread before exposing it to the UI. @@ -219,6 +226,7 @@ public final class ScanViewModel: ObservableObject { var needsLayout = false for dirPath in dirPaths { guard let node = Self.findNode(path: dirPath, in: root) else { continue } + let sizeBefore = node.size let changed = await Task.detached(priority: .userInitiated) { Self.refreshDirectory(node: node) }.value @@ -226,6 +234,10 @@ public final class ScanViewModel: ObservableObject { Self.bubbleUpSizes(from: node) needsLayout = true } + if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { + let line = "REFRESH changed=\(changed) nodeBefore=\(sizeBefore) nodeAfter=\(node.size) rootAfter=\(root.size) path=\(dirPath)\n" + FileHandle.standardError.write(line.data(using: .utf8)!) + } } if needsLayout { diff --git a/Tests/DiagScanTests.swift b/Tests/DiagScanTests.swift new file mode 100644 index 0000000..fab1859 --- /dev/null +++ b/Tests/DiagScanTests.swift @@ -0,0 +1,41 @@ +import XCTest +@testable import MacDirStat + +// Diagnostic harness, not a CI test: scans a real directory with the production +// FileScanner and prints per-child totals for comparison against `du`. +// Enable with MDS_DIAG_PATH=/some/path. +final class DiagScanTests: XCTestCase { + + func test_diag_scan_prints_top_level_sizes() async throws { + guard let target = ProcessInfo.processInfo.environment["MDS_DIAG_PATH"] else { + throw XCTSkip("set MDS_DIAG_PATH to run the diagnostic scan") + } + + let scanner = FileScanner() + var completedRoot: FSNode? + var lastItems = 0 + var lastBytes: Int64 = 0 + for await progress in await scanner.scan(url: URL(fileURLWithPath: target)) { + switch progress { + case .update(let items, let bytes): + lastItems = items + lastBytes = bytes + case .completed(let root): + completedRoot = root + case .failed(let msg): + XCTFail("scan failed: \(msg)") + } + } + + guard let root = completedRoot else { + XCTFail("no completed root (last progress: \(lastItems) items, \(lastBytes) bytes)") + return + } + + func gb(_ v: Int64) -> String { String(format: "%.1f GB", Double(v) / 1_000_000_000) } + print("DIAG root=\(target) total=\(gb(root.size)) rawBytes=\(root.size) progressBytes=\(gb(lastBytes)) items=\(lastItems)") + for child in root.children.sorted(by: { $0.size > $1.size }).prefix(25) { + print("DIAG child \(gb(child.size)) \(child.name)\(child.isDirectory ? "/" : "")") + } + } +} From 319117c8df4494ed3f8b4f826e2f3d6ba26d7b56 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 6 Jul 2026 03:12:24 +0300 Subject: [PATCH 07/41] build(release): drop App Sandbox so Full Disk Access works (Developer ID app) --- MacDirStat/MacDirStat.entitlements | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/MacDirStat/MacDirStat.entitlements b/MacDirStat/MacDirStat.entitlements index 18aff0c..e89b7f3 100644 --- a/MacDirStat/MacDirStat.entitlements +++ b/MacDirStat/MacDirStat.entitlements @@ -3,8 +3,6 @@ com.apple.security.app-sandbox - - com.apple.security.files.user-selected.read-only - + From 876412acd549abcacd46d596244d8560283bcd3f Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 6 Jul 2026 03:19:44 +0300 Subject: [PATCH 08/41] feat(trust): surface access-denied folders + FDA banner + hidden-space reconciliation node --- Sources/App/ContentView.swift | 21 ++++-- Sources/Duplicates/DuplicateDetector.swift | 5 +- Sources/Layout/TreemapLayout.swift | 5 ++ Sources/Safety/SafetyAnalyzer.swift | 7 ++ Sources/Scanner/FSNode.swift | 2 + Sources/Scanner/FileScanner.swift | 26 ++++++- Sources/Scanner/ScanProgress.swift | 2 +- Sources/ViewModels/ScanViewModel.swift | 59 ++++++++++++++-- .../DirectoryTree/DirectoryTreeView.swift | 7 ++ Tests/FileScannerTests.swift | 35 +++++++++- Tests/HiddenSpaceTests.swift | 69 +++++++++++++++++++ 11 files changed, 223 insertions(+), 15 deletions(-) create mode 100644 Tests/HiddenSpaceTests.swift diff --git a/Sources/App/ContentView.swift b/Sources/App/ContentView.swift index c209b3e..4c2fdd2 100644 --- a/Sources/App/ContentView.swift +++ b/Sources/App/ContentView.swift @@ -190,8 +190,8 @@ struct ContentView: View { @ViewBuilder private var detailContent: some View { VStack(spacing: 0) { - if !vm.hasFullDiskAccess { - FullDiskAccessBanner() + if !vm.hasFullDiskAccess || (vm.deniedCount > 0 && !vm.isScanning) { + FullDiskAccessBanner(hasFullDiskAccess: vm.hasFullDiskAccess, deniedCount: vm.deniedCount) } if vm.root == nil && !vm.isScanning && !vm.isComputingLayout { WelcomeView() @@ -374,6 +374,19 @@ private struct FolderTitleView: View { // MARK: - Full Disk Access banner private struct FullDiskAccessBanner: View { + let hasFullDiskAccess: Bool + let deniedCount: Int + + private let title = "Full Disk Access required for complete results" + + private var subtitle: String { + if !hasFullDiskAccess { + return "Go to System Settings → Privacy & Security → Full Disk Access and add DirStat." + } + let folders = deniedCount == 1 ? "1 folder" : "\(deniedCount) folders" + return "\(folders) couldn't be read. Grant Full Disk Access for a complete scan." + } + var body: some View { HStack(spacing: 10) { Image(systemName: "lock.shield") @@ -381,9 +394,9 @@ private struct FullDiskAccessBanner: View { .foregroundStyle(.orange) VStack(alignment: .leading, spacing: 1) { - Text("Full Disk Access required for complete results") + Text(title) .font(.system(size: 12, weight: .semibold)) - Text("Go to System Settings → Privacy & Security → Full Disk Access and add DirStat.") + Text(subtitle) .font(.system(size: 11)) .foregroundStyle(.secondary) } diff --git a/Sources/Duplicates/DuplicateDetector.swift b/Sources/Duplicates/DuplicateDetector.swift index 5f9ce3c..bdab4ac 100644 --- a/Sources/Duplicates/DuplicateDetector.swift +++ b/Sources/Duplicates/DuplicateDetector.swift @@ -50,7 +50,10 @@ public actor DuplicateDetector { private func collect(node: FSNode, into list: inout [FSNode]) { var stack: [FSNode] = [node] while let current = stack.popLast() { - if !current.isDirectory && current.size >= minSize && current.size <= maxSize { + // Synthetic nodes (e.g. the hidden-space reconciliation entry) have no + // real file behind their URL — hashing them would just fail harmlessly, + // but skip them outright rather than waste the attempt. + if !current.isSynthetic && !current.isDirectory && current.size >= minSize && current.size <= maxSize { list.append(current) } stack.append(contentsOf: current.children) diff --git a/Sources/Layout/TreemapLayout.swift b/Sources/Layout/TreemapLayout.swift index 85afbb5..50a9eff 100644 --- a/Sources/Layout/TreemapLayout.swift +++ b/Sources/Layout/TreemapLayout.swift @@ -96,6 +96,11 @@ public struct TreemapLayout { // ── Color assignment ───────────────────────────────────────────────────── private static func color(for node: FSNode, depth: Int, colorMap: ExtensionColorMap) -> Color { + if node.isSynthetic { + // Distinct muted gray so the hidden-space reconciliation node reads as + // "not a real file" at a glance, rather than blending into the chart. + return Color(hue: 0, saturation: 0, brightness: 0.35) + } if node.isDirectory { // Directories: muted tinted containers — hue from name hash, low saturation let hash = abs(node.name.hashValue) % 360 diff --git a/Sources/Safety/SafetyAnalyzer.swift b/Sources/Safety/SafetyAnalyzer.swift index b4422c6..98986b5 100644 --- a/Sources/Safety/SafetyAnalyzer.swift +++ b/Sources/Safety/SafetyAnalyzer.swift @@ -16,6 +16,10 @@ public struct SafetyAnalyzer { // MARK: - Bulk tagging (no string allocations — just enum assignment) public static func level(for node: FSNode) -> SafetyLevel { + // Synthetic nodes (e.g. the hidden-space reconciliation entry) don't point + // at a real, deletable file — always treat them as the most protective + // level so nothing ever attempts to trash/move them. + if node.isSynthetic { return .danger } let path = node.url.path let name = node.name if isDanger(path: path, name: name) { return .danger } @@ -26,6 +30,9 @@ public struct SafetyAnalyzer { // MARK: - On-demand reason (called only when UI needs to display it) public static func reason(for node: FSNode) -> String? { + if node.isSynthetic { + return "Represents space macOS reports as used but the scanner can't read or enumerate (snapshots, purgeable space, protected folders)" + } let path = node.url.path let name = node.name return dangerReason(path: path, name: name) diff --git a/Sources/Scanner/FSNode.swift b/Sources/Scanner/FSNode.swift index da4699e..f5fd5f1 100644 --- a/Sources/Scanner/FSNode.swift +++ b/Sources/Scanner/FSNode.swift @@ -11,6 +11,8 @@ public final class FSNode: Identifiable, @unchecked Sendable { public let fileExtension: String public var duplicateGroupID: UUID? public var safetyLevel: SafetyLevel = .caution + public var isAccessDenied: Bool = false + public var isSynthetic: Bool = false public init(url: URL, name: String, isDirectory: Bool, size: Int64, fileExtension: String, parent: FSNode? = nil) { self.url = url diff --git a/Sources/Scanner/FileScanner.swift b/Sources/Scanner/FileScanner.swift index a9ad5db..fec2d07 100644 --- a/Sources/Scanner/FileScanner.swift +++ b/Sources/Scanner/FileScanner.swift @@ -26,6 +26,7 @@ private final class ProgressCounter: @unchecked Sendable { private var lock = os_unfair_lock() private(set) var items: Int = 0 private(set) var bytes: Int64 = 0 + private(set) var denied: Int = 0 func add(items: Int, bytes: Int64) { os_unfair_lock_lock(&lock) @@ -34,12 +35,25 @@ private final class ProgressCounter: @unchecked Sendable { os_unfair_lock_unlock(&lock) } + func addDenied() { + os_unfair_lock_lock(&lock) + self.denied += 1 + os_unfair_lock_unlock(&lock) + } + var snapshot: (items: Int, bytes: Int64) { os_unfair_lock_lock(&lock) let result = (items, bytes) os_unfair_lock_unlock(&lock) return result } + + var deniedCount: Int { + os_unfair_lock_lock(&lock) + let result = denied + os_unfair_lock_unlock(&lock) + return result + } } public actor FileScanner { @@ -74,7 +88,7 @@ public actor FileScanner { do { let root = try await _buildTree(path: url.path, url: url, parent: nil, rootDev: rootDev, counter: counter, visited: visited) progressTask.cancel() - continuation.yield(.completed(root: root)) + continuation.yield(.completed(root: root, deniedCount: counter.deniedCount)) } catch is CancellationError { progressTask.cancel() } catch { @@ -125,6 +139,8 @@ private func _buildTree( // Accumulate direct file sizes immediately node.size = listing.totalSize node.children = listing.children + node.isAccessDenied = listing.accessDenied + if listing.accessDenied { counter.addDenied() } // Recurse into subdirectories in parallel if !listing.subdirPaths.isEmpty { @@ -171,12 +187,18 @@ private struct DirectoryContents { var subdirPaths: [(String, URL)] // subdirectory (path, url) pairs for parallel recursion var totalSize: Int64 // sum of immediate file sizes var itemCount: Int // count of items processed here + var accessDenied: Bool = false // true when opendir failed due to permissions (EACCES/EPERM) } private func _listDirectory(path: String, url: URL, rootDev: dev_t?, node: FSNode, counter: ProgressCounter, visited: VisitedSet) -> DirectoryContents { var result = DirectoryContents(children: [], subdirPaths: [], totalSize: 0, itemCount: 0) - guard let dir = opendir(path) else { return result } + guard let dir = opendir(path) else { + if errno == EACCES || errno == EPERM { + result.accessDenied = true + } + return result + } defer { closedir(dir) } let directoryFD = dirfd(dir) diff --git a/Sources/Scanner/ScanProgress.swift b/Sources/Scanner/ScanProgress.swift index e10931a..3a93f84 100644 --- a/Sources/Scanner/ScanProgress.swift +++ b/Sources/Scanner/ScanProgress.swift @@ -2,6 +2,6 @@ import Foundation public enum ScanProgress: Sendable { case update(itemsScanned: Int, bytesFound: Int64) - case completed(root: FSNode) + case completed(root: FSNode, deniedCount: Int) case failed(String) } diff --git a/Sources/ViewModels/ScanViewModel.swift b/Sources/ViewModels/ScanViewModel.swift index 904acce..fe489d3 100644 --- a/Sources/ViewModels/ScanViewModel.swift +++ b/Sources/ViewModels/ScanViewModel.swift @@ -20,6 +20,7 @@ public final class ScanViewModel: ObservableObject { @Published public var duplicateGroups: [[FSNode]] = [] @Published public var hasFullDiskAccess: Bool = true @Published public var isWatching: Bool = false + @Published public var deniedCount: Int = 0 public var treemapRoot: FSNode? { drillStack.last ?? root } @@ -109,6 +110,7 @@ public final class ScanViewModel: ObservableObject { bytesFound = 0 isComputingLayout = false errorMessage = nil + deniedCount = 0 scanTask = Task { for await progress in await scanner.scan(url: url) { @@ -116,8 +118,15 @@ public final class ScanViewModel: ObservableObject { case .update(let items, let bytes): self.itemsScanned = items self.bytesFound = bytes - case .completed(let node): + case .completed(let node, let denied): self.isScanning = false + self.deniedCount = denied + // If the scanned root is a volume mount point, the file total will + // always fall short of Finder's "used" figure (APFS snapshots, + // purgeable space, excluded/unreadable folders). Make that gap + // visible instead of silently under-reporting. Must run before the + // sort pass below so the synthetic node sorts into place. + Self.appendHiddenSpaceNodeIfNeeded(root: node, scannedURL: url) self.isComputingLayout = true // keep spinner until treemap is ready // Sort + safety-tag the entire tree off-thread before exposing it to the UI. await Task.detached(priority: .userInitiated) { @@ -252,7 +261,7 @@ public final class ScanViewModel: ObservableObject { // Re-stat the directory on disk and update children to match. // Returns true if anything changed (additions/removals/size changes). @discardableResult - private nonisolated static func refreshDirectory(node: FSNode) -> Bool { + nonisolated static func refreshDirectory(node: FSNode) -> Bool { guard node.isDirectory else { return false } let fm = FileManager.default guard let entries = try? fm.contentsOfDirectory( @@ -264,9 +273,11 @@ public final class ScanViewModel: ObservableObject { let onDisk = Dictionary(uniqueKeysWithValues: entries.map { ($0.lastPathComponent, $0) }) var changed = false - // Remove children that no longer exist + // Remove children that no longer exist on disk — but never remove synthetic + // nodes (e.g. the "Hidden & Unreadable Space" reconciliation entry), which + // never correspond to a real path and would otherwise be deleted on refresh. let before = node.children.count - node.children.removeAll { !onDisk.keys.contains($0.name) } + node.children.removeAll { !$0.isSynthetic && !onDisk.keys.contains($0.name) } if node.children.count != before { changed = true } // Add or update children @@ -300,6 +311,46 @@ public final class ScanViewModel: ObservableObject { return changed } + // Computes the gap between what the volume reports as used (total - available) + // and what the scanner actually accounted for. On APFS volumes this gap is + // never zero: snapshots, purgeable space, and unreadable/excluded areas all + // count toward "used" without ever appearing as a scannable file. Returns nil + // when inputs are invalid or the gap is small enough to be measurement noise. + nonisolated static func hiddenSpaceBytes(volumeTotal: Int64, volumeAvailable: Int64, scannedTotal: Int64) -> Int64? { + guard volumeTotal > 0 else { return nil } + let hidden = max(0, volumeTotal - volumeAvailable - scannedTotal) + let oneGB: Int64 = 1_000_000_000 + return hidden >= oneGB ? hidden : nil + } + + // When the scanned URL is itself a volume's mount point, appends a synthetic + // "Hidden & Unreadable Space" child representing the portion of the volume's + // used space that the scanner could never account for. No-op for non-volume + // scans (e.g. scanning a subfolder) or when the gap is negligible. + private nonisolated static func appendHiddenSpaceNodeIfNeeded(root: FSNode, scannedURL: URL) { + guard let values = try? scannedURL.resourceValues(forKeys: [.volumeURLKey]), + let volumeURL = values.volume, + volumeURL.standardizedFileURL.path == scannedURL.standardizedFileURL.path + else { return } + + guard let volumeValues = try? scannedURL.resourceValues(forKeys: [.volumeTotalCapacityKey, .volumeAvailableCapacityKey]), + let totalCapacity = volumeValues.volumeTotalCapacity, + let availableCapacity = volumeValues.volumeAvailableCapacity + else { return } + + guard let hidden = hiddenSpaceBytes( + volumeTotal: Int64(totalCapacity), + volumeAvailable: Int64(availableCapacity), + scannedTotal: root.size + ) else { return } + + let syntheticURL = scannedURL.appendingPathComponent("#hidden-space") + let synthetic = FSNode(url: syntheticURL, name: "Hidden & Unreadable Space", isDirectory: false, size: hidden, fileExtension: "", parent: root) + synthetic.isSynthetic = true + root.children.append(synthetic) + root.size += hidden + } + // Walk up the parent chain recalculating folder sizes from their children. private nonisolated static func bubbleUpSizes(from node: FSNode) { var current: FSNode? = node diff --git a/Sources/Views/DirectoryTree/DirectoryTreeView.swift b/Sources/Views/DirectoryTree/DirectoryTreeView.swift index 3513428..2bf68fc 100644 --- a/Sources/Views/DirectoryTree/DirectoryTreeView.swift +++ b/Sources/Views/DirectoryTree/DirectoryTreeView.swift @@ -131,6 +131,13 @@ private struct NodeRow: View { .help(SafetyAnalyzer.reason(for: node) ?? "Do not delete") } + if node.isAccessDenied { + Image(systemName: "lock.fill") + .font(.system(size: 9)) + .foregroundStyle(.secondary) + .help("Contents couldn't be read — Full Disk Access may be required") + } + // Size Text(ByteFormatter.string(from: node.size)) .font(.system(size: 11, design: .monospaced)) diff --git a/Tests/FileScannerTests.swift b/Tests/FileScannerTests.swift index c4ebd6a..9ba43e0 100644 --- a/Tests/FileScannerTests.swift +++ b/Tests/FileScannerTests.swift @@ -44,7 +44,7 @@ final class FileScannerTests: XCTestCase { let scanner = FileScanner() var root: FSNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node) = progress { root = node } + if case .completed(let node, _) = progress { root = node } } XCTAssertNotNil(root) @@ -67,7 +67,7 @@ final class FileScannerTests: XCTestCase { let scanner = FileScanner() var root: FSNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node) = progress { root = node } + if case .completed(let node, _) = progress { root = node } } XCTAssertEqual(root?.size ?? 0, root?.children.first?.size ?? -1) @@ -86,10 +86,39 @@ final class FileScannerTests: XCTestCase { let scanner = FileScanner() var root: FSNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node) = progress { root = node } + if case .completed(let node, _) = progress { root = node } } XCTAssertEqual(root?.children.count, 1, "symlink should be skipped") XCTAssertEqual(root?.children.first?.name, "real.txt") } + + func test_scanner_marks_unreadable_directory_as_denied() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + let locked = tmp.appendingPathComponent("locked") + try FileManager.default.createDirectory(at: locked, withIntermediateDirectories: true) + try Data(repeating: 0, count: 4096).write(to: locked.appendingPathComponent("secret.bin")) + + chmod(locked.path, 0) + defer { + chmod(locked.path, 0o755) + try? FileManager.default.removeItem(at: tmp) + } + + let scanner = FileScanner() + var root: FSNode? + var deniedCount = 0 + for await progress in await scanner.scan(url: tmp) { + if case .completed(let node, let denied) = progress { + root = node + deniedCount = denied + } + } + + let lockedNode = root?.children.first { $0.name == "locked" } + XCTAssertNotNil(lockedNode) + XCTAssertTrue(lockedNode?.isAccessDenied ?? false, "locked directory should be marked as access denied") + XCTAssertGreaterThanOrEqual(deniedCount, 1) + } } diff --git a/Tests/HiddenSpaceTests.swift b/Tests/HiddenSpaceTests.swift new file mode 100644 index 0000000..5cd1d45 --- /dev/null +++ b/Tests/HiddenSpaceTests.swift @@ -0,0 +1,69 @@ +import XCTest +@testable import MacDirStat + +@MainActor +final class HiddenSpaceTests: XCTestCase { + + // MARK: - hiddenSpaceBytes math + + func test_hidden_space_math() { + // Simple case: 100 GB total, 40 GB available, 50 GB scanned -> 10 GB hidden + let hundredGB: Int64 = 100_000_000_000 + let fortyGB: Int64 = 40_000_000_000 + let fiftyGB: Int64 = 50_000_000_000 + XCTAssertEqual( + ScanViewModel.hiddenSpaceBytes(volumeTotal: hundredGB, volumeAvailable: fortyGB, scannedTotal: fiftyGB), + 10_000_000_000 + ) + } + + func test_hidden_space_below_1gb_is_nil() { + // Gap is under 1 GB — treated as noise, not worth surfacing. + let total: Int64 = 100_000_000_000 + let available: Int64 = 50_000_000_000 + let scanned: Int64 = 49_500_000_000 // gap = 500,000,000 (0.5 GB) + XCTAssertNil(ScanViewModel.hiddenSpaceBytes(volumeTotal: total, volumeAvailable: available, scannedTotal: scanned)) + } + + func test_hidden_space_zero_total_is_nil() { + XCTAssertNil(ScanViewModel.hiddenSpaceBytes(volumeTotal: 0, volumeAvailable: 0, scannedTotal: 0)) + } + + func test_hidden_space_negative_total_is_nil() { + XCTAssertNil(ScanViewModel.hiddenSpaceBytes(volumeTotal: -1, volumeAvailable: 0, scannedTotal: 0)) + } + + func test_hidden_space_negative_gap_clamps_to_nil() { + // scanned + available exceeds total (e.g. race/measurement skew) -> clamps to 0, below threshold -> nil + let total: Int64 = 100_000_000_000 + let available: Int64 = 60_000_000_000 + let scanned: Int64 = 60_000_000_000 + XCTAssertNil(ScanViewModel.hiddenSpaceBytes(volumeTotal: total, volumeAvailable: available, scannedTotal: scanned)) + } + + // MARK: - refresh preserves synthetic children + + func test_refresh_preserves_synthetic_children() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let realFileURL = tmp.appendingPathComponent("real.bin") + try Data(repeating: 0, count: 4096).write(to: realFileURL) + + let root = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + let realChild = FSNode(url: realFileURL, name: "real.bin", isDirectory: false, size: 4096, fileExtension: "bin", parent: root) + + let syntheticURL = tmp.appendingPathComponent("#hidden-space") + let syntheticChild = FSNode(url: syntheticURL, name: "Hidden & Unreadable Space", isDirectory: false, size: 5_000_000_000, fileExtension: "", parent: root) + syntheticChild.isSynthetic = true + + root.children = [realChild, syntheticChild] + root.size = realChild.size + syntheticChild.size + + let changed = ScanViewModel.refreshDirectory(node: root) + + XCTAssertTrue(root.children.contains { $0.isSynthetic }, "synthetic child must survive a refresh pass") + _ = changed + } +} From e180618df3a7ec8eb3f4c119468b79beeb2e0e6d Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 6 Jul 2026 06:35:33 +0300 Subject: [PATCH 09/41] perf(duplicates): honor cancellation inside chunked hash loops --- Sources/Duplicates/DuplicateDetector.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sources/Duplicates/DuplicateDetector.swift b/Sources/Duplicates/DuplicateDetector.swift index 2fe7dcd..a2a8c63 100644 --- a/Sources/Duplicates/DuplicateDetector.swift +++ b/Sources/Duplicates/DuplicateDetector.swift @@ -140,6 +140,7 @@ public actor DuplicateDetector { var remaining = maxBytes let chunkSize = min(65_536, maxBytes) while remaining > 0 { + if Task.isCancelled { return nil } let toRead = min(chunkSize, remaining) let chunk: Data? = autoreleasepool { try? handle.read(upToCount: toRead) } guard let chunk, !chunk.isEmpty else { break } @@ -157,6 +158,7 @@ public actor DuplicateDetector { var hasher = SHA256() let chunkSize = 1024 * 1024 while true { + if Task.isCancelled { return nil } let chunk: Data? = autoreleasepool { try? handle.read(upToCount: chunkSize) } guard let chunk, !chunk.isEmpty else { break } hasher.update(data: chunk) From 87c17125cde508acf4fbc3c23b913bdc3b2bf30d Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 6 Jul 2026 11:31:35 +0300 Subject: [PATCH 10/41] fix(refresh): inode-identity hardlink handling (no double-count on new links, promote survivor on winner deletion) --- Sources/Scanner/FSNode.swift | 14 ++++ Sources/Scanner/FileScanner.swift | 8 +++ Sources/ViewModels/ScanViewModel.swift | 79 +++++++++++++++++++--- Tests/FileScannerTests.swift | 26 ++++++++ Tests/ScanRefreshTests.swift | 91 ++++++++++++++++++++++++++ 5 files changed, 209 insertions(+), 9 deletions(-) diff --git a/Sources/Scanner/FSNode.swift b/Sources/Scanner/FSNode.swift index 2017378..b6ff272 100644 --- a/Sources/Scanner/FSNode.swift +++ b/Sources/Scanner/FSNode.swift @@ -1,5 +1,17 @@ import Foundation +// Identity of a hardlinked inode: lets the refresh path recognize that two +// directory entries are the same underlying file, the way the scanner's +// scan-time VisitedSet does. +public struct HardLinkRef: Hashable, Sendable { + public let dev: UInt64 + public let ino: UInt64 + public init(dev: UInt64, ino: UInt64) { + self.dev = dev + self.ino = ino + } +} + public final class FSNode: Identifiable, @unchecked Sendable { public let id: UUID = UUID() public let url: URL @@ -13,6 +25,8 @@ public final class FSNode: Identifiable, @unchecked Sendable { public var safetyLevel: SafetyLevel = .caution public var isAccessDenied: Bool = false public var isSynthetic: Bool = false + // Set only for files with st_nlink > 1 (both the size-carrying node and its 0-size siblings). + public var hardLinkRef: HardLinkRef? public init(url: URL, name: String, isDirectory: Bool, size: Int64, fileExtension: String, parent: FSNode? = nil) { self.url = url diff --git a/Sources/Scanner/FileScanner.swift b/Sources/Scanner/FileScanner.swift index 43da5c0..26273a8 100644 --- a/Sources/Scanner/FileScanner.swift +++ b/Sources/Scanner/FileScanner.swift @@ -297,6 +297,7 @@ private func _listDirectory(path: String, url: URL, rootDev: dev_t?, node: FSNod let allocSize = allocatedSize(st: st, visited: visited) let ext = childURL.pathExtension.lowercased() let fileNode = FSNode(url: childURL, name: name, isDirectory: false, size: allocSize, fileExtension: ext, parent: node) + if st.st_nlink > 1 { fileNode.hardLinkRef = hardLinkRef(of: st) } result.children.append(fileNode) result.totalSize += allocSize result.itemCount += 1 @@ -320,6 +321,7 @@ private func _listDirectory(path: String, url: URL, rootDev: dev_t?, node: FSNod let allocSize = allocatedSize(st: st, visited: visited) let ext = childURL.pathExtension.lowercased() let fileNode = FSNode(url: childURL, name: name, isDirectory: false, size: allocSize, fileExtension: ext, parent: node) + if st.st_nlink > 1 { fileNode.hardLinkRef = hardLinkRef(of: st) } result.children.append(fileNode) result.totalSize += allocSize result.itemCount += 1 @@ -331,6 +333,12 @@ private func _listDirectory(path: String, url: URL, rootDev: dev_t?, node: FSNod return result } +// Builds the inode identity for a hardlinked file. dev_t is a signed 32-bit value +// (can be negative for synthetic filesystems), so use bit-pattern conversion. +private func hardLinkRef(of st: stat) -> HardLinkRef { + HardLinkRef(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) +} + // Returns the file's allocated disk bytes, deduplicating hardlinks via the visited set. // Files with nlink == 1 skip the set entirely (fast path for the common case). // Hardlinked files (nlink > 1) are counted only on their first encounter. diff --git a/Sources/ViewModels/ScanViewModel.swift b/Sources/ViewModels/ScanViewModel.swift index bd907a2..eacc371 100644 --- a/Sources/ViewModels/ScanViewModel.swift +++ b/Sources/ViewModels/ScanViewModel.swift @@ -272,14 +272,34 @@ public final class ScanViewModel: ObservableObject { // Returns the allocated disk size (st_blocks * 512) for a path via lstat, or nil if // the path can't be stat'd or is a symlink. - private nonisolated static func lstatInfo(path: String) -> (isDir: Bool, isSymlink: Bool, allocatedSize: Int64, linkCount: Int)? { + private nonisolated static func lstatInfo(path: String) -> (isDir: Bool, isSymlink: Bool, allocatedSize: Int64, linkCount: Int, ref: HardLinkRef)? { var st = stat() guard lstat(path, &st) == 0 else { return nil } let mode = st.st_mode & S_IFMT let isSymlink = mode == S_IFLNK let isDir = mode == S_IFDIR let allocatedSize = Int64(st.st_blocks) * 512 - return (isDir, isSymlink, allocatedSize, Int(st.st_nlink)) + let ref = HardLinkRef(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) + return (isDir, isSymlink, allocatedSize, Int(st.st_nlink), ref) + } + + // Climbs the parent chain to the tree's root node. + private nonisolated static func rootNode(of node: FSNode) -> FSNode { + var current = node + while let parent = current.parent { current = parent } + return current + } + + // Iterative whole-tree search for a node representing the given inode. + // Only invoked when hardlinked entries (st_nlink > 1) appear or disappear, + // which is rare per refresh, so the O(tree) walk is acceptable. + private nonisolated static func firstNode(withRef ref: HardLinkRef, in root: FSNode, requireZeroSize: Bool = false) -> FSNode? { + var stack = [root] + while let n = stack.popLast() { + if n.hardLinkRef == ref, !requireZeroSize || n.size == 0 { return n } + stack.append(contentsOf: n.children) + } + return nil } // Parses the excludedFolderNames default the same way FileScanner does. @@ -305,7 +325,7 @@ public final class ScanViewModel: ObservableObject { // Filter out excluded folder names and symlinks up front, so both the // removal pass and the add/update pass agree on what's "on disk". - var onDisk: [String: (url: URL, info: (isDir: Bool, isSymlink: Bool, allocatedSize: Int64, linkCount: Int))] = [:] + var onDisk: [String: (url: URL, info: (isDir: Bool, isSymlink: Bool, allocatedSize: Int64, linkCount: Int, ref: HardLinkRef))] = [:] for url in entries { let name = url.lastPathComponent if excludedNames.contains(name) { continue } @@ -319,16 +339,34 @@ public final class ScanViewModel: ObservableObject { // Remove children that no longer exist on disk (or are now excluded/symlinks), // but never remove synthetic nodes (e.g. the "Hidden & Unreadable Space" // reconciliation entry), which never correspond to a real path. + let removedSizeCarriers = node.children.filter { + !$0.isSynthetic && !onDisk.keys.contains($0.name) && $0.hardLinkRef != nil && $0.size > 0 + } let before = node.children.count node.children.removeAll { !$0.isSynthetic && !onDisk.keys.contains($0.name) } if node.children.count != before { changed = true } + // If a removed entry carried the representative size for a hardlinked inode + // that still exists via other links, promote a surviving 0-size link + // (anywhere in the tree) to carry the size, or the bytes vanish forever. + for removed in removedSizeCarriers { + guard let ref = removed.hardLinkRef else { continue } + let root = rootNode(of: node) + guard let survivor = firstNode(withRef: ref, in: root, requireZeroSize: true), + let survivorInfo = lstatInfo(path: survivor.url.path), !survivorInfo.isDir + else { continue } + survivor.size = survivorInfo.allocatedSize + bubbleUpSizes(from: survivor.parent ?? survivor) + changed = true + } + // Add or update children for (name, entry) in onDisk { let (url, info) = entry if let existing = node.children.first(where: { $0.name == name }) { // Update size for files (directories update via recursive bubble) if !existing.isDirectory { + if info.linkCount > 1 && existing.hardLinkRef == nil { existing.hardLinkRef = info.ref } // A 0-byte node for a multi-link inode is a hardlink the initial scan // already counted elsewhere — re-statting it would double-count. if info.linkCount > 1 && existing.size == 0 { continue } @@ -340,13 +378,22 @@ public final class ScanViewModel: ObservableObject { } } else if info.isDir { // New directory — scan its whole subtree so it isn't left as a 0-byte leaf. - let child = scanSubtree(url: url, parent: node, showHiddenFiles: showHiddenFiles, excludedNames: excludedNames) + var seenRefs = Set() + let child = scanSubtree(url: url, parent: node, showHiddenFiles: showHiddenFiles, excludedNames: excludedNames, treeRoot: rootNode(of: node), seenRefs: &seenRefs) node.children.append(child) changed = true } else { - // New file + // New file. A new name for an inode the tree already accounts for + // (a hardlink created after the scan) must contribute 0 bytes. let ext = url.pathExtension.lowercased() - let child = FSNode(url: url, name: name, isDirectory: false, size: info.allocatedSize, fileExtension: ext, parent: node) + var size = info.allocatedSize + var linkRef: HardLinkRef? + if info.linkCount > 1 { + linkRef = info.ref + if firstNode(withRef: info.ref, in: rootNode(of: node)) != nil { size = 0 } + } + let child = FSNode(url: url, name: name, isDirectory: false, size: size, fileExtension: ext, parent: node) + child.hardLinkRef = linkRef child.safetyLevel = SafetyAnalyzer.level(for: child) node.children.append(child) changed = true @@ -366,7 +413,9 @@ public final class ScanViewModel: ObservableObject { url: URL, parent: FSNode?, showHiddenFiles: Bool, - excludedNames: Set + excludedNames: Set, + treeRoot: FSNode?, + seenRefs: inout Set ) -> FSNode { let name = url.lastPathComponent guard let info = lstatInfo(path: url.path), info.isDir else { @@ -395,10 +444,22 @@ public final class ScanViewModel: ObservableObject { let child: FSNode if childInfo.isDir { - child = scanSubtree(url: childURL, parent: node, showHiddenFiles: showHiddenFiles, excludedNames: excludedNames) + child = scanSubtree(url: childURL, parent: node, showHiddenFiles: showHiddenFiles, excludedNames: excludedNames, treeRoot: treeRoot, seenRefs: &seenRefs) } else { let ext = childURL.pathExtension.lowercased() - child = FSNode(url: childURL, name: childName, isDirectory: false, size: childInfo.allocatedSize, fileExtension: ext, parent: node) + var size = childInfo.allocatedSize + var linkRef: HardLinkRef? + if childInfo.linkCount > 1 { + linkRef = childInfo.ref + if seenRefs.contains(childInfo.ref) { + size = 0 + } else { + seenRefs.insert(childInfo.ref) + if let treeRoot, firstNode(withRef: childInfo.ref, in: treeRoot) != nil { size = 0 } + } + } + child = FSNode(url: childURL, name: childName, isDirectory: false, size: size, fileExtension: ext, parent: node) + child.hardLinkRef = linkRef child.safetyLevel = SafetyAnalyzer.level(for: child) } children.append(child) diff --git a/Tests/FileScannerTests.swift b/Tests/FileScannerTests.swift index 7a892aa..c4dffc6 100644 --- a/Tests/FileScannerTests.swift +++ b/Tests/FileScannerTests.swift @@ -102,6 +102,32 @@ final class FileScannerTests: XCTestCase { XCTAssertEqual(root?.children.first?.name, "real.txt") } + func test_scanner_sets_hardlink_refs() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let hard1 = tmp.appendingPathComponent("hard1.bin") + let hard2 = tmp.appendingPathComponent("hard2.bin") + try Data(repeating: 7, count: 262_144).write(to: hard1) + try FileManager.default.linkItem(at: hard1, to: hard2) + var st = stat() + XCTAssertEqual(lstat(hard1.path, &st), 0) + let fullSize = Int64(st.st_blocks) * 512 + + let scanner = FileScanner() + var root: FSNode? + for await progress in await scanner.scan(url: tmp) { + if case .completed(let node, _) = progress { root = node } + } + + let n1 = root?.children.first { $0.name == "hard1.bin" } + let n2 = root?.children.first { $0.name == "hard2.bin" } + XCTAssertNotNil(n1?.hardLinkRef, "hardlinked file must carry its inode identity") + XCTAssertEqual(n1?.hardLinkRef, n2?.hardLinkRef, "both links to one inode share the same ref") + XCTAssertEqual(root?.size, fullSize, "hardlinked inode counted exactly once") + } + func test_scanner_marks_unreadable_directory_as_denied() async throws { let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) diff --git a/Tests/ScanRefreshTests.swift b/Tests/ScanRefreshTests.swift index aecf11d..203aba0 100644 --- a/Tests/ScanRefreshTests.swift +++ b/Tests/ScanRefreshTests.swift @@ -32,6 +32,97 @@ final class ScanRefreshTests: XCTestCase { XCTAssertEqual(updated?.size, expectedSize, "refresh must use allocated size (st_blocks * 512), not logical size") } + // Returns the (dev, ino) identity for a path the way the scanner records it. + private func ref(at path: String) -> HardLinkRef { + var st = stat() + precondition(lstat(path, &st) == 0) + return HardLinkRef(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) + } + + func test_refresh_new_hardlink_not_double_counted() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let orig = tmp.appendingPathComponent("orig.bin") + try Data(repeating: 1, count: 262_144).write(to: orig) + let fullSize = allocatedSize(at: orig.path) + + // Tree state as the initial scan left it: only orig.bin existed then. + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: fullSize, fileExtension: "", parent: nil) + let origNode = FSNode(url: orig, name: "orig.bin", isDirectory: false, size: fullSize, fileExtension: "bin", parent: dirNode) + origNode.hardLinkRef = ref(at: orig.path) + dirNode.children = [origNode] + + // A new hardlink appears after the scan. + let newLink = tmp.appendingPathComponent("newlink.bin") + try FileManager.default.linkItem(at: orig, to: newLink) + + ScanViewModel.refreshDirectory(node: dirNode) + + let linkNode = dirNode.children.first { $0.name == "newlink.bin" } + XCTAssertNotNil(linkNode) + XCTAssertEqual(linkNode?.size, 0, "a new name for an already-counted inode must not add size") + XCTAssertNotNil(linkNode?.hardLinkRef) + let total = dirNode.children.reduce(Int64(0)) { $0 + $1.size } + XCTAssertEqual(total, fullSize, "directory total must not double-count the inode") + } + + func test_refresh_promotes_survivor_when_winner_deleted() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let hard1 = tmp.appendingPathComponent("hard1.bin") + let hard2 = tmp.appendingPathComponent("hard2.bin") + let hard3 = tmp.appendingPathComponent("hard3.bin") + try Data(repeating: 2, count: 262_144).write(to: hard1) + try FileManager.default.linkItem(at: hard1, to: hard2) + try FileManager.default.linkItem(at: hard1, to: hard3) + let fullSize = allocatedSize(at: hard1.path) + let inodeRef = ref(at: hard1.path) + + // Tree as the initial scan left it: hard1 carries the size, the others are 0. + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: fullSize, fileExtension: "", parent: nil) + let winner = FSNode(url: hard1, name: "hard1.bin", isDirectory: false, size: fullSize, fileExtension: "bin", parent: dirNode) + let loser2 = FSNode(url: hard2, name: "hard2.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirNode) + let loser3 = FSNode(url: hard3, name: "hard3.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirNode) + for n in [winner, loser2, loser3] { n.hardLinkRef = inodeRef } + dirNode.children = [winner, loser2, loser3] + + // The size-carrying link is deleted; the inode still exists via the survivors. + try FileManager.default.removeItem(at: hard1) + + ScanViewModel.refreshDirectory(node: dirNode) + + XCTAssertEqual(dirNode.children.count, 2) + let sizes = dirNode.children.map(\.size).sorted(by: >) + XCTAssertEqual(sizes, [fullSize, 0], "one survivor must be promoted to carry the inode's size") + } + + func test_scan_subtree_dedupes_internal_hardlinks() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + // Tree as scanned: empty directory. + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + + // A new directory appears containing two links to one inode. + let newDir = tmp.appendingPathComponent("newdir") + try FileManager.default.createDirectory(at: newDir, withIntermediateDirectories: true) + let a = newDir.appendingPathComponent("a.bin") + try Data(repeating: 3, count: 262_144).write(to: a) + try FileManager.default.linkItem(at: a, to: newDir.appendingPathComponent("b.bin")) + let fullSize = allocatedSize(at: a.path) + + ScanViewModel.refreshDirectory(node: dirNode) + + let newDirNode = dirNode.children.first { $0.name == "newdir" } + XCTAssertNotNil(newDirNode) + XCTAssertEqual(newDirNode?.size, fullSize, "hardlinked pair inside a new directory must count once") + } + func test_refresh_preserves_hardlink_dedup() throws { let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) From d04a2dea668adcd314e49ba5f4a6a910c33bd500 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 6 Jul 2026 11:53:08 +0300 Subject: [PATCH 11/41] =?UTF-8?q?feat(fda):=20guided=20Full=20Disk=20Acces?= =?UTF-8?q?s=20onboarding=20=E2=80=94=20explainer=20sheet,=20live=20grant?= =?UTF-8?q?=20detection,=20relaunch=20+=20auto-rescan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/App/ContentView.swift | 18 +-- Sources/App/FullDiskAccessSheet.swift | 151 +++++++++++++++++++++++++ Sources/App/MacDirStatApp.swift | 5 + Sources/ViewModels/ScanViewModel.swift | 65 ++++++++++- Tests/FDAPromptTests.swift | 48 ++++++++ 5 files changed, 279 insertions(+), 8 deletions(-) create mode 100644 Sources/App/FullDiskAccessSheet.swift create mode 100644 Tests/FDAPromptTests.swift diff --git a/Sources/App/ContentView.swift b/Sources/App/ContentView.swift index 4c2fdd2..5ef85bf 100644 --- a/Sources/App/ContentView.swift +++ b/Sources/App/ContentView.swift @@ -110,6 +110,9 @@ struct ContentView: View { .onReceive(NotificationCenter.default.publisher(for: .exportCSV)) { _ in vm.exportCSV() } + .sheet(isPresented: $vm.showFDASheet) { + FullDiskAccessSheet() + } } // MARK: - Tab picker @@ -191,7 +194,9 @@ struct ContentView: View { private var detailContent: some View { VStack(spacing: 0) { if !vm.hasFullDiskAccess || (vm.deniedCount > 0 && !vm.isScanning) { - FullDiskAccessBanner(hasFullDiskAccess: vm.hasFullDiskAccess, deniedCount: vm.deniedCount) + FullDiskAccessBanner(hasFullDiskAccess: vm.hasFullDiskAccess, deniedCount: vm.deniedCount) { + vm.showFDASheet = true + } } if vm.root == nil && !vm.isScanning && !vm.isComputingLayout { WelcomeView() @@ -376,6 +381,7 @@ private struct FolderTitleView: View { private struct FullDiskAccessBanner: View { let hasFullDiskAccess: Bool let deniedCount: Int + let onOpenSheet: () -> Void private let title = "Full Disk Access required for complete results" @@ -403,12 +409,10 @@ private struct FullDiskAccessBanner: View { Spacer() - Button("Open Settings") { - NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_AllFiles")!) - } - .font(.system(size: 11, weight: .medium)) - .buttonStyle(.bordered) - .controlSize(.small) + Button("Open Settings", action: onOpenSheet) + .font(.system(size: 11, weight: .medium)) + .buttonStyle(.bordered) + .controlSize(.small) } .padding(.horizontal, 14) .padding(.vertical, 9) diff --git a/Sources/App/FullDiskAccessSheet.swift b/Sources/App/FullDiskAccessSheet.swift new file mode 100644 index 0000000..70826e7 --- /dev/null +++ b/Sources/App/FullDiskAccessSheet.swift @@ -0,0 +1,151 @@ +import SwiftUI +import AppKit + +/// Guided Full Disk Access onboarding. macOS provides no API to grant FDA +/// programmatically, so the best achievable flow is: explain what's blocked and why, +/// jump straight to the right System Settings pane, detect the grant while the sheet +/// is open, then offer to relaunch (required for the new permission to take effect) +/// and auto-resume the scan the user was on. +struct FullDiskAccessSheet: View { + @EnvironmentObject private var vm: ScanViewModel + @Environment(\.dismiss) private var dismiss + + var body: some View { + VStack(spacing: 22) { + if vm.hasFullDiskAccess { + grantedState + } else { + missingState + } + } + .padding(28) + .frame(width: 460) + .task { + // Poll for a live permission change while the sheet is on screen — macOS + // has no notification for TCC grants, so this is the only way to react + // to the user flipping the toggle in System Settings without closing us. + while !Task.isCancelled { + try? await Task.sleep(for: .seconds(1.5)) + guard !Task.isCancelled else { break } + vm.recheckFullDiskAccess() + } + } + } + + // MARK: - State A: access missing + + private var missingState: some View { + VStack(spacing: 20) { + Image(systemName: "lock.shield") + .font(.system(size: 44, weight: .medium)) + .foregroundStyle(.tint) + .symbolRenderingMode(.hierarchical) + .padding(22) + .glassTintedCard(tint: .accentColor, cornerRadius: 200) + + VStack(spacing: 8) { + Text("See your whole disk") + .font(.system(size: 20, weight: .bold, design: .rounded)) + Text(explainerBody) + .font(.system(size: 12.5)) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .fixedSize(horizontal: false, vertical: true) + } + + steps + + VStack(spacing: 10) { + Button("Open System Settings") { + NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_AllFiles")!) + } + .keyboardShortcut(.defaultAction) + .glassProminentButton() + .controlSize(.large) + .frame(maxWidth: .infinity) + + Button("Not Now") { dismiss() } + .glassButton() + .controlSize(.large) + .frame(maxWidth: .infinity) + + Button("Don't ask again") { + UserDefaults.standard.set(true, forKey: "fdaPromptSuppressed") + dismiss() + } + .buttonStyle(.plain) + .font(.system(size: 11)) + .foregroundStyle(.tertiary) + } + } + } + + private var explainerBody: String { + let base = "macOS protects some folders (Documents, Desktop, other apps' data) until you grant Full Disk Access. MacDirStat reads sizes only — nothing is modified, collected, or sent anywhere." + guard vm.deniedCount > 0 else { return base } + let folders = vm.deniedCount == 1 ? "1 folder was" : "\(vm.deniedCount) folders were" + return "\(base) \(folders) blocked during your last scan." + } + + private var steps: some View { + VStack(alignment: .leading, spacing: 10) { + stepRow(1, "Open System Settings") + stepRow(2, "Find MacDirStat in the list and switch it on") + stepRow(3, "Come back here — we'll take it from there") + } + .padding(14) + .frame(maxWidth: .infinity, alignment: .leading) + .glassCard(cornerRadius: 14) + } + + private func stepRow(_ number: Int, _ text: String) -> some View { + HStack(alignment: .top, spacing: 10) { + Text("\(number)") + .font(.system(size: 11, weight: .bold, design: .rounded)) + .foregroundStyle(.white) + .frame(width: 18, height: 18) + .background(Circle().fill(.tint)) + Text(text) + .font(.system(size: 12)) + .foregroundStyle(.primary) + } + } + + // MARK: - State B: access granted, needs relaunch + + private var grantedState: some View { + VStack(spacing: 20) { + Image(systemName: "checkmark.shield.fill") + .font(.system(size: 44, weight: .medium)) + .foregroundStyle(.green) + .symbolRenderingMode(.hierarchical) + .padding(22) + .glassTintedCard(tint: .green, cornerRadius: 200) + + VStack(spacing: 8) { + Text("Access granted!") + .font(.system(size: 20, weight: .bold, design: .rounded)) + Text("MacDirStat needs to relaunch for macOS to apply the new permission. It will rescan automatically.") + .font(.system(size: 12.5)) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .fixedSize(horizontal: false, vertical: true) + } + + VStack(spacing: 10) { + Button("Relaunch & Rescan") { + vm.relaunchForFullDiskAccess() + } + .keyboardShortcut(.defaultAction) + .glassProminentButton() + .controlSize(.large) + .frame(maxWidth: .infinity) + + Button("Later") { dismiss() } + .glassButton() + .controlSize(.large) + .frame(maxWidth: .infinity) + } + } + } +} diff --git a/Sources/App/MacDirStatApp.swift b/Sources/App/MacDirStatApp.swift index 1b72cc3..739bb6e 100644 --- a/Sources/App/MacDirStatApp.swift +++ b/Sources/App/MacDirStatApp.swift @@ -40,6 +40,11 @@ struct MacDirStatApp: App { NSWorkspace.shared.open(URL(string: "https://ti-03.github.io/MacDirStat/")!) } } + CommandGroup(after: .help) { + Button("Grant Full Disk Access…") { + vm.showFDASheet = true + } + } } } } diff --git a/Sources/ViewModels/ScanViewModel.swift b/Sources/ViewModels/ScanViewModel.swift index eacc371..efff548 100644 --- a/Sources/ViewModels/ScanViewModel.swift +++ b/Sources/ViewModels/ScanViewModel.swift @@ -21,6 +21,7 @@ public final class ScanViewModel: ObservableObject { @Published public var hasFullDiskAccess: Bool = true @Published public var isWatching: Bool = false @Published public var deniedCount: Int = 0 + @Published public var showFDASheet: Bool = false public var treemapRoot: FSNode? { drillStack.last ?? root } @@ -35,6 +36,7 @@ public final class ScanViewModel: ObservableObject { private var layoutGeneration: Int = 0 private var securityScopedURL: URL? private var memoryPressureSource: DispatchSourceMemoryPressure? + private var fdaSheetShownThisLaunch = false public init() { UserDefaults.standard.register(defaults: [ @@ -49,7 +51,21 @@ public final class ScanViewModel: ObservableObject { ]) setupMemoryPressureHandler() checkFullDiskAccess() - if UserDefaults.standard.bool(forKey: "autoScanLastFolder"), + + // A relaunch triggered from the Full Disk Access flow leaves behind the path + // that was being scanned, so the new instance can resume right where the user + // left off instead of landing back on the welcome screen. + var resumedPendingRescan = false + if let pending = UserDefaults.standard.string(forKey: "fdaPendingRescanPath") { + UserDefaults.standard.removeObject(forKey: "fdaPendingRescanPath") + if FileManager.default.fileExists(atPath: pending) { + scan(url: URL(fileURLWithPath: pending)) + resumedPendingRescan = true + } + } + + if !resumedPendingRescan, + UserDefaults.standard.bool(forKey: "autoScanLastFolder"), let path = UserDefaults.standard.string(forKey: "lastScannedPath"), FileManager.default.fileExists(atPath: path) { scan(url: URL(fileURLWithPath: path)) @@ -66,6 +82,44 @@ public final class ScanViewModel: ObservableObject { hasFullDiskAccess = FileManager.default.isReadableFile(atPath: probe) } + /// Public wrapper so the onboarding sheet can poll for a live permission change + /// without exposing the private TCC probe itself. + public func recheckFullDiskAccess() { + checkFullDiskAccess() + } + + /// Pure decision logic for whether the guided Full Disk Access sheet should be + /// offered after a scan completes: only when access is actually missing, the scan + /// hit blocked folders, and the user hasn't opted out. + nonisolated static func shouldOfferFullDiskAccess(deniedCount: Int, hasFullDiskAccess: Bool, suppressed: Bool) -> Bool { + !hasFullDiskAccess && deniedCount > 0 && !suppressed + } + + /// Relaunches the app so macOS re-evaluates the Full Disk Access grant, and leaves + /// a breadcrumb so the new instance automatically resumes the scan the user was on. + public func relaunchForFullDiskAccess() { + let pathToResume = scanURL?.path ?? UserDefaults.standard.string(forKey: "lastScannedPath") + if let pathToResume { + UserDefaults.standard.set(pathToResume, forKey: "fdaPendingRescanPath") + } + + guard Bundle.main.bundlePath.hasSuffix(".app") else { + // Debug binary, not an app bundle — there's nothing to relaunch + // programmatically. The developer restarts the process by hand. + NSApp.terminate(nil) + return + } + + let configuration = NSWorkspace.OpenConfiguration() + configuration.createsNewApplicationInstance = true + NSWorkspace.shared.openApplication(at: Bundle.main.bundleURL, configuration: configuration) { _, _ in + // Give the new instance a moment to spawn before this one exits. + DispatchQueue.main.asyncAfter(deadline: .now() + 0.4) { + NSApp.terminate(nil) + } + } + } + private func setupMemoryPressureHandler() { let source = DispatchSource.makeMemoryPressureSource(eventMask: [.warning, .critical], queue: .main) source.setEventHandler { [weak self] in @@ -121,6 +175,15 @@ public final class ScanViewModel: ObservableObject { case .completed(let node, let denied): self.isScanning = false self.deniedCount = denied + if !self.fdaSheetShownThisLaunch, + Self.shouldOfferFullDiskAccess( + deniedCount: denied, + hasFullDiskAccess: self.hasFullDiskAccess, + suppressed: UserDefaults.standard.bool(forKey: "fdaPromptSuppressed") + ) { + self.showFDASheet = true + self.fdaSheetShownThisLaunch = true + } // If the scanned root is a volume mount point, the file total will // always fall short of Finder's "used" figure (APFS snapshots, // purgeable space, excluded/unreadable folders). Make that gap diff --git a/Tests/FDAPromptTests.swift b/Tests/FDAPromptTests.swift new file mode 100644 index 0000000..ab29d44 --- /dev/null +++ b/Tests/FDAPromptTests.swift @@ -0,0 +1,48 @@ +import XCTest +@testable import MacDirStat + +@MainActor +final class FDAPromptTests: XCTestCase { + + // MARK: - shouldOfferFullDiskAccess + + func test_offers_when_fda_missing_and_folders_denied() { + XCTAssertTrue( + ScanViewModel.shouldOfferFullDiskAccess( + deniedCount: 241, + hasFullDiskAccess: false, + suppressed: false + ) + ) + } + + func test_does_not_offer_when_fda_already_granted() { + XCTAssertFalse( + ScanViewModel.shouldOfferFullDiskAccess( + deniedCount: 241, + hasFullDiskAccess: true, + suppressed: false + ) + ) + } + + func test_does_not_offer_when_no_folders_denied() { + XCTAssertFalse( + ScanViewModel.shouldOfferFullDiskAccess( + deniedCount: 0, + hasFullDiskAccess: false, + suppressed: false + ) + ) + } + + func test_does_not_offer_when_suppressed() { + XCTAssertFalse( + ScanViewModel.shouldOfferFullDiskAccess( + deniedCount: 241, + hasFullDiskAccess: false, + suppressed: true + ) + ) + } +} From 462717b89e1117cea49fa01d8f8aea012a4f5aa8 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 6 Jul 2026 12:10:52 +0300 Subject: [PATCH 12/41] build(xcode): add FullDiskAccessSheet.swift to the app target --- MacDirStat.xcodeproj/project.pbxproj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MacDirStat.xcodeproj/project.pbxproj b/MacDirStat.xcodeproj/project.pbxproj index 7f0bde7..5aa18fe 100644 --- a/MacDirStat.xcodeproj/project.pbxproj +++ b/MacDirStat.xcodeproj/project.pbxproj @@ -16,6 +16,7 @@ 3881FC74433C93187BF941AF /* FSNode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 35F5AEBBF9DECB5CB793339A /* FSNode.swift */; }; 3B190FDF9A024126B4ED2872 /* GlassWindow.swift in Sources */ = {isa = PBXBuildFile; fileRef = 08B5316639CA4AC8990FBDD4 /* GlassWindow.swift */; }; D1E2F3A4B5C607182930AABD /* GlassCompat.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F607182930AABC /* GlassCompat.swift */; }; + FDA0FDA0FDA0FDA0FDA00002 /* FullDiskAccessSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = FDA0FDA0FDA0FDA0FDA00001 /* FullDiskAccessSheet.swift */; }; 3F67AACE53A6C32467A5191C /* TreemapRenderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 670402F6A7AEB43C1D764E04 /* TreemapRenderer.swift */; }; 7AE6B3CB6CDE22EAA3AA46B3 /* DuplicateDetector.swift in Sources */ = {isa = PBXBuildFile; fileRef = 42B58422D704C6E5265DA664 /* DuplicateDetector.swift */; }; 86B25923164A00AB10D72C9B /* ScanProgress.swift in Sources */ = {isa = PBXBuildFile; fileRef = DBCAD22DB54E68077B842421 /* ScanProgress.swift */; }; @@ -40,6 +41,7 @@ 074CBBA0222600D63C734574 /* ExtensionColorMap.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExtensionColorMap.swift; sourceTree = ""; }; 08B5316639CA4AC8990FBDD4 /* GlassWindow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GlassWindow.swift; sourceTree = ""; }; A1B2C3D4E5F607182930AABC /* GlassCompat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GlassCompat.swift; sourceTree = ""; }; + FDA0FDA0FDA0FDA0FDA00001 /* FullDiskAccessSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FullDiskAccessSheet.swift; sourceTree = ""; }; 2767B8AD8F6829C36A3576E6 /* DuplicatesView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DuplicatesView.swift; sourceTree = ""; }; 35F5AEBBF9DECB5CB793339A /* FSNode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FSNode.swift; sourceTree = ""; }; 42B58422D704C6E5265DA664 /* DuplicateDetector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DuplicateDetector.swift; sourceTree = ""; }; @@ -182,6 +184,7 @@ 7F8FDBB44CE9624311841E36 /* ContentView.swift */, 08B5316639CA4AC8990FBDD4 /* GlassWindow.swift */, A1B2C3D4E5F607182930AABC /* GlassCompat.swift */, + FDA0FDA0FDA0FDA0FDA00001 /* FullDiskAccessSheet.swift */, 783902530DE4EA8040806BC9 /* SettingsView.swift */, ); path = App; @@ -323,6 +326,7 @@ 3864E5104A08A42FF56D7880 /* ContentView.swift in Sources */, 3B190FDF9A024126B4ED2872 /* GlassWindow.swift in Sources */, D1E2F3A4B5C607182930AABD /* GlassCompat.swift in Sources */, + FDA0FDA0FDA0FDA0FDA00002 /* FullDiskAccessSheet.swift in Sources */, E5BB63BED418F08CF4727F72 /* SettingsView.swift in Sources */, 7AE6B3CB6CDE22EAA3AA46B3 /* DuplicateDetector.swift in Sources */, C001072776950E6399A45B18 /* ExtensionColorMap.swift in Sources */, From 27a334c26a1f2bbb2ccf3b095d8198df631fe86f Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 12:16:23 +0300 Subject: [PATCH 13/41] perf(scanner): getattrlistbulk enumeration + iterative bounded traversal Replace per-entry readdir+fstatat with getattrlistbulk(2) batch enumeration (name + metadata per syscall) with a readdir fallback for volumes that don't support it. Traversal is now an iterative work queue drained by a bounded worker pool (min(max(2,cores/2),8)) instead of unbounded recursive task fan-out. Parity guaranteed by a fingerprint test comparing bulk vs forced-fallback scans. Bench on /Applications: 1.63s bulk vs 2.02s fallback. All scan semantics preserved: allocated sizes, symlink/hidden/exclusion skips, mount-point and firmlink (dev,ino) dedup, hardlink dedup, access-denied surfacing. Public FileScanner/FSNode/ScanProgress API unchanged. --- Sources/Scanner/BulkDirectoryEnumerator.swift | 286 +++++++++++ Sources/Scanner/FileScanner.swift | 461 ++++++++++-------- Tests/BulkScannerTests.swift | 346 +++++++++++++ 3 files changed, 902 insertions(+), 191 deletions(-) create mode 100644 Sources/Scanner/BulkDirectoryEnumerator.swift create mode 100644 Tests/BulkScannerTests.swift diff --git a/Sources/Scanner/BulkDirectoryEnumerator.swift b/Sources/Scanner/BulkDirectoryEnumerator.swift new file mode 100644 index 0000000..17e65ac --- /dev/null +++ b/Sources/Scanner/BulkDirectoryEnumerator.swift @@ -0,0 +1,286 @@ +import Darwin +import Foundation + +// The kind of a directory entry as reported by either enumeration path. +// Symlinks and special files (fifo/char/block/socket) are surfaced so the +// caller can apply the "skip entirely" rule (see FileScanner.swift semantics); +// this module never resolves symlink targets. +enum BulkEntryKind: Sendable { + case directory + case file + case symlink + case other +} + +// One immediate child of a directory, with just enough metadata for the +// scanner to apply its rules (mount-point / hardlink / dedup checks) without +// an additional per-entry stat call. `dev`/`ino`/`linkCount`/`allocatedSize` +// are meaningful for `.directory` (dev, ino only) and `.file` (all four); +// they are zero for `.symlink`/`.other`, which the scanner always skips. +struct BulkDirEntry: Sendable { + let name: String + let kind: BulkEntryKind + let dev: UInt64 + let ino: UInt64 + let linkCount: UInt32 + let allocatedSize: Int64 +} + +// Thrown when `getattrlistbulk` can't be used for a directory at all (the +// filesystem doesn't support it) or when the packed attribute buffer can't be +// parsed with confidence (unexpected layout, truncated entry, non-UTF8 name). +// Callers should retry the same directory with `fallbackEnumerateDirectory(fd:)`. +struct BulkEnumerationUnavailable: Error {} + +private let bulkBufferCapacity = 64 * 1_024 +private let unsupportedBulkErrors: Set = [EINVAL, ENOTSUP, ENOSYS] + +// Only the attributes FileScanner actually needs: identity (dev/ino via +// DEVID+FILEID), name, and object type for every entry, plus link count and +// allocated size for regular files. Keeping the request minimal keeps the +// packed buffer layout small and simple to parse. +private var requestedAttrList: attrlist { + var list = attrlist() + list.bitmapcount = UInt16(ATTR_BIT_MAP_COUNT) + list.commonattr = attrgroup_t(ATTR_CMN_RETURNED_ATTRS) + | attrgroup_t(ATTR_CMN_ERROR) + | attrgroup_t(ATTR_CMN_NAME) + | attrgroup_t(ATTR_CMN_OBJTYPE) + | attrgroup_t(ATTR_CMN_DEVID) + | attrgroup_t(ATTR_CMN_FILEID) + list.fileattr = attrgroup_t(ATTR_FILE_LINKCOUNT) | attrgroup_t(ATTR_FILE_ALLOCSIZE) + return list +} + +private let requiredCommonAttrs = attrgroup_t(ATTR_CMN_NAME) + | attrgroup_t(ATTR_CMN_OBJTYPE) + | attrgroup_t(ATTR_CMN_DEVID) + | attrgroup_t(ATTR_CMN_FILEID) +private let requiredFileAttrs = attrgroup_t(ATTR_FILE_LINKCOUNT) | attrgroup_t(ATTR_FILE_ALLOCSIZE) + +// Enumerates the immediate children of an already-open directory descriptor +// via getattrlistbulk(2): one syscall per ~64 KB batch returns name + metadata +// together for every entry in the batch, instead of one syscall per entry +// (readdir) plus one more (fstatat) to get its metadata. +// +// Does not close `fd`; the caller owns its lifetime. +func enumerateDirectoryBulk(fd: Int32) throws -> [BulkDirEntry] { + var attrs = requestedAttrList + var results: [BulkDirEntry] = [] + let buffer = UnsafeMutableRawBufferPointer.allocate(byteCount: bulkBufferCapacity, alignment: 8) + defer { buffer.deallocate() } + guard let bufferBase = buffer.baseAddress else { throw BulkEnumerationUnavailable() } + + while true { + let count = getattrlistbulk(fd, &attrs, bufferBase, buffer.count, UInt64(FSOPT_PACK_INVAL_ATTRS)) + if count < 0 { + if unsupportedBulkErrors.contains(errno) { throw BulkEnumerationUnavailable() } + throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) + } + if count == 0 { break } + + try parseBulkBatch( + bufferBase: UnsafeRawPointer(bufferBase), + bufferByteCount: buffer.count, + entryCount: Int(count), + into: &results + ) + } + + return results +} + +// Fallback listing used when bulk enumeration is unavailable for a directory, +// or when `MDS_FORCE_FALLBACK_ENUM=1` forces it everywhere (parity testing). +// Uses the traditional readdir + fstatat path, duplicating `fd` so the +// caller's descriptor is unaffected by fdopendir's ownership rules. +func fallbackEnumerateDirectory(fd: Int32) throws -> [BulkDirEntry] { + let duped = dup(fd) + guard duped >= 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + guard let dir = fdopendir(duped) else { + let openErrno = errno + close(duped) + throw NSError(domain: NSPOSIXErrorDomain, code: Int(openErrno)) + } + defer { closedir(dir) } + let dfd = dirfd(dir) + + var results: [BulkDirEntry] = [] + while let entry = readdir(dir) { + let nameBytes = entry.pointee.d_name + let name: String = withUnsafeBytes(of: nameBytes) { ptr in + let bytes = ptr.bindMemory(to: CChar.self) + return String(cString: bytes.baseAddress!) + } + guard name != "." && name != ".." else { continue } + + let dtype = entry.pointee.d_type + // Fast path: these types never need a stat call, the scanner skips them regardless. + if dtype == DT_LNK { + results.append(BulkDirEntry(name: name, kind: .symlink, dev: 0, ino: 0, linkCount: 0, allocatedSize: 0)) + continue + } + if dtype == DT_FIFO || dtype == DT_CHR || dtype == DT_BLK || dtype == DT_SOCK { + results.append(BulkDirEntry(name: name, kind: .other, dev: 0, ino: 0, linkCount: 0, allocatedSize: 0)) + continue + } + + // DT_DIR / DT_REG / DT_UNKNOWN (some network/FUSE filesystems don't + // populate d_type): resolve the real type via fstatat. + var st = stat() + guard fstatat(dfd, name, &st, AT_SYMLINK_NOFOLLOW) == 0 else { continue } + let mode = st.st_mode & S_IFMT + let kind: BulkEntryKind + switch mode { + case S_IFDIR: kind = .directory + case S_IFREG: kind = .file + case S_IFLNK: kind = .symlink + default: kind = .other + } + let allocatedSize = kind == .file ? Int64(st.st_blocks) * 512 : 0 + results.append(BulkDirEntry( + name: name, + kind: kind, + dev: UInt64(bitPattern: Int64(st.st_dev)), + ino: UInt64(st.st_ino), + linkCount: UInt32(st.st_nlink), + allocatedSize: allocatedSize + )) + } + return results +} + +// MARK: - getattrlistbulk buffer parsing + +// Reads fixed-size fields out of one packed getattrlistbulk entry, advancing +// past each field's 4-byte-aligned footprint (matches the kernel's packing). +private struct AttributeCursor { + var current: UnsafeRawPointer + let end: UnsafeRawPointer + + mutating func read() -> T? { + let size = MemoryLayout.size + let alignedSize = (size + 3) & ~3 + guard alignedSize <= current.distance(to: end) else { return nil } + let value = current.loadUnaligned(as: T.self) + current = current.advanced(by: alignedSize) + return value + } +} + +private func parseBulkBatch( + bufferBase: UnsafeRawPointer, + bufferByteCount: Int, + entryCount: Int, + into results: inout [BulkDirEntry] +) throws { + let bufferEnd = bufferBase.advanced(by: bufferByteCount) + var entryAddress = bufferBase + + for _ in 0...size <= entryAddress.distance(to: bufferEnd) else { + throw BulkEnumerationUnavailable() + } + let entryLength = Int(entryAddress.loadUnaligned(as: UInt32.self)) + guard entryLength >= MemoryLayout.size, + entryLength <= entryAddress.distance(to: bufferEnd) else { + throw BulkEnumerationUnavailable() + } + let entryEnd = entryAddress.advanced(by: entryLength) + + if let entry = try parseBulkEntry(entryStart: entryAddress, entryEnd: entryEnd) { + results.append(entry) + } + // else: this single entry had a per-entry error (ATTR_CMN_ERROR set, + // e.g. a race with deletion) - skip it, keep parsing the batch. + entryAddress = entryEnd + } +} + +// Parses one packed entry. Field order (for the attributes requested above) +// follows getattrlist(2)'s fixed declaration order: ATTR_CMN_RETURNED_ATTRS, +// ATTR_CMN_ERROR, ATTR_CMN_NAME (attrreference_t), ATTR_CMN_DEVID, +// ATTR_CMN_OBJTYPE, ATTR_CMN_FILEID, then (regular files only) +// ATTR_FILE_LINKCOUNT, ATTR_FILE_ALLOCSIZE. +private func parseBulkEntry(entryStart: UnsafeRawPointer, entryEnd: UnsafeRawPointer) throws -> BulkDirEntry? { + var cursor = AttributeCursor(current: entryStart.advanced(by: MemoryLayout.size), end: entryEnd) + + guard let returned: attribute_set_t = cursor.read(), + let entryError: UInt32 = cursor.read() else { + throw BulkEnumerationUnavailable() + } + + let nameRefAddress = cursor.current + guard let nameRef: attrreference_t = cursor.read() else { throw BulkEnumerationUnavailable() } + guard let name = parseBulkName(reference: nameRef, referenceAddress: nameRefAddress, entryEnd: entryEnd) else { + throw BulkEnumerationUnavailable() + } + + guard let deviceID: dev_t = cursor.read(), + let objectType: fsobj_type_t = cursor.read(), + let fileID: UInt64 = cursor.read() else { + throw BulkEnumerationUnavailable() + } + + // A per-entry error means the filesystem couldn't produce metadata for + // this one child (e.g. deleted mid-listing); skip only this entry. + if entryError != 0 { return nil } + + guard returned.commonattr & requiredCommonAttrs == requiredCommonAttrs else { + throw BulkEnumerationUnavailable() + } + + let kind: BulkEntryKind + if objectType == VDIR.rawValue { kind = .directory } + else if objectType == VREG.rawValue { kind = .file } + else if objectType == VLNK.rawValue { kind = .symlink } + else { kind = .other } + + var linkCount: UInt32 = 1 + var allocatedSize: Int64 = 0 + if kind == .file { + // File attributes are only present in the buffer for regular files. + guard returned.fileattr & requiredFileAttrs == requiredFileAttrs else { + throw BulkEnumerationUnavailable() + } + guard let readLinkCount: UInt32 = cursor.read(), + let readAllocSize: off_t = cursor.read() else { + throw BulkEnumerationUnavailable() + } + linkCount = readLinkCount + allocatedSize = Int64(readAllocSize) + } + + return BulkDirEntry( + name: name, + kind: kind, + dev: UInt64(bitPattern: Int64(deviceID)), + ino: fileID, + linkCount: linkCount, + allocatedSize: max(allocatedSize, 0) + ) +} + +// `attr_dataoffset` is relative to the attrreference_t's own address. +private func parseBulkName( + reference: attrreference_t, + referenceAddress: UnsafeRawPointer, + entryEnd: UnsafeRawPointer +) -> String? { + guard reference.attr_dataoffset >= 0 else { return nil } + let dataOffset = Int(reference.attr_dataoffset) + guard dataOffset <= referenceAddress.distance(to: entryEnd) else { return nil } + let start = referenceAddress.advanced(by: dataOffset) + let byteCount = Int(reference.attr_length) + guard byteCount > 0, byteCount <= start.distance(to: entryEnd) else { return nil } + + let bytes = UnsafeRawBufferPointer(start: start, count: byteCount) + // The name is NUL-terminated; trim the trailing NUL before decoding. + let stringByteCount = bytes.last == 0 ? byteCount - 1 : byteCount + guard stringByteCount > 0 else { return nil } + let nameBytes = UnsafeRawBufferPointer(start: start, count: stringByteCount).bindMemory(to: UInt8.self) + guard let name = String(bytes: nameBytes, encoding: .utf8), name.utf8.count == stringByteCount else { + return nil + } + return name +} diff --git a/Sources/Scanner/FileScanner.swift b/Sources/Scanner/FileScanner.swift index 26273a8..7a6b5f3 100644 --- a/Sources/Scanner/FileScanner.swift +++ b/Sources/Scanner/FileScanner.swift @@ -2,17 +2,20 @@ import Foundation // Thread-safe set tracking visited (dev, ino) pairs to prevent double-counting. // On macOS, /System/Volumes/Data/Applications shares the same inode as /Applications, etc. +// Shared by both directory dedup (firmlinks/mount aliases) and hardlinked-file +// dedup: a directory's inode and a file's inode never collide on one device, +// so the two uses safely share one (dev, ino) namespace, exactly as before. private final class VisitedSet: @unchecked Sendable { private var lock = os_unfair_lock() private var set = Set() private struct DevIno: Hashable { - let dev: dev_t - let ino: ino_t + let dev: UInt64 + let ino: UInt64 } // Returns true if this (dev, ino) was NOT previously seen (and marks it seen). - func visit(dev: dev_t, ino: ino_t) -> Bool { + func visit(dev: UInt64, ino: UInt64) -> Bool { let key = DevIno(dev: dev, ino: ino) os_unfair_lock_lock(&lock) let inserted = set.insert(key).inserted @@ -56,45 +59,23 @@ private final class ProgressCounter: @unchecked Sendable { } } -// Thread-safe counter bounding the number of concurrently-spawned subtree tasks. -// Prevents unbounded task-group fan-out (and the associated flood of open dirfds) -// on very deep/wide directory trees. -private final class TaskBudget: @unchecked Sendable { - private var lock = os_unfair_lock() - private var remaining: Int - - init(limit: Int) { - self.remaining = limit - } - - // Returns true if a slot was reserved (caller must call release() when done). - func tryAcquire() -> Bool { - os_unfair_lock_lock(&lock) - defer { os_unfair_lock_unlock(&lock) } - guard remaining > 0 else { return false } - remaining -= 1 - return true - } - - func release() { - os_unfair_lock_lock(&lock) - remaining += 1 - os_unfair_lock_unlock(&lock) - } -} - // Snapshot of scan-time settings, read once per scan (not per directory) to avoid -// UserDefaults overhead on the hot path. +// UserDefaults / environment overhead on the hot path. struct ScanConfig: Sendable { let excludedNames: Set let showHiddenFiles: Bool + // Forces the readdir+fstatat fallback path for every directory in the scan, + // bypassing getattrlistbulk entirely. Used by parity tests to compare the + // two enumeration strategies against each other. + let forceFallbackEnum: Bool static func loadFromUserDefaults() -> ScanConfig { let rawExcluded = UserDefaults.standard.string(forKey: "excludedFolderNames") ?? ".git,node_modules,DerivedData,.Trash" let excludedNames = Set(rawExcluded.components(separatedBy: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty }) let showHiddenFiles = UserDefaults.standard.bool(forKey: "showHiddenFiles") - return ScanConfig(excludedNames: excludedNames, showHiddenFiles: showHiddenFiles) + let forceFallbackEnum = ProcessInfo.processInfo.environment["MDS_FORCE_FALLBACK_ENUM"] == "1" + return ScanConfig(excludedNames: excludedNames, showHiddenFiles: showHiddenFiles, forceFallbackEnum: forceFallbackEnum) } } @@ -115,10 +96,6 @@ public actor FileScanner { let counter = ProgressCounter() let visited = VisitedSet() let config = ScanConfig.loadFromUserDefaults() - let taskBudget = TaskBudget(limit: min(max(4, ProcessInfo.processInfo.activeProcessorCount), 16)) - // Get the root device ID to detect mount points - var rootStat = stat() - let rootDev: dev_t? = (stat(url.path, &rootStat) == 0) ? rootStat.st_dev : nil // Emit periodic progress updates every 0.2s let progressTask = Task { @@ -130,7 +107,7 @@ public actor FileScanner { } do { - let root = try await _buildTree(path: url.path, url: url, parent: nil, rootDev: rootDev, counter: counter, visited: visited, config: config, taskBudget: taskBudget) + let root = try await _buildTree(rootPath: url.path, rootURL: url, counter: counter, visited: visited, config: config) progressTask.cancel() continuation.yield(.completed(root: root, deniedCount: counter.deniedCount)) } catch is CancellationError { @@ -146,191 +123,282 @@ public actor FileScanner { } } -// MARK: - POSIX-based parallel tree builder (free functions, not actor-isolated) +// MARK: - Iterative work-stack tree builder (free functions, not actor-isolated) +// +// Concurrency invariant: a directory's `FSNode.children` array is written by +// exactly one worker (the one that processes that directory's work item), and +// every child FSNode is created by the parent's worker before being handed to +// the shared queue as a new work item. So no two workers ever touch the same +// node's mutable state concurrently, even though FSNode is `@unchecked Sendable`. + +private struct DirWorkItem { + let path: String + let url: URL + let node: FSNode + // Identity recorded at discovery time (nil only for the scan root, which + // was just lstat'd immediately before being opened). Re-checked via fstat + // right after opening the directory, so a directory replaced in-between + // (TOCTOU) is detected and skipped rather than silently scanned wrong. + let expectedDev: UInt64? + let expectedIno: UInt64? +} + +// Bounded-concurrency work queue: a LIFO stack plus an in-flight counter. +// `pop()` returning nil doesn't mean "done" by itself - workers must also +// check `isFinished` (stack empty AND nothing in flight) before exiting, since +// another worker's current item may still push more work. +private final class WorkQueue: @unchecked Sendable { + private var lock = os_unfair_lock() + private var stack: [DirWorkItem] + private var inFlight: Int + + init(seed: DirWorkItem) { + stack = [seed] + inFlight = 1 + } + + func push(_ item: DirWorkItem) { + os_unfair_lock_lock(&lock) + stack.append(item) + inFlight += 1 + os_unfair_lock_unlock(&lock) + } + + func pop() -> DirWorkItem? { + os_unfair_lock_lock(&lock) + let item = stack.popLast() + os_unfair_lock_unlock(&lock) + return item + } + + // Call exactly once per item that was popped, after it has been fully + // processed (including pushing any children it discovered). + func markDone() { + os_unfair_lock_lock(&lock) + inFlight -= 1 + os_unfair_lock_unlock(&lock) + } + + var isFinished: Bool { + os_unfair_lock_lock(&lock) + let finished = stack.isEmpty && inFlight == 0 + os_unfair_lock_unlock(&lock) + return finished + } +} private func _buildTree( - path: String, - url: URL, - parent: FSNode?, - rootDev: dev_t?, + rootPath: String, + rootURL: URL, counter: ProgressCounter, visited: VisitedSet, - config: ScanConfig, - taskBudget: TaskBudget + config: ScanConfig ) async throws -> FSNode { try Task.checkCancellation() var st = stat() - guard lstat(path, &st) == 0 else { return FSNode(url: url, name: url.lastPathComponent, isDirectory: false, size: 0, fileExtension: "", parent: parent) } + guard lstat(rootPath, &st) == 0 else { + return FSNode(url: rootURL, name: rootURL.lastPathComponent, isDirectory: false, size: 0, fileExtension: "", parent: nil) + } - // Skip symlinks + // Skip symlinks (including a symlink scan root). if st.st_mode & S_IFMT == S_IFLNK { throw SkipError() } let isDir = st.st_mode & S_IFMT == S_IFDIR - - // Deduplicate directories by (dev, ino) — prevents double-counting paths like - // /Applications and /System/Volumes/Data/Applications (same inode, different paths). - if isDir { - guard visited.visit(dev: st.st_dev, ino: st.st_ino) else { - if ProcessInfo.processInfo.environment["MDS_DEBUG_SKIPS"] != nil { - FileHandle.standardError.write("DEDUP_SKIP dev=\(st.st_dev) ino=\(st.st_ino) \(path)\n".data(using: .utf8)!) - } - throw SkipError() - } + let name = rootURL.lastPathComponent + + guard isDir else { + // A file was passed directly as the scan root. + let ext = rootURL.pathExtension.lowercased() + let allocSize = allocatedSize(st: st, visited: visited) + let node = FSNode(url: rootURL, name: name, isDirectory: false, size: allocSize, fileExtension: ext, parent: nil) + if st.st_nlink > 1 { node.hardLinkRef = hardLinkRef(of: st) } + counter.add(items: 1, bytes: allocSize) + return node } - let name = url.lastPathComponent - let ext = isDir ? "" : url.pathExtension.lowercased() - - let node = FSNode(url: url, name: name, isDirectory: isDir, size: 0, fileExtension: ext, parent: parent) - - if isDir { - let listing = _listDirectory(path: path, url: url, rootDev: rootDev, node: node, counter: counter, visited: visited, config: config) - - // Accumulate direct file sizes immediately - node.size = listing.totalSize - node.children = listing.children - node.isAccessDenied = listing.accessDenied - if listing.accessDenied { counter.addDenied() } - - // Recurse into subdirectories, in parallel up to the task budget; beyond - // that, recurse inline in the current task to bound total concurrency. - if !listing.subdirPaths.isEmpty { - var subdirSize: Int64 = 0 - try await withThrowingTaskGroup(of: FSNode?.self) { group in - for (subPath, subURL) in listing.subdirPaths { - if taskBudget.tryAcquire() { - group.addTask { - defer { taskBudget.release() } - // Propagate cancellation; silently skip symlinks / unreadable entries. - try Task.checkCancellation() - do { - return try await _buildTree(path: subPath, url: subURL, parent: node, rootDev: rootDev, counter: counter, visited: visited, config: config, taskBudget: taskBudget) - } catch is SkipError { - return nil - } - } - } else { - // Budget exhausted: recurse inline (no new task spawned) to - // bound concurrency without blocking on any lock or semaphore. - try Task.checkCancellation() - do { - let child = try await _buildTree(path: subPath, url: subURL, parent: node, rootDev: rootDev, counter: counter, visited: visited, config: config, taskBudget: taskBudget) - node.children.append(child) - subdirSize += child.size - } catch is SkipError { - // skip - } - } - } - for try await child in group { - guard let child else { continue } - node.children.append(child) - subdirSize += child.size - } + let rootDevKey = UInt64(bitPattern: Int64(st.st_dev)) + let rootInoKey = UInt64(st.st_ino) + // First visit of this scan always succeeds (fresh VisitedSet); kept for + // symmetry with the dedup check every subdirectory goes through below. + _ = visited.visit(dev: rootDevKey, ino: rootInoKey) + + let rootNode = FSNode(url: rootURL, name: name, isDirectory: true, size: 0, fileExtension: "", parent: nil) + let seed = DirWorkItem(path: rootPath, url: rootURL, node: rootNode, expectedDev: rootDevKey, expectedIno: rootInoKey) + let queue = WorkQueue(seed: seed) + + let workerCount = min(max(2, ProcessInfo.processInfo.activeProcessorCount / 2), 8) + + try await withThrowingTaskGroup(of: Void.self) { group in + for _ in 0.. DirectoryContents { - var result = DirectoryContents(children: [], subdirPaths: [], totalSize: 0, itemCount: 0) +// Processes exactly one directory: opens it, lists its immediate children +// (bulk enumeration with fallback), applies all scan semantics, records +// direct file children + their sizes on `item.node`, and pushes any +// subdirectories as new work items. Always calls `queue.markDone()` exactly +// once, even on early return. +private func _processDirectory( + item: DirWorkItem, + rootDevKey: UInt64, + counter: ProgressCounter, + visited: VisitedSet, + config: ScanConfig, + queue: WorkQueue +) throws { + defer { queue.markDone() } + try Task.checkCancellation() - guard let dir = opendir(path) else { + let fd = open(item.path, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW) + guard fd >= 0 else { if errno == EACCES || errno == EPERM { - result.accessDenied = true + item.node.isAccessDenied = true + counter.addDenied() } - return result + counter.add(items: 1, bytes: 0) + return } - defer { closedir(dir) } - let directoryFD = dirfd(dir) - - while let entry = readdir(dir) { - let nameBytes = entry.pointee.d_name - let name: String = withUnsafeBytes(of: nameBytes) { ptr in - let bytes = ptr.bindMemory(to: CChar.self) - return String(cString: bytes.baseAddress!) + defer { close(fd) } + + if let expectedDev = item.expectedDev, let expectedIno = item.expectedIno { + var st = stat() + guard fstat(fd, &st) == 0, + UInt64(bitPattern: Int64(st.st_dev)) == expectedDev, + UInt64(st.st_ino) == expectedIno else { + // The directory at this path was replaced between discovery and + // open (TOCTOU race); drop it silently rather than scan the wrong thing. + return } - guard name != "." && name != ".." else { continue } - if name.hasPrefix("."), !config.showHiddenFiles { continue } - if config.excludedNames.contains(name) { continue } - - let childURL = url.appendingPathComponent(name, isDirectory: entry.pointee.d_type == DT_DIR) - let dtype = entry.pointee.d_type - - // Fast type check using d_type from dirent (avoids extra stat call for most entries) - if dtype == DT_LNK { continue } // skip symlinks - - if dtype == DT_DIR || dtype == DT_UNKNOWN { - // For DT_UNKNOWN (e.g. some network filesystems), use lstat - var st = stat() - let childPath = path.hasSuffix("/") ? path + name : path + "/" + name - if dtype == DT_UNKNOWN { - guard fstatat(directoryFD, name, &st, AT_SYMLINK_NOFOLLOW) == 0 else { continue } - let mode = st.st_mode & S_IFMT - if mode == S_IFLNK { continue } - if mode != S_IFDIR { - // It's a regular file with DT_UNKNOWN - let allocSize = allocatedSize(st: st, visited: visited) - let ext = childURL.pathExtension.lowercased() - let fileNode = FSNode(url: childURL, name: name, isDirectory: false, size: allocSize, fileExtension: ext, parent: node) - if st.st_nlink > 1 { fileNode.hardLinkRef = hardLinkRef(of: st) } - result.children.append(fileNode) - result.totalSize += allocSize - result.itemCount += 1 - counter.add(items: 1, bytes: allocSize) - continue - } - // Check mount point via st_dev - if let rootDev, st.st_dev != rootDev { continue } - result.subdirPaths.append((childPath, childURL)) - } else { - // DT_DIR — check mount point via a quick stat - if let rootDev { - guard fstatat(directoryFD, name, &st, 0) == 0 else { continue } - if st.st_dev != rootDev { continue } - } - result.subdirPaths.append((childPath, childURL)) - } - } else if dtype == DT_REG { - var st = stat() - guard fstatat(directoryFD, name, &st, AT_SYMLINK_NOFOLLOW) == 0 else { continue } - let allocSize = allocatedSize(st: st, visited: visited) + } + + let entries: [BulkDirEntry] + do { + entries = try listDirectoryEntries(path: item.path, fd: fd, forceFallback: config.forceFallbackEnum) + } catch { + // Enumeration failed even after falling back: treat as empty, not denied. + counter.add(items: 1, bytes: 0) + return + } + + var directSize: Int64 = 0 + var children: [FSNode] = [] + children.reserveCapacity(entries.count) + + for (index, entry) in entries.enumerated() { + if index % 256 == 0 { + try Task.checkCancellation() + } + guard entry.name != "." && entry.name != ".." else { continue } + if entry.name.hasPrefix("."), !config.showHiddenFiles { continue } + if config.excludedNames.contains(entry.name) { continue } + + switch entry.kind { + case .symlink, .other: + continue + + case .directory: + // Mount point: skip directories on a different device than the scan root. + if entry.dev != rootDevKey { continue } + // Dedup by (dev, ino): protects against firmlink aliases like + // /Applications vs /System/Volumes/Data/Applications. + guard visited.visit(dev: entry.dev, ino: entry.ino) else { continue } + + let childURL = item.url.appendingPathComponent(entry.name, isDirectory: true) + let childNode = FSNode(url: childURL, name: entry.name, isDirectory: true, size: 0, fileExtension: "", parent: item.node) + children.append(childNode) + let childPath = item.path.hasSuffix("/") ? item.path + entry.name : item.path + "/" + entry.name + queue.push(DirWorkItem(path: childPath, url: childURL, node: childNode, expectedDev: entry.dev, expectedIno: entry.ino)) + + case .file: + let childURL = item.url.appendingPathComponent(entry.name, isDirectory: false) + let allocSize = bulkAllocatedSize(entry: entry, visited: visited) let ext = childURL.pathExtension.lowercased() - let fileNode = FSNode(url: childURL, name: name, isDirectory: false, size: allocSize, fileExtension: ext, parent: node) - if st.st_nlink > 1 { fileNode.hardLinkRef = hardLinkRef(of: st) } - result.children.append(fileNode) - result.totalSize += allocSize - result.itemCount += 1 + let fileNode = FSNode(url: childURL, name: entry.name, isDirectory: false, size: allocSize, fileExtension: ext, parent: item.node) + if entry.linkCount > 1 { fileNode.hardLinkRef = HardLinkRef(dev: entry.dev, ino: entry.ino) } + children.append(fileNode) + directSize += allocSize counter.add(items: 1, bytes: allocSize) } - // DT_FIFO, DT_CHR, DT_BLK, DT_SOCK — skip } - return result + item.node.children = children + item.node.size = directSize + counter.add(items: 1, bytes: 0) +} + +// Picks bulk vs. fallback enumeration for one directory. If bulk enumeration +// throws partway through (having already consumed some of `fd`'s kernel-side +// listing position), the fallback re-opens the directory fresh by path so it +// always sees the complete, unconsumed listing rather than a partial remainder. +private func listDirectoryEntries(path: String, fd: Int32, forceFallback: Bool) throws -> [BulkDirEntry] { + if forceFallback { + return try fallbackEnumerateDirectory(fd: fd) + } + do { + return try enumerateDirectoryBulk(fd: fd) + } catch is BulkEnumerationUnavailable { + let freshFD = open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW) + guard freshFD >= 0 else { return [] } + defer { close(freshFD) } + return try fallbackEnumerateDirectory(fd: freshFD) + } +} + +// Iterative post-order pass: folds each directory's descendant directory +// sizes into its own `size` (which already holds its direct file sum from +// `_processDirectory`). Runs once, after all workers finish, so there is no +// concurrent mutation of node.size during traversal. +private func aggregateDirectorySizes(root: FSNode) { + guard root.isDirectory else { return } + + // `order` ends up a valid pre-order (a node always precedes its own + // descendants, since children are only pushed after their parent is + // appended). Processing it in reverse guarantees every directory's + // children are fully aggregated before the directory itself is folded + // into its own parent. + var order: [FSNode] = [] + var stack: [FSNode] = [root] + while let node = stack.popLast() { + order.append(node) + for child in node.children where child.isDirectory { + stack.append(child) + } + } + + for node in order.reversed() { + var subtreeAddition: Int64 = 0 + for child in node.children where child.isDirectory { + subtreeAddition += child.size + } + node.size += subtreeAddition + } } // Builds the inode identity for a hardlinked file. dev_t is a signed 32-bit value @@ -342,11 +410,22 @@ private func hardLinkRef(of st: stat) -> HardLinkRef { // Returns the file's allocated disk bytes, deduplicating hardlinks via the visited set. // Files with nlink == 1 skip the set entirely (fast path for the common case). // Hardlinked files (nlink > 1) are counted only on their first encounter. +// Used only for a file passed directly as the scan root; regular directory +// listings use `bulkAllocatedSize` on the enumerator's own metadata instead. private func allocatedSize(st: stat, visited: VisitedSet) -> Int64 { if st.st_nlink > 1 { - guard visited.visit(dev: st.st_dev, ino: st.st_ino) else { return 0 } + guard visited.visit(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) else { return 0 } } return Int64(st.st_blocks) * 512 } +// Same dedup rule as `allocatedSize(st:visited:)` above, but sourced from a +// `BulkDirEntry` (either enumeration path) instead of a raw `stat`. +private func bulkAllocatedSize(entry: BulkDirEntry, visited: VisitedSet) -> Int64 { + if entry.linkCount > 1 { + guard visited.visit(dev: entry.dev, ino: entry.ino) else { return 0 } + } + return entry.allocatedSize +} + private struct SkipError: Error {} diff --git a/Tests/BulkScannerTests.swift b/Tests/BulkScannerTests.swift new file mode 100644 index 0000000..42cf5cf --- /dev/null +++ b/Tests/BulkScannerTests.swift @@ -0,0 +1,346 @@ +import XCTest +@testable import MacDirStat + +final class BulkScannerTests: XCTestCase { + + // MARK: - Helpers + + // Sorted (relativePath, isDirectory, size) fingerprint of an entire tree, + // used to compare the bulk and fallback enumeration paths against each other. + // + // Hardlinked files need special handling: which specific link "wins" the + // non-zero size (VisitedSet.visit is a first-come-first-served race across + // concurrent workers) is real, expected nondeterminism of concurrent + // traversal, not a correctness bug - verified directly, it can differ + // between two runs of the *same* enumeration method on a hardlink-heavy + // real tree (e.g. Xcode.app, which hardlinks tens of thousands of files + // across sibling bundles), not just bulk vs fallback. Two consequences: + // - a hardlinked file's own size is normalized to 0 here and instead + // checked as a group: the same set of paths must share one inode, and + // that group's total allocated size must match. + // - a *directory's* aggregated size becomes nondeterministic too, if a + // hardlink's two links sit under different parents (the bytes get + // attributed to whichever parent's link happened to win). Only the + // grand total is guaranteed invariant. So directory entries carry no + // size at all here; only file entries and the root total do. + private struct FileFingerprintEntry: Hashable { + let relativePath: String + let size: Int64 + } + + private struct HardlinkGroupFingerprint: Hashable { + let paths: [String] + let total: Int64 + } + + private struct Fingerprint { + let fileEntries: [FileFingerprintEntry] + let directoryPaths: [String] + let hardlinkGroups: [HardlinkGroupFingerprint] + let total: Int64 + } + + private func fingerprint(root: FSNode, base: URL) -> Fingerprint { + var fileEntries: [FileFingerprintEntry] = [] + var directoryPaths: [String] = [] + var groups: [HardLinkRef: (paths: Set, total: Int64)] = [:] + + func visit(_ node: FSNode) { + let relativePath = String(node.url.path.dropFirst(base.path.count)) + if node.isDirectory { + directoryPaths.append(relativePath) + } else if let ref = node.hardLinkRef { + var group = groups[ref] ?? (paths: [], total: 0) + group.paths.insert(relativePath) + group.total += node.size + groups[ref] = group + fileEntries.append(FileFingerprintEntry(relativePath: relativePath, size: 0)) + } else { + fileEntries.append(FileFingerprintEntry(relativePath: relativePath, size: node.size)) + } + for child in node.children { visit(child) } + } + visit(root) + fileEntries.sort { $0.relativePath < $1.relativePath } + directoryPaths.sort() + + let hardlinkGroups = groups.values + .map { HardlinkGroupFingerprint(paths: $0.paths.sorted(), total: $0.total) } + .sorted { $0.paths.first ?? "" < $1.paths.first ?? "" } + + return Fingerprint(fileEntries: fileEntries, directoryPaths: directoryPaths, hardlinkGroups: hardlinkGroups, total: root.size) + } + + private func scanTree(at url: URL) async -> FSNode? { + let scanner = FileScanner() + var root: FSNode? + for await progress in await scanner.scan(url: url) { + if case .completed(let node, _) = progress { root = node } + } + return root + } + + // Builds a tree exercising every scan semantic: nested dirs, hidden files, + // an excluded dir name, a symlink, a hardlink pair, and a file > 4 KB. + private func buildFixtureTree(at tmp: URL) throws { + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + + let sub = tmp.appendingPathComponent("sub") + let nested = sub.appendingPathComponent("nested") + try FileManager.default.createDirectory(at: nested, withIntermediateDirectories: true) + + try Data(repeating: 0xAB, count: 8192).write(to: tmp.appendingPathComponent("big.bin")) + try Data(repeating: 0, count: 16).write(to: sub.appendingPathComponent("small.txt")) + try Data(repeating: 0, count: 16).write(to: nested.appendingPathComponent("deep.txt")) + try Data(repeating: 0, count: 8).write(to: tmp.appendingPathComponent(".hidden")) + + let excludedDir = tmp.appendingPathComponent("node_modules") + try FileManager.default.createDirectory(at: excludedDir, withIntermediateDirectories: true) + try Data(repeating: 0, count: 4096).write(to: excludedDir.appendingPathComponent("junk.bin")) + + let real = tmp.appendingPathComponent("real.bin") + try Data(repeating: 1, count: 4096).write(to: real) + let link = tmp.appendingPathComponent("link.bin") + try FileManager.default.createSymbolicLink(at: link, withDestinationURL: real) + + let hard1 = sub.appendingPathComponent("hard1.bin") + let hard2 = sub.appendingPathComponent("hard2.bin") + try Data(repeating: 3, count: 262_144).write(to: hard1) + try FileManager.default.linkItem(at: hard1, to: hard2) + } + + // MARK: - 1. Parity fingerprint test + + func test_bulk_and_fallback_enumeration_produce_identical_fingerprint() async throws { + let priorValue = UserDefaults.standard.string(forKey: "excludedFolderNames") + UserDefaults.standard.set(".git,node_modules,DerivedData,.Trash", forKey: "excludedFolderNames") + defer { + if let priorValue { UserDefaults.standard.set(priorValue, forKey: "excludedFolderNames") } + else { UserDefaults.standard.removeObject(forKey: "excludedFolderNames") } + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try buildFixtureTree(at: tmp) + defer { try? FileManager.default.removeItem(at: tmp) } + + guard let bulkRoot = await scanTree(at: tmp) else { + return XCTFail("bulk scan produced no root") + } + let bulkFingerprint = fingerprint(root: bulkRoot, base: tmp) + + setenv("MDS_FORCE_FALLBACK_ENUM", "1", 1) + let fallbackRoot = await scanTree(at: tmp) + unsetenv("MDS_FORCE_FALLBACK_ENUM") + + guard let fallbackRoot else { + return XCTFail("fallback scan produced no root") + } + let fallbackFingerprint = fingerprint(root: fallbackRoot, base: tmp) + + XCTAssertEqual(bulkFingerprint.fileEntries, fallbackFingerprint.fileEntries, "bulk and fallback enumeration must see the same files with the same sizes") + XCTAssertEqual(bulkFingerprint.directoryPaths, fallbackFingerprint.directoryPaths, "bulk and fallback enumeration must see the same directories") + XCTAssertEqual(bulkFingerprint.hardlinkGroups, fallbackFingerprint.hardlinkGroups, "hardlink groupings and their total sizes must match") + XCTAssertEqual(bulkFingerprint.total, fallbackFingerprint.total, "bulk and fallback totals must match") + XCTAssertGreaterThan(bulkFingerprint.total, 0) + } + + // MARK: - 2. Bulk enumerator unit test + + func test_bulk_enumerator_matches_lstat_ground_truth() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let fileA = tmp.appendingPathComponent("a.bin") + let fileB = tmp.appendingPathComponent("b.bin") + let dirC = tmp.appendingPathComponent("c_dir") + try Data(repeating: 1, count: 12_345).write(to: fileA) + try Data(repeating: 2, count: 42).write(to: fileB) + try FileManager.default.createDirectory(at: dirC, withIntermediateDirectories: true) + + let fd = open(tmp.path, O_RDONLY | O_DIRECTORY | O_CLOEXEC) + XCTAssertGreaterThanOrEqual(fd, 0) + defer { close(fd) } + + let entries = try enumerateDirectoryBulk(fd: fd) + let byName = Dictionary(uniqueKeysWithValues: entries.map { ($0.name, $0) }) + + XCTAssertEqual(entries.count, 3) + + for name in ["a.bin", "b.bin", "c_dir"] { + guard let entry = byName[name] else { + XCTFail("missing entry for \(name)") + continue + } + var st = stat() + XCTAssertEqual(lstat(tmp.appendingPathComponent(name).path, &st), 0) + XCTAssertEqual(entry.dev, UInt64(bitPattern: Int64(st.st_dev)), "dev mismatch for \(name)") + XCTAssertEqual(entry.ino, UInt64(st.st_ino), "ino mismatch for \(name)") + if name == "c_dir" { + XCTAssertEqual(entry.kind, .directory) + } else { + XCTAssertEqual(entry.kind, .file) + XCTAssertEqual(entry.linkCount, UInt32(st.st_nlink), "linkCount mismatch for \(name)") + XCTAssertEqual(entry.allocatedSize, Int64(st.st_blocks) * 512, "allocatedSize mismatch for \(name)") + } + } + } + + // MARK: - 3. Hardlink dedup test + + func test_hardlinked_file_deduped_across_two_directories() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let dirA = tmp.appendingPathComponent("dirA") + let dirB = tmp.appendingPathComponent("dirB") + try FileManager.default.createDirectory(at: dirA, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: dirB, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let original = dirA.appendingPathComponent("orig.bin") + try Data(repeating: 9, count: 262_144).write(to: original) + let linked = dirB.appendingPathComponent("linked.bin") + try FileManager.default.linkItem(at: original, to: linked) + + var st = stat() + XCTAssertEqual(lstat(original.path, &st), 0) + let fullSize = Int64(st.st_blocks) * 512 + + guard let root = await scanTree(at: tmp) else { + return XCTFail("scan produced no root") + } + + let aNode = root.children.first { $0.name == "dirA" } + let bNode = root.children.first { $0.name == "dirB" } + let origNode = aNode?.children.first { $0.name == "orig.bin" } + let linkedNode = bNode?.children.first { $0.name == "linked.bin" } + + XCTAssertNotNil(origNode?.hardLinkRef) + XCTAssertNotNil(linkedNode?.hardLinkRef) + XCTAssertEqual(origNode?.hardLinkRef, linkedNode?.hardLinkRef) + + let sizes = [origNode?.size ?? -1, linkedNode?.size ?? -1].sorted() + XCTAssertEqual(sizes, [0, fullSize], "one link carries the full size, the other is zero") + XCTAssertEqual(root.size, fullSize, "the shared inode must be counted exactly once across both directories") + } + + // MARK: - 4. Access-denied test + + func test_unreadable_subdirectory_marked_access_denied() async throws { + guard getuid() != 0 else { + throw XCTSkip("running as root: chmod 000 does not block access") + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + let locked = tmp.appendingPathComponent("locked") + try FileManager.default.createDirectory(at: locked, withIntermediateDirectories: true) + try Data(repeating: 0, count: 4096).write(to: locked.appendingPathComponent("secret.bin")) + + chmod(locked.path, 0) + defer { + chmod(locked.path, 0o755) + try? FileManager.default.removeItem(at: tmp) + } + + var deniedCount = 0 + let scanner = FileScanner() + var root: FSNode? + for await progress in await scanner.scan(url: tmp) { + if case .completed(let node, let denied) = progress { + root = node + deniedCount = denied + } + } + + let lockedNode = root?.children.first { $0.name == "locked" } + XCTAssertNotNil(lockedNode) + XCTAssertTrue(lockedNode?.isAccessDenied ?? false) + XCTAssertEqual(lockedNode?.children.count, 0) + XCTAssertGreaterThanOrEqual(deniedCount, 1) + } + + // MARK: - 5. Cancellation test + + func test_cancellation_ends_stream_without_completed() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + // A moderately large synthetic tree so the scan is very unlikely to + // finish before cancellation takes effect. + for i in 0..<50 { + let dir = tmp.appendingPathComponent("d\(i)") + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + for j in 0..<50 { + try Data(repeating: 0, count: 256).write(to: dir.appendingPathComponent("f\(j).bin")) + } + } + + let scanner = FileScanner() + var sawCompleted = false + let stream = await scanner.scan(url: tmp) + let consumeTask = Task { + for await progress in stream { + if case .completed = progress { sawCompleted = true } + } + } + await scanner.cancel() + _ = await consumeTask.value + + XCTAssertFalse(sawCompleted, "a cancelled scan must not emit .completed") + } + + // MARK: - 7. Env-gated benchmark + + func test_benchmark_bulk_vs_fallback() async throws { + guard ProcessInfo.processInfo.environment["MDS_BENCH"] == "1" else { + throw XCTSkip("set MDS_BENCH=1 to run the benchmark") + } + let path = ProcessInfo.processInfo.environment["MDS_BENCH_PATH"] ?? "/Applications" + let url = URL(fileURLWithPath: path) + + let bulkStart = DispatchTime.now() + guard let bulkRoot = await scanTree(at: url) else { + return XCTFail("bulk benchmark scan produced no root") + } + let bulkElapsed = Double(DispatchTime.now().uptimeNanoseconds - bulkStart.uptimeNanoseconds) / 1_000_000_000 + + setenv("MDS_FORCE_FALLBACK_ENUM", "1", 1) + let fallbackStart = DispatchTime.now() + let fallbackRoot = await scanTree(at: url) + let fallbackElapsed = Double(DispatchTime.now().uptimeNanoseconds - fallbackStart.uptimeNanoseconds) / 1_000_000_000 + unsetenv("MDS_FORCE_FALLBACK_ENUM") + + guard let fallbackRoot else { + return XCTFail("fallback benchmark scan produced no root") + } + + let bulkFingerprint = fingerprint(root: bulkRoot, base: url) + let fallbackFingerprint = fingerprint(root: fallbackRoot, base: url) + + print("MDS_BENCH elapsed_bulk=\(bulkElapsed) elapsed_fallback=\(fallbackElapsed) path=\(path)") + + if bulkFingerprint.fileEntries != fallbackFingerprint.fileEntries { + let bulkSet = Set(bulkFingerprint.fileEntries.map { "\($0.relativePath)|\($0.size)" }) + let fallbackSet = Set(fallbackFingerprint.fileEntries.map { "\($0.relativePath)|\($0.size)" }) + let onlyBulk = bulkSet.subtracting(fallbackSet) + let onlyFallback = fallbackSet.subtracting(bulkSet) + print("MDS_BENCH_DIFF onlyBulk=\(onlyBulk.count) onlyFallback=\(onlyFallback.count)") + for line in onlyBulk.sorted().prefix(20) { print("MDS_BENCH_ONLY_BULK \(line)") } + for line in onlyFallback.sorted().prefix(20) { print("MDS_BENCH_ONLY_FALLBACK \(line)") } + } + if bulkFingerprint.directoryPaths != fallbackFingerprint.directoryPaths { + let bulkSet = Set(bulkFingerprint.directoryPaths) + let fallbackSet = Set(fallbackFingerprint.directoryPaths) + print("MDS_BENCH_DIFF onlyBulkDirs=\(bulkSet.subtracting(fallbackSet).count) onlyFallbackDirs=\(fallbackSet.subtracting(bulkSet).count)") + } + if bulkFingerprint.hardlinkGroups != fallbackFingerprint.hardlinkGroups { + print("MDS_BENCH_DIFF hardlinkGroups bulk=\(bulkFingerprint.hardlinkGroups.count) fallback=\(fallbackFingerprint.hardlinkGroups.count)") + } + + XCTAssertEqual(bulkFingerprint.fileEntries, fallbackFingerprint.fileEntries, "bulk and fallback enumeration must see the same files with the same sizes") + XCTAssertEqual(bulkFingerprint.directoryPaths, fallbackFingerprint.directoryPaths, "bulk and fallback enumeration must see the same directories") + XCTAssertEqual(bulkFingerprint.hardlinkGroups, fallbackFingerprint.hardlinkGroups, "hardlink groupings and their total sizes must match") + XCTAssertEqual(bulkFingerprint.total, fallbackFingerprint.total) + } +} From 0fe30aea33cef16db58eba655f0e5207c3ae4c45 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 12:34:09 +0300 Subject: [PATCH 14/41] perf(scanner): auto-summarize tiny-file directories (node_modules etc.) --- Sources/Scanner/AtomicDirectorySummary.swift | 153 ++++++++++ Sources/Scanner/FSNode.swift | 7 + Sources/Scanner/FileScanner.swift | 49 +++- .../DirectoryTree/DirectoryTreeView.swift | 9 + Tests/AutoSummaryTests.swift | 269 ++++++++++++++++++ 5 files changed, 481 insertions(+), 6 deletions(-) create mode 100644 Sources/Scanner/AtomicDirectorySummary.swift create mode 100644 Tests/AutoSummaryTests.swift diff --git a/Sources/Scanner/AtomicDirectorySummary.swift b/Sources/Scanner/AtomicDirectorySummary.swift new file mode 100644 index 0000000..ff4ead8 --- /dev/null +++ b/Sources/Scanner/AtomicDirectorySummary.swift @@ -0,0 +1,153 @@ +import Darwin +import Foundation + +// Auto-summarization: when a directory is clearly a generated / tiny-file +// tree (node_modules, huge flat caches, ...), the scanner collapses its +// entire subtree into ONE leaf FSNode carrying the deep allocated size and a +// descendant file count, instead of materializing an FSNode per descendant. +// This is the biggest remaining scan-speed/node-count win: no FSNode +// allocation, no tree-building overhead for a subtree the user never wants +// to browse file-by-file. +// +// v1 scope is deliberately simple: a named-layout shortcut plus an +// immediate-file-count/average-size heuristic, walked single-threaded by the +// one worker that already owns the candidate directory. See the TODOs below +// for what a later phase could add. + +// Directory names that are, by convention, generated dependency/package +// trees not worth browsing file-by-file. Matched against a directory's own +// name; bypasses the depth gate below. +private let knownGeneratedDirectoryNames: Set = ["node_modules"] + +enum AtomicSummaryThresholds { + // Overridable via env var so tests can exercise the general heuristic + // without creating thousands of real files. + static var minFileCount: Int { + if let raw = ProcessInfo.processInfo.environment["MDS_SUMMARY_MIN_FILES"], let value = Int(raw) { + return value + } + return 5_000 + } + static let maxAverageFileSize: Int64 = 4_096 + static let minDepth = 2 +} + +// Cheap, allocation-free gate: decides whether `item`'s directory (already +// enumerated into `entries`, no extra I/O) should be collapsed into a single +// summarized FSNode instead of being expanded normally. +func shouldAutoSummarize(entries: [BulkDirEntry], name: String, depth: Int) -> Bool { + // 1. Named-layout shortcut: known generated trees (e.g. node_modules) + // often keep few files at their own top level (most live several levels + // deep, e.g. node_modules//dist/...), so the immediate-file + // heuristic below would miss them. Name match alone is enough for v1. + if knownGeneratedDirectoryNames.contains(name), depth >= 1 { + return true + } + + // 2. General heuristic: lots of small immediate files, deep enough in + // the tree that summarizing won't collapse something the user is + // directly looking at. + // TODO(phase3+): bounded descendant probe for ambiguous dirs (e.g. a + // directory whose files are deeper than its immediate listing, but that + // doesn't match a known name) - see Radix's AtomicDirectorySummarizer + // for the pattern; out of scope here to keep the summary walk simple + // and single-threaded. + guard depth >= AtomicSummaryThresholds.minDepth else { return false } + + var fileCount = 0 + var totalAllocated: Int64 = 0 + for entry in entries where entry.kind == .file { + fileCount += 1 + totalAllocated += entry.allocatedSize + } + guard fileCount > 0, fileCount >= AtomicSummaryThresholds.minFileCount else { return false } + let averageAllocated = totalAllocated / Int64(fileCount) + return averageAllocated < AtomicSummaryThresholds.maxAverageFileSize +} + +// Iterative, single-threaded walk of the subtree rooted at a directory +// already deemed a summarization candidate. Applies the exact same scan +// semantics as the main traversal (symlink/hidden/exclusion/mount-point +// skips, hardlink + directory dedup via the shared `visited` set) so the +// aggregate size/file-count it returns matches what a full, non-summarized +// scan of the same subtree would have produced. +// +// `rootEntries` is the already-decoded listing of the candidate directory +// itself (the caller enumerated it once to run `shouldAutoSummarize`); reused +// here instead of re-enumerating so summarization costs zero extra I/O for +// the root of the collapsed subtree. Descendant directories are opened by +// path as the walk descends (this candidate directory is already being +// processed by one worker, so keeping the walk single-threaded here adds no +// contention - see TODO below for a future parallel version). +// +// TODO(phase3+): parallel summary pool - fan the descendant walk of very +// large summarized subtrees out across multiple workers, the way the main +// traversal does. Left single-threaded for v1: correctness and the +// node-count/scan-time win both come from not materializing FSNodes, which +// a single-threaded walk already delivers. +func summarizeSubtree( + rootEntries: [BulkDirEntry], + rootPath: String, + rootDev: UInt64, + config: ScanConfig, + visited: VisitedSet, + cancel: () throws -> Void +) throws -> (allocatedSize: Int64, fileCount: Int) { + var totalAllocated: Int64 = 0 + var fileCount = 0 + var entriesSeen = 0 + + // Directories still to be listed, identified by their full path (their + // (dev, ino) dedup check already happened when they were discovered and + // pushed here). + var pendingDirs: [String] = [] + + func consume(_ entries: [BulkDirEntry], dirPath: String) throws { + for entry in entries { + entriesSeen += 1 + if entriesSeen % 256 == 0 { try cancel() } + + guard entry.name != "." && entry.name != ".." else { continue } + if entry.name.hasPrefix("."), !config.showHiddenFiles { continue } + if config.excludedNames.contains(entry.name) { continue } + + switch entry.kind { + case .symlink, .other: + continue + + case .directory: + // Mount point: skip directories on a different device than the scan root. + if entry.dev != rootDev { continue } + // Dedup by (dev, ino): same firmlink/hardlink protection as the main scanner. + guard visited.visit(dev: entry.dev, ino: entry.ino) else { continue } + let childPath = dirPath.hasSuffix("/") ? dirPath + entry.name : dirPath + "/" + entry.name + pendingDirs.append(childPath) + + case .file: + totalAllocated += bulkAllocatedSize(entry: entry, visited: visited) + fileCount += 1 + } + } + } + + try consume(rootEntries, dirPath: rootPath) + + while let dirPath = pendingDirs.popLast() { + try cancel() + let fd = open(dirPath, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW) + guard fd >= 0 else { continue } + defer { close(fd) } + + let entries: [BulkDirEntry] + do { + entries = try listDirectoryEntries(path: dirPath, fd: fd, forceFallback: config.forceFallbackEnum) + } catch { + // Enumeration failed even after falling back: contributes + // nothing further, matches the main scanner's "treat as empty" rule. + continue + } + try consume(entries, dirPath: dirPath) + } + + return (totalAllocated, fileCount) +} diff --git a/Sources/Scanner/FSNode.swift b/Sources/Scanner/FSNode.swift index b6ff272..516a5a3 100644 --- a/Sources/Scanner/FSNode.swift +++ b/Sources/Scanner/FSNode.swift @@ -27,6 +27,13 @@ public final class FSNode: Identifiable, @unchecked Sendable { public var isSynthetic: Bool = false // Set only for files with st_nlink > 1 (both the size-carrying node and its 0-size siblings). public var hardLinkRef: HardLinkRef? + // True when the traversal collapsed this directory's entire subtree into + // this single node instead of materializing its descendants (see + // AtomicDirectorySummary.swift). `children` is always empty in that case. + public var isAutoSummarized: Bool = false + // Total number of files collapsed under this node by auto-summarization. + // 0 for normal (non-summarized) nodes. + public var descendantFileCount: Int = 0 public init(url: URL, name: String, isDirectory: Bool, size: Int64, fileExtension: String, parent: FSNode? = nil) { self.url = url diff --git a/Sources/Scanner/FileScanner.swift b/Sources/Scanner/FileScanner.swift index 7a6b5f3..e247ee3 100644 --- a/Sources/Scanner/FileScanner.swift +++ b/Sources/Scanner/FileScanner.swift @@ -5,7 +5,10 @@ import Foundation // Shared by both directory dedup (firmlinks/mount aliases) and hardlinked-file // dedup: a directory's inode and a file's inode never collide on one device, // so the two uses safely share one (dev, ino) namespace, exactly as before. -private final class VisitedSet: @unchecked Sendable { +// Shared with AtomicDirectorySummary.swift (same module) so the summarization +// walk dedups directories/hardlinks against the exact same set as the main +// traversal. +final class VisitedSet: @unchecked Sendable { private var lock = os_unfair_lock() private var set = Set() @@ -68,6 +71,10 @@ struct ScanConfig: Sendable { // bypassing getattrlistbulk entirely. Used by parity tests to compare the // two enumeration strategies against each other. let forceFallbackEnum: Bool + // Collapses tiny-file directories (node_modules, etc.) into one summarized + // leaf FSNode instead of materializing every descendant. See + // AtomicDirectorySummary.swift. + let autoSummarizeEnabled: Bool static func loadFromUserDefaults() -> ScanConfig { let rawExcluded = UserDefaults.standard.string(forKey: "excludedFolderNames") @@ -75,7 +82,12 @@ struct ScanConfig: Sendable { let excludedNames = Set(rawExcluded.components(separatedBy: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty }) let showHiddenFiles = UserDefaults.standard.bool(forKey: "showHiddenFiles") let forceFallbackEnum = ProcessInfo.processInfo.environment["MDS_FORCE_FALLBACK_ENUM"] == "1" - return ScanConfig(excludedNames: excludedNames, showHiddenFiles: showHiddenFiles, forceFallbackEnum: forceFallbackEnum) + // Default-true feature flag: absent key means "on" (unlike the other + // UserDefaults-backed flags above, which default to false/absent-Bool). + let autoSummarizeEnabled = UserDefaults.standard.object(forKey: "autoSummarizeEnabled") == nil + ? true + : UserDefaults.standard.bool(forKey: "autoSummarizeEnabled") + return ScanConfig(excludedNames: excludedNames, showHiddenFiles: showHiddenFiles, forceFallbackEnum: forceFallbackEnum, autoSummarizeEnabled: autoSummarizeEnabled) } } @@ -141,6 +153,9 @@ private struct DirWorkItem { // (TOCTOU) is detected and skipped rather than silently scanned wrong. let expectedDev: UInt64? let expectedIno: UInt64? + // Root is depth 0; each child work item is its parent's depth + 1. Used + // by the auto-summarization depth gate (see AtomicDirectorySummary.swift). + let depth: Int } // Bounded-concurrency work queue: a LIFO stack plus an in-flight counter. @@ -224,7 +239,7 @@ private func _buildTree( _ = visited.visit(dev: rootDevKey, ino: rootInoKey) let rootNode = FSNode(url: rootURL, name: name, isDirectory: true, size: 0, fileExtension: "", parent: nil) - let seed = DirWorkItem(path: rootPath, url: rootURL, node: rootNode, expectedDev: rootDevKey, expectedIno: rootInoKey) + let seed = DirWorkItem(path: rootPath, url: rootURL, node: rootNode, expectedDev: rootDevKey, expectedIno: rootInoKey, depth: 0) let queue = WorkQueue(seed: seed) let workerCount = min(max(2, ProcessInfo.processInfo.activeProcessorCount / 2), 8) @@ -307,6 +322,26 @@ private func _processDirectory( return } + if config.autoSummarizeEnabled, + shouldAutoSummarize(entries: entries, name: item.url.lastPathComponent, depth: item.depth) { + let summary = try summarizeSubtree( + rootEntries: entries, + rootPath: item.path, + rootDev: rootDevKey, + config: config, + visited: visited + ) { try Task.checkCancellation() } + item.node.isAutoSummarized = true + item.node.descendantFileCount = summary.fileCount + // Deep allocated total for the whole collapsed subtree. + // `aggregateDirectorySizes` only adds up directory *children*'s sizes, + // and this node's children are empty, so it will not be touched again. + item.node.size = summary.allocatedSize + item.node.children = [] + counter.add(items: summary.fileCount + 1, bytes: summary.allocatedSize) + return + } + var directSize: Int64 = 0 var children: [FSNode] = [] children.reserveCapacity(entries.count) @@ -334,7 +369,7 @@ private func _processDirectory( let childNode = FSNode(url: childURL, name: entry.name, isDirectory: true, size: 0, fileExtension: "", parent: item.node) children.append(childNode) let childPath = item.path.hasSuffix("/") ? item.path + entry.name : item.path + "/" + entry.name - queue.push(DirWorkItem(path: childPath, url: childURL, node: childNode, expectedDev: entry.dev, expectedIno: entry.ino)) + queue.push(DirWorkItem(path: childPath, url: childURL, node: childNode, expectedDev: entry.dev, expectedIno: entry.ino, depth: item.depth + 1)) case .file: let childURL = item.url.appendingPathComponent(entry.name, isDirectory: false) @@ -357,7 +392,8 @@ private func _processDirectory( // throws partway through (having already consumed some of `fd`'s kernel-side // listing position), the fallback re-opens the directory fresh by path so it // always sees the complete, unconsumed listing rather than a partial remainder. -private func listDirectoryEntries(path: String, fd: Int32, forceFallback: Bool) throws -> [BulkDirEntry] { +// Shared with AtomicDirectorySummary.swift (same module). +func listDirectoryEntries(path: String, fd: Int32, forceFallback: Bool) throws -> [BulkDirEntry] { if forceFallback { return try fallbackEnumerateDirectory(fd: fd) } @@ -421,7 +457,8 @@ private func allocatedSize(st: stat, visited: VisitedSet) -> Int64 { // Same dedup rule as `allocatedSize(st:visited:)` above, but sourced from a // `BulkDirEntry` (either enumeration path) instead of a raw `stat`. -private func bulkAllocatedSize(entry: BulkDirEntry, visited: VisitedSet) -> Int64 { +// Shared with AtomicDirectorySummary.swift (same module). +func bulkAllocatedSize(entry: BulkDirEntry, visited: VisitedSet) -> Int64 { if entry.linkCount > 1 { guard visited.visit(dev: entry.dev, ino: entry.ino) else { return 0 } } diff --git a/Sources/Views/DirectoryTree/DirectoryTreeView.swift b/Sources/Views/DirectoryTree/DirectoryTreeView.swift index 2bf68fc..60a0028 100644 --- a/Sources/Views/DirectoryTree/DirectoryTreeView.swift +++ b/Sources/Views/DirectoryTree/DirectoryTreeView.swift @@ -138,6 +138,15 @@ private struct NodeRow: View { .help("Contents couldn't be read — Full Disk Access may be required") } + if node.isAutoSummarized { + Text("(\(node.descendantFileCount) files, summarized)") + .font(.system(size: 9.5)) + .foregroundStyle(isSelected ? Color.white.opacity(0.55) : Color.secondary.opacity(0.65)) + .lineLimit(1) + .fixedSize() + .help("Collapsed to keep the scan fast: contents aren't browsable individually") + } + // Size Text(ByteFormatter.string(from: node.size)) .font(.system(size: 11, design: .monospaced)) diff --git a/Tests/AutoSummaryTests.swift b/Tests/AutoSummaryTests.swift new file mode 100644 index 0000000..96f3cb3 --- /dev/null +++ b/Tests/AutoSummaryTests.swift @@ -0,0 +1,269 @@ +import XCTest +@testable import MacDirStat + +final class AutoSummaryTests: XCTestCase { + + // MARK: - Helpers + + private func withAutoSummarize(_ enabled: Bool, _ body: () async throws -> T) async rethrows -> T { + let prior = UserDefaults.standard.object(forKey: "autoSummarizeEnabled") + UserDefaults.standard.set(enabled, forKey: "autoSummarizeEnabled") + defer { + if let prior { UserDefaults.standard.set(prior, forKey: "autoSummarizeEnabled") } + else { UserDefaults.standard.removeObject(forKey: "autoSummarizeEnabled") } + } + return try await body() + } + + private func withMinFileCountOverride(_ value: Int, _ body: () async throws -> T) async rethrows -> T { + setenv("MDS_SUMMARY_MIN_FILES", "\(value)", 1) + defer { unsetenv("MDS_SUMMARY_MIN_FILES") } + return try await body() + } + + // The app's default excludedFolderNames already contains "node_modules" + // (it is fully skipped, not just collapsed - see ScanConfig's default in + // FileScanner.swift), so tests exercising the node_modules named-layout + // shortcut must use a config where it is NOT excluded, exactly like a + // user who removed it from their exclusion list would see. + private func withExcludedFolderNames(_ names: String, _ body: () async throws -> T) async rethrows -> T { + let prior = UserDefaults.standard.string(forKey: "excludedFolderNames") + UserDefaults.standard.set(names, forKey: "excludedFolderNames") + defer { + if let prior { UserDefaults.standard.set(prior, forKey: "excludedFolderNames") } + else { UserDefaults.standard.removeObject(forKey: "excludedFolderNames") } + } + return try await body() + } + + private func scanTree(at url: URL) async -> FSNode? { + let scanner = FileScanner() + var root: FSNode? + for await progress in await scanner.scan(url: url) { + if case .completed(let node, _) = progress { root = node } + } + return root + } + + // Recursively counts every non-directory FSNode (normal leaf files) plus + // every isAutoSummarized node's descendantFileCount, so summarized and + // non-summarized scans of the same tree can be compared on a "how many + // files did we account for" basis. + private func totalAccountedFiles(_ node: FSNode) -> Int { + if node.isAutoSummarized { return node.descendantFileCount } + if !node.isDirectory { return 1 } + return node.children.reduce(0) { $0 + totalAccountedFiles($1) } + } + + private func findNode(_ root: FSNode, path: [String]) -> FSNode? { + var current = root + for name in path { + guard let next = current.children.first(where: { $0.name == name }) else { return nil } + current = next + } + return current + } + + // Creates `count` small files spread across a couple of nested + // subdirectories under `dir`, each `bytes` bytes long. Returns the total + // allocated size (via lstat) of every file created, so tests can assert + // an exact expected total without hardcoding filesystem block behavior. + @discardableResult + private func makeFiles(count: Int, bytes: Int, under dir: URL, nestedEvery: Int = 7) throws -> Int64 { + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + var total: Int64 = 0 + var currentDir = dir + for i in 0.. 0, i > 0, i % nestedEvery == 0 { + currentDir = currentDir.appendingPathComponent("nested\(i)") + try FileManager.default.createDirectory(at: currentDir, withIntermediateDirectories: true) + } + let file = currentDir.appendingPathComponent("f\(i).bin") + try Data(repeating: UInt8(i % 251), count: bytes).write(to: file) + var st = stat() + XCTAssertEqual(lstat(file.path, &st), 0) + total += Int64(st.st_blocks) * 512 + } + return total + } + + // MARK: - 1. node_modules named-layout shortcut + + func test_node_modules_is_summarized() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: tmp) } + + let nodeModules = tmp.appendingPathComponent("proj").appendingPathComponent("node_modules") + let expectedTotal = try makeFiles(count: 50, bytes: 16, under: nodeModules) + + guard let onRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(true, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan (autoSummarize on) produced no root") + } + guard let nmNode = findNode(onRoot, path: ["proj", "node_modules"]) else { + return XCTFail("node_modules node not found") + } + + XCTAssertTrue(nmNode.isAutoSummarized) + XCTAssertTrue(nmNode.children.isEmpty) + XCTAssertEqual(nmNode.descendantFileCount, 50) + XCTAssertEqual(nmNode.size, expectedTotal) + + guard let offRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(false, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan (autoSummarize off) produced no root") + } + XCTAssertEqual(onRoot.size, offRoot.size, "collapsing node_modules must not change the root's total size") + + guard let offNmNode = findNode(offRoot, path: ["proj", "node_modules"]) else { + return XCTFail("node_modules node not found in the non-summarized scan") + } + XCTAssertFalse(offNmNode.isAutoSummarized) + XCTAssertEqual(offNmNode.children.isEmpty, false) + } + + // MARK: - 2. General immediate-file-count heuristic (env-overridden threshold) + + func test_threshold_directory_summarized() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: tmp) } + + // depth: tmp=0, level1=1, tinyDir/bigDir=2 (>= minDepth). + let level1 = tmp.appendingPathComponent("level1") + let tinyDir = level1.appendingPathComponent("tinyDir") + let bigDir = level1.appendingPathComponent("bigDir") + + // Empty files: allocatedSize 0, average 0 < maxAverageFileSize. + try makeFiles(count: 10, bytes: 0, under: tinyDir, nestedEvery: 0) + // 8 KB files: average clearly above maxAverageFileSize (4096). + try makeFiles(count: 10, bytes: 8192, under: bigDir, nestedEvery: 0) + + guard let root = await withMinFileCountOverride(10, { + await withAutoSummarize(true, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan produced no root") + } + + guard let tinyNode = findNode(root, path: ["level1", "tinyDir"]) else { + return XCTFail("tinyDir node not found") + } + XCTAssertTrue(tinyNode.isAutoSummarized, "many-tiny-file directory below the average-size threshold should be summarized") + XCTAssertEqual(tinyNode.descendantFileCount, 10) + + guard let bigNode = findNode(root, path: ["level1", "bigDir"]) else { + return XCTFail("bigDir node not found") + } + XCTAssertFalse(bigNode.isAutoSummarized, "directory whose average file size exceeds the threshold must not be summarized") + XCTAssertEqual(bigNode.children.count, 10) + } + + // MARK: - 3. autoSummarize off matches a full scan's sizes and file accounting + + func test_summarize_off_matches_full_scan_sizes() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: tmp) } + + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + try Data(repeating: 1, count: 4096).write(to: tmp.appendingPathComponent("plain.bin")) + + let nodeModules = tmp.appendingPathComponent("node_modules") + try makeFiles(count: 20, bytes: 16, under: nodeModules) + + guard let onRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(true, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan (autoSummarize on) produced no root") + } + guard let offRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(false, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan (autoSummarize off) produced no root") + } + + XCTAssertEqual(onRoot.size, offRoot.size, "root size must be identical whether or not subtrees got collapsed") + XCTAssertEqual(totalAccountedFiles(onRoot), totalAccountedFiles(offRoot), "every file must be accounted for exactly once in both modes") + // 1 plain file + 20 files under node_modules. + XCTAssertEqual(totalAccountedFiles(offRoot), 21) + } + + // MARK: - 4. Hardlinks inside a summarized subtree are not double-counted + + func test_hardlinks_not_double_counted_in_summary() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: tmp) } + + let nodeModules = tmp.appendingPathComponent("node_modules") + try FileManager.default.createDirectory(at: nodeModules, withIntermediateDirectories: true) + + let hard1 = nodeModules.appendingPathComponent("hard1.bin") + let hard2 = nodeModules.appendingPathComponent("hard2.bin") + try Data(repeating: 3, count: 262_144).write(to: hard1) + try FileManager.default.linkItem(at: hard1, to: hard2) + try Data(repeating: 4, count: 16).write(to: nodeModules.appendingPathComponent("other.bin")) + + var st = stat() + XCTAssertEqual(lstat(hard1.path, &st), 0) + let hardlinkSize = Int64(st.st_blocks) * 512 + XCTAssertEqual(lstat(nodeModules.appendingPathComponent("other.bin").path, &st), 0) + let otherSize = Int64(st.st_blocks) * 512 + + guard let root = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(true, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan produced no root") + } + guard let nmNode = findNode(root, path: ["node_modules"]) else { + return XCTFail("node_modules node not found") + } + + XCTAssertTrue(nmNode.isAutoSummarized) + XCTAssertEqual(nmNode.descendantFileCount, 3, "both hardlink names and the unrelated file are each one descendant file") + XCTAssertEqual(nmNode.size, hardlinkSize + otherSize, "the hardlinked pair must contribute its allocated bytes exactly once") + } + + // MARK: - 5. Env-gated benchmark + + func test_benchmark_autosummary() async throws { + guard ProcessInfo.processInfo.environment["MDS_SUMMARY_BENCH"] == "1" else { + throw XCTSkip("set MDS_SUMMARY_BENCH=1 to run the benchmark") + } + guard let path = ProcessInfo.processInfo.environment["MDS_SUMMARY_BENCH_PATH"] else { + return XCTFail("set MDS_SUMMARY_BENCH_PATH to a directory containing a node_modules-heavy tree") + } + let url = URL(fileURLWithPath: path) + + func countNodes(_ node: FSNode) -> Int { + 1 + node.children.reduce(0) { $0 + countNodes($1) } + } + + // The app's default excludedFolderNames already fully excludes + // "node_modules" (see ScanConfig); drop it here so the benchmark + // actually exercises the named-layout summarization path instead of + // both runs skipping the tree identically. + let onStart = DispatchTime.now() + guard let onRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(true, { await scanTree(at: url) }) + }) else { + return XCTFail("summarize-on benchmark scan produced no root") + } + let onElapsed = Double(DispatchTime.now().uptimeNanoseconds - onStart.uptimeNanoseconds) / 1_000_000_000 + + let offStart = DispatchTime.now() + guard let offRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(false, { await scanTree(at: url) }) + }) else { + return XCTFail("summarize-off benchmark scan produced no root") + } + let offElapsed = Double(DispatchTime.now().uptimeNanoseconds - offStart.uptimeNanoseconds) / 1_000_000_000 + + let nodesOn = countNodes(onRoot) + let nodesOff = countNodes(offRoot) + let rootBytesEqual = onRoot.size == offRoot.size + + print("MDS_SUMMARY_BENCH on=\(onElapsed) off=\(offElapsed) nodes_on=\(nodesOn) nodes_off=\(nodesOff) rootBytesEqual=\(rootBytesEqual)") + + XCTAssertTrue(rootBytesEqual, "auto-summarization must not change the total scanned size") + } +} From d3f809dfb8f1a717df8b3e4700c9128f18ba36ea Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 12:49:26 +0300 Subject: [PATCH 15/41] perf(scanner): parallel auto-summary walk --- Sources/Scanner/AtomicDirectorySummary.swift | 339 +++++++++++++++---- Tests/AutoSummaryTests.swift | 55 ++- 2 files changed, 330 insertions(+), 64 deletions(-) diff --git a/Sources/Scanner/AtomicDirectorySummary.swift b/Sources/Scanner/AtomicDirectorySummary.swift index ff4ead8..97967b3 100644 --- a/Sources/Scanner/AtomicDirectorySummary.swift +++ b/Sources/Scanner/AtomicDirectorySummary.swift @@ -1,4 +1,5 @@ import Darwin +import Dispatch import Foundation // Auto-summarization: when a directory is clearly a generated / tiny-file @@ -10,9 +11,10 @@ import Foundation // to browse file-by-file. // // v1 scope is deliberately simple: a named-layout shortcut plus an -// immediate-file-count/average-size heuristic, walked single-threaded by the -// one worker that already owns the candidate directory. See the TODOs below -// for what a later phase could add. +// immediate-file-count/average-size heuristic. Phase 3.5 replaced the walk +// itself with a bounded parallel pool (see `summarizeSubtree` below) so that +// summarizing a huge subtree is a scan-speed win too, not just a node-count +// win. // Directory names that are, by convention, generated dependency/package // trees not worth browsing file-by-file. Matched against a directory's own @@ -50,8 +52,7 @@ func shouldAutoSummarize(entries: [BulkDirEntry], name: String, depth: Int) -> B // TODO(phase3+): bounded descendant probe for ambiguous dirs (e.g. a // directory whose files are deeper than its immediate listing, but that // doesn't match a known name) - see Radix's AtomicDirectorySummarizer - // for the pattern; out of scope here to keep the summary walk simple - // and single-threaded. + // for the pattern; out of scope here to keep the summarization gate simple. guard depth >= AtomicSummaryThresholds.minDepth else { return false } var fileCount = 0 @@ -65,89 +66,301 @@ func shouldAutoSummarize(entries: [BulkDirEntry], name: String, depth: Int) -> B return averageAllocated < AtomicSummaryThresholds.maxAverageFileSize } -// Iterative, single-threaded walk of the subtree rooted at a directory -// already deemed a summarization candidate. Applies the exact same scan -// semantics as the main traversal (symlink/hidden/exclusion/mount-point -// skips, hardlink + directory dedup via the shared `visited` set) so the -// aggregate size/file-count it returns matches what a full, non-summarized -// scan of the same subtree would have produced. +// Thread-safe pending-directory queue for the summarization walk, analogous +// to `WorkQueue` in FileScanner.swift but holding directory PATHS (a +// directory's (dev, ino) dedup check already happened when it was +// discovered and pushed here, so the queue itself needs no identity checks). +// `pop()` returning nil doesn't mean "done" - callers must also check +// `isFinished` (stack empty AND nothing in flight), since another worker's +// current item may still push more work. +private final class SummaryDirQueue: @unchecked Sendable { + private var lock = os_unfair_lock() + private var stack: [String] + private var inFlight: Int + + init(seed: [String]) { + stack = seed + inFlight = seed.count + } + + func push(_ path: String) { + os_unfair_lock_lock(&lock) + stack.append(path) + inFlight += 1 + os_unfair_lock_unlock(&lock) + } + + func pop() -> String? { + os_unfair_lock_lock(&lock) + let item = stack.popLast() + os_unfair_lock_unlock(&lock) + return item + } + + // Call exactly once per path that was popped, after it has been fully + // processed (including pushing any subdirectories it discovered). + func markDone() { + os_unfair_lock_lock(&lock) + inFlight -= 1 + os_unfair_lock_unlock(&lock) + } + + var isFinished: Bool { + os_unfair_lock_lock(&lock) + let finished = stack.isEmpty && inFlight == 0 + os_unfair_lock_unlock(&lock) + return finished + } +} + +// Thread-safe running total for the summarization walk's aggregate result. +private final class SummaryAccumulator: @unchecked Sendable { + private var lock = os_unfair_lock() + private var totalAllocated: Int64 + private var totalFileCount: Int + + init(allocatedSize: Int64, fileCount: Int) { + totalAllocated = allocatedSize + totalFileCount = fileCount + } + + func add(files: Int, bytes: Int64) { + os_unfair_lock_lock(&lock) + totalAllocated += bytes + totalFileCount += files + os_unfair_lock_unlock(&lock) + } + + var snapshot: (allocatedSize: Int64, fileCount: Int) { + os_unfair_lock_lock(&lock) + let result = (totalAllocated, totalFileCount) + os_unfair_lock_unlock(&lock) + return result + } +} + +// Tiny lock-guarded box used only to carry the first error thrown by the +// worker pool back across the sync/async bridge in `summarizeSubtree`. +private final class SummaryResultBox: @unchecked Sendable { + private var lock = os_unfair_lock() + private var storedError: Error? + + func setErrorIfAbsent(_ error: Error) { + os_unfair_lock_lock(&lock) + if storedError == nil { storedError = error } + os_unfair_lock_unlock(&lock) + } + + var error: Error? { + os_unfair_lock_lock(&lock) + let result = storedError + os_unfair_lock_unlock(&lock) + return result + } +} + +// Parallel walk of the subtree rooted at a directory already deemed a +// summarization candidate. Applies the exact same scan semantics as the main +// traversal (symlink/hidden/exclusion/mount-point skips, hardlink + +// directory dedup via the shared `visited` set) so the aggregate size/file +// count it returns matches what a full, non-summarized scan of the same +// subtree would have produced - regardless of how many workers race to +// discover each directory/file, since `visited.visit` is atomic. That +// invariant (count each (dev, ino) exactly once, no matter which worker sees +// it first) is exactly why this parallel result equals the single-threaded +// result computed before Phase 3.5. // // `rootEntries` is the already-decoded listing of the candidate directory // itself (the caller enumerated it once to run `shouldAutoSummarize`); reused // here instead of re-enumerating so summarization costs zero extra I/O for -// the root of the collapsed subtree. Descendant directories are opened by -// path as the walk descends (this candidate directory is already being -// processed by one worker, so keeping the walk single-threaded here adds no -// contention - see TODO below for a future parallel version). +// the root of the collapsed subtree. That root listing is consumed +// synchronously, on the calling thread, exactly as before Phase 3.5 - only +// once there is more than one discovered subdirectory do we spin up the +// worker pool below. // -// TODO(phase3+): parallel summary pool - fan the descendant walk of very -// large summarized subtrees out across multiple workers, the way the main -// traversal does. Left single-threaded for v1: correctness and the -// node-count/scan-time win both come from not materializing FSNodes, which -// a single-threaded walk already delivers. +// No FSNode objects are built here, so there is no shared-mutable-node +// hazard to worry about; the only shared mutable state is the queue and the +// accumulator, both lock-guarded. +// +// Concurrency bridge: this function's signature stays synchronous (`throws`, +// not `async`) so its one call site in FileScanner.swift is unchanged - it +// already runs on one of the main scan's worker threads, itself inside an +// async Task. To drive `withThrowingTaskGroup` from here, it hands the pool +// off to a detached Task and blocks this thread on a semaphore, polling with +// a short timeout so it can keep re-checking the ORIGINAL caller's +// cancellation via the passed-in `cancel` closure (`Task.checkCancellation` +// is dynamic-scoped to whatever Task is executing at the call site, so +// checking it here - still on the calling worker's own Task - reflects the +// real scan's cancellation; the detached pool Task is a separate Task, so +// checking cancellation from inside it would never see the outer scan being +// cancelled unless we explicitly forward it, which is what the polling loop +// below does via `poolTask.cancel()`). +// +// This runs nested inside one main-scan worker's task; a nested worker pool +// is fine here. Oversubscription (main pool x summary pool) is bounded (both +// pools cap at 8 workers) and self-limiting: only directories big enough to +// trip auto-summarization spin up a nested pool at all, and macOS's +// thread-pool scheduling handles the resulting oversubscription gracefully. func summarizeSubtree( rootEntries: [BulkDirEntry], rootPath: String, rootDev: UInt64, config: ScanConfig, visited: VisitedSet, - cancel: () throws -> Void + cancel: @Sendable @escaping () throws -> Void ) throws -> (allocatedSize: Int64, fileCount: Int) { var totalAllocated: Int64 = 0 var fileCount = 0 + var seedDirs: [String] = [] var entriesSeen = 0 - // Directories still to be listed, identified by their full path (their - // (dev, ino) dedup check already happened when they were discovered and - // pushed here). - var pendingDirs: [String] = [] - - func consume(_ entries: [BulkDirEntry], dirPath: String) throws { - for entry in entries { - entriesSeen += 1 - if entriesSeen % 256 == 0 { try cancel() } - - guard entry.name != "." && entry.name != ".." else { continue } - if entry.name.hasPrefix("."), !config.showHiddenFiles { continue } - if config.excludedNames.contains(entry.name) { continue } - - switch entry.kind { - case .symlink, .other: - continue - - case .directory: - // Mount point: skip directories on a different device than the scan root. - if entry.dev != rootDev { continue } - // Dedup by (dev, ino): same firmlink/hardlink protection as the main scanner. - guard visited.visit(dev: entry.dev, ino: entry.ino) else { continue } - let childPath = dirPath.hasSuffix("/") ? dirPath + entry.name : dirPath + "/" + entry.name - pendingDirs.append(childPath) - - case .file: - totalAllocated += bulkAllocatedSize(entry: entry, visited: visited) - fileCount += 1 - } + for entry in rootEntries { + entriesSeen += 1 + if entriesSeen % 256 == 0 { try cancel() } + + guard entry.name != "." && entry.name != ".." else { continue } + if entry.name.hasPrefix("."), !config.showHiddenFiles { continue } + if config.excludedNames.contains(entry.name) { continue } + + switch entry.kind { + case .symlink, .other: + continue + + case .directory: + // Mount point: skip directories on a different device than the scan root. + if entry.dev != rootDev { continue } + // Dedup by (dev, ino): same firmlink/hardlink protection as the main scanner. + guard visited.visit(dev: entry.dev, ino: entry.ino) else { continue } + let childPath = rootPath.hasSuffix("/") ? rootPath + entry.name : rootPath + "/" + entry.name + seedDirs.append(childPath) + + case .file: + totalAllocated += bulkAllocatedSize(entry: entry, visited: visited) + fileCount += 1 } } - try consume(rootEntries, dirPath: rootPath) + // No subdirectories discovered at the candidate root: nothing left to + // walk, so skip spinning up a worker pool entirely. + guard !seedDirs.isEmpty else { + return (totalAllocated, fileCount) + } - while let dirPath = pendingDirs.popLast() { - try cancel() - let fd = open(dirPath, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW) - guard fd >= 0 else { continue } - defer { close(fd) } + let queue = SummaryDirQueue(seed: seedDirs) + let accumulator = SummaryAccumulator(allocatedSize: totalAllocated, fileCount: fileCount) + let resultBox = SummaryResultBox() + let workerCount = min(max(2, ProcessInfo.processInfo.activeProcessorCount / 2), 8) - let entries: [BulkDirEntry] + let semaphore = DispatchSemaphore(value: 0) + let poolTask = Task.detached(priority: .userInitiated) { do { - entries = try listDirectoryEntries(path: dirPath, fd: fd, forceFallback: config.forceFallbackEnum) + try await withThrowingTaskGroup(of: Void.self) { group in + for _ in 0.. Void +) async throws { + while true { + try cancel() + if let dirPath = queue.pop() { + try _processSummaryDirectory(dirPath: dirPath, rootDev: rootDev, config: config, visited: visited, accumulator: accumulator, queue: queue, cancel: cancel) + continue + } + if queue.isFinished { return } + await Task.yield() + } +} + +// Processes exactly one directory of the summarized subtree: opens it, lists +// its immediate children (bulk enumeration with fallback), applies all scan +// semantics, adds direct file bytes/count to the shared accumulator, and +// pushes any subdirectories as new queue entries. Always calls +// `queue.markDone()` exactly once, even on early return - mirrors +// `_processDirectory` in FileScanner.swift. +private func _processSummaryDirectory( + dirPath: String, + rootDev: UInt64, + config: ScanConfig, + visited: VisitedSet, + accumulator: SummaryAccumulator, + queue: SummaryDirQueue, + cancel: @Sendable () throws -> Void +) throws { + defer { queue.markDone() } + try cancel() + + let fd = open(dirPath, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW) + guard fd >= 0 else { return } + defer { close(fd) } + + let entries: [BulkDirEntry] + do { + entries = try listDirectoryEntries(path: dirPath, fd: fd, forceFallback: config.forceFallbackEnum) + } catch { + // Enumeration failed even after falling back: contributes nothing + // further, matches the main scanner's "treat as empty" rule. + return + } + + var entriesSeen = 0 + var localFiles = 0 + var localBytes: Int64 = 0 + + for entry in entries { + entriesSeen += 1 + if entriesSeen % 256 == 0 { try cancel() } + + guard entry.name != "." && entry.name != ".." else { continue } + if entry.name.hasPrefix("."), !config.showHiddenFiles { continue } + if config.excludedNames.contains(entry.name) { continue } + + switch entry.kind { + case .symlink, .other: continue + + case .directory: + // Mount point: skip directories on a different device than the scan root. + if entry.dev != rootDev { continue } + // Dedup by (dev, ino): same firmlink/hardlink protection as the main scanner. + guard visited.visit(dev: entry.dev, ino: entry.ino) else { continue } + let childPath = dirPath.hasSuffix("/") ? dirPath + entry.name : dirPath + "/" + entry.name + queue.push(childPath) + + case .file: + localBytes += bulkAllocatedSize(entry: entry, visited: visited) + localFiles += 1 } - try consume(entries, dirPath: dirPath) } - return (totalAllocated, fileCount) + // Batch this directory's file bytes/count into a single lock acquisition + // rather than one per file. + if localFiles > 0 { accumulator.add(files: localFiles, bytes: localBytes) } } diff --git a/Tests/AutoSummaryTests.swift b/Tests/AutoSummaryTests.swift index 96f3cb3..8e61586 100644 --- a/Tests/AutoSummaryTests.swift +++ b/Tests/AutoSummaryTests.swift @@ -223,7 +223,60 @@ final class AutoSummaryTests: XCTestCase { XCTAssertEqual(nmNode.size, hardlinkSize + otherSize, "the hardlinked pair must contribute its allocated bytes exactly once") } - // MARK: - 5. Env-gated benchmark + // MARK: - 5. Parallel walk matches a serial (autoSummarize off) scan + + // Builds a moderately deep/wide tree that trips summarization (via a low + // MDS_SUMMARY_MIN_FILES override), scans it with autoSummarize on (which + // now drives the parallel worker pool in `summarizeSubtree`) and off + // (full, per-file walk), and asserts the summarized node's aggregate + // size and descendantFileCount exactly match the full scan's totals. + // This is a determinism/parity check for the parallel accumulator: since + // multiple workers race to discover directories/hardlinks, this is the + // test that would catch any double-counting or dropped entries the + // parallelization could introduce. + func test_parallel_summary_matches_serial_for_nested_tree() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: tmp) } + + // Use the node_modules named-layout shortcut (rather than the + // immediate-file-count heuristic) so the tree can be wide - many + // independent "package" subdirectories, each with its own shallow + // nested chain - without needing files directly inside node_modules + // itself. Wide-and-shallow, rather than one long chained path, is + // what actually makes the parallel pool fan out across many workers + // while staying well under macOS's path-length limit. + let nodeModules = tmp.appendingPathComponent("proj").appendingPathComponent("node_modules") + var expectedTotal: Int64 = 0 + for pkg in 0..<20 { + expectedTotal += try makeFiles(count: 20, bytes: 32, under: nodeModules.appendingPathComponent("pkg\(pkg)"), nestedEvery: 3) + } + + guard let onRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(true, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan (autoSummarize on) produced no root") + } + guard let nmNode = findNode(onRoot, path: ["proj", "node_modules"]) else { + return XCTFail("node_modules node not found") + } + XCTAssertTrue(nmNode.isAutoSummarized, "node_modules-named directory should trip summarization") + + guard let offRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(false, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan (autoSummarize off) produced no root") + } + guard let offNmNode = findNode(offRoot, path: ["proj", "node_modules"]) else { + return XCTFail("node_modules node not found in the non-summarized scan") + } + + XCTAssertEqual(nmNode.size, expectedTotal, "parallel summary walk must total exactly the bytes on disk") + XCTAssertEqual(nmNode.descendantFileCount, 400, "parallel summary walk must count every file exactly once") + XCTAssertEqual(nmNode.size, offNmNode.size, "parallel summary result must match a full serial scan's size") + XCTAssertEqual(nmNode.descendantFileCount, totalAccountedFiles(offNmNode), "parallel summary result must match a full serial scan's file count") + } + + // MARK: - 6. Env-gated benchmark func test_benchmark_autosummary() async throws { guard ProcessInfo.processInfo.environment["MDS_SUMMARY_BENCH"] == "1" else { From f7de7513983c29081c1a1f189033b55574394ed3 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 13:11:55 +0300 Subject: [PATCH 16/41] =?UTF-8?q?WIP(phase2):=20flat=20FileTree=20store=20?= =?UTF-8?q?=E2=80=94=20app=20target=20green,=20tests=20not=20yet=20migrate?= =?UTF-8?q?d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New Sources/Model/ types (FileTree, FileNodeRecord, FileNode, FileTreeBuilder) replace FSNode as the app-facing tree representation. FSNode remains the scanner's internal build type; FileScanner assembles a FileTree at the end of a scan via FileTreeBuilder and yields it through the changed ScanProgress.completed(tree:deniedCount:) case. Migrated to FileNode: ScanViewModel (tree: FileTree? replaces root: FSNode?, with `root` now a computed FileNode?), DuplicateDetector (detect(in: FileTree) over flat indices), SafetyAnalyzer (added FileNode overloads alongside the existing FSNode ones, which the live-refresh FSNode helpers still use), ExtensionColorMap, TreemapLayout, TreemapCell, and all Views (DirectoryTree, Duplicates, ExtensionList, Treemap, FileTypeIcon, HapticEngine). Live-refresh path: interim full-rescan-on-change (handleFileSystemChanges now calls scan(url:) again) rather than the splice-based approach, marked TODO(phase4). The FSNode-based refreshDirectory/scanSubtree/bubbleUpSizes/ findNode/firstNode helpers in ScanViewModel are kept completely unchanged (unused by the live path for now) so they keep compiling and so Phase 4 can reuse them as the per-directory rescan step of a real splice. `swift build` (app target) is green. Tests/*.swift not yet migrated to the new FileTree/FileNode types — that's next. --- Sources/Duplicates/DuplicateDetector.swift | 100 ++++---- Sources/Layout/ExtensionColorMap.swift | 2 +- Sources/Layout/TreemapCell.swift | 2 +- Sources/Layout/TreemapLayout.swift | 6 +- Sources/Model/FileNode.swift | 76 ++++++ Sources/Model/FileNodeRecord.swift | 47 ++++ Sources/Model/FileTree.swift | 135 ++++++++++ Sources/Model/FileTreeBuilder.swift | 81 ++++++ Sources/Safety/SafetyAnalyzer.swift | 31 ++- Sources/Scanner/FileScanner.swift | 13 +- Sources/Scanner/ScanProgress.swift | 2 +- Sources/ViewModels/ScanViewModel.swift | 230 ++++++++---------- .../DirectoryTree/DirectoryTreeView.swift | 2 +- Sources/Views/Duplicates/DuplicatesView.swift | 14 +- .../ExtensionList/ExtensionListView.swift | 2 +- Sources/Views/Shared/FileTypeIcon.swift | 4 +- Sources/Views/Treemap/HapticEngine.swift | 6 +- Sources/Views/Treemap/TreemapRenderer.swift | 4 +- Sources/Views/Treemap/TreemapView.swift | 6 +- 19 files changed, 550 insertions(+), 213 deletions(-) create mode 100644 Sources/Model/FileNode.swift create mode 100644 Sources/Model/FileNodeRecord.swift create mode 100644 Sources/Model/FileTree.swift create mode 100644 Sources/Model/FileTreeBuilder.swift diff --git a/Sources/Duplicates/DuplicateDetector.swift b/Sources/Duplicates/DuplicateDetector.swift index a2a8c63..d271050 100644 --- a/Sources/Duplicates/DuplicateDetector.swift +++ b/Sources/Duplicates/DuplicateDetector.swift @@ -13,111 +13,113 @@ public actor DuplicateDetector { public init() {} - public func detect(in root: FSNode) async { - var candidates: [FSNode] = [] - collect(node: root, into: &candidates) + // Operates over flat FileTree indices rather than an FSNode tree — a + // simple loop over `tree.records` reaches every node without recursion, + // since the array already covers the whole tree regardless of hierarchy. + public func detect(in tree: FileTree) async { + var candidates: [Int] = [] + collect(tree: tree, into: &candidates) // Group by size first — eliminates the vast majority of files cheaply - let bySize = Dictionary(grouping: candidates) { $0.size } + let bySize = Dictionary(grouping: candidates) { tree.records[$0].size } .filter { $0.value.count > 1 } let quickHashBytes = self.quickHashBytes // Phase 1: quick hash (first 64 KB) to rule out non-duplicates without reading entire files. // Hashing is I/O-bound and embarrassingly parallel, so run it with bounded concurrency. - var byQuickHash: [String: [FSNode]] = [:] - let quickHashResults: [(FSNode, String)]? = await Self.hashInParallel( - nodes: bySize.values.flatMap { $0 } - ) { node in - Self.partialHash(url: node.url, maxBytes: quickHashBytes) + var byQuickHash: [String: [Int]] = [:] + let quickHashResults: [(Int, String)]? = await Self.hashInParallel( + indices: bySize.values.flatMap { $0 } + ) { index in + Self.partialHash(url: FileNode(tree: tree, index: index).url, maxBytes: quickHashBytes) } guard let quickHashResults else { return } - for (node, qh) in quickHashResults { - let key = "\(node.size)-\(qh)" - byQuickHash[key, default: []].append(node) + for (index, qh) in quickHashResults { + let key = "\(tree.records[index].size)-\(qh)" + byQuickHash[key, default: []].append(index) } // Small-file shortcut: if a file's whole content fits within the quick-hash window, // the quick hash already IS a full-content hash, so those groups are final as-is. // Only groups whose files exceed the quick-hash window need a full-file hash pass. - var hashGroups: [String: [FSNode]] = [:] - var toFullHash: [FSNode] = [] - for (key, nodes) in byQuickHash where nodes.count > 1 { - if nodes[0].size <= Int64(quickHashBytes) { - hashGroups[key] = nodes + var hashGroups: [String: [Int]] = [:] + var toFullHash: [Int] = [] + for (key, indices) in byQuickHash where indices.count > 1 { + if tree.records[indices[0]].size <= Int64(quickHashBytes) { + hashGroups[key] = indices } else { - toFullHash.append(contentsOf: nodes) + toFullHash.append(contentsOf: indices) } } // Phase 2: full hash only for groups that survived the quick-hash filter and are // larger than the quick-hash window (their quick hash alone is not conclusive). if !toFullHash.isEmpty { - let fullHashResults: [(FSNode, String)]? = await Self.hashInParallel(nodes: toFullHash) { node in - Self.fullHash(url: node.url) + let fullHashResults: [(Int, String)]? = await Self.hashInParallel(indices: toFullHash) { index in + Self.fullHash(url: FileNode(tree: tree, index: index).url) } guard let fullHashResults else { return } - for (node, hash) in fullHashResults { - let key = "\(node.size)-\(hash)" - hashGroups[key, default: []].append(node) + for (index, hash) in fullHashResults { + let key = "\(tree.records[index].size)-\(hash)" + hashGroups[key, default: []].append(index) } } // Assign group IDs to genuine duplicates - for (_, nodes) in hashGroups where nodes.count > 1 { + for (_, indices) in hashGroups where indices.count > 1 { guard !Task.isCancelled else { return } let groupID = UUID() - for node in nodes { node.duplicateGroupID = groupID } + for index in indices { tree.setDuplicateGroupID(groupID, at: index) } } } - private func collect(node: FSNode, into list: inout [FSNode]) { - var stack: [FSNode] = [node] - while let current = stack.popLast() { + private func collect(tree: FileTree, into list: inout [Int]) { + for index in 0..= minSize && current.size <= maxSize { - list.append(current) + if !record.isSynthetic && !record.isDirectory && record.size >= minSize && record.size <= maxSize { + list.append(index) } - stack.append(contentsOf: current.children) } } - // Runs `hash` over `nodes` with bounded sliding-window concurrency, returning nil if the - // task was cancelled before completion. Nodes for which `hash` returns nil are dropped. + // Runs `hash` over `indices` with bounded sliding-window concurrency, returning nil if the + // task was cancelled before completion. Indices for which `hash` returns nil are dropped. private static func hashInParallel( - nodes: [FSNode], - hash: @escaping @Sendable (FSNode) -> String? - ) async -> [(FSNode, String)]? { - guard !nodes.isEmpty else { return [] } + indices: [Int], + hash: @escaping @Sendable (Int) -> String? + ) async -> [(Int, String)]? { + guard !indices.isEmpty else { return [] } guard !Task.isCancelled else { return nil } - var results: [(FSNode, String)] = [] - results.reserveCapacity(nodes.count) + var results: [(Int, String)] = [] + results.reserveCapacity(indices.count) - await withTaskGroup(of: (FSNode, String?).self) { group in - var index = 0 + await withTaskGroup(of: (Int, String?).self) { group in + var cursor = 0 let limit = maxConcurrency func launchNext() { - guard index < nodes.count else { return } - let node = nodes[index] - index += 1 + guard cursor < indices.count else { return } + let index = indices[cursor] + cursor += 1 group.addTask { - guard !Task.isCancelled else { return (node, nil) } - return (node, hash(node)) + guard !Task.isCancelled else { return (index, nil) } + return (index, hash(index)) } } // Prime the sliding window. - while index < limit && index < nodes.count { + while cursor < limit && cursor < indices.count { launchNext() } - while let (node, key) = await group.next() { + while let (index, key) = await group.next() { if let key { - results.append((node, key)) + results.append((index, key)) } if Task.isCancelled { // Drain remaining in-flight tasks without launching more. diff --git a/Sources/Layout/ExtensionColorMap.swift b/Sources/Layout/ExtensionColorMap.swift index baaea70..056e0a8 100644 --- a/Sources/Layout/ExtensionColorMap.swift +++ b/Sources/Layout/ExtensionColorMap.swift @@ -33,7 +33,7 @@ public struct ExtensionColorMap: Equatable, Sendable { let scheme: String - public init(root: FSNode) { + public init(root: FileNode) { self.scheme = UserDefaults.standard.string(forKey: "treemapColorScheme") ?? "byType" } diff --git a/Sources/Layout/TreemapCell.swift b/Sources/Layout/TreemapCell.swift index c055577..92414bc 100644 --- a/Sources/Layout/TreemapCell.swift +++ b/Sources/Layout/TreemapCell.swift @@ -3,7 +3,7 @@ import SwiftUI /// Arc-based cell for the sunburst visualization. public struct TreemapCell: Identifiable { public let id: UUID = UUID() - public let node: FSNode + public let node: FileNode public let startAngle: Double // radians; 0 = right, increases clockwise on screen public let endAngle: Double public let innerRadius: CGFloat diff --git a/Sources/Layout/TreemapLayout.swift b/Sources/Layout/TreemapLayout.swift index 14a975b..e1fd1db 100644 --- a/Sources/Layout/TreemapLayout.swift +++ b/Sources/Layout/TreemapLayout.swift @@ -21,7 +21,7 @@ public struct TreemapLayout { // Gentle per-depth darkening so deep files stay recognisable private static let depthDarken: [Double] = [0.0, 0.10, 0.18, 0.26, 0.32, 0.38] - public static func compute(root: FSNode, in rect: CGRect, colorMap: ExtensionColorMap) -> [TreemapCell] { + public static func compute(root: FileNode, in rect: CGRect, colorMap: ExtensionColorMap) -> [TreemapCell] { var cells: [TreemapCell] = [] guard rect.width > 1, rect.height > 1, root.size > 0 else { return cells } @@ -41,7 +41,7 @@ public struct TreemapLayout { } private static func layout( - _ children: [FSNode], + _ children: [FileNode], parentStart: Double, parentEnd: Double, depth: Int, @@ -100,7 +100,7 @@ public struct TreemapLayout { return Double(hash % 360) / 360.0 } - private static func color(for node: FSNode, depth: Int, colorMap: ExtensionColorMap) -> Color { + private static func color(for node: FileNode, depth: Int, colorMap: ExtensionColorMap) -> Color { if node.isSynthetic { // Distinct muted gray so the hidden-space reconciliation node reads as // "not a real file" at a glance, rather than blending into the chart. diff --git a/Sources/Model/FileNode.swift b/Sources/Model/FileNode.swift new file mode 100644 index 0000000..86246b7 --- /dev/null +++ b/Sources/Model/FileNode.swift @@ -0,0 +1,76 @@ +import Foundation + +// Lightweight, app-facing handle into a `FileTree` — the replacement for +// `FSNode` everywhere outside the scanner's own internal build step. A +// `FileNode` is just a (tree, index) pair; every property reads through to +// the tree's flat storage, so copying a `FileNode` is cheap and many can +// exist for the same underlying record. +public struct FileNode: Identifiable, Hashable, Sendable { + public let tree: FileTree + public let index: Int + + public init(tree: FileTree, index: Int) { + self.tree = tree + self.index = index + } + + // Stable within one tree instance — sufficient for SwiftUI identity and + // selection. Not globally unique across trees (e.g. before/after a + // rescan replaces `tree` entirely), which mirrors how the rest of the UI + // already treats a rescan as a clean slate. + public var id: Int { index } + + public var name: String { tree.records[index].name } + public var size: Int64 { tree.records[index].size } + public var isDirectory: Bool { tree.records[index].isDirectory } + public var fileExtension: String { tree.records[index].fileExtension } + public var isAccessDenied: Bool { tree.records[index].isAccessDenied } + public var isSynthetic: Bool { tree.records[index].isSynthetic } + public var isAutoSummarized: Bool { tree.records[index].isAutoSummarized } + public var descendantFileCount: Int { tree.records[index].descendantFileCount } + public var hardLinkRef: HardLinkRef? { tree.records[index].hardLinkRef } + public var duplicateGroupID: UUID? { tree.records[index].duplicateGroupID } + public var safetyLevel: SafetyLevel { tree.records[index].safetyLevel } + + public var url: URL { URL(fileURLWithPath: tree.path(of: index)) } + + public var parent: FileNode? { + let p = tree.parentIndex[index] + return p >= 0 ? FileNode(tree: tree, index: p) : nil + } + + // Pre-sorted size-desc — the builder (and the synthetic-node splice) + // guarantee this, so no consumer ever needs to sort children itself. + public var children: [FileNode] { + let start = tree.childStart[index] + let count = tree.childCount[index] + guard count > 0 else { return [] } + return (0.. 0 else { return nil } + return children + } + + // MARK: - Mutation (writes through to the shared tree) + + public func setDuplicateGroupID(_ id: UUID?) { + tree.setDuplicateGroupID(id, at: index) + } + + public func setSafety(_ level: SafetyLevel) { + tree.setSafety(level, at: index) + } + + // MARK: - Hashable / Equatable + + public static func == (lhs: FileNode, rhs: FileNode) -> Bool { + ObjectIdentifier(lhs.tree) == ObjectIdentifier(rhs.tree) && lhs.index == rhs.index + } + + public func hash(into hasher: inout Hasher) { + hasher.combine(ObjectIdentifier(tree)) + hasher.combine(index) + } +} diff --git a/Sources/Model/FileNodeRecord.swift b/Sources/Model/FileNodeRecord.swift new file mode 100644 index 0000000..7a0770d --- /dev/null +++ b/Sources/Model/FileNodeRecord.swift @@ -0,0 +1,47 @@ +import Foundation + +// Per-node payload stored contiguously in `FileTree.records`. This is the +// flat, struct-of-arrays replacement for `FSNode`'s per-node instance fields: +// no URL and no UUID are stored per record (that was FSNode's memory cost on +// huge scans) — the absolute path is reconstructed on demand from names + +// parentIndex (see `FileTree.path(of:)` / `FileNode.url`), and node identity +// for SwiftUI purposes is the (tree, index) pair (see `FileNode.id`). +public struct FileNodeRecord: Sendable { + public let name: String + public let isDirectory: Bool + public var size: Int64 + public let fileExtension: String + public var isAccessDenied: Bool + public var isSynthetic: Bool + public var isAutoSummarized: Bool + public var descendantFileCount: Int + public var hardLinkRef: HardLinkRef? + public var duplicateGroupID: UUID? + public var safetyLevel: SafetyLevel + + public init( + name: String, + isDirectory: Bool, + size: Int64, + fileExtension: String, + isAccessDenied: Bool = false, + isSynthetic: Bool = false, + isAutoSummarized: Bool = false, + descendantFileCount: Int = 0, + hardLinkRef: HardLinkRef? = nil, + duplicateGroupID: UUID? = nil, + safetyLevel: SafetyLevel = .caution + ) { + self.name = name + self.isDirectory = isDirectory + self.size = size + self.fileExtension = fileExtension + self.isAccessDenied = isAccessDenied + self.isSynthetic = isSynthetic + self.isAutoSummarized = isAutoSummarized + self.descendantFileCount = descendantFileCount + self.hardLinkRef = hardLinkRef + self.duplicateGroupID = duplicateGroupID + self.safetyLevel = safetyLevel + } +} diff --git a/Sources/Model/FileTree.swift b/Sources/Model/FileTree.swift new file mode 100644 index 0000000..36cca5c --- /dev/null +++ b/Sources/Model/FileTree.swift @@ -0,0 +1,135 @@ +import Foundation + +// Flat struct-of-arrays store for an entire scanned tree — the app-facing +// replacement for the `FSNode` class tree. One array entry per node, indexed +// by a plain `Int` (root is always index 0). This is the memory win over +// per-node class instances: no per-node URL/UUID/weak-parent-pointer +// overhead, and everything lives in a handful of contiguous arrays. +// +// Mutability split: `records` holds per-node fields that legitimately change +// after the initial scan assembly (duplicateGroupID, safetyLevel, size from a +// future refresh) and is mutated in place via the `set...(at:)` methods +// below. The topology arrays (parentIndex/childStart/childCount/childIndices) +// are immutable for the life of one `FileTree` instance — anything that needs +// to change the shape of the tree (e.g. appending the synthetic hidden-space +// node) produces a NEW `FileTree` instead (see `appendingSyntheticRootChild`). +public final class FileTree: @unchecked Sendable { + public private(set) var records: [FileNodeRecord] + let parentIndex: [Int] + let childStart: [Int] + let childCount: [Int] + let childIndices: [Int] + public let rootIndex: Int + // The scanned root's own absolute path (not derivable from its name + // alone) — `path(of:)` is this, plus the joined names of every node + // between the root and the target index. + public let rootPath: String + + public init( + records: [FileNodeRecord], + parentIndex: [Int], + childStart: [Int], + childCount: [Int], + childIndices: [Int], + rootIndex: Int, + rootPath: String + ) { + self.records = records + self.parentIndex = parentIndex + self.childStart = childStart + self.childCount = childCount + self.childIndices = childIndices + self.rootIndex = rootIndex + self.rootPath = rootPath + } + + // MARK: - Mutation (per-node fields only; topology never changes in place) + + public func setDuplicateGroupID(_ id: UUID?, at index: Int) { + records[index].duplicateGroupID = id + } + + public func setSafety(_ level: SafetyLevel, at index: Int) { + records[index].safetyLevel = level + } + + public func setSize(_ size: Int64, at index: Int) { + records[index].size = size + } + + // MARK: - Path reconstruction + + // Rebuilds the absolute path of `index` by walking parentIndex up to the + // root and joining names, since individual records don't store a URL. + public func path(of index: Int) -> String { + guard index != rootIndex else { return rootPath } + + var components: [String] = [] + var current = index + while current != rootIndex { + components.append(records[current].name) + let parent = parentIndex[current] + guard parent >= 0 else { break } + current = parent + } + + let suffix = components.reversed().joined(separator: "/") + return rootPath.hasSuffix("/") ? rootPath + suffix : rootPath + "/" + suffix + } + + // MARK: - Synthetic hidden-space node + + // Returns a NEW tree with one extra leaf record appended as a child of + // root, keeping root's children sorted size-desc (the invariant every + // `FileNode.children` accessor relies on). Used for the "Hidden & + // Unreadable Space" reconciliation entry appended after a volume-root + // scan; see `ScanViewModel.appendHiddenSpaceNodeIfNeeded`. + public func appendingSyntheticRootChild(name: String, size: Int64) -> FileTree { + var newRecords = records + newRecords[rootIndex].size += size + let newNodeIndex = newRecords.count + newRecords.append(FileNodeRecord( + name: name, + isDirectory: false, + size: size, + fileExtension: "", + isSynthetic: true, + safetyLevel: .danger + )) + + var newParentIndex = parentIndex + newParentIndex.append(rootIndex) + + // Root is index 0 by construction, and the builder lays out child + // spans in index order, so root's span is always the very first one + // in `childIndices` — inserting into it only requires shifting every + // later span's start by one; root's own start never moves. + let rootStart = childStart[rootIndex] + let rootCount = childCount[rootIndex] + let rootChildrenEnd = rootStart + rootCount + + var rootChildren = Array(childIndices[rootStart..= rootChildrenEnd { + newChildStart[i] += 1 + } + + return FileTree( + records: newRecords, + parentIndex: newParentIndex, + childStart: newChildStart, + childCount: newChildCount, + childIndices: newChildIndices, + rootIndex: rootIndex, + rootPath: rootPath + ) + } +} diff --git a/Sources/Model/FileTreeBuilder.swift b/Sources/Model/FileTreeBuilder.swift new file mode 100644 index 0000000..0419f02 --- /dev/null +++ b/Sources/Model/FileTreeBuilder.swift @@ -0,0 +1,81 @@ +import Foundation + +// Converts a scanner-internal `FSNode` class tree into a flat `FileTree`. +// `FSNode` remains the scanner's own build type (see FileScanner.swift) so +// none of the traversal logic there had to change; this is the one place +// that walks an already-fully-built `FSNode` tree and assembles the +// compact, app-facing store from it. +// +// Public so tests (and, in principle, any future consumer that still builds +// an `FSNode` tree by hand — e.g. the live-refresh rescan helpers) can drive +// the rest of the app's data flow (layout, view model, duplicate detection) +// from a small hand-built fixture without going through a real disk scan. +public enum FileTreeBuilder { + public static func build(from root: FSNode, rootPath: String) -> FileTree { + var records: [FileNodeRecord] = [] + var parentIndex: [Int] = [] + // Children discovered so far for each index, built up as nodes are + // visited; flattened into `childIndices`/`childStart`/`childCount` + // once the full traversal is done. + var childrenOf: [[Int]] = [] + + // Iterative pre-order walk (explicit stack, not recursion) so + // assembly can't stack-overflow on a very deep real-world tree — + // mirrors the style of the scanner's own iterative traversal. + // + // Children are pushed in ascending-size order so the stack (LIFO) + // pops them back off in descending-size order; since a child is + // appended to its parent's `childrenOf` entry at the moment it's + // popped, this guarantees `childrenOf[i]` ends up sorted size-desc — + // the invariant `FileNode.children` promises its callers. + var stack: [(node: FSNode, parent: Int)] = [(root, -1)] + while let (node, parent) = stack.popLast() { + let index = records.count + records.append(FileNodeRecord( + name: node.name, + isDirectory: node.isDirectory, + size: node.size, + fileExtension: node.fileExtension, + isAccessDenied: node.isAccessDenied, + isSynthetic: node.isSynthetic, + isAutoSummarized: node.isAutoSummarized, + descendantFileCount: node.descendantFileCount, + hardLinkRef: node.hardLinkRef, + // Filled post-assembly: safety tagging and duplicate + // detection both run as a pass over the finished FileTree. + duplicateGroupID: nil, + safetyLevel: .caution + )) + parentIndex.append(parent) + childrenOf.append([]) + if parent >= 0 { + childrenOf[parent].append(index) + } + + let sortedChildren = node.children.sorted { $0.size > $1.size } + for child in sortedChildren.reversed() { + stack.append((child, index)) + } + } + + var childIndices: [Int] = [] + childIndices.reserveCapacity(records.count) + var childStart = [Int](repeating: 0, count: records.count) + var childCount = [Int](repeating: 0, count: records.count) + for i in 0.. SafetyLevel { + level(path: node.url.path, name: node.name, isSynthetic: node.isSynthetic) + } + + // `FileNode` overload: used by the app-facing FileTree tagging pass and + // by every view that reads `safetyLevel`. + public static func level(for node: FileNode) -> SafetyLevel { + level(path: node.url.path, name: node.name, isSynthetic: node.isSynthetic) + } + + private static func level(path: String, name: String, isSynthetic: Bool) -> SafetyLevel { // Synthetic nodes (e.g. the hidden-space reconciliation entry) don't point // at a real, deletable file — always treat them as the most protective // level so nothing ever attempts to trash/move them. - if node.isSynthetic { return .danger } - let path = node.url.path - let name = node.name + if isSynthetic { return .danger } if isDanger(path: path, name: name) { return .danger } if isSafe(path: path, name: name) { return .safe } return .caution @@ -30,11 +41,17 @@ public struct SafetyAnalyzer { // MARK: - On-demand reason (called only when UI needs to display it) public static func reason(for node: FSNode) -> String? { - if node.isSynthetic { + reason(path: node.url.path, name: node.name, isSynthetic: node.isSynthetic) + } + + public static func reason(for node: FileNode) -> String? { + reason(path: node.url.path, name: node.name, isSynthetic: node.isSynthetic) + } + + private static func reason(path: String, name: String, isSynthetic: Bool) -> String? { + if isSynthetic { return "Represents space macOS reports as used but the scanner can't read or enumerate (snapshots, purgeable space, protected folders)" } - let path = node.url.path - let name = node.name return dangerReason(path: path, name: name) ?? safeReason(path: path, name: name) } diff --git a/Sources/Scanner/FileScanner.swift b/Sources/Scanner/FileScanner.swift index e247ee3..c426b7a 100644 --- a/Sources/Scanner/FileScanner.swift +++ b/Sources/Scanner/FileScanner.swift @@ -119,9 +119,9 @@ public actor FileScanner { } do { - let root = try await _buildTree(rootPath: url.path, rootURL: url, counter: counter, visited: visited, config: config) + let tree = try await _buildTree(rootPath: url.path, rootURL: url, counter: counter, visited: visited, config: config) progressTask.cancel() - continuation.yield(.completed(root: root, deniedCount: counter.deniedCount)) + continuation.yield(.completed(tree: tree, deniedCount: counter.deniedCount)) } catch is CancellationError { progressTask.cancel() } catch { @@ -208,12 +208,13 @@ private func _buildTree( counter: ProgressCounter, visited: VisitedSet, config: ScanConfig -) async throws -> FSNode { +) async throws -> FileTree { try Task.checkCancellation() var st = stat() guard lstat(rootPath, &st) == 0 else { - return FSNode(url: rootURL, name: rootURL.lastPathComponent, isDirectory: false, size: 0, fileExtension: "", parent: nil) + let node = FSNode(url: rootURL, name: rootURL.lastPathComponent, isDirectory: false, size: 0, fileExtension: "", parent: nil) + return FileTreeBuilder.build(from: node, rootPath: rootPath) } // Skip symlinks (including a symlink scan root). @@ -229,7 +230,7 @@ private func _buildTree( let node = FSNode(url: rootURL, name: name, isDirectory: false, size: allocSize, fileExtension: ext, parent: nil) if st.st_nlink > 1 { node.hardLinkRef = hardLinkRef(of: st) } counter.add(items: 1, bytes: allocSize) - return node + return FileTreeBuilder.build(from: node, rootPath: rootPath) } let rootDevKey = UInt64(bitPattern: Int64(st.st_dev)) @@ -254,7 +255,7 @@ private func _buildTree( } aggregateDirectorySizes(root: rootNode) - return rootNode + return FileTreeBuilder.build(from: rootNode, rootPath: rootPath) } private func _runWorker( diff --git a/Sources/Scanner/ScanProgress.swift b/Sources/Scanner/ScanProgress.swift index 3a93f84..3bc10c6 100644 --- a/Sources/Scanner/ScanProgress.swift +++ b/Sources/Scanner/ScanProgress.swift @@ -2,6 +2,6 @@ import Foundation public enum ScanProgress: Sendable { case update(itemsScanned: Int, bytesFound: Int64) - case completed(root: FSNode, deniedCount: Int) + case completed(tree: FileTree, deniedCount: Int) case failed(String) } diff --git a/Sources/ViewModels/ScanViewModel.swift b/Sources/ViewModels/ScanViewModel.swift index efff548..ff41b2c 100644 --- a/Sources/ViewModels/ScanViewModel.swift +++ b/Sources/ViewModels/ScanViewModel.swift @@ -3,27 +3,31 @@ import SwiftUI @MainActor public final class ScanViewModel: ObservableObject { - @Published public var root: FSNode? + // The flat store the scanner hands back (see Sources/Model/). `root` is + // a computed FileNode handle onto it, kept under the old name so views + // that just read `vm.root` didn't need to change. + @Published public var tree: FileTree? @Published public var cells: [TreemapCell] = [] @Published public var colorMap: ExtensionColorMap? - @Published public var selectedNode: FSNode? + @Published public var selectedNode: FileNode? @Published public var isScanning: Bool = false @Published public var itemsScanned: Int = 0 @Published public var bytesFound: Int64 = 0 @Published public var errorMessage: String? @Published public var duplicatesReady: Bool = false - @Published public var drillStack: [FSNode] = [] + @Published public var drillStack: [FileNode] = [] @Published public var highlightedExtension: String? @Published public var isComputingLayout: Bool = false @Published public var scanURL: URL? @Published public var extensionSummaries: [ExtensionSummary] = [] - @Published public var duplicateGroups: [[FSNode]] = [] + @Published public var duplicateGroups: [[FileNode]] = [] @Published public var hasFullDiskAccess: Bool = true @Published public var isWatching: Bool = false @Published public var deniedCount: Int = 0 @Published public var showFDASheet: Bool = false - public var treemapRoot: FSNode? { drillStack.last ?? root } + public var root: FileNode? { tree.map { FileNode(tree: $0, index: $0.rootIndex) } } + public var treemapRoot: FileNode? { drillStack.last ?? root } private let scanner = FileScanner() private let fileWatcher = FileWatcher() @@ -150,7 +154,7 @@ public final class ScanViewModel: ObservableObject { layoutGeneration += 1 // invalidate any in-progress layout scanURL = url UserDefaults.standard.set(url.path, forKey: "lastScannedPath") - root = nil + tree = nil cells = [] colorMap = nil selectedNode = nil @@ -172,7 +176,7 @@ public final class ScanViewModel: ObservableObject { case .update(let items, let bytes): self.itemsScanned = items self.bytesFound = bytes - case .completed(let node, let denied): + case .completed(let scannedTree, let denied): self.isScanning = false self.deniedCount = denied if !self.fdaSheetShownThisLaunch, @@ -187,24 +191,29 @@ public final class ScanViewModel: ObservableObject { // If the scanned root is a volume mount point, the file total will // always fall short of Finder's "used" figure (APFS snapshots, // purgeable space, excluded/unreadable folders). Make that gap - // visible instead of silently under-reporting. Must run before the - // sort pass below so the synthetic node sorts into place. - Self.appendHiddenSpaceNodeIfNeeded(root: node, scannedURL: url) + // visible instead of silently under-reporting. Produces a NEW tree + // (topology is immutable) with the synthetic child already in its + // sorted place, so no separate sort pass is needed afterward. + let finalTree = Self.appendHiddenSpaceNodeIfNeeded(tree: scannedTree, scannedURL: url) ?? scannedTree if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { - var dump = "TREE_COMPLETED total=\(node.size) denied=\(denied)\n" - for c in node.children.sorted(by: { $0.size > $1.size }).prefix(15) { + let rootNode = FileNode(tree: finalTree, index: finalTree.rootIndex) + var dump = "TREE_COMPLETED total=\(rootNode.size) denied=\(denied)\n" + for c in rootNode.children.prefix(15) { dump += "TREE_CHILD \(c.size) \(c.name)\(c.isSynthetic ? " [synthetic]" : "")\n" } FileHandle.standardError.write(dump.data(using: .utf8)!) } self.isComputingLayout = true // keep spinner until treemap is ready - // Sort + safety-tag the entire tree off-thread before exposing it to the UI. + // Safety-tag the entire tree off-thread before exposing it to the UI. + // (Children are already size-sorted by FileTreeBuilder/the synthetic + // splice above, so unlike the old FSNode path there is no separate + // sort pass to run here.) await Task.detached(priority: .userInitiated) { - Self.sortAllChildren(node: node) - Self.tagSafetyLevels(node: node) + Self.tagSafetyLevels(tree: finalTree) }.value - self.root = node - let map = ExtensionColorMap(root: node) + self.tree = finalTree + let rootNode = FileNode(tree: finalTree, index: finalTree.rootIndex) + let map = ExtensionColorMap(root: rootNode) self.colorMap = map await self.recomputeLayout() // isComputingLayout set to false inside recomputeLayout @@ -215,18 +224,18 @@ public final class ScanViewModel: ObservableObject { } // Extension summaries: potentially millions of nodes — run off main actor - self.extensionTask = Task.detached(priority: .userInitiated) { [node, map, weak self] in - let summaries = Self.buildExtensionSummaries(root: node, map: map) + self.extensionTask = Task.detached(priority: .userInitiated) { [finalTree, map, weak self] in + let summaries = Self.buildExtensionSummaries(tree: finalTree, map: map) guard !Task.isCancelled else { return } let vm = self await MainActor.run { vm?.extensionSummaries = summaries } } // Duplicate detection: lower priority, also off main actor - self.duplicateTask = Task.detached(priority: .utility) { [node, weak self] in + self.duplicateTask = Task.detached(priority: .utility) { [finalTree, weak self] in let detector = DuplicateDetector() - await detector.detect(in: node) + await detector.detect(in: finalTree) guard !Task.isCancelled else { return } - let groups = Self.buildDuplicateGroups(root: node) + let groups = Self.buildDuplicateGroups(tree: finalTree) guard !Task.isCancelled else { return } let vm = self await MainActor.run { @@ -281,40 +290,37 @@ public final class ScanViewModel: ObservableObject { } } + // TODO(phase4): incremental splice refresh. + // + // `FileTree`'s topology (parent/child arrays) is immutable for the life + // of one instance (that immutability is exactly what makes the flat + // arena cheap), so the old in-place FSNode mutation this method used to + // do (find the live node, patch its children/sizes) can no longer work + // directly against the live tree. The intended replacement (see the + // Phase 2 plan) is: for each changed directory, look up its index via a + // path->index map, re-scan just that directory into a fresh FSNode + // subtree (reusing `refreshDirectory`/`scanSubtree` below, which already + // do exactly this kind of on-disk rescan), convert it with + // `FileTreeBuilder`, and splice the result into a NEW `FileTree` that + // shares every untouched record/edge with the old one. + // + // That splice is a meaningful chunk of work on its own, so for this + // migration the interim (explicitly allowed by the plan) is simpler and + // still correct: any filesystem change under the watched root triggers a + // full rescan of the currently-scanned root, exactly like the "Move to + // Trash" actions elsewhere in the app already do after a delete. The + // FSNode-based helpers below (refreshDirectory/scanSubtree/bubbleUpSizes/ + // findNode/firstNode) are kept exactly as they were — unused by this + // method for now, but still exercised directly by ScanRefreshTests / + // HiddenSpaceTests, and ready to be reused as the per-directory rescan + // step of the real splice in Phase 4. private func handleFileSystemChanges(_ paths: [String]) async { - guard let root else { return } - - // Collect the unique directory paths that changed - var dirPaths = Set() - for path in paths { - var isDir: ObjCBool = false - if FileManager.default.fileExists(atPath: path, isDirectory: &isDir), isDir.boolValue { - dirPaths.insert(path) - } else { - dirPaths.insert((path as NSString).deletingLastPathComponent) - } - } - - var needsLayout = false - for dirPath in dirPaths { - guard let node = Self.findNode(path: dirPath, in: root) else { continue } - let sizeBefore = node.size - let changed = await Task.detached(priority: .userInitiated) { - Self.refreshDirectory(node: node) - }.value - if changed { - Self.bubbleUpSizes(from: node) - needsLayout = true - } - if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { - let line = "REFRESH changed=\(changed) nodeBefore=\(sizeBefore) nodeAfter=\(node.size) rootAfter=\(root.size) path=\(dirPath)\n" - FileHandle.standardError.write(line.data(using: .utf8)!) - } - } - - if needsLayout { - await recomputeLayout() + guard let scanURL else { return } + if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { + let line = "REFRESH full-rescan changedPaths=\(paths.count) root=\(scanURL.path)\n" + FileHandle.standardError.write(line.data(using: .utf8)!) } + scan(url: scanURL) } // Walk the tree by path components to find the FSNode for a given path. @@ -547,32 +553,30 @@ public final class ScanViewModel: ObservableObject { return hidden >= oneGB ? hidden : nil } - // When the scanned URL is itself a volume's mount point, appends a synthetic - // "Hidden & Unreadable Space" child representing the portion of the volume's - // used space that the scanner could never account for. No-op for non-volume - // scans (e.g. scanning a subfolder) or when the gap is negligible. - private nonisolated static func appendHiddenSpaceNodeIfNeeded(root: FSNode, scannedURL: URL) { + // When the scanned URL is itself a volume's mount point, returns a NEW + // tree with a synthetic "Hidden & Unreadable Space" child representing + // the portion of the volume's used space the scanner could never + // account for (nil for non-volume scans, e.g. scanning a subfolder, or + // when the gap is negligible). Topology is immutable on `FileTree`, so + // this can't append in place the way the old FSNode version did. + private nonisolated static func appendHiddenSpaceNodeIfNeeded(tree: FileTree, scannedURL: URL) -> FileTree? { guard let values = try? scannedURL.resourceValues(forKeys: [.volumeURLKey]), let volumeURL = values.volume, volumeURL.standardizedFileURL.path == scannedURL.standardizedFileURL.path - else { return } + else { return nil } guard let volumeValues = try? scannedURL.resourceValues(forKeys: [.volumeTotalCapacityKey, .volumeAvailableCapacityKey]), let totalCapacity = volumeValues.volumeTotalCapacity, let availableCapacity = volumeValues.volumeAvailableCapacity - else { return } + else { return nil } guard let hidden = hiddenSpaceBytes( volumeTotal: Int64(totalCapacity), volumeAvailable: Int64(availableCapacity), - scannedTotal: root.size - ) else { return } + scannedTotal: tree.records[tree.rootIndex].size + ) else { return nil } - let syntheticURL = scannedURL.appendingPathComponent("#hidden-space") - let synthetic = FSNode(url: syntheticURL, name: "Hidden & Unreadable Space", isDirectory: false, size: hidden, fileExtension: "", parent: root) - synthetic.isSynthetic = true - root.children.append(synthetic) - root.size += hidden + return tree.appendingSyntheticRootChild(name: "Hidden & Unreadable Space", size: hidden) } // Walk up the parent chain recalculating folder sizes from their children. @@ -599,7 +603,7 @@ public final class ScanViewModel: ObservableObject { } } - public func drillDown(into node: FSNode) { + public func drillDown(into node: FileNode) { guard node.isDirectory else { return } drillStack.append(node) Task { await recomputeLayout() } @@ -611,7 +615,7 @@ public final class ScanViewModel: ObservableObject { Task { await recomputeLayout() } } - public func select(_ node: FSNode?) { + public func select(_ node: FileNode?) { selectedNode = node } @@ -625,9 +629,11 @@ public final class ScanViewModel: ObservableObject { Task { await recomputeLayout() } } - private nonisolated static func buildDuplicateGroups(root: FSNode) -> [[FSNode]] { - var all: [FSNode] = [] - collectAll(node: root, into: &all) + // Whole-tree aggregate: a flat loop over `tree.records` reaches every + // node without recursion, since the array already covers the entire + // tree regardless of hierarchy. + private nonisolated static func buildDuplicateGroups(tree: FileTree) -> [[FileNode]] { + let all = (0.. 1 } @@ -647,10 +653,13 @@ public final class ScanViewModel: ObservableObject { public let percentage: Double } - private nonisolated static func buildExtensionSummaries(root: FSNode, map: ExtensionColorMap) -> [ExtensionSummary] { + private nonisolated static func buildExtensionSummaries(tree: FileTree, map: ExtensionColorMap) -> [ExtensionSummary] { var groups: [String: (count: Int, size: Int64)] = [:] - collectExtensions(node: root, into: &groups) - let total = Double(root.size) + for record in tree.records where !record.isDirectory { + groups[record.fileExtension, default: (0, 0)].count += 1 + groups[record.fileExtension, default: (0, 0)].size += record.size + } + let total = Double(tree.records[tree.rootIndex].size) return groups.map { ext, stats in ExtensionSummary( id: ext, @@ -680,56 +689,26 @@ public final class ScanViewModel: ObservableObject { self.isComputingLayout = false } - private nonisolated static func tagSafetyLevels(node: FSNode) { - var stack: [FSNode] = [node] - while !stack.isEmpty { - let n = stack.removeLast() - n.safetyLevel = SafetyAnalyzer.level(for: n) - stack.append(contentsOf: n.children) - } - } - - private nonisolated static func sortAllChildren(node: FSNode) { - var stack: [FSNode] = [node] - while !stack.isEmpty { - let n = stack.removeLast() - guard !n.children.isEmpty else { continue } - n.children.sort { $0.size > $1.size } - stack.append(contentsOf: n.children) - } - } - - private nonisolated static func collectAll(node: FSNode, into list: inout [FSNode]) { - var stack = [node] - while !stack.isEmpty { - let n = stack.removeLast() - list.append(n) - stack.append(contentsOf: n.children) - } - } - - private nonisolated static func collectExtensions(node: FSNode, into groups: inout [String: (count: Int, size: Int64)]) { - var stack = [node] - while !stack.isEmpty { - let n = stack.removeLast() - if !n.isDirectory { - groups[n.fileExtension, default: (0, 0)].count += 1 - groups[n.fileExtension, default: (0, 0)].size += n.size - } - stack.append(contentsOf: n.children) + // Flat loop over every index — order doesn't matter (each node's safety + // level only depends on its own reconstructed path/name, both already + // fully populated by the builder before this runs). + private nonisolated static func tagSafetyLevels(tree: FileTree) { + for index in 0.. some View { + private func summaryBar(groups: [[FileNode]]) -> some View { let totalWasted = groups.reduce(Int64(0)) { $0 + $1[0].size * Int64($1.count - 1) } let totalFiles = groups.reduce(0) { $0 + $1.count - 1 } @@ -73,7 +73,7 @@ struct DuplicatesView: View { // MARK: - Delete helpers - private func deleteAll(groups: [[FSNode]]) { + private func deleteAll(groups: [[FileNode]]) { guard let rootURL = vm.root?.url else { return } var deleted = false for group in groups { @@ -86,7 +86,7 @@ struct DuplicatesView: View { if deleted { vm.scan(url: rootURL) } } - private func deleteGroup(_ group: [FSNode]) { + private func deleteGroup(_ group: [FileNode]) { guard let rootURL = vm.root?.url else { return } var deleted = false for node in group.dropFirst() { @@ -97,7 +97,7 @@ struct DuplicatesView: View { if deleted { vm.scan(url: rootURL) } } - private func deleteNode(_ node: FSNode) { + private func deleteNode(_ node: FileNode) { guard let rootURL = vm.root?.url else { return } if (try? FileManager.default.trashItem(at: node.url, resultingItemURL: nil)) != nil { vm.scan(url: rootURL) @@ -109,11 +109,11 @@ struct DuplicatesView: View { private struct GroupSection: View { let index: Int - let group: [FSNode] + let group: [FileNode] let isExpanded: Bool let onToggle: () -> Void let onDeleteGroup: () -> Void - let onDeleteNode: (FSNode) -> Void + let onDeleteNode: (FileNode) -> Void private var wasted: Int64 { group[0].size * Int64(group.count - 1) } @@ -189,7 +189,7 @@ private struct GroupSection: View { // MARK: - File row private struct FileRow: View { - let node: FSNode + let node: FileNode let isKeep: Bool let onDelete: () -> Void @State private var isHovered = false diff --git a/Sources/Views/ExtensionList/ExtensionListView.swift b/Sources/Views/ExtensionList/ExtensionListView.swift index 54e299e..afd42a6 100644 --- a/Sources/Views/ExtensionList/ExtensionListView.swift +++ b/Sources/Views/ExtensionList/ExtensionListView.swift @@ -8,7 +8,7 @@ struct ExtensionListView: View { private let topN = 5 // Show selected item, or fall back to the current chart root - private var displayNode: FSNode? { vm.selectedNode ?? vm.treemapRoot } + private var displayNode: FileNode? { vm.selectedNode ?? vm.treemapRoot } var body: some View { VStack(spacing: 0) { diff --git a/Sources/Views/Shared/FileTypeIcon.swift b/Sources/Views/Shared/FileTypeIcon.swift index 08737a8..ded2b46 100644 --- a/Sources/Views/Shared/FileTypeIcon.swift +++ b/Sources/Views/Shared/FileTypeIcon.swift @@ -1,7 +1,7 @@ import SwiftUI enum FileTypeIcon { - static func systemName(for node: FSNode) -> String { + static func systemName(for node: FileNode) -> String { node.isDirectory ? "folder.fill" : systemName(forExt: node.fileExtension) } @@ -48,7 +48,7 @@ enum FileTypeIcon { } } - static func color(for node: FSNode) -> Color { + static func color(for node: FileNode) -> Color { node.isDirectory ? .yellow : color(forExt: node.fileExtension) } diff --git a/Sources/Views/Treemap/HapticEngine.swift b/Sources/Views/Treemap/HapticEngine.swift index f36bf22..b72aebf 100644 --- a/Sources/Views/Treemap/HapticEngine.swift +++ b/Sources/Views/Treemap/HapticEngine.swift @@ -7,7 +7,7 @@ final class HapticEngine { static let shared = HapticEngine() private let p = NSHapticFeedbackManager.defaultPerformer - private var lastID: UUID? + private var lastID: Int? private var isEnabled: Bool { UserDefaults.standard.object(forKey: "hapticFeedbackEnabled") as? Bool ?? true @@ -15,7 +15,7 @@ final class HapticEngine { // ── Hover ──────────────────────────────────────────────────────────────── - func hoverEntered(_ node: FSNode) { + func hoverEntered(_ node: FileNode) { guard isEnabled, node.id != lastID else { return } lastID = node.id @@ -51,7 +51,7 @@ final class HapticEngine { /// Choose pattern + tap-count based on node size. /// Directories always feel like a firm "snap" (alignment). - private func feedback(for node: FSNode) -> (NSHapticFeedbackManager.FeedbackPattern, Int) { + private func feedback(for node: FileNode) -> (NSHapticFeedbackManager.FeedbackPattern, Int) { if node.isDirectory { return (.alignment, 1) } diff --git a/Sources/Views/Treemap/TreemapRenderer.swift b/Sources/Views/Treemap/TreemapRenderer.swift index c34eb7e..ec8864d 100644 --- a/Sources/Views/Treemap/TreemapRenderer.swift +++ b/Sources/Views/Treemap/TreemapRenderer.swift @@ -6,8 +6,8 @@ struct TreemapRenderer { static func draw( cells: [TreemapCell], - hoveredNode: FSNode?, - selectedNode: FSNode?, + hoveredNode: FileNode?, + selectedNode: FileNode?, highlightedExtension: String?, duplicatesReady: Bool, pulsePhase: Double, diff --git a/Sources/Views/Treemap/TreemapView.swift b/Sources/Views/Treemap/TreemapView.swift index cf7ba53..fabc76a 100644 --- a/Sources/Views/Treemap/TreemapView.swift +++ b/Sources/Views/Treemap/TreemapView.swift @@ -158,7 +158,7 @@ struct TreemapView: View { @ViewBuilder private func contextMenuContent() -> some View { let c = chartCenter - let node: FSNode? = hoveredCell?.node + let node: FileNode? = hoveredCell?.node ?? (c.x > 0 ? TreemapRenderer.cell(at: cursorPos, center: c, in: vm.cells)?.node : nil) ?? vm.selectedNode @@ -217,7 +217,7 @@ struct TreemapView: View { // MARK: - Center label - private func centerLabel(for root: FSNode) -> some View { + private func centerLabel(for root: FileNode) -> some View { VStack(spacing: 4) { if !vm.drillStack.isEmpty { Image(systemName: "chevron.backward.circle.fill") @@ -304,7 +304,7 @@ struct TreemapView: View { // MARK: - Hover tooltip private struct HoverTooltip: View { - let node: FSNode + let node: FileNode var body: some View { HStack(alignment: .top, spacing: 10) { From 7441b49c86c3c94747164af9772ad518d6503d16 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 13:23:02 +0300 Subject: [PATCH 17/41] perf(model): flat FileTree store replacing FSNode class tree Migrate all Tests/*.swift to the new FileTree/FileNode types, add Tests/FileTreeTests.swift (builder correctness, path reconstruction, record carry-over, mutation, and the synthetic-root-child splice) and Tests/ScanViewModelIntegrationTests.swift (end-to-end scan -> tree -> safety tagging -> layout -> extension summaries -> duplicate detection, since no other test exercised ScanViewModel.scan(url:) directly), and mirror the new Sources/Model/ files into MacDirStat.xcodeproj's target membership. swift build + swift test are fully green (77 tests, 3 env-gated skips, 0 failures). This is the final commit for the Phase 2 flat-store migration that WIP f7de751 started. --- MacDirStat.xcodeproj/project.pbxproj | 24 +++ Tests/AutoSummaryTests.swift | 14 +- Tests/BulkScannerTests.swift | 16 +- Tests/DiagScanTests.swift | 6 +- Tests/DuplicateDetectorTests.swift | 81 ++++---- Tests/FileScannerTests.swift | 40 ++-- Tests/FileTreeTests.swift | 214 ++++++++++++++++++++++ Tests/ScanViewModelIntegrationTests.swift | 92 ++++++++++ Tests/TreemapLayoutTests.swift | 10 +- 9 files changed, 424 insertions(+), 73 deletions(-) create mode 100644 Tests/FileTreeTests.swift create mode 100644 Tests/ScanViewModelIntegrationTests.swift diff --git a/MacDirStat.xcodeproj/project.pbxproj b/MacDirStat.xcodeproj/project.pbxproj index 5aa18fe..937848e 100644 --- a/MacDirStat.xcodeproj/project.pbxproj +++ b/MacDirStat.xcodeproj/project.pbxproj @@ -34,6 +34,10 @@ F81DF36922942E2F442814A3 /* TreemapLayout.swift in Sources */ = {isa = PBXBuildFile; fileRef = BAB4247FB8E3E3FB5AF053F9 /* TreemapLayout.swift */; }; FB3E660E0C4E74F2A9EFBEF0 /* FileScanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = DDD56FD4714436315629F989 /* FileScanner.swift */; }; FD11F6ED4164F05503706848 /* ExtensionListView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 987EDA67820F162556A7CA53 /* ExtensionListView.swift */; }; + 84449F67C7DF40C737411C80 /* FileTree.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1B5ADD782F265EBF4B1037EE /* FileTree.swift */; }; + A498AC088B1E3C6E1B255BC2 /* FileNode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9429B8A593856D8B12941F49 /* FileNode.swift */; }; + 26B861D7144BFB6B2FD00F6B /* FileNodeRecord.swift in Sources */ = {isa = PBXBuildFile; fileRef = F03CE9327807EE8357D3678B /* FileNodeRecord.swift */; }; + E7ED11C27A082009974B4706 /* FileTreeBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4FB7871D4D4A1E73B7984396 /* FileTreeBuilder.swift */; }; SPARK003000000000000003AA /* Sparkle in Frameworks */ = {isa = PBXBuildFile; productRef = SPARK002000000000000002AA /* Sparkle */; }; /* End PBXBuildFile section */ @@ -68,6 +72,10 @@ DDD56FD4714436315629F989 /* FileScanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileScanner.swift; sourceTree = ""; }; EE11223344556677889900EE /* FileWatcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileWatcher.swift; sourceTree = ""; }; F6ADF08FD012926F8704A51C /* MacDirStat.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = MacDirStat.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 1B5ADD782F265EBF4B1037EE /* FileTree.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileTree.swift; sourceTree = ""; }; + 9429B8A593856D8B12941F49 /* FileNode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileNode.swift; sourceTree = ""; }; + F03CE9327807EE8357D3678B /* FileNodeRecord.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileNodeRecord.swift; sourceTree = ""; }; + 4FB7871D4D4A1E73B7984396 /* FileTreeBuilder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileTreeBuilder.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -136,6 +144,17 @@ path = Scanner; sourceTree = ""; }; + 794E4678AAB2F65C291BAFA3 /* Model */ = { + isa = PBXGroup; + children = ( + 1B5ADD782F265EBF4B1037EE /* FileTree.swift */, + 9429B8A593856D8B12941F49 /* FileNode.swift */, + F03CE9327807EE8357D3678B /* FileNodeRecord.swift */, + 4FB7871D4D4A1E73B7984396 /* FileTreeBuilder.swift */, + ); + path = Model; + sourceTree = ""; + }; 943FF7A0195AF9CFFDCE7D83 /* DirectoryTree */ = { isa = PBXGroup; children = ( @@ -238,6 +257,7 @@ AB8D408CB1881C666AAC0832 /* App */, 2401B46468064D5D254B09BA /* Duplicates */, D6C0DA3A2FC09B3404E91BF6 /* Layout */, + 794E4678AAB2F65C291BAFA3 /* Model */, CC11223344556677889900CC /* Safety */, 7332ECE572A5B7E9E9A8F57F /* Scanner */, EE1E649591DBDF61DE2E4396 /* ViewModels */, @@ -347,6 +367,10 @@ 09F2331AB12AD6C9FB63744A /* HapticEngine.swift in Sources */, 3F67AACE53A6C32467A5191C /* TreemapRenderer.swift in Sources */, 20E67F1EEADF74D28CDF6191 /* TreemapView.swift in Sources */, + 84449F67C7DF40C737411C80 /* FileTree.swift in Sources */, + A498AC088B1E3C6E1B255BC2 /* FileNode.swift in Sources */, + 26B861D7144BFB6B2FD00F6B /* FileNodeRecord.swift in Sources */, + E7ED11C27A082009974B4706 /* FileTreeBuilder.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Tests/AutoSummaryTests.swift b/Tests/AutoSummaryTests.swift index 8e61586..9c5cb05 100644 --- a/Tests/AutoSummaryTests.swift +++ b/Tests/AutoSummaryTests.swift @@ -36,26 +36,26 @@ final class AutoSummaryTests: XCTestCase { return try await body() } - private func scanTree(at url: URL) async -> FSNode? { + private func scanTree(at url: URL) async -> FileNode? { let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: url) { - if case .completed(let node, _) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } return root } - // Recursively counts every non-directory FSNode (normal leaf files) plus + // Recursively counts every non-directory FileNode (normal leaf files) plus // every isAutoSummarized node's descendantFileCount, so summarized and // non-summarized scans of the same tree can be compared on a "how many // files did we account for" basis. - private func totalAccountedFiles(_ node: FSNode) -> Int { + private func totalAccountedFiles(_ node: FileNode) -> Int { if node.isAutoSummarized { return node.descendantFileCount } if !node.isDirectory { return 1 } return node.children.reduce(0) { $0 + totalAccountedFiles($1) } } - private func findNode(_ root: FSNode, path: [String]) -> FSNode? { + private func findNode(_ root: FileNode, path: [String]) -> FileNode? { var current = root for name in path { guard let next = current.children.first(where: { $0.name == name }) else { return nil } @@ -287,7 +287,7 @@ final class AutoSummaryTests: XCTestCase { } let url = URL(fileURLWithPath: path) - func countNodes(_ node: FSNode) -> Int { + func countNodes(_ node: FileNode) -> Int { 1 + node.children.reduce(0) { $0 + countNodes($1) } } diff --git a/Tests/BulkScannerTests.swift b/Tests/BulkScannerTests.swift index 42cf5cf..fdbe762 100644 --- a/Tests/BulkScannerTests.swift +++ b/Tests/BulkScannerTests.swift @@ -40,12 +40,12 @@ final class BulkScannerTests: XCTestCase { let total: Int64 } - private func fingerprint(root: FSNode, base: URL) -> Fingerprint { + private func fingerprint(root: FileNode, base: URL) -> Fingerprint { var fileEntries: [FileFingerprintEntry] = [] var directoryPaths: [String] = [] var groups: [HardLinkRef: (paths: Set, total: Int64)] = [:] - func visit(_ node: FSNode) { + func visit(_ node: FileNode) { let relativePath = String(node.url.path.dropFirst(base.path.count)) if node.isDirectory { directoryPaths.append(relativePath) @@ -71,11 +71,11 @@ final class BulkScannerTests: XCTestCase { return Fingerprint(fileEntries: fileEntries, directoryPaths: directoryPaths, hardlinkGroups: hardlinkGroups, total: root.size) } - private func scanTree(at url: URL) async -> FSNode? { + private func scanTree(at url: URL) async -> FileNode? { let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: url) { - if case .completed(let node, _) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } return root } @@ -244,10 +244,10 @@ final class BulkScannerTests: XCTestCase { var deniedCount = 0 let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node, let denied) = progress { - root = node + if case .completed(let tree, let denied) = progress { + root = FileNode(tree: tree, index: tree.rootIndex) deniedCount = denied } } diff --git a/Tests/DiagScanTests.swift b/Tests/DiagScanTests.swift index 5819ebf..c0061a3 100644 --- a/Tests/DiagScanTests.swift +++ b/Tests/DiagScanTests.swift @@ -12,7 +12,7 @@ final class DiagScanTests: XCTestCase { } let scanner = FileScanner() - var completedRoot: FSNode? + var completedRoot: FileNode? var lastItems = 0 var lastBytes: Int64 = 0 for await progress in await scanner.scan(url: URL(fileURLWithPath: target)) { @@ -20,8 +20,8 @@ final class DiagScanTests: XCTestCase { case .update(let items, let bytes): lastItems = items lastBytes = bytes - case .completed(let root, _): - completedRoot = root + case .completed(let tree, _): + completedRoot = FileNode(tree: tree, index: tree.rootIndex) case .failed(let msg): XCTFail("scan failed: \(msg)") } diff --git a/Tests/DuplicateDetectorTests.swift b/Tests/DuplicateDetectorTests.swift index 171f7f4..7912c2e 100644 --- a/Tests/DuplicateDetectorTests.swift +++ b/Tests/DuplicateDetectorTests.swift @@ -3,6 +3,13 @@ import XCTest final class DuplicateDetectorTests: XCTestCase { + // Finds the record index for a given file name — the FSNode -> FileTree + // conversion doesn't preserve any positional ordering guarantees the + // tests can rely on, so tests look nodes up by name. + private func index(in tree: FileTree, named name: String) -> Int? { + tree.records.firstIndex { $0.name == name } + } + func test_detects_identical_files() async throws { let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) @@ -24,18 +31,19 @@ final class DuplicateDetectorTests: XCTestCase { root.children.append(child) root.size += child.size } + let tree = FileTreeBuilder.build(from: root, rootPath: tmp.path) let detector = DuplicateDetector() - await detector.detect(in: root) + await detector.detect(in: tree) - let copy1 = root.children.first { $0.name == "copy1.bin" }! - let copy2 = root.children.first { $0.name == "copy2.bin" }! - let unique = root.children.first { $0.name == "unique.bin" }! + let copy1 = index(in: tree, named: "copy1.bin")! + let copy2 = index(in: tree, named: "copy2.bin")! + let unique = index(in: tree, named: "unique.bin")! - XCTAssertNotNil(copy1.duplicateGroupID) - XCTAssertNotNil(copy2.duplicateGroupID) - XCTAssertEqual(copy1.duplicateGroupID, copy2.duplicateGroupID) - XCTAssertNil(unique.duplicateGroupID, "unique file must not be grouped") + XCTAssertNotNil(tree.records[copy1].duplicateGroupID) + XCTAssertNotNil(tree.records[copy2].duplicateGroupID) + XCTAssertEqual(tree.records[copy1].duplicateGroupID, tree.records[copy2].duplicateGroupID) + XCTAssertNil(tree.records[unique].duplicateGroupID, "unique file must not be grouped") } func test_small_files_below_threshold_are_skipped() async throws { @@ -54,12 +62,15 @@ final class DuplicateDetectorTests: XCTestCase { let child = FSNode(url: url, name: url.lastPathComponent, isDirectory: false, size: Int64(tinyData.count), fileExtension: "txt", parent: root) root.children.append(child) } + let tree = FileTreeBuilder.build(from: root, rootPath: tmp.path) let detector = DuplicateDetector() - await detector.detect(in: root) + await detector.detect(in: tree) - XCTAssertNil(root.children[0].duplicateGroupID, "files below threshold should not be grouped") - XCTAssertNil(root.children[1].duplicateGroupID) + let tiny1 = index(in: tree, named: "tiny1.txt")! + let tiny2 = index(in: tree, named: "tiny2.txt")! + XCTAssertNil(tree.records[tiny1].duplicateGroupID, "files below threshold should not be grouped") + XCTAssertNil(tree.records[tiny2].duplicateGroupID) } func test_same_prefix_different_tail_not_duplicates() async throws { @@ -84,12 +95,15 @@ final class DuplicateDetectorTests: XCTestCase { let child = FSNode(url: url, name: url.lastPathComponent, isDirectory: false, size: Int64(data1.count), fileExtension: "bin", parent: root) root.children.append(child) } + let tree = FileTreeBuilder.build(from: root, rootPath: tmp.path) let detector = DuplicateDetector() - await detector.detect(in: root) + await detector.detect(in: tree) - XCTAssertNil(root.children[0].duplicateGroupID, "files sharing only a quick-hash prefix must not be grouped") - XCTAssertNil(root.children[1].duplicateGroupID) + let a = index(in: tree, named: "a.bin")! + let b = index(in: tree, named: "b.bin")! + XCTAssertNil(tree.records[a].duplicateGroupID, "files sharing only a quick-hash prefix must not be grouped") + XCTAssertNil(tree.records[b].duplicateGroupID) } func test_small_file_duplicates_detected() async throws { @@ -112,18 +126,19 @@ final class DuplicateDetectorTests: XCTestCase { let child = FSNode(url: url, name: url.lastPathComponent, isDirectory: false, size: Int64(data.count), fileExtension: "bin", parent: root) root.children.append(child) } + let tree = FileTreeBuilder.build(from: root, rootPath: tmp.path) let detector = DuplicateDetector() - await detector.detect(in: root) + await detector.detect(in: tree) - let s1 = root.children.first { $0.name == "s1.bin" }! - let s2 = root.children.first { $0.name == "s2.bin" }! - let s3 = root.children.first { $0.name == "s3.bin" }! + let s1 = index(in: tree, named: "s1.bin")! + let s2 = index(in: tree, named: "s2.bin")! + let s3 = index(in: tree, named: "s3.bin")! - XCTAssertNotNil(s1.duplicateGroupID) - XCTAssertNotNil(s2.duplicateGroupID) - XCTAssertEqual(s1.duplicateGroupID, s2.duplicateGroupID) - XCTAssertNil(s3.duplicateGroupID) + XCTAssertNotNil(tree.records[s1].duplicateGroupID) + XCTAssertNotNil(tree.records[s2].duplicateGroupID) + XCTAssertEqual(tree.records[s1].duplicateGroupID, tree.records[s2].duplicateGroupID) + XCTAssertNil(tree.records[s3].duplicateGroupID) } func test_many_duplicate_pairs_all_detected() async throws { @@ -134,7 +149,7 @@ final class DuplicateDetectorTests: XCTestCase { let root = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) let pairCount = 20 - var pairFiles: [[FSNode]] = [] + var pairNames: [(String, String)] = [] for i in 0..() - for pair in pairFiles { - let a = pair[0] - let b = pair[1] - XCTAssertNotNil(a.duplicateGroupID, "\(a.name) should be grouped") - XCTAssertNotNil(b.duplicateGroupID, "\(b.name) should be grouped") - XCTAssertEqual(a.duplicateGroupID, b.duplicateGroupID, "\(a.name) and \(b.name) should share a group") - if let gid = a.duplicateGroupID { + for (nameA, nameB) in pairNames { + let a = index(in: tree, named: nameA)! + let b = index(in: tree, named: nameB)! + XCTAssertNotNil(tree.records[a].duplicateGroupID, "\(nameA) should be grouped") + XCTAssertNotNil(tree.records[b].duplicateGroupID, "\(nameB) should be grouped") + XCTAssertEqual(tree.records[a].duplicateGroupID, tree.records[b].duplicateGroupID, "\(nameA) and \(nameB) should share a group") + if let gid = tree.records[a].duplicateGroupID { XCTAssertFalse(seenGroupIDs.contains(gid), "group ID \(gid) reused across pairs") seenGroupIDs.insert(gid) } diff --git a/Tests/FileScannerTests.swift b/Tests/FileScannerTests.swift index c4dffc6..8b84cab 100644 --- a/Tests/FileScannerTests.swift +++ b/Tests/FileScannerTests.swift @@ -51,9 +51,9 @@ final class FileScannerTests: XCTestCase { try Data(repeating: 0, count: 2048).write(to: file2) let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node, _) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } XCTAssertNotNil(root) @@ -74,9 +74,9 @@ final class FileScannerTests: XCTestCase { try Data(repeating: 0, count: 4096).write(to: sub.appendingPathComponent("big.bin")) let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node, _) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } XCTAssertEqual(root?.size ?? 0, root?.children.first?.size ?? -1) @@ -93,9 +93,9 @@ final class FileScannerTests: XCTestCase { try FileManager.default.createSymbolicLink(at: link, withDestinationURL: real) let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node, _) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } XCTAssertEqual(root?.children.count, 1, "symlink should be skipped") @@ -116,9 +116,9 @@ final class FileScannerTests: XCTestCase { let fullSize = Int64(st.st_blocks) * 512 let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node, _) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } let n1 = root?.children.first { $0.name == "hard1.bin" } @@ -142,11 +142,11 @@ final class FileScannerTests: XCTestCase { } let scanner = FileScanner() - var root: FSNode? + var root: FileNode? var deniedCount = 0 for await progress in await scanner.scan(url: tmp) { - if case .completed(let node, let denied) = progress { - root = node + if case .completed(let tree, let denied) = progress { + root = FileNode(tree: tree, index: tree.rootIndex) deniedCount = denied } } @@ -181,9 +181,9 @@ final class FileScannerTests: XCTestCase { try Data(repeating: 0, count: 4096).write(to: keepDir.appendingPathComponent("file.bin")) let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node, _) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } XCTAssertNotNil(root) @@ -216,9 +216,9 @@ final class FileScannerTests: XCTestCase { UserDefaults.standard.set(false, forKey: "showHiddenFiles") do { let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node, _) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } let names = Set(root?.children.map { $0.name } ?? []) XCTAssertFalse(names.contains(".hidden"), "hidden file should be excluded when showHiddenFiles is false") @@ -228,9 +228,9 @@ final class FileScannerTests: XCTestCase { UserDefaults.standard.set(true, forKey: "showHiddenFiles") do { let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node, _) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } let names = Set(root?.children.map { $0.name } ?? []) XCTAssertTrue(names.contains(".hidden"), "hidden file should be included when showHiddenFiles is true") @@ -277,15 +277,15 @@ final class FileScannerTests: XCTestCase { } let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node, _) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } XCTAssertNotNil(root) XCTAssertEqual(root?.size, expectedSize, "total allocated size must match regardless of task-group vs inline recursion") - func countNodes(_ node: FSNode) -> (dirs: Int, files: Int) { + func countNodes(_ node: FileNode) -> (dirs: Int, files: Int) { if node.isDirectory { var dirs = 1 var files = 0 diff --git a/Tests/FileTreeTests.swift b/Tests/FileTreeTests.swift new file mode 100644 index 0000000..0edee01 --- /dev/null +++ b/Tests/FileTreeTests.swift @@ -0,0 +1,214 @@ +import XCTest +@testable import MacDirStat + +final class FileTreeTests: XCTestCase { + + // Builds a small FSNode fixture: + // root (/scan) + // ├── big.bin (500) + // └── sub (dir) + // ├── a.txt (300) + // └── b.txt (100) + private func makeFixture() -> (root: FSNode, big: FSNode, sub: FSNode, a: FSNode, b: FSNode) { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big = FSNode(url: URL(fileURLWithPath: "/scan/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: root) + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 400, fileExtension: "", parent: root) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 300, fileExtension: "txt", parent: sub) + let b = FSNode(url: URL(fileURLWithPath: "/scan/sub/b.txt"), name: "b.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: sub) + sub.children = [a, b] // deliberately unsorted (a=300 before b=100 is fine, already desc) + root.children = [sub, big] // deliberately NOT size-desc (big=500 should come first) + root.size = big.size + sub.size + return (root, big, sub, a, b) + } + + // MARK: - Builder correctness + + func test_builder_root_index_is_zero() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + XCTAssertEqual(tree.rootIndex, 0) + XCTAssertEqual(tree.records[tree.rootIndex].name, "scan") + XCTAssertEqual(tree.parentIndex[tree.rootIndex], -1) + } + + func test_builder_parent_links_are_correct() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + for child in rootNode.children { + XCTAssertEqual(child.parent?.index, tree.rootIndex) + } + let subNode = rootNode.children.first { $0.name == "sub" }! + for grandchild in subNode.children { + XCTAssertEqual(grandchild.parent?.index, subNode.index) + } + } + + func test_builder_children_are_sorted_size_desc_regardless_of_fsnode_order() { + let (root, _, _, _, _) = makeFixture() + // FSNode fixture deliberately has children in [sub(400), big(500)] order. + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + let sizes = rootNode.children.map(\.size) + XCTAssertEqual(sizes, sizes.sorted(by: >), "builder must sort each node's children size-desc") + XCTAssertEqual(rootNode.children.first?.name, "big.bin") + + let subNode = rootNode.children.first { $0.name == "sub" }! + XCTAssertEqual(subNode.children.map(\.name), ["a.txt", "b.txt"], "a.txt (300) must sort before b.txt (100)") + } + + func test_builder_child_spans_do_not_overlap() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + // Every index reachable from root's children arrays must be unique and + // within bounds — a cheap proxy for "spans don't alias each other". + var seen = Set() + for index in 0..= 0 && childIndex < tree.records.count) + XCTAssertTrue(seen.insert(childIndex).inserted, "child index \(childIndex) referenced by more than one parent") + } + } + } + + func test_builder_leaf_has_no_children() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + let bigNode = rootNode.children.first { $0.name == "big.bin" }! + XCTAssertEqual(bigNode.children.count, 0) + XCTAssertNil(bigNode.optionalChildren) + } + + // MARK: - Path reconstruction + + func test_path_of_root_is_root_path() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + XCTAssertEqual(tree.path(of: tree.rootIndex), "/scan") + } + + func test_path_of_nested_node_joins_names() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + let subNode = rootNode.children.first { $0.name == "sub" }! + let aNode = subNode.children.first { $0.name == "a.txt" }! + XCTAssertEqual(subNode.url.path, "/scan/sub") + XCTAssertEqual(aNode.url.path, "/scan/sub/a.txt") + } + + func test_path_handles_trailing_slash_root_path() { + let root = FSNode(url: URL(fileURLWithPath: "/"), name: "/", isDirectory: true, size: 10, fileExtension: "", parent: nil) + let child = FSNode(url: URL(fileURLWithPath: "/Users"), name: "Users", isDirectory: true, size: 10, fileExtension: "", parent: root) + root.children = [child] + let tree = FileTreeBuilder.build(from: root, rootPath: "/") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + XCTAssertEqual(rootNode.children.first?.url.path, "/Users", "must not produce a double slash when rootPath already ends in '/'") + } + + // MARK: - Record field carry-over + + func test_builder_carries_over_hardlink_ref_and_synthetic_and_autosummarized_flags() { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + + let hardlinked = FSNode(url: URL(fileURLWithPath: "/scan/h.bin"), name: "h.bin", isDirectory: false, size: 200, fileExtension: "bin", parent: root) + hardlinked.hardLinkRef = HardLinkRef(dev: 1, ino: 42) + + let summarized = FSNode(url: URL(fileURLWithPath: "/scan/node_modules"), name: "node_modules", isDirectory: true, size: 900, fileExtension: "", parent: root) + summarized.isAutoSummarized = true + summarized.descendantFileCount = 123 + + let synthetic = FSNode(url: URL(fileURLWithPath: "/scan/synthetic"), name: "synthetic", isDirectory: false, size: 50, fileExtension: "", parent: root) + synthetic.isSynthetic = true + + let denied = FSNode(url: URL(fileURLWithPath: "/scan/denied"), name: "denied", isDirectory: true, size: 0, fileExtension: "", parent: root) + denied.isAccessDenied = true + + root.children = [hardlinked, summarized, synthetic, denied] + root.size = 1200 + + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + + let hNode = rootNode.children.first { $0.name == "h.bin" }! + XCTAssertEqual(hNode.hardLinkRef, HardLinkRef(dev: 1, ino: 42)) + + let nmNode = rootNode.children.first { $0.name == "node_modules" }! + XCTAssertTrue(nmNode.isAutoSummarized) + XCTAssertEqual(nmNode.descendantFileCount, 123) + + let synNode = rootNode.children.first { $0.name == "synthetic" }! + XCTAssertTrue(synNode.isSynthetic) + + let deniedNode = rootNode.children.first { $0.name == "denied" }! + XCTAssertTrue(deniedNode.isAccessDenied) + + // Per the design, safetyLevel/duplicateGroupID always start at their + // defaults at assembly time — they're filled by a separate pass + // (tagSafetyLevels / DuplicateDetector) over the finished FileTree. + XCTAssertNil(hNode.duplicateGroupID) + XCTAssertEqual(hNode.safetyLevel, .caution) + } + + // MARK: - Mutation + + func test_set_duplicate_group_id_and_safety_and_size() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + let bigNode = rootNode.children.first { $0.name == "big.bin" }! + + let groupID = UUID() + tree.setDuplicateGroupID(groupID, at: bigNode.index) + XCTAssertEqual(FileNode(tree: tree, index: bigNode.index).duplicateGroupID, groupID) + + tree.setSafety(.danger, at: bigNode.index) + XCTAssertEqual(FileNode(tree: tree, index: bigNode.index).safetyLevel, .danger) + + tree.setSize(999, at: bigNode.index) + XCTAssertEqual(FileNode(tree: tree, index: bigNode.index).size, 999) + } + + // MARK: - Synthetic root child (topology-changing splice used for hidden-space) + + func test_appending_synthetic_root_child_inserts_in_sorted_position() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + // Root's children before: [big.bin(500), sub(400)]. A synthetic entry + // of 450 bytes should land between them. + let newTree = tree.appendingSyntheticRootChild(name: "Hidden & Unreadable Space", size: 450) + let newRoot = FileNode(tree: newTree, index: newTree.rootIndex) + + XCTAssertEqual(newRoot.children.map(\.name), ["big.bin", "Hidden & Unreadable Space", "sub"]) + XCTAssertTrue(newRoot.children[1].isSynthetic) + XCTAssertEqual(newRoot.children[1].safetyLevel, .danger) + XCTAssertEqual(newRoot.size, 500 + 400 + 450, "root's aggregate size must include the synthetic child") + } + + func test_appending_synthetic_root_child_leaves_original_tree_untouched() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let originalChildCount = FileNode(tree: tree, index: tree.rootIndex).children.count + let originalSize = tree.records[tree.rootIndex].size + + _ = tree.appendingSyntheticRootChild(name: "Hidden & Unreadable Space", size: 450) + + XCTAssertEqual(FileNode(tree: tree, index: tree.rootIndex).children.count, originalChildCount, "original tree's topology must not mutate") + XCTAssertEqual(tree.records[tree.rootIndex].size, originalSize, "original tree's root size must not mutate") + } + + func test_appending_synthetic_root_child_preserves_other_subtrees() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let newTree = tree.appendingSyntheticRootChild(name: "Hidden & Unreadable Space", size: 450) + let newRoot = FileNode(tree: newTree, index: newTree.rootIndex) + + let subNode = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(subNode.children.map(\.name), ["a.txt", "b.txt"], "unrelated subtree's children/order must survive the splice") + XCTAssertEqual(subNode.url.path, "/scan/sub") + } +} diff --git a/Tests/ScanViewModelIntegrationTests.swift b/Tests/ScanViewModelIntegrationTests.swift new file mode 100644 index 0000000..478d179 --- /dev/null +++ b/Tests/ScanViewModelIntegrationTests.swift @@ -0,0 +1,92 @@ +import XCTest +@testable import MacDirStat + +// End-to-end coverage for ScanViewModel.scan(url:) itself — the biggest +// rewritten code path in the Phase 2 flat-store migration (FileScanner -> +// FileTree -> safety tagging -> extension summaries -> duplicate detection -> +// layout), which none of the other test files exercise directly (they either +// drive FileScanner/FileTree/DuplicateDetector in isolation, or call the +// FSNode-based static refresh helpers directly). +@MainActor +final class ScanViewModelIntegrationTests: XCTestCase { + + private func withRealtimeMonitoringDisabled(_ body: () async throws -> T) async rethrows -> T { + let prior = UserDefaults.standard.object(forKey: "realtimeMonitoring") as? Bool + UserDefaults.standard.set(false, forKey: "realtimeMonitoring") + defer { + if let prior { UserDefaults.standard.set(prior, forKey: "realtimeMonitoring") } + else { UserDefaults.standard.removeObject(forKey: "realtimeMonitoring") } + } + return try await body() + } + + // Polls a condition instead of a fixed sleep, since scan completion runs + // through several hops (scanner -> Task.detached tagging -> recomputeLayout). + private func waitUntil(timeout: TimeInterval = 5, _ condition: () -> Bool) async { + let deadline = Date().addingTimeInterval(timeout) + while !condition(), Date() < deadline { + try? await Task.sleep(nanoseconds: 10_000_000) + } + } + + func test_scan_populates_tree_layout_and_extension_summaries() async throws { + try await withRealtimeMonitoringDisabled { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + try Data(repeating: 1, count: 8192).write(to: tmp.appendingPathComponent("a.bin")) + try Data(repeating: 2, count: 4096).write(to: tmp.appendingPathComponent("b.txt")) + let sub = tmp.appendingPathComponent("sub") + try FileManager.default.createDirectory(at: sub, withIntermediateDirectories: true) + try Data(repeating: 3, count: 2048).write(to: sub.appendingPathComponent("c.bin")) + + let vm = ScanViewModel() + vm.updateLayoutSize(CGSize(width: 400, height: 400)) + vm.scan(url: tmp) + + await waitUntil { !vm.isScanning && !vm.isComputingLayout } + + XCTAssertNotNil(vm.tree) + guard let root = vm.root else { return XCTFail("scan did not populate a root") } + XCTAssertTrue(root.isDirectory) + XCTAssertEqual(root.children.map(\.name).sorted(), ["a.bin", "b.txt", "sub"]) + // Children must be size-desc (a.bin=8192 > sub=2048 > b.txt=4096... sub aggregates + // c.bin's 2048, so expected order is a.bin(8192), b.txt(4096), sub(2048)). + XCTAssertEqual(root.children.map(\.name), ["a.bin", "b.txt", "sub"]) + + // Safety tagging ran over the whole tree: a plain temp-dir file/subdir + // matches none of the danger/safe path rules, so it settles at .caution + // (the default an untagged record would also read as — the real signal + // this checks is that tagging ran without crashing across a synthetic-free + // tree with directories and files mixed together). + for child in root.children { + XCTAssertEqual(child.safetyLevel, .caution) + } + + // Layout: a non-trivial layoutSize was set before scanning, so recomputeLayout + // should have produced cells once the scan settled. + await waitUntil { !vm.cells.isEmpty } + XCTAssertFalse(vm.cells.isEmpty, "treemap layout should be computed after a scan completes") + + // Extension summaries run off-thread; wait for them to land. + await waitUntil(timeout: 5) { !vm.extensionSummaries.isEmpty } + XCTAssertFalse(vm.extensionSummaries.isEmpty) + let binSummary = vm.extensionSummaries.first { $0.ext == ".bin" } + XCTAssertNotNil(binSummary) + XCTAssertEqual(binSummary?.fileCount, 2, "both a.bin and sub/c.bin should be counted") + } + } + + func test_scan_of_deleted_root_reports_failure_not_crash() async throws { + let missing = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + // Never created — the scanner should treat it as a nonexistent leaf, + // not throw or crash the view model. + let vm = ScanViewModel() + vm.scan(url: missing) + await waitUntil { !vm.isScanning } + // Whatever the scanner decides (empty leaf vs failure), the view model + // must reach a settled, non-scanning state without crashing. + XCTAssertFalse(vm.isScanning) + } +} diff --git a/Tests/TreemapLayoutTests.swift b/Tests/TreemapLayoutTests.swift index 8c4925c..ff0cc79 100644 --- a/Tests/TreemapLayoutTests.swift +++ b/Tests/TreemapLayoutTests.swift @@ -54,7 +54,10 @@ final class TreemapLayoutTests: XCTestCase { // MARK: - Helpers - func makeTree(_ files: [(String, Int64)]) -> FSNode { + // Builds an FSNode fixture (as before) and converts it to a FileTree, + // returning a FileNode handle onto its root — TreemapLayout/ExtensionColorMap + // are now FileNode-based, but the fixture construction itself is unchanged. + func makeTree(_ files: [(String, Int64)]) -> FileNode { let root = FSNode(url: URL(fileURLWithPath: "/"), name: "/", isDirectory: true, size: 0, fileExtension: "", parent: nil) for (name, size) in files { let ext = (name as NSString).pathExtension.lowercased() @@ -62,7 +65,8 @@ final class TreemapLayoutTests: XCTestCase { root.children.append(child) root.size += size } - return root + let tree = FileTreeBuilder.build(from: root, rootPath: "/") + return FileNode(tree: tree, index: tree.rootIndex) } // The layout is a radial sunburst: a node's children fill the angular range @@ -120,7 +124,7 @@ final class TreemapLayoutTests: XCTestCase { } func test_layout_empty_children_returns_no_cells() { - let root = FSNode(url: URL(fileURLWithPath: "/"), name: "/", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let root = makeTree([]) let map = ExtensionColorMap(root: root) let cells = TreemapLayout.compute(root: root, in: CGRect(x: 0, y: 0, width: 400, height: 400), colorMap: map) XCTAssertTrue(cells.isEmpty) From 32dcd70bee2df21ba0ca67cb7b4394c14c6b4b48 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 13:27:18 +0300 Subject: [PATCH 18/41] build(xcode): add scanner files to app target BulkDirectoryEnumerator.swift and AtomicDirectorySummary.swift (added in the Phase 1/3 scanner work) were only picked up by SwiftPM, never added to the Xcode app target, so xcodebuild failed to compile the app. Add both to the Scanner group and Sources build phase. --- MacDirStat.xcodeproj/project.pbxproj | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/MacDirStat.xcodeproj/project.pbxproj b/MacDirStat.xcodeproj/project.pbxproj index 937848e..dfeef1a 100644 --- a/MacDirStat.xcodeproj/project.pbxproj +++ b/MacDirStat.xcodeproj/project.pbxproj @@ -33,6 +33,8 @@ E5BB63BED418F08CF4727F72 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 783902530DE4EA8040806BC9 /* SettingsView.swift */; }; F81DF36922942E2F442814A3 /* TreemapLayout.swift in Sources */ = {isa = PBXBuildFile; fileRef = BAB4247FB8E3E3FB5AF053F9 /* TreemapLayout.swift */; }; FB3E660E0C4E74F2A9EFBEF0 /* FileScanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = DDD56FD4714436315629F989 /* FileScanner.swift */; }; + A1B2C3D4E5F600112233445A /* BulkDirectoryEnumerator.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F600112233445B /* BulkDirectoryEnumerator.swift */; }; + A1B2C3D4E5F600112233445C /* AtomicDirectorySummary.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F600112233445D /* AtomicDirectorySummary.swift */; }; FD11F6ED4164F05503706848 /* ExtensionListView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 987EDA67820F162556A7CA53 /* ExtensionListView.swift */; }; 84449F67C7DF40C737411C80 /* FileTree.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1B5ADD782F265EBF4B1037EE /* FileTree.swift */; }; A498AC088B1E3C6E1B255BC2 /* FileNode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9429B8A593856D8B12941F49 /* FileNode.swift */; }; @@ -70,6 +72,8 @@ C40A6B24C517964825BA4B85 /* TreemapView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TreemapView.swift; sourceTree = ""; }; DBCAD22DB54E68077B842421 /* ScanProgress.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScanProgress.swift; sourceTree = ""; }; DDD56FD4714436315629F989 /* FileScanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileScanner.swift; sourceTree = ""; }; + A1B2C3D4E5F600112233445B /* BulkDirectoryEnumerator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BulkDirectoryEnumerator.swift; sourceTree = ""; }; + A1B2C3D4E5F600112233445D /* AtomicDirectorySummary.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AtomicDirectorySummary.swift; sourceTree = ""; }; EE11223344556677889900EE /* FileWatcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileWatcher.swift; sourceTree = ""; }; F6ADF08FD012926F8704A51C /* MacDirStat.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = MacDirStat.app; sourceTree = BUILT_PRODUCTS_DIR; }; 1B5ADD782F265EBF4B1037EE /* FileTree.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileTree.swift; sourceTree = ""; }; @@ -137,6 +141,8 @@ isa = PBXGroup; children = ( DDD56FD4714436315629F989 /* FileScanner.swift */, + A1B2C3D4E5F600112233445B /* BulkDirectoryEnumerator.swift */, + A1B2C3D4E5F600112233445D /* AtomicDirectorySummary.swift */, 35F5AEBBF9DECB5CB793339A /* FSNode.swift */, EE11223344556677889900EE /* FileWatcher.swift */, DBCAD22DB54E68077B842421 /* ScanProgress.swift */, @@ -355,6 +361,8 @@ AA11223344556677889900AA /* SafetyAnalyzer.swift in Sources */, DD11223344556677889900DD /* FileWatcher.swift in Sources */, FB3E660E0C4E74F2A9EFBEF0 /* FileScanner.swift in Sources */, + A1B2C3D4E5F600112233445A /* BulkDirectoryEnumerator.swift in Sources */, + A1B2C3D4E5F600112233445C /* AtomicDirectorySummary.swift in Sources */, 3881FC74433C93187BF941AF /* FSNode.swift in Sources */, 86B25923164A00AB10D72C9B /* ScanProgress.swift in Sources */, 114A022A9D8A4529EC91BC12 /* ScanViewModel.swift in Sources */, From bf9bedc320cd1d93e5003d9c1ac8d809509fa3a4 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 21:22:49 +0300 Subject: [PATCH 19/41] feat(cleanup): prune trashed items from the tree instead of rescanning --- Sources/Model/FileTree.swift | 119 +++++++ Sources/ViewModels/ScanViewModel.swift | 116 +++++++ .../DirectoryTree/DirectoryTreeView.swift | 5 +- Sources/Views/Duplicates/DuplicatesView.swift | 25 +- Sources/Views/Treemap/TreemapView.swift | 5 +- Tests/FileTreePruneTests.swift | 300 ++++++++++++++++++ 6 files changed, 540 insertions(+), 30 deletions(-) create mode 100644 Tests/FileTreePruneTests.swift diff --git a/Sources/Model/FileTree.swift b/Sources/Model/FileTree.swift index 36cca5c..78db867 100644 --- a/Sources/Model/FileTree.swift +++ b/Sources/Model/FileTree.swift @@ -132,4 +132,123 @@ public final class FileTree: @unchecked Sendable { rootPath: rootPath ) } + + // MARK: - Pruning (Move to Trash without a full rescan) + + // Returns a NEW tree with the subtree rooted at `index` removed (the root + // itself can never be removed — returns `self` unchanged if asked to). + // Thin wrapper over `removingSubtrees(at:)`, which already handles the + // single-index case without any extra cost. + public func removingSubtree(at index: Int) -> FileTree { + removingSubtrees(at: [index]) + } + + // Returns a NEW tree with every subtree rooted at each of `indices` + // removed — the "Delete All Duplicates" / "keep 1, delete N" case, where + // several unrelated nodes are trashed at once. + // + // Single pass over all indices (not a fold of `removingSubtree` calls one + // at a time): every seed's whole subtree is marked in one iterative DFS, + // then `records`/`parentIndex`/children are rebuilt once from the + // resulting old->new index map. This is O(n) total rather than O(n·k) + // for k removed subtrees, while remaining just as simple to reason about + // as folding would be (a fold is also correct here — just slower). + public func removingSubtrees(at indices: [Int]) -> FileTree { + let count = records.count + + // Mark every node in each seed's subtree as removed via an iterative + // DFS over childIndices (explicit stack, not recursion, so this can't + // stack-overflow on a very deep real-world tree). + var removed = [Bool](repeating: false, count: count) + var stack: [Int] = [] + for seed in indices where seed != rootIndex && seed >= 0 && seed < count && !removed[seed] { + stack.append(seed) + } + while let i = stack.popLast() { + if removed[i] { continue } + removed[i] = true + let start = childStart[i] + let cnt = childCount[i] + for offset in 0.. new index map over kept nodes, built in ascending old-index + // order — this is what keeps every child span's relative order + // (already size-desc) intact after dropping the removed entries. + var oldToNew = [Int](repeating: -1, count: count) + var newRecords: [FileNodeRecord] = [] + newRecords.reserveCapacity(count) + for old in 0..= 0 ? oldToNew[oldParent] : -1 + } + + // Subtract each removed subtree's root size from every one of its + // ancestors. Only process seeds whose immediate parent is NOT itself + // removed — if the parent is removed too, this seed is a descendant + // of some other (higher) removed subtree root, and its size was + // already folded into the ancestor chain when that higher seed was + // processed, so subtracting again here would double-count. Also + // de-dupe in case the same index appears more than once in `indices`. + var processedRoots = Set() + for seed in indices { + guard seed != rootIndex, seed >= 0, seed < count, removed[seed] else { continue } + let parent = parentIndex[seed] + if parent >= 0 && removed[parent] { continue } + guard processedRoots.insert(seed).inserted else { continue } + + let removedSize = records[seed].size + var ancestorOld = parent + while ancestorOld >= 0 { + newRecords[oldToNew[ancestorOld]].size -= removedSize + ancestorOld = parentIndex[ancestorOld] + } + } + + // Rebuild children arrays, dropping removed entries from each + // surviving parent's span but keeping the rest in their original + // (size-desc) relative order. + var newChildIndices: [Int] = [] + newChildIndices.reserveCapacity(childIndices.count) + var newChildStart = [Int](repeating: 0, count: newRecords.count) + var newChildCount = [Int](repeating: 0, count: newRecords.count) + for old in 0.. Bool { + trashNodes([node]) + } + + // Moves several nodes to the Trash (the "keep 1, delete N" / "Delete All + // Duplicates" case) and prunes every successfully-trashed one out of the + // tree in a single splice, rather than rescanning once per node. + @discardableResult + public func trashNodes(_ nodes: [FileNode]) -> Bool { + guard let currentTree = tree, !nodes.isEmpty else { return false } + + var trashedIndices: [Int] = [] + trashedIndices.reserveCapacity(nodes.count) + for node in nodes where ObjectIdentifier(node.tree) == ObjectIdentifier(currentTree) { + do { + try FileManager.default.trashItem(at: node.url, resultingItemURL: nil) + trashedIndices.append(node.index) + } catch { + // Swallow, same as the old per-call-site `try?` behavior — + // one failed delete (e.g. permissions) shouldn't block the + // rest of the batch or surface a blocking alert. + } + } + guard !trashedIndices.isEmpty else { return false } + + pruneTree(afterTrashing: trashedIndices, from: currentTree) + return true + } + + // Splices the trashed nodes out of `tree` and repairs everything that + // referenced the old topology: selection, drill stack, color map, + // extension summaries, duplicate groups, and layout. Indices shift on + // every prune, so selection/drill state is captured as paths beforehand + // and resolved back to indices in the new tree afterward. + private func pruneTree(afterTrashing indices: [Int], from oldTree: FileTree) { + let selectedPath = selectedNode.map { oldTree.path(of: $0.index) } + let drillPaths = drillStack.map { oldTree.path(of: $0.index) } + + let newTree = oldTree.removingSubtrees(at: indices) + self.tree = newTree + + if let selectedPath, let idx = Self.findIndex(forPath: selectedPath, in: newTree) { + selectedNode = FileNode(tree: newTree, index: idx) + } else { + selectedNode = nil + } + + // Keep every prefix of the drill stack that still resolves; the + // first missing entry (the trashed directory itself, if it was + // drilled into) truncates the stack back to its nearest surviving + // ancestor instead of leaving stale/dangling entries. + var newDrillStack: [FileNode] = [] + for path in drillPaths { + guard let idx = Self.findIndex(forPath: path, in: newTree) else { break } + newDrillStack.append(FileNode(tree: newTree, index: idx)) + } + drillStack = newDrillStack + + let rootNode = FileNode(tree: newTree, index: newTree.rootIndex) + let map = ExtensionColorMap(root: rootNode) + colorMap = map + + // Safety tags and duplicateGroupIDs are per-node fields that were + // already computed on the surviving records and carry over as-is + // through the splice (removingSubtrees copies whole `FileNodeRecord` + // values), so only the two aggregate derived passes need to re-run — + // and neither needs a full re-detect, just a re-group/re-bucket over + // the pruned record set. + extensionTask?.cancel() + extensionTask = Task.detached(priority: .userInitiated) { [weak self] in + let summaries = Self.buildExtensionSummaries(tree: newTree, map: map) + guard !Task.isCancelled else { return } + await MainActor.run { self?.extensionSummaries = summaries } + } + + duplicateTask?.cancel() + duplicateTask = Task.detached(priority: .utility) { [weak self] in + let groups = Self.buildDuplicateGroups(tree: newTree) + guard !Task.isCancelled else { return } + await MainActor.run { self?.duplicateGroups = groups } + } + + Task { await recomputeLayout() } + } + + // Resolves an absolute path back to an index in `tree` by walking + // root->leaf name components (no full-tree path map to build — this is + // only ever called for a handful of paths per prune: selection + drill + // stack). Returns nil if the path no longer exists (it was the node + // that got trashed, or a descendant of it). + private nonisolated static func findIndex(forPath path: String, in tree: FileTree) -> Int? { + if path == tree.rootPath { return tree.rootIndex } + let rootPrefix = tree.rootPath.hasSuffix("/") ? tree.rootPath : tree.rootPath + "/" + guard path.hasPrefix(rootPrefix) else { return nil } + + let relative = String(path.dropFirst(rootPrefix.count)) + let components = relative.split(separator: "/").map(String.init) + var current = tree.rootIndex + for component in components { + let start = tree.childStart[current] + let count = tree.childCount[current] + guard let offset = (0.. (root: FSNode, big: FSNode, sub: FSNode, a: FSNode, b: FSNode) { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big = FSNode(url: URL(fileURLWithPath: "/scan/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: root) + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 400, fileExtension: "", parent: root) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 300, fileExtension: "txt", parent: sub) + let b = FSNode(url: URL(fileURLWithPath: "/scan/sub/b.txt"), name: "b.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: sub) + sub.children = [a, b] + root.children = [big, sub] + root.size = big.size + sub.size + return (root, big, sub, a, b) + } + + private func node(named name: String, in tree: FileTree) -> FileNode { + for i in 0..= 0 && start + cnt <= tree.childIndices.count, "child span out of range at \(i)", file: file, line: line) + var previousSize: Int64? = nil + for offset in 0..= 0 && child < count, "child index \(child) out of range", file: file, line: line) + XCTAssertEqual(tree.parentIndex[child], i, "child \(child)'s parentIndex must point back to \(i)", file: file, line: line) + let size = tree.records[child].size + if let previousSize { + XCTAssertGreaterThanOrEqual(previousSize, size, "children must stay sorted size-desc", file: file, line: line) + } + previousSize = size + } + } + XCTAssertEqual(tree.parentIndex[tree.rootIndex], -1, "root must have no parent", file: file, line: line) + } + + // MARK: - removingSubtree: single removal + + func test_removingSubtree_leaf_reduces_record_count_by_one() { + let (root, _, sub, a, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let aNode = node(named: "a.txt", in: tree) + _ = sub; _ = a + + let pruned = tree.removingSubtree(at: aNode.index) + + XCTAssertEqual(pruned.records.count, tree.records.count - 1) + assertValidTopology(pruned) + } + + func test_removingSubtree_reduces_every_ancestor_by_exactly_the_removed_size() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let aNode = node(named: "a.txt", in: tree) + let removedSize = aNode.size // 300 + + let pruned = tree.removingSubtree(at: aNode.index) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(newSub.size, 400 - removedSize) + XCTAssertEqual(newRoot.size, 900 - removedSize) + } + + func test_removingSubtree_directory_removes_whole_subtree_and_reduces_only_direct_ancestors() { + let (root, _, sub, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let subNode = node(named: "sub", in: tree) + _ = sub + + let pruned = tree.removingSubtree(at: subNode.index) + + // sub + a.txt + b.txt all gone -> 3 fewer records than the original 5. + XCTAssertEqual(pruned.records.count, tree.records.count - 3) + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.children.map(\.name), ["big.bin"]) + XCTAssertEqual(newRoot.size, 500, "root size must drop by the whole removed subtree's size (400), not just sub's direct children") + assertValidTopology(pruned) + } + + func test_removingSubtree_preserves_sort_order_and_sibling_subtrees() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let aNode = node(named: "a.txt", in: tree) + + let pruned = tree.removingSubtree(at: aNode.index) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + // big.bin(500) must still sort ahead of the shrunk sub(100). + XCTAssertEqual(newRoot.children.map(\.name), ["big.bin", "sub"]) + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(newSub.children.map(\.name), ["b.txt"]) + } + + func test_removingSubtree_path_reconstruction_still_correct_for_survivors() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let aNode = node(named: "a.txt", in: tree) + + let pruned = tree.removingSubtree(at: aNode.index) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.url.path, "/scan") + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(newSub.url.path, "/scan/sub") + let newB = newSub.children.first { $0.name == "b.txt" }! + XCTAssertEqual(newB.url.path, "/scan/sub/b.txt") + let newBig = newRoot.children.first { $0.name == "big.bin" }! + XCTAssertEqual(newBig.url.path, "/scan/big.bin") + } + + func test_removingSubtree_leaves_original_tree_untouched() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let originalCount = tree.records.count + let originalRootSize = tree.records[tree.rootIndex].size + let aNode = node(named: "a.txt", in: tree) + + _ = tree.removingSubtree(at: aNode.index) + + XCTAssertEqual(tree.records.count, originalCount, "original tree must not mutate") + XCTAssertEqual(tree.records[tree.rootIndex].size, originalRootSize, "original tree's root size must not mutate") + } + + func test_removingSubtree_at_root_returns_self_unchanged() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + + let result = tree.removingSubtree(at: tree.rootIndex) + + XCTAssertTrue(result === tree, "removing the root must be a no-op, returning the same instance") + } + + func test_removingSubtree_carries_over_duplicate_group_and_safety_fields_on_survivors() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let bigNode = node(named: "big.bin", in: tree) + let groupID = UUID() + tree.setDuplicateGroupID(groupID, at: bigNode.index) + tree.setSafety(.safe, at: bigNode.index) + let aNode = node(named: "a.txt", in: tree) + + let pruned = tree.removingSubtree(at: aNode.index) + + let prunedBig = node(named: "big.bin", in: pruned) + XCTAssertEqual(prunedBig.duplicateGroupID, groupID) + XCTAssertEqual(prunedBig.safetyLevel, .safe) + } + + // MARK: - removingSubtrees: multiple removals + + func test_removingSubtrees_disjoint_nodes_reduces_each_ancestor_chain_independently() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let aNode = node(named: "a.txt", in: tree) // under sub + let bigNode = node(named: "big.bin", in: tree) // under root directly + + let pruned = tree.removingSubtrees(at: [aNode.index, bigNode.index]) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.children.map(\.name), ["sub"]) + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(newSub.size, 400 - 300, "sub must lose exactly a.txt's size") + XCTAssertEqual(newRoot.size, 900 - 300 - 500, "root must lose both removed subtrees' sizes, once each") + assertValidTopology(pruned) + } + + func test_removingSubtrees_nested_indices_does_not_double_count_ancestor_size() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let subNode = node(named: "sub", in: tree) + let aNode = node(named: "a.txt", in: tree) // already inside sub's subtree + + // Passing both sub and one of its own descendants must behave exactly + // like removing sub alone — a.txt's size must not be subtracted twice. + let pruned = tree.removingSubtrees(at: [subNode.index, aNode.index]) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.children.map(\.name), ["big.bin"]) + XCTAssertEqual(newRoot.size, 900 - 400) + assertValidTopology(pruned) + } + + func test_removingSubtrees_empty_indices_returns_self() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + + let result = tree.removingSubtrees(at: []) + + XCTAssertTrue(result === tree) + } + + func test_removingSubtrees_out_of_range_index_is_ignored() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + + let result = tree.removingSubtrees(at: [999]) + + XCTAssertTrue(result === tree, "an out-of-range seed must be a no-op, not a crash") + } + + func test_removingSubtrees_duplicate_index_in_list_removed_once() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let aNode = node(named: "a.txt", in: tree) + + let pruned = tree.removingSubtrees(at: [aNode.index, aNode.index]) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.size, 900 - 300, "duplicate seeds in the list must not subtract twice") + assertValidTopology(pruned) + } +} + +// MARK: - ScanViewModel-level integration + +// Unlike the pure-tree tests above, this one exercises the real +// `ScanViewModel.trashNode` entry point end to end: a real scan of a temp +// directory, a real `FileManager.trashItem` call, and the resulting prune +// updating `tree`/`selectedNode`/`extensionSummaries`/`duplicateGroups` +// in place instead of the old "trash then full rescan" behavior. +@MainActor +final class ScanViewModelPruneTests: XCTestCase { + + private func waitUntil(timeout: TimeInterval = 5, _ condition: () -> Bool) async { + let deadline = Date().addingTimeInterval(timeout) + while !condition(), Date() < deadline { + try? await Task.sleep(nanoseconds: 10_000_000) + } + } + + func test_trashNode_prunes_tree_in_place_without_rescanning() async throws { + let prior = UserDefaults.standard.object(forKey: "realtimeMonitoring") as? Bool + UserDefaults.standard.set(false, forKey: "realtimeMonitoring") + defer { + if let prior { UserDefaults.standard.set(prior, forKey: "realtimeMonitoring") } + else { UserDefaults.standard.removeObject(forKey: "realtimeMonitoring") } + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + try Data(repeating: 1, count: 4096).write(to: tmp.appendingPathComponent("keep.bin")) + let toDeleteURL = tmp.appendingPathComponent("delete-me.bin") + try Data(repeating: 2, count: 2048).write(to: toDeleteURL) + + let vm = ScanViewModel() + vm.updateLayoutSize(CGSize(width: 400, height: 400)) + vm.scan(url: tmp) + await waitUntil { !vm.isScanning && !vm.isComputingLayout } + + guard let root = vm.root else { return XCTFail("scan should populate a root") } + let originalTotal = root.size + guard let target = root.children.first(where: { $0.name == "delete-me.bin" }) else { + return XCTFail("expected delete-me.bin among the scanned children") + } + let targetSize = target.size + vm.select(target) + + let trashed = vm.trashNode(target) + XCTAssertTrue(trashed, "trashNode should report success for a real, trashable file") + + // The prune itself (tree/selection/drillStack/colorMap) is synchronous; + // only the off-thread extension-summary/duplicate-group passes need + // waiting for. + XCTAssertFalse(FileManager.default.fileExists(atPath: toDeleteURL.path), "trashItem should have actually moved the file out of tmp") + guard let newRoot = vm.root else { return XCTFail("tree must still exist after a prune") } + XCTAssertEqual(newRoot.children.map(\.name), ["keep.bin"], "the trashed node must be gone from the live tree") + XCTAssertEqual(newRoot.size, originalTotal - targetSize, "root size must shrink by exactly the trashed node's size") + XCTAssertNil(vm.selectedNode, "the node that was selected and then trashed must be deselected, not left dangling") + + await waitUntil { vm.extensionSummaries.first(where: { $0.ext == ".bin" })?.fileCount == 1 } + let binSummary = vm.extensionSummaries.first { $0.ext == ".bin" } + XCTAssertEqual(binSummary?.fileCount, 1, "extension summaries must be recomputed against the pruned tree") + } +} From 4b44ccbb42230d51813b0c5a1126f06bd09b2a34 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 21:31:16 +0300 Subject: [PATCH 20/41] feat(archive): save and reopen scans as .mdscan snapshots Persist a completed FileTree to a JSON .mdscan archive (FileNodeRecord/ HardLinkRef/SafetyLevel made Codable) and reopen it later as a read-only snapshot: File > Save Scan.../Open Scan... drive ScanViewModel.saveScan(to:)/ openArchive(from:), which validates topology (bounds, parent/child consistency, reachability) before installing the tree so a corrupted or doctored archive surfaces as an error instead of crashing or hanging. isReadOnlySnapshot disables all trash/delete actions and skips FSEvents watching, with a banner surfacing the snapshot date in the UI. --- MacDirStat.xcodeproj/project.pbxproj | 4 + Sources/App/ContentView.swift | 43 ++++ Sources/App/MacDirStatApp.swift | 44 ++++ Sources/Model/FileNodeRecord.swift | 2 +- Sources/Model/ScanArchive.swift | 165 +++++++++++++ Sources/Safety/SafetyAnalyzer.swift | 2 +- Sources/Scanner/FSNode.swift | 2 +- Sources/ViewModels/ScanViewModel.swift | 116 ++++++++- .../DirectoryTree/DirectoryTreeView.swift | 12 +- Sources/Views/Duplicates/DuplicatesView.swift | 41 ++-- Sources/Views/Treemap/TreemapView.swift | 12 +- Tests/ScanArchiveTests.swift | 232 ++++++++++++++++++ 12 files changed, 644 insertions(+), 31 deletions(-) create mode 100644 Sources/Model/ScanArchive.swift create mode 100644 Tests/ScanArchiveTests.swift diff --git a/MacDirStat.xcodeproj/project.pbxproj b/MacDirStat.xcodeproj/project.pbxproj index dfeef1a..1b549a0 100644 --- a/MacDirStat.xcodeproj/project.pbxproj +++ b/MacDirStat.xcodeproj/project.pbxproj @@ -40,6 +40,7 @@ A498AC088B1E3C6E1B255BC2 /* FileNode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9429B8A593856D8B12941F49 /* FileNode.swift */; }; 26B861D7144BFB6B2FD00F6B /* FileNodeRecord.swift in Sources */ = {isa = PBXBuildFile; fileRef = F03CE9327807EE8357D3678B /* FileNodeRecord.swift */; }; E7ED11C27A082009974B4706 /* FileTreeBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4FB7871D4D4A1E73B7984396 /* FileTreeBuilder.swift */; }; + A7CECAA54EC4419CA4B847D2 /* ScanArchive.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5B0C663AD1A1417DA8FC6942 /* ScanArchive.swift */; }; SPARK003000000000000003AA /* Sparkle in Frameworks */ = {isa = PBXBuildFile; productRef = SPARK002000000000000002AA /* Sparkle */; }; /* End PBXBuildFile section */ @@ -80,6 +81,7 @@ 9429B8A593856D8B12941F49 /* FileNode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileNode.swift; sourceTree = ""; }; F03CE9327807EE8357D3678B /* FileNodeRecord.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileNodeRecord.swift; sourceTree = ""; }; 4FB7871D4D4A1E73B7984396 /* FileTreeBuilder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileTreeBuilder.swift; sourceTree = ""; }; + 5B0C663AD1A1417DA8FC6942 /* ScanArchive.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScanArchive.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -157,6 +159,7 @@ 9429B8A593856D8B12941F49 /* FileNode.swift */, F03CE9327807EE8357D3678B /* FileNodeRecord.swift */, 4FB7871D4D4A1E73B7984396 /* FileTreeBuilder.swift */, + 5B0C663AD1A1417DA8FC6942 /* ScanArchive.swift */, ); path = Model; sourceTree = ""; @@ -379,6 +382,7 @@ A498AC088B1E3C6E1B255BC2 /* FileNode.swift in Sources */, 26B861D7144BFB6B2FD00F6B /* FileNodeRecord.swift in Sources */, E7ED11C27A082009974B4706 /* FileTreeBuilder.swift in Sources */, + A7CECAA54EC4419CA4B847D2 /* ScanArchive.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Sources/App/ContentView.swift b/Sources/App/ContentView.swift index 5ef85bf..29310c1 100644 --- a/Sources/App/ContentView.swift +++ b/Sources/App/ContentView.swift @@ -193,6 +193,9 @@ struct ContentView: View { @ViewBuilder private var detailContent: some View { VStack(spacing: 0) { + if vm.isReadOnlySnapshot { + SnapshotBanner(date: vm.snapshotDate) + } if !vm.hasFullDiskAccess || (vm.deniedCount > 0 && !vm.isScanning) { FullDiskAccessBanner(hasFullDiskAccess: vm.hasFullDiskAccess, deniedCount: vm.deniedCount) { vm.showFDASheet = true @@ -421,6 +424,46 @@ private struct FullDiskAccessBanner: View { } } +// MARK: - Read-only snapshot banner + +// Shown whenever `vm.tree` was loaded from a `.mdscan` archive instead of a +// live scan. Trash/delete actions are already disabled at the ScanViewModel +// level (`trashNodes` no-ops while `isReadOnlySnapshot` is set); this is +// just the visible explanation of why. +private struct SnapshotBanner: View { + let date: Date? + + private var subtitle: String { + guard let date else { return "Opened from a saved scan. Delete actions are disabled." } + let formatter = DateFormatter() + formatter.dateStyle = .medium + formatter.timeStyle = .short + return "Saved \(formatter.string(from: date)). Delete actions are disabled." + } + + var body: some View { + HStack(spacing: 10) { + Image(systemName: "camera.aperture") + .font(.system(size: 15, weight: .medium)) + .foregroundStyle(.blue) + + VStack(alignment: .leading, spacing: 1) { + Text("Read-only snapshot") + .font(.system(size: 12, weight: .semibold)) + Text(subtitle) + .font(.system(size: 11)) + .foregroundStyle(.secondary) + } + + Spacer() + } + .padding(.horizontal, 14) + .padding(.vertical, 9) + .background(.blue.opacity(0.15)) + .overlay(alignment: .bottom) { Divider().overlay(.blue.opacity(0.3)) } + } +} + // MARK: - Dashboard settings popover private struct DashboardSettingsView: View { diff --git a/Sources/App/MacDirStatApp.swift b/Sources/App/MacDirStatApp.swift index 739bb6e..b55d5c9 100644 --- a/Sources/App/MacDirStatApp.swift +++ b/Sources/App/MacDirStatApp.swift @@ -1,5 +1,6 @@ import SwiftUI import Sparkle +import UniformTypeIdentifiers @main struct MacDirStatApp: App { @@ -25,8 +26,19 @@ struct MacDirStatApp: App { openFolderPicker(vm: vm) } .keyboardShortcut("o", modifiers: .command) + + Button("Open Scan…") { + openArchivePicker(vm: vm) + } + .keyboardShortcut("o", modifiers: [.command, .shift]) } CommandGroup(after: .newItem) { + Button("Save Scan…") { + saveScanPicker(vm: vm) + } + .keyboardShortcut("s", modifiers: .command) + .disabled(vm.tree == nil) + Button("Export CSV…") { NotificationCenter.default.post(name: .exportCSV, object: nil) } @@ -61,3 +73,35 @@ private func openFolderPicker(vm: ScanViewModel) { Task { @MainActor in vm.scan(url: url) } } } + +// Not registered as a system-wide document type (no Info.plist exported-type +// entry) — `UTType(filenameExtension:)` gives macOS a dynamic UTI that's +// perfectly sufficient for filtering these two panels by the `.mdscan` +// extension without touching the app's document/type registration. +private let mdscanType = UTType(filenameExtension: "mdscan") ?? .data + +@MainActor +private func saveScanPicker(vm: ScanViewModel) { + guard let tree = vm.tree else { return } + let panel = NSSavePanel() + panel.allowedContentTypes = [mdscanType] + panel.nameFieldStringValue = "\(tree.records[tree.rootIndex].name).mdscan" + panel.message = "Save this scan to reopen later as a read-only snapshot" + if panel.runModal() == .OK, let url = panel.url { + vm.saveScan(to: url) + } +} + +@MainActor +private func openArchivePicker(vm: ScanViewModel) { + let panel = NSOpenPanel() + panel.canChooseDirectories = false + panel.canChooseFiles = true + panel.allowsMultipleSelection = false + panel.allowedContentTypes = [mdscanType] + panel.prompt = "Open" + panel.message = "Choose a saved scan to reopen as a read-only snapshot" + if panel.runModal() == .OK, let url = panel.url { + vm.openArchive(from: url) + } +} diff --git a/Sources/Model/FileNodeRecord.swift b/Sources/Model/FileNodeRecord.swift index 7a0770d..2ec8c97 100644 --- a/Sources/Model/FileNodeRecord.swift +++ b/Sources/Model/FileNodeRecord.swift @@ -6,7 +6,7 @@ import Foundation // huge scans) — the absolute path is reconstructed on demand from names + // parentIndex (see `FileTree.path(of:)` / `FileNode.url`), and node identity // for SwiftUI purposes is the (tree, index) pair (see `FileNode.id`). -public struct FileNodeRecord: Sendable { +public struct FileNodeRecord: Sendable, Codable { public let name: String public let isDirectory: Bool public var size: Int64 diff --git a/Sources/Model/ScanArchive.swift b/Sources/Model/ScanArchive.swift new file mode 100644 index 0000000..0e255d4 --- /dev/null +++ b/Sources/Model/ScanArchive.swift @@ -0,0 +1,165 @@ +import Foundation + +// Persists a completed `FileTree` to disk and reloads it later as a +// read-only snapshot ("Save and Reopen Results"). The flat store's arrays +// are trivially Codable (see the conformances added to `FileNodeRecord`, +// `HardLinkRef`, and `SafetyLevel`), so this is a thin envelope: the raw +// topology arrays plus a small metadata block describing where/when the +// scan happened. +// +// Format is plain JSON (not gzipped) behind a `.mdscan` extension — +// `formatVersion` is carried so a future revision (e.g. adding compression, +// or new per-node fields) can still read old archives or reject ones it +// doesn't understand. +public struct ScanArchive: Codable, Sendable { + public static let currentFormatVersion = 1 + + public struct Metadata: Codable, Sendable { + public let scannedPath: String + public let scanDate: Date + public let deniedCount: Int + public let appVersion: String + public let formatVersion: Int + + public init(scannedPath: String, scanDate: Date, deniedCount: Int, appVersion: String, formatVersion: Int = ScanArchive.currentFormatVersion) { + self.scannedPath = scannedPath + self.scanDate = scanDate + self.deniedCount = deniedCount + self.appVersion = appVersion + self.formatVersion = formatVersion + } + } + + public let records: [FileNodeRecord] + public let parentIndex: [Int] + public let childStart: [Int] + public let childCount: [Int] + public let childIndices: [Int] + public let rootIndex: Int + public let rootPath: String + public let metadata: Metadata + + public init(tree: FileTree, metadata: Metadata) { + self.records = tree.records + self.parentIndex = tree.parentIndex + self.childStart = tree.childStart + self.childCount = tree.childCount + self.childIndices = tree.childIndices + self.rootIndex = tree.rootIndex + self.rootPath = tree.rootPath + self.metadata = metadata + } + + // Rebuilds a live `FileTree` from the archived arrays. Callers MUST run + // `validate()` first — this initializer trusts the topology is sound. + public func makeTree() -> FileTree { + FileTree( + records: records, + parentIndex: parentIndex, + childStart: childStart, + childCount: childCount, + childIndices: childIndices, + rootIndex: rootIndex, + rootPath: rootPath + ) + } + + // MARK: - Validation + + // A hostile or corrupted archive must never crash the app or hang it — + // every check here is a bounds/consistency check performed with plain + // array indexing, and the whole-tree walk is iterative (explicit stack) + // and visits each node at most once, so it terminates even if the + // topology is cyclic. + public func validate() throws { + let count = records.count + + guard count > 0 else { throw ScanArchiveError.corrupted("Archive contains no records") } + // A generous upper bound: guards against a maliciously/corruptly + // huge archive trying to exhaust memory, while never limiting any + // real scan (tens of millions of files is already far beyond what + // this app scans in practice). + guard count <= 50_000_000 else { throw ScanArchiveError.corrupted("Archive is implausibly large (\(count) records)") } + + guard parentIndex.count == count, childStart.count == count, childCount.count == count else { + throw ScanArchiveError.corrupted("Archive arrays have mismatched lengths") + } + + guard rootIndex >= 0, rootIndex < count else { + throw ScanArchiveError.corrupted("Root index out of range") + } + guard parentIndex[rootIndex] < 0 else { + throw ScanArchiveError.corrupted("Root node has a parent") + } + + // Every child span must be in range before touching childIndices. + var totalSpanned = 0 + for i in 0..= 0, start >= 0 else { + throw ScanArchiveError.corrupted("Negative child span at index \(i)") + } + guard start + cnt <= childIndices.count else { + throw ScanArchiveError.corrupted("Child span out of range at index \(i)") + } + totalSpanned += cnt + } + guard totalSpanned == childIndices.count else { + throw ScanArchiveError.corrupted("Child spans don't cover childIndices exactly") + } + + // Every entry in childIndices must reference a valid, non-self node, + // and must agree with that child's own parentIndex. + for i in 0..= 0, child < count else { + throw ScanArchiveError.corrupted("Child index out of range under parent \(i)") + } + guard child != i else { + throw ScanArchiveError.corrupted("Node \(i) lists itself as its own child") + } + guard parentIndex[child] == i else { + throw ScanArchiveError.corrupted("parentIndex/childIndices disagree for node \(child)") + } + } + } + + // Whole-tree reachability walk from root, iterative so a cycle can't + // hang the app — a node already marked visited is simply skipped + // rather than re-pushed, and if a cycle exists among unreached + // nodes, they just never get visited and the final count check + // below catches it. + var visited = [Bool](repeating: false, count: count) + var stack = [rootIndex] + var visitedCount = 0 + while let i = stack.popLast() { + if visited[i] { continue } + visited[i] = true + visitedCount += 1 + let start = childStart[i] + let cnt = childCount[i] + for offset in 0.. Bool { - guard let currentTree = tree, !nodes.isEmpty else { return false } + guard !isReadOnlySnapshot, let currentTree = tree, !nodes.isEmpty else { return false } var trashedIndices: [Int] = [] trashedIndices.reserveCapacity(nodes.count) @@ -815,6 +828,107 @@ public final class ScanViewModel: ObservableObject { } } + // MARK: - Save / reopen scans (.mdscan archive) + + // Encodes the current tree + a small metadata block and writes it to + // `url` off the main actor (encoding a multi-million-record tree to + // JSON is real work). Errors surface through `errorMessage`, same as + // `exportCSV`. + public func saveScan(to url: URL) { + guard let tree else { return } + let metadata = ScanArchive.Metadata( + scannedPath: tree.rootPath, + scanDate: Date(), + deniedCount: deniedCount, + appVersion: Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "unknown" + ) + let archive = ScanArchive(tree: tree, metadata: metadata) + + Task.detached(priority: .utility) { [weak self] in + do { + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + let data = try encoder.encode(archive) + try data.write(to: url, options: .atomic) + } catch { + await MainActor.run { self?.errorMessage = "Couldn't save scan: \(error.localizedDescription)" } + } + } + } + + // Decodes and validates a `.mdscan` archive off the main actor, then + // installs it as a read-only snapshot. A malformed or doctored archive + // surfaces as `errorMessage` rather than crashing or hanging — see + // `ScanArchive.validate()`. + public func openArchive(from url: URL) { + scanTask?.cancel() + extensionTask?.cancel() + duplicateTask?.cancel() + watchTask?.cancel() + fileWatcher.stop() + isWatching = false + layoutGeneration += 1 + securityScopedURL?.stopAccessingSecurityScopedResource() + securityScopedURL = nil + errorMessage = nil + isScanning = true + isComputingLayout = false + + Task.detached(priority: .userInitiated) { [weak self] in + do { + let data = try Data(contentsOf: url) + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + let archive = try decoder.decode(ScanArchive.self, from: data) + try archive.validate() + let tree = archive.makeTree() + await MainActor.run { self?.applyOpenedArchive(tree: tree, metadata: archive.metadata) } + } catch { + await MainActor.run { + self?.isScanning = false + self?.errorMessage = "Couldn't open scan: \(error.localizedDescription)" + } + } + } + } + + // Installs a validated, freshly-decoded tree as the active read-only + // snapshot: no live watching is started (this never calls `scan(url:)` + // or `startWatching`), and `isReadOnlySnapshot` gates trash actions off + // for the rest of this tree's lifetime. + private func applyOpenedArchive(tree: FileTree, metadata: ScanArchive.Metadata) { + isScanning = false + isReadOnlySnapshot = true + snapshotDate = metadata.scanDate + scanURL = URL(fileURLWithPath: metadata.scannedPath) + deniedCount = metadata.deniedCount + selectedNode = nil + drillStack = [] + highlightedExtension = nil + errorMessage = nil + UserDefaults.standard.set(metadata.scannedPath, forKey: "lastScannedPath") + + self.tree = tree + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + let map = ExtensionColorMap(root: rootNode) + colorMap = map + isComputingLayout = true + Task { await recomputeLayout() } + + // Safety tags and duplicateGroupIDs were already computed before the + // original scan was saved and travel with the archived records, so + // only the two aggregate derived views need to be rebuilt. + extensionTask?.cancel() + extensionTask = Task.detached(priority: .userInitiated) { [tree, map, weak self] in + let summaries = Self.buildExtensionSummaries(tree: tree, map: map) + guard !Task.isCancelled else { return } + await MainActor.run { self?.extensionSummaries = summaries } + } + + duplicateGroups = Self.buildDuplicateGroups(tree: tree) + duplicatesReady = true + } + public func exportCSV() { guard let tree else { return } let panel = NSSavePanel() diff --git a/Sources/Views/DirectoryTree/DirectoryTreeView.swift b/Sources/Views/DirectoryTree/DirectoryTreeView.swift index dd0e709..187c675 100644 --- a/Sources/Views/DirectoryTree/DirectoryTreeView.swift +++ b/Sources/Views/DirectoryTree/DirectoryTreeView.swift @@ -40,11 +40,13 @@ struct DirectoryTreeView: View { Label("Open in Chart", systemImage: "arrow.down.right.circle") } } - Divider() - Button(role: .destructive) { - vm.trashNode(node) - } label: { - Label("Move to Trash", systemImage: "trash") + if !vm.isReadOnlySnapshot { + Divider() + Button(role: .destructive) { + vm.trashNode(node) + } label: { + Label("Move to Trash", systemImage: "trash") + } } } } diff --git a/Sources/Views/Duplicates/DuplicatesView.swift b/Sources/Views/Duplicates/DuplicatesView.swift index 24b8b91..1a64fc5 100644 --- a/Sources/Views/Duplicates/DuplicatesView.swift +++ b/Sources/Views/Duplicates/DuplicatesView.swift @@ -24,6 +24,7 @@ struct DuplicatesView: View { index: index, group: group, isExpanded: expanded.contains(index), + isReadOnly: vm.isReadOnlySnapshot, onToggle: { withAnimation(.easeInOut(duration: 0.15)) { if expanded.contains(index) { expanded.remove(index) } @@ -58,13 +59,15 @@ struct DuplicatesView: View { systemImage: "externaldrive.badge.minus") .font(.caption).foregroundStyle(.orange) Spacer() - Button(role: .destructive) { deleteAll(groups: groups) } label: { - Label("Delete All Duplicates", systemImage: "trash") - .font(.caption.weight(.medium)) + if !vm.isReadOnlySnapshot { + Button(role: .destructive) { deleteAll(groups: groups) } label: { + Label("Delete All Duplicates", systemImage: "trash") + .font(.caption.weight(.medium)) + } + .glassButton() + .tint(.red) + .controlSize(.small) } - .glassButton() - .tint(.red) - .controlSize(.small) } .padding(.horizontal, 12) .padding(.vertical, 8) @@ -92,6 +95,7 @@ private struct GroupSection: View { let index: Int let group: [FileNode] let isExpanded: Bool + let isReadOnly: Bool let onToggle: () -> Void let onDeleteGroup: () -> Void let onDeleteNode: (FileNode) -> Void @@ -134,15 +138,17 @@ private struct GroupSection: View { .frame(maxWidth: 180, alignment: .trailing) } - Button(role: .destructive, action: onDeleteGroup) { - Label("Keep 1, Delete \(group.count - 1)", systemImage: "trash") - .font(.system(size: 11, weight: .medium)) + if !isReadOnly { + Button(role: .destructive, action: onDeleteGroup) { + Label("Keep 1, Delete \(group.count - 1)", systemImage: "trash") + .font(.system(size: 11, weight: .medium)) + } + .glassButton() + .tint(.red) + .controlSize(.mini) + // Don't let the delete button trigger the expand toggle + .onTapGesture {} } - .glassButton() - .tint(.red) - .controlSize(.mini) - // Don't let the delete button trigger the expand toggle - .onTapGesture {} } .padding(.horizontal, 12) .padding(.vertical, 7) @@ -154,7 +160,7 @@ private struct GroupSection: View { if isExpanded { VStack(spacing: 0) { ForEach(Array(group.enumerated()), id: \.element.id) { i, node in - FileRow(node: node, isKeep: i == 0) { onDeleteNode(node) } + FileRow(node: node, isKeep: i == 0, isReadOnly: isReadOnly) { onDeleteNode(node) } if i < group.count - 1 { Divider().padding(.leading, 52) } @@ -172,6 +178,7 @@ private struct GroupSection: View { private struct FileRow: View { let node: FileNode let isKeep: Bool + let isReadOnly: Bool let onDelete: () -> Void @State private var isHovered = false @@ -209,7 +216,7 @@ private struct FileRow: View { .foregroundStyle(.secondary) .fixedSize() - if isKeep { + if isKeep || isReadOnly { Color.clear.frame(width: 24, height: 24) } else { Button(action: onDelete) { @@ -234,7 +241,7 @@ private struct FileRow: View { NSPasteboard.general.clearContents() NSPasteboard.general.setString(node.url.path, forType: .string) } - if !isKeep { + if !isKeep && !isReadOnly { Divider() Button(role: .destructive, action: onDelete) { Label("Move to Trash", systemImage: "trash") diff --git a/Sources/Views/Treemap/TreemapView.swift b/Sources/Views/Treemap/TreemapView.swift index 30d7ed7..edb0828 100644 --- a/Sources/Views/Treemap/TreemapView.swift +++ b/Sources/Views/Treemap/TreemapView.swift @@ -202,12 +202,14 @@ struct TreemapView: View { } } - Divider() + if !vm.isReadOnlySnapshot { + Divider() - Button(role: .destructive) { - vm.trashNode(node) - } label: { - Label("Move to Trash", systemImage: "trash") + Button(role: .destructive) { + vm.trashNode(node) + } label: { + Label("Move to Trash", systemImage: "trash") + } } } } diff --git a/Tests/ScanArchiveTests.swift b/Tests/ScanArchiveTests.swift new file mode 100644 index 0000000..c49dc9a --- /dev/null +++ b/Tests/ScanArchiveTests.swift @@ -0,0 +1,232 @@ +import XCTest +@testable import MacDirStat + +final class ScanArchiveTests: XCTestCase { + + // Same shape as the FileTreeTests fixture: + // root (/scan) + // ├── big.bin (500) + // └── sub (dir) + // ├── a.txt (300) + // └── b.txt (100) + private func makeFixtureTree() -> FileTree { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big = FSNode(url: URL(fileURLWithPath: "/scan/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: root) + big.hardLinkRef = HardLinkRef(dev: 1, ino: 42) + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 400, fileExtension: "", parent: root) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 300, fileExtension: "txt", parent: sub) + let b = FSNode(url: URL(fileURLWithPath: "/scan/sub/b.txt"), name: "b.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: sub) + sub.children = [a, b] + root.children = [sub, big] + root.size = big.size + sub.size + + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + // Give a couple of fields their post-scan-pass values so the round + // trip test actually exercises non-default data. + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + let bigNode = rootNode.children.first { $0.name == "big.bin" }! + let aNode = rootNode.children.first { $0.name == "sub" }!.children.first { $0.name == "a.txt" }! + tree.setSafety(.danger, at: bigNode.index) + let groupID = UUID() + tree.setDuplicateGroupID(groupID, at: aNode.index) + return tree + } + + private func makeMetadata() -> ScanArchive.Metadata { + ScanArchive.Metadata(scannedPath: "/scan", scanDate: Date(timeIntervalSince1970: 1_700_000_000), deniedCount: 3, appVersion: "1.1") + } + + // MARK: - Round trip + + func test_round_trip_preserves_records_topology_and_sizes() throws { + let tree = makeFixtureTree() + let archive = ScanArchive(tree: tree, metadata: makeMetadata()) + + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + let data = try encoder.encode(archive) + + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + let decoded = try decoder.decode(ScanArchive.self, from: data) + try decoded.validate() + + XCTAssertEqual(decoded.records.count, tree.records.count) + XCTAssertEqual(decoded.parentIndex, tree.parentIndex) + XCTAssertEqual(decoded.childStart, tree.childStart) + XCTAssertEqual(decoded.childCount, tree.childCount) + XCTAssertEqual(decoded.childIndices, tree.childIndices) + XCTAssertEqual(decoded.rootIndex, tree.rootIndex) + XCTAssertEqual(decoded.rootPath, tree.rootPath) + + for i in 0.. 0 }) else { + return XCTFail("fixture must have at least one node with children") + } + childCount[parentWithChildren] -= 1 + let doctored = ScanArchive( + records: archive.records, parentIndex: archive.parentIndex, + childStart: archive.childStart, childCount: childCount, + childIndices: archive.childIndices, rootIndex: archive.rootIndex, + rootPath: archive.rootPath, metadata: archive.metadata + ) + XCTAssertThrowsError(try doctored.validate()) + } + + func test_validate_rejects_empty_archive() { + let archive = ScanArchive( + records: [], parentIndex: [], childStart: [], childCount: [], + childIndices: [], rootIndex: 0, rootPath: "/scan", metadata: makeMetadata() + ) + XCTAssertThrowsError(try archive.validate()) + } +} + +// Test-only initializer mirroring every stored property, so validation +// tests can construct a deliberately-doctored archive without going through +// `ScanArchive(tree:metadata:)`. +private extension ScanArchive { + init( + records: [FileNodeRecord], parentIndex: [Int], childStart: [Int], + childCount: [Int], childIndices: [Int], rootIndex: Int, + rootPath: String, metadata: ScanArchive.Metadata + ) { + self = try! JSONDecoder().decode(ScanArchive.self, from: JSONEncoder().encode( + ScanArchiveTestPayload( + records: records, parentIndex: parentIndex, childStart: childStart, + childCount: childCount, childIndices: childIndices, rootIndex: rootIndex, + rootPath: rootPath, metadata: metadata + ) + )) + } +} + +// `ScanArchive`'s memberwise fields aren't independently settable (its only +// public initializer takes a `FileTree`), so this mirrors its `Codable` +// shape exactly and round-trips through JSON to construct arbitrary/doctored +// instances for the validator tests above. +private struct ScanArchiveTestPayload: Codable { + let records: [FileNodeRecord] + let parentIndex: [Int] + let childStart: [Int] + let childCount: [Int] + let childIndices: [Int] + let rootIndex: Int + let rootPath: String + let metadata: ScanArchive.Metadata +} From 1aded6b2d3d95c1eb42106a903ac39f02e290aaa Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 21:38:50 +0300 Subject: [PATCH 21/41] feat(compare): diff two scans (added/removed/grown/shrank) --- MacDirStat.xcodeproj/project.pbxproj | 16 ++ Sources/App/ContentView.swift | 3 + Sources/App/MacDirStatApp.swift | 21 ++ Sources/Model/ScanComparison.swift | 151 +++++++++++++ Sources/ViewModels/ScanViewModel.swift | 53 +++++ Sources/Views/Comparison/ComparisonView.swift | 210 ++++++++++++++++++ Tests/ScanComparisonTests.swift | 128 +++++++++++ 7 files changed, 582 insertions(+) create mode 100644 Sources/Model/ScanComparison.swift create mode 100644 Sources/Views/Comparison/ComparisonView.swift create mode 100644 Tests/ScanComparisonTests.swift diff --git a/MacDirStat.xcodeproj/project.pbxproj b/MacDirStat.xcodeproj/project.pbxproj index 1b549a0..fb1956b 100644 --- a/MacDirStat.xcodeproj/project.pbxproj +++ b/MacDirStat.xcodeproj/project.pbxproj @@ -41,6 +41,8 @@ 26B861D7144BFB6B2FD00F6B /* FileNodeRecord.swift in Sources */ = {isa = PBXBuildFile; fileRef = F03CE9327807EE8357D3678B /* FileNodeRecord.swift */; }; E7ED11C27A082009974B4706 /* FileTreeBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4FB7871D4D4A1E73B7984396 /* FileTreeBuilder.swift */; }; A7CECAA54EC4419CA4B847D2 /* ScanArchive.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5B0C663AD1A1417DA8FC6942 /* ScanArchive.swift */; }; + 4A4F4C3CFEF84A1B827AB6EE /* ScanComparison.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4542CEEA3CDE4292A6661596 /* ScanComparison.swift */; }; + 57FD4B40D3F541AD98E734A1 /* ComparisonView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1A060E7C6E1C405F81CE31FA /* ComparisonView.swift */; }; SPARK003000000000000003AA /* Sparkle in Frameworks */ = {isa = PBXBuildFile; productRef = SPARK002000000000000002AA /* Sparkle */; }; /* End PBXBuildFile section */ @@ -82,6 +84,8 @@ F03CE9327807EE8357D3678B /* FileNodeRecord.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileNodeRecord.swift; sourceTree = ""; }; 4FB7871D4D4A1E73B7984396 /* FileTreeBuilder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileTreeBuilder.swift; sourceTree = ""; }; 5B0C663AD1A1417DA8FC6942 /* ScanArchive.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScanArchive.swift; sourceTree = ""; }; + 4542CEEA3CDE4292A6661596 /* ScanComparison.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScanComparison.swift; sourceTree = ""; }; + 1A060E7C6E1C405F81CE31FA /* ComparisonView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ComparisonView.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -130,6 +134,14 @@ path = Duplicates; sourceTree = ""; }; + A240638C83684A03A61256D5 /* Comparison */ = { + isa = PBXGroup; + children = ( + 1A060E7C6E1C405F81CE31FA /* ComparisonView.swift */, + ); + path = Comparison; + sourceTree = ""; + }; 71D2E56247FE75C3DCE9B609 = { isa = PBXGroup; children = ( @@ -160,6 +172,7 @@ F03CE9327807EE8357D3678B /* FileNodeRecord.swift */, 4FB7871D4D4A1E73B7984396 /* FileTreeBuilder.swift */, 5B0C663AD1A1417DA8FC6942 /* ScanArchive.swift */, + 4542CEEA3CDE4292A6661596 /* ScanComparison.swift */, ); path = Model; sourceTree = ""; @@ -176,6 +189,7 @@ isa = PBXGroup; children = ( 943FF7A0195AF9CFFDCE7D83 /* DirectoryTree */, + A240638C83684A03A61256D5 /* Comparison */, 546A8B7B0B821FBB727B5691 /* Duplicates */, B4F8EFC4604E6BCB45034AC5 /* ExtensionList */, A6644EB28BF6F3D30059D92F /* Shared */, @@ -383,6 +397,8 @@ 26B861D7144BFB6B2FD00F6B /* FileNodeRecord.swift in Sources */, E7ED11C27A082009974B4706 /* FileTreeBuilder.swift in Sources */, A7CECAA54EC4419CA4B847D2 /* ScanArchive.swift in Sources */, + 4A4F4C3CFEF84A1B827AB6EE /* ScanComparison.swift in Sources */, + 57FD4B40D3F541AD98E734A1 /* ComparisonView.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Sources/App/ContentView.swift b/Sources/App/ContentView.swift index 29310c1..2252444 100644 --- a/Sources/App/ContentView.swift +++ b/Sources/App/ContentView.swift @@ -113,6 +113,9 @@ struct ContentView: View { .sheet(isPresented: $vm.showFDASheet) { FullDiskAccessSheet() } + .sheet(isPresented: $vm.showComparisonSheet) { + ComparisonView() + } } // MARK: - Tab picker diff --git a/Sources/App/MacDirStatApp.swift b/Sources/App/MacDirStatApp.swift index b55d5c9..d76d2a1 100644 --- a/Sources/App/MacDirStatApp.swift +++ b/Sources/App/MacDirStatApp.swift @@ -43,6 +43,12 @@ struct MacDirStatApp: App { NotificationCenter.default.post(name: .exportCSV, object: nil) } .keyboardShortcut("e", modifiers: [.command, .shift]) + + Button("Compare With Saved Scan…") { + compareWithSavedScanPicker(vm: vm) + } + .keyboardShortcut("d", modifiers: [.command, .shift]) + .disabled(vm.tree == nil) } CommandGroup(after: .appInfo) { Button("Check for Updates…") { @@ -105,3 +111,18 @@ private func openArchivePicker(vm: ScanViewModel) { vm.openArchive(from: url) } } + +@MainActor +private func compareWithSavedScanPicker(vm: ScanViewModel) { + guard vm.tree != nil else { return } + let panel = NSOpenPanel() + panel.canChooseDirectories = false + panel.canChooseFiles = true + panel.allowsMultipleSelection = false + panel.allowedContentTypes = [mdscanType] + panel.prompt = "Compare" + panel.message = "Choose a saved scan to compare against the current one" + if panel.runModal() == .OK, let url = panel.url { + vm.compareWithSavedScan(archiveURL: url) + } +} diff --git a/Sources/Model/ScanComparison.swift b/Sources/Model/ScanComparison.swift new file mode 100644 index 0000000..8303eca --- /dev/null +++ b/Sources/Model/ScanComparison.swift @@ -0,0 +1,151 @@ +import Foundation + +// A single reported difference between two scans of (nominally) the same +// root, keyed by the node's path relative to that root so it's meaningful +// even when `before`/`after` are two entirely separate `FileTree` instances +// (e.g. one loaded from a `.mdscan` archive, one the live in-memory scan). +public struct ScanChange: Identifiable, Hashable, Sendable { + public enum Kind: String, Sendable { + case added + case removed + case grew + case shrank + } + + public let relativePath: String + public let name: String + public let isDirectory: Bool + public let kind: Kind + public let beforeSize: Int64 + public let afterSize: Int64 + + public var delta: Int64 { afterSize - beforeSize } + public var id: String { relativePath } + + public init(relativePath: String, name: String, isDirectory: Bool, kind: Kind, beforeSize: Int64, afterSize: Int64) { + self.relativePath = relativePath + self.name = name + self.isDirectory = isDirectory + self.kind = kind + self.beforeSize = beforeSize + self.afterSize = afterSize + } +} + +// Diffs two `FileTree`s ("Compare Scans Over Time"): what was added, removed, +// grew, or shrank between them. Read-only — never mutates either tree. +public enum ScanComparison { + // Builds an index of every node in `tree`, keyed by its path relative to + // `tree.rootPath` (root itself is the empty string), and diffs the two + // resulting maps. Each tree is indexed exactly once via a single + // iterative DFS (see `relativePathIndex`), so the whole comparison is + // O(nBefore + nAfter) — no nested tree-against-tree scanning — which + // keeps it cheap even on multi-million-node trees. + public static func compare(before: FileTree, after: FileTree) -> [ScanChange] { + let beforeIndex = relativePathIndex(of: before) + let afterIndex = relativePathIndex(of: after) + + var changes: [ScanChange] = [] + + // Added: present in `after`, absent from `before`. Collapsed to the + // topmost new node in each new subtree — a node is only reported if + // its parent already existed in `before` (i.e. the parent is NOT + // itself newly added); every descendant of a newly-added directory + // is skipped here, since it's implied by the directory's own + // "added" row, exactly like Radix's diff-row suppression. + for (relativePath, afterNodeIndex) in afterIndex { + guard beforeIndex[relativePath] == nil else { continue } + if let parentPath = parentRelativePath(of: relativePath), beforeIndex[parentPath] == nil { + continue + } + let record = after.records[afterNodeIndex] + changes.append(ScanChange( + relativePath: relativePath, + name: record.name, + isDirectory: record.isDirectory, + kind: .added, + beforeSize: 0, + afterSize: record.size + )) + } + + // Removed: present in `before`, absent from `after`. Same collapsing + // rule, mirrored: skip a node whose parent is also removed (not + // present in `after`) — only the topmost removed node per subtree is + // reported. + for (relativePath, beforeNodeIndex) in beforeIndex { + guard afterIndex[relativePath] == nil else { continue } + if let parentPath = parentRelativePath(of: relativePath), afterIndex[parentPath] == nil { + continue + } + let record = before.records[beforeNodeIndex] + changes.append(ScanChange( + relativePath: relativePath, + name: record.name, + isDirectory: record.isDirectory, + kind: .removed, + beforeSize: record.size, + afterSize: 0 + )) + } + + // Grew/shrank: only for *files* present in both trees whose size + // differs. Directories present in both are deliberately never + // reported here — a directory's size is entirely derived from its + // descendants, so any real change under it already surfaces as an + // added/removed/grew/shrank row for that descendant; reporting the + // containing directories too would just be noisy duplication of the + // same information at every level of the path. + for (relativePath, beforeNodeIndex) in beforeIndex { + guard let afterNodeIndex = afterIndex[relativePath] else { continue } + let beforeRecord = before.records[beforeNodeIndex] + let afterRecord = after.records[afterNodeIndex] + guard !beforeRecord.isDirectory, !afterRecord.isDirectory else { continue } + guard beforeRecord.size != afterRecord.size else { continue } + changes.append(ScanChange( + relativePath: relativePath, + name: afterRecord.name, + isDirectory: false, + kind: afterRecord.size > beforeRecord.size ? .grew : .shrank, + beforeSize: beforeRecord.size, + afterSize: afterRecord.size + )) + } + + return changes.sorted { abs($0.delta) > abs($1.delta) } + } + + // Iterative DFS (explicit stack, mirroring `FileTree.removingSubtrees`'s + // style) building every node's root-relative path incrementally from its + // parent's — O(n) total, unlike calling `FileTree.path(of:)` per node + // (which walks the parent chain from scratch each time and would be + // O(n·depth) over a whole tree). + private static func relativePathIndex(of tree: FileTree) -> [String: Int] { + var index: [String: Int] = [:] + index.reserveCapacity(tree.records.count) + + var stack: [(index: Int, relativePath: String)] = [(tree.rootIndex, "")] + while let (nodeIndex, relativePath) = stack.popLast() { + index[relativePath] = nodeIndex + let start = tree.childStart[nodeIndex] + let count = tree.childCount[nodeIndex] + for offset in 0.. String? { + guard let lastSlash = relativePath.lastIndex(of: "/") else { + return relativePath.isEmpty ? nil : "" + } + return String(relativePath[.. String { + let formatter = DateFormatter() + formatter.dateStyle = .medium + formatter.timeStyle = .short + return formatter.string(from: date) + } + + // MARK: - Summary bar + + private func summaryBar(changes: [ScanChange]) -> some View { + let added = changes.filter { $0.kind == .added }.count + let removed = changes.filter { $0.kind == .removed }.count + let grew = changes.filter { $0.kind == .grew }.count + let shrank = changes.filter { $0.kind == .shrank }.count + let netDelta = changes.reduce(Int64(0)) { $0 + $1.delta } + + return HStack(spacing: 14) { + summaryPill(count: added, label: "added", color: .green, icon: "plus.circle") + summaryPill(count: removed, label: "removed", color: .red, icon: "minus.circle") + summaryPill(count: grew, label: "grew", color: .orange, icon: "arrow.up.circle") + summaryPill(count: shrank, label: "shrank", color: .teal, icon: "arrow.down.circle") + + Spacer() + + Label(signedByteString(netDelta), systemImage: netDelta >= 0 ? "arrow.up.right" : "arrow.down.right") + .font(.system(size: 11, weight: .semibold, design: .monospaced)) + .foregroundStyle(netDelta >= 0 ? .orange : .teal) + } + .padding(.horizontal, 12) + .padding(.vertical, 8) + .background(.ultraThinMaterial) + } + + @ViewBuilder + private func summaryPill(count: Int, label: String, color: Color, icon: String) -> some View { + if count > 0 { + Label("\(count) \(label)", systemImage: icon) + .font(.caption) + .foregroundStyle(color) + } + } + + private func signedByteString(_ delta: Int64) -> String { + let sign = delta > 0 ? "+" : (delta < 0 ? "-" : "") + return sign + ByteFormatter.string(from: abs(delta)) + } + + // MARK: - Change list + + private func changeList(changes: [ScanChange]) -> some View { + ScrollView { + LazyVStack(spacing: 0) { + ForEach(changes) { change in + ChangeRow(change: change) + Divider().padding(.leading, 34) + } + } + } + } +} + +// MARK: - Change row + +private struct ChangeRow: View { + let change: ScanChange + + private var icon: String { + switch change.kind { + case .added: return "plus.circle.fill" + case .removed: return "minus.circle.fill" + case .grew: return "arrow.up.circle.fill" + case .shrank: return "arrow.down.circle.fill" + } + } + + private var color: Color { + switch change.kind { + case .added: return .green + case .removed: return .red + case .grew: return .orange + case .shrank: return .teal + } + } + + private var deltaString: String { + let sign = change.delta > 0 ? "+" : (change.delta < 0 ? "-" : "") + return sign + ByteFormatter.string(from: abs(change.delta)) + } + + var body: some View { + HStack(spacing: 10) { + Image(systemName: icon) + .font(.system(size: 13)) + .foregroundStyle(color) + .frame(width: 16) + + Image(systemName: change.isDirectory ? "folder" : "doc") + .font(.system(size: 11)) + .foregroundStyle(.secondary) + .frame(width: 14) + + VStack(alignment: .leading, spacing: 1) { + Text(change.name) + .font(.system(size: 12.5)) + .lineLimit(1) + Text(change.relativePath) + .font(.system(size: 10.5)) + .foregroundStyle(.tertiary) + .lineLimit(1) + .truncationMode(.middle) + } + + Spacer(minLength: 8) + + switch change.kind { + case .added: + Text(ByteFormatter.string(from: change.afterSize)) + .font(.system(size: 11, design: .monospaced)) + .foregroundStyle(.secondary) + case .removed: + Text(ByteFormatter.string(from: change.beforeSize)) + .font(.system(size: 11, design: .monospaced)) + .foregroundStyle(.secondary) + case .grew, .shrank: + Text("\(ByteFormatter.string(from: change.beforeSize)) → \(ByteFormatter.string(from: change.afterSize))") + .font(.system(size: 11, design: .monospaced)) + .foregroundStyle(.secondary) + } + + Text(deltaString) + .font(.system(size: 11, weight: .semibold, design: .monospaced)) + .foregroundStyle(color) + .frame(width: 70, alignment: .trailing) + } + .padding(.horizontal, 12) + .padding(.vertical, 6) + .contentShape(Rectangle()) + } +} diff --git a/Tests/ScanComparisonTests.swift b/Tests/ScanComparisonTests.swift new file mode 100644 index 0000000..a98a6df --- /dev/null +++ b/Tests/ScanComparisonTests.swift @@ -0,0 +1,128 @@ +import XCTest +@testable import MacDirStat + +final class ScanComparisonTests: XCTestCase { + + // "Before" fixture: + // root (/scan) + // ├── big.bin (500) -- shrinks to 100 in "after" + // ├── sub (dir, 400) + // │ ├── a.txt (300) -- grows to 350 in "after" + // │ └── b.txt (100) -- unchanged + // └── oldDir (dir, 200) -- entirely removed in "after" + // └── oldFile.txt (200) + private func makeBeforeTree() -> FileTree { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big = FSNode(url: URL(fileURLWithPath: "/scan/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: root) + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 400, fileExtension: "", parent: root) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 300, fileExtension: "txt", parent: sub) + let b = FSNode(url: URL(fileURLWithPath: "/scan/sub/b.txt"), name: "b.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: sub) + sub.children = [a, b] + + let oldDir = FSNode(url: URL(fileURLWithPath: "/scan/oldDir"), name: "oldDir", isDirectory: true, size: 200, fileExtension: "", parent: root) + let oldFile = FSNode(url: URL(fileURLWithPath: "/scan/oldDir/oldFile.txt"), name: "oldFile.txt", isDirectory: false, size: 200, fileExtension: "txt", parent: oldDir) + oldDir.children = [oldFile] + + root.children = [big, sub, oldDir] + root.size = big.size + sub.size + oldDir.size + return FileTreeBuilder.build(from: root, rootPath: "/scan") + } + + // "After" fixture: big.bin shrank, sub/a.txt grew, oldDir is gone + // entirely, and a brand-new top-level file appeared. + private func makeAfterTree() -> FileTree { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big = FSNode(url: URL(fileURLWithPath: "/scan/big.bin"), name: "big.bin", isDirectory: false, size: 100, fileExtension: "bin", parent: root) + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 450, fileExtension: "", parent: root) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 350, fileExtension: "txt", parent: sub) + let b = FSNode(url: URL(fileURLWithPath: "/scan/sub/b.txt"), name: "b.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: sub) + sub.children = [a, b] + + let newFile = FSNode(url: URL(fileURLWithPath: "/scan/newFile.txt"), name: "newFile.txt", isDirectory: false, size: 250, fileExtension: "txt", parent: root) + + root.children = [big, sub, newFile] + root.size = big.size + sub.size + newFile.size + return FileTreeBuilder.build(from: root, rootPath: "/scan") + } + + func test_compare_reports_exactly_the_four_expected_changes() { + let before = makeBeforeTree() + let after = makeAfterTree() + + let changes = ScanComparison.compare(before: before, after: after) + + XCTAssertEqual(changes.count, 4, "must not list oldDir's descendant separately, or unrelated unchanged nodes") + + let byPath = Dictionary(uniqueKeysWithValues: changes.map { ($0.relativePath, $0) }) + + let added = byPath["newFile.txt"] + XCTAssertEqual(added?.kind, .added) + XCTAssertEqual(added?.isDirectory, false) + XCTAssertEqual(added?.beforeSize, 0) + XCTAssertEqual(added?.afterSize, 250) + XCTAssertEqual(added?.delta, 250) + + let removed = byPath["oldDir"] + XCTAssertEqual(removed?.kind, .removed) + XCTAssertEqual(removed?.isDirectory, true) + XCTAssertEqual(removed?.beforeSize, 200) + XCTAssertEqual(removed?.afterSize, 0) + XCTAssertEqual(removed?.delta, -200) + XCTAssertNil(byPath["oldDir/oldFile.txt"], "descendant of a removed directory must be collapsed into the directory's own row") + + let grew = byPath["sub/a.txt"] + XCTAssertEqual(grew?.kind, .grew) + XCTAssertEqual(grew?.beforeSize, 300) + XCTAssertEqual(grew?.afterSize, 350) + XCTAssertEqual(grew?.delta, 50) + + let shrank = byPath["big.bin"] + XCTAssertEqual(shrank?.kind, .shrank) + XCTAssertEqual(shrank?.beforeSize, 500) + XCTAssertEqual(shrank?.afterSize, 100) + XCTAssertEqual(shrank?.delta, -400) + + // Unchanged node must not appear at all. + XCTAssertNil(byPath["sub/b.txt"]) + // The directory that merely contains a changed file must not be + // reported itself (its size delta is implied by sub/a.txt's row). + XCTAssertNil(byPath["sub"]) + } + + func test_compare_sorts_by_absolute_delta_descending() { + let changes = ScanComparison.compare(before: makeBeforeTree(), after: makeAfterTree()) + let deltas = changes.map { abs($0.delta) } + XCTAssertEqual(deltas, deltas.sorted(by: >)) + // big.bin (400) > newFile.txt (250) > oldDir (200) > sub/a.txt (50) + XCTAssertEqual(changes.map(\.relativePath), ["big.bin", "newFile.txt", "oldDir", "sub/a.txt"]) + } + + func test_compare_identical_trees_reports_no_changes() { + let tree = makeBeforeTree() + XCTAssertTrue(ScanComparison.compare(before: tree, after: tree).isEmpty) + } + + func test_compare_collapses_a_newly_added_directory_to_a_single_row() { + // "before": just a root with one file. + let beforeRoot = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 100, fileExtension: "", parent: nil) + let keep = FSNode(url: URL(fileURLWithPath: "/scan/keep.txt"), name: "keep.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: beforeRoot) + beforeRoot.children = [keep] + let before = FileTreeBuilder.build(from: beforeRoot, rootPath: "/scan") + + // "after": same file, plus a whole new directory with two files inside. + let afterRoot = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 400, fileExtension: "", parent: nil) + let keep2 = FSNode(url: URL(fileURLWithPath: "/scan/keep.txt"), name: "keep.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: afterRoot) + let newDir = FSNode(url: URL(fileURLWithPath: "/scan/newDir"), name: "newDir", isDirectory: true, size: 300, fileExtension: "", parent: afterRoot) + let n1 = FSNode(url: URL(fileURLWithPath: "/scan/newDir/n1.txt"), name: "n1.txt", isDirectory: false, size: 200, fileExtension: "txt", parent: newDir) + let n2 = FSNode(url: URL(fileURLWithPath: "/scan/newDir/n2.txt"), name: "n2.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: newDir) + newDir.children = [n1, n2] + afterRoot.children = [keep2, newDir] + let after = FileTreeBuilder.build(from: afterRoot, rootPath: "/scan") + + let changes = ScanComparison.compare(before: before, after: after) + XCTAssertEqual(changes.count, 1, "adding a whole directory tree must produce exactly one collapsed row") + XCTAssertEqual(changes[0].relativePath, "newDir") + XCTAssertEqual(changes[0].kind, .added) + XCTAssertEqual(changes[0].afterSize, 300) + } +} From 3c9d29cf5ffb1b6ebd6e2ce83afd85a24bc521af Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 21:52:09 +0300 Subject: [PATCH 22/41] perf(refresh): incremental subtree splice instead of full rescan Replace the Phase 2 interim (any FSEvents change triggers a full rescan of the root) with a targeted rescan-and-splice: FileTree.replacingSubtree(at:with:) swaps a changed directory's stale subtree for a freshly-rescanned one, rebuilding ancestor sizes and re-sorting only the affected ancestor chain. ScanViewModel.splicedTree(afterChangeAt:in:) resolves each changed path, reuses the kept scanSubtree helper to rescan just that directory, and falls back to a full rescan only when the splice can't be trusted (root changed/vanished, path unresolvable, or an auto-summarized target). --- Sources/Model/FileTree.swift | 147 ++++++++++ Sources/ViewModels/ScanViewModel.swift | 216 ++++++++++++-- Tests/IncrementalRefreshTests.swift | 379 +++++++++++++++++++++++++ 3 files changed, 714 insertions(+), 28 deletions(-) create mode 100644 Tests/IncrementalRefreshTests.swift diff --git a/Sources/Model/FileTree.swift b/Sources/Model/FileTree.swift index 78db867..879f506 100644 --- a/Sources/Model/FileTree.swift +++ b/Sources/Model/FileTree.swift @@ -251,4 +251,151 @@ public final class FileTree: @unchecked Sendable { rootPath: rootPath ) } + + // MARK: - Splice (incremental live-refresh, replacing the full-rescan interim) + + // Returns a NEW tree where the subtree rooted at `index` is replaced + // wholesale by `subtree`'s own nodes, reparented under `index`'s former + // parent in the very same child slot. This is the live-refresh + // counterpart to `removingSubtrees` above: instead of dropping a stale + // subtree, it swaps in a freshly-rescanned replacement for it (produced + // by re-walking just that one directory on disk — see + // `ScanViewModel.splicedTree(afterChangeAt:in:)`), so a filesystem + // change deep inside a huge tree only costs a rescan of the changed + // directory, not the whole root. + // + // The root itself can never be replaced this way (mirrors + // `removingSubtree`'s own root guard, returning `self` unchanged) — a + // changed root is the caller's responsibility to detect ahead of time + // and fall back to a full rescan for instead. + public func replacingSubtree(at index: Int, with subtree: FileTree) -> FileTree { + let count = records.count + guard index != rootIndex, index >= 0, index < count else { return self } + let oldParent = parentIndex[index] + guard oldParent >= 0 else { return self } // unreachable given index != rootIndex, but defensive + + // Mark the stale subtree rooted at `index` (itself plus every + // descendant) via the same iterative DFS `removingSubtrees` uses. + var removed = [Bool](repeating: false, count: count) + var stack: [Int] = [index] + while let i = stack.popLast() { + if removed[i] { continue } + removed[i] = true + let start = childStart[i] + let cnt = childCount[i] + for offset in 0.. new index map over surviving (kept) nodes, ascending + // old-index order — identical compaction to `removingSubtrees`. + var oldToNew = [Int](repeating: -1, count: count) + var newRecords: [FileNodeRecord] = [] + newRecords.reserveCapacity(count + subtree.records.count) + for old in 0..= 0 ? oldToNew[op] : -1 + } + for i in 0..= 0 ? offset + subParent : oldToNew[oldParent] + } + + // Ancestor sizes: fold in the difference between the new subtree + // root's size and the stale one it's replacing, all the way up to + // the root (positive or negative — the changed directory may have + // grown or shrunk). + let delta = subtree.records[subtree.rootIndex].size - records[index].size + var ancestorOld = oldParent + while ancestorOld >= 0 { + newRecords[oldToNew[ancestorOld]].size += delta + ancestorOld = parentIndex[ancestorOld] + } + + // Children spans: every kept node keeps its surviving children in + // their original relative order, except the replaced node's own + // slot in its parent's span (which used to point at `index`), which + // now points at the new subtree's root instead. + var newChildIndices: [Int] = [] + newChildIndices.reserveCapacity(childIndices.count + subtree.childIndices.count) + var newChildStart = [Int](repeating: 0, count: newRecords.count) + var newChildCount = [Int](repeating: 0, count: newRecords.count) + for old in 0..= 0 { + ancestorNewIndices.append(oldToNew[a]) + a = parentIndex[a] + } + for newIdx in ancestorNewIndices { + let start = newChildStart[newIdx] + let cnt = newChildCount[newIdx] + guard cnt > 1 else { continue } + var slice = Array(newChildIndices[start..<(start + cnt)]) + slice.sort { newRecords[$0].size > newRecords[$1].size } + newChildIndices.replaceSubrange(start..<(start + cnt), with: slice) + } + + return FileTree( + records: newRecords, + parentIndex: newParentIndex, + childStart: newChildStart, + childCount: newChildCount, + childIndices: newChildIndices, + rootIndex: oldToNew[rootIndex], + rootPath: rootPath + ) + } } diff --git a/Sources/ViewModels/ScanViewModel.swift b/Sources/ViewModels/ScanViewModel.swift index ea752d6..65c6cab 100644 --- a/Sources/ViewModels/ScanViewModel.swift +++ b/Sources/ViewModels/ScanViewModel.swift @@ -304,37 +304,197 @@ public final class ScanViewModel: ObservableObject { } } - // TODO(phase4): incremental splice refresh. + // Incremental splice refresh: for each directory FSEvents reports as + // changed, rescan just that directory from disk and splice the result + // into `tree` (see `Self.splicedTree(afterChangeAt:in:)` / + // `FileTree.replacingSubtree(at:with:)`) instead of rescanning the whole + // root, exactly like the "Move to Trash" prune path already avoids a + // full rescan for deletes. Multiple changed paths in one FSEvents batch + // are folded sequentially — each splice's result feeds the next lookup — + // which the plan explicitly allows as simpler than a single combined + // multi-directory splice, at the cost of a little redundant rescanning + // when a batch contains both a directory and one of its own descendants. // - // `FileTree`'s topology (parent/child arrays) is immutable for the life - // of one instance (that immutability is exactly what makes the flat - // arena cheap), so the old in-place FSNode mutation this method used to - // do (find the live node, patch its children/sizes) can no longer work - // directly against the live tree. The intended replacement (see the - // Phase 2 plan) is: for each changed directory, look up its index via a - // path->index map, re-scan just that directory into a fresh FSNode - // subtree (reusing `refreshDirectory`/`scanSubtree` below, which already - // do exactly this kind of on-disk rescan), convert it with - // `FileTreeBuilder`, and splice the result into a NEW `FileTree` that - // shares every untouched record/edge with the old one. - // - // That splice is a meaningful chunk of work on its own, so for this - // migration the interim (explicitly allowed by the plan) is simpler and - // still correct: any filesystem change under the watched root triggers a - // full rescan of the currently-scanned root, exactly like the "Move to - // Trash" actions elsewhere in the app already do after a delete. The - // FSNode-based helpers below (refreshDirectory/scanSubtree/bubbleUpSizes/ - // findNode/firstNode) are kept exactly as they were — unused by this - // method for now, but still exercised directly by ScanRefreshTests / - // HiddenSpaceTests, and ready to be reused as the per-directory rescan - // step of the real splice in Phase 4. - private func handleFileSystemChanges(_ paths: [String]) async { - guard let scanURL else { return } + // Falls back to a full rescan (unchanged from before) only for the cases + // a splice can't safely handle — see `splicedTree`'s doc comment: the + // root itself changed/vanished, a changed path no longer resolves + // anywhere in the tree, or it resolves into an auto-summarized node. + // Not `private`: exercised directly by IncrementalRefreshTests (via + // `@testable import`) to simulate an FSEvents batch deterministically, + // without needing a real FSEventStream round trip. + func handleFileSystemChanges(_ paths: [String]) async { + guard let scanURL, let startingTree = tree, !isReadOnlySnapshot else { return } + + guard FileManager.default.fileExists(atPath: scanURL.path) else { + // The scanned root itself is gone (deleted/renamed/unmounted) — + // no subtree splice can recover from that. + if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { + FileHandle.standardError.write("REFRESH fallback-full-rescan root-vanished root=\(scanURL.path)\n".data(using: .utf8)!) + } + scan(url: scanURL) + return + } + + var workingTree = startingTree + for changedPath in paths { + guard let spliced = Self.splicedTree(afterChangeAt: changedPath, in: workingTree) else { + if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { + FileHandle.standardError.write("REFRESH fallback-full-rescan path=\(changedPath)\n".data(using: .utf8)!) + } + scan(url: scanURL) + return + } + workingTree = spliced + } + + guard workingTree !== startingTree else { return } // nothing actually spliceable in this batch + if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { - let line = "REFRESH full-rescan changedPaths=\(paths.count) root=\(scanURL.path)\n" - FileHandle.standardError.write(line.data(using: .utf8)!) + FileHandle.standardError.write("REFRESH spliced changedPaths=\(paths.count) root=\(scanURL.path)\n".data(using: .utf8)!) + } + await applySplicedTree(workingTree, from: startingTree) + } + + // Rescans exactly the on-disk directory at `changedPath` (a full, + // synchronous walk via `scanSubtree`, the same per-directory rescan step + // the pre-Phase-2 refresh path used) and splices the result into `tree`, + // replacing its stale subtree in place — the cheap alternative to + // rescanning the whole root on every FSEvents notification. + // + // Returns nil when the splice can't be trusted, and the caller must fall + // back to a full rescan of the root instead: + // - `changedPath` (after normalizing away a trailing slash) resolves + // to the tree's own root — a changed root might mean the scanned + // directory itself was replaced or renamed, which no subtree splice + // can recover from. + // - `changedPath` doesn't resolve to any node in `tree` at all: it (or + // an ancestor) was deleted/renamed since the last refresh, or it + // lives inside an auto-summarized directory, whose children were + // never materialized in the first place — the path-component walk + // simply runs out of children to match partway down. + // - the resolved node is itself auto-summarized: it has no children + // array to splice into (see AtomicDirectorySummary.swift) — + // re-summarizing it in place is future work; falling back to a full + // rescan is correct and simple for now. + // - the freshly-rescanned replacement contains a directory the real + // scanner's auto-summarization would have collapsed (named + // "node_modules", mirroring `knownGeneratedDirectoryNames` in + // AtomicDirectorySummary.swift) — `scanSubtree` below doesn't + // implement that heuristic at all, so materializing it here would + // both be slow and disagree with the rest of the tree's + // summarization policy. + nonisolated static func splicedTree(afterChangeAt changedPath: String, in tree: FileTree) -> FileTree? { + let normalized = (changedPath.hasSuffix("/") && changedPath != "/") ? String(changedPath.dropLast()) : changedPath + + guard var index = findIndex(forPath: normalized, in: tree) else { return nil } + // FSEvents (without the FileEvents flag, which this app doesn't + // request) reports directories, but be defensive: if this ever + // resolves to a file, the directory that actually needs rescanning + // is its parent. + if !tree.records[index].isDirectory { + let parent = tree.parentIndex[index] + guard parent >= 0 else { return nil } + index = parent + } + guard index != tree.rootIndex else { return nil } + guard !tree.records[index].isAutoSummarized else { return nil } + + let node = FileNode(tree: tree, index: index) + let showHiddenFiles = UserDefaults.standard.bool(forKey: "showHiddenFiles") + let excludedNames = parseExcludedNames() + var seenRefs = Set() + let freshNode = scanSubtree( + url: node.url, + parent: nil, + showHiddenFiles: showHiddenFiles, + excludedNames: excludedNames, + treeRoot: nil, + seenRefs: &seenRefs + ) + + guard freshNode.name != "node_modules", !containsUnsummarizedGeneratedDirectory(freshNode) else { return nil } + + let subtree = FileTreeBuilder.build(from: freshNode, rootPath: node.url.path) + return tree.replacingSubtree(at: index, with: subtree) + } + + // See the last bullet of `splicedTree`'s doc comment above. + private nonisolated static func containsUnsummarizedGeneratedDirectory(_ node: FSNode) -> Bool { + for child in node.children where child.isDirectory { + if child.name == "node_modules" || containsUnsummarizedGeneratedDirectory(child) { + return true + } } - scan(url: scanURL) + return false + } + + // Repairs everything that referenced the old topology after one or more + // splices, mirroring `pruneTree(afterTrashing:from:)`: selection/drill + // stack are captured as paths beforehand and resolved back to indices + // afterward, since indices shift on every splice. The synthetic + // "Hidden & Unreadable Space" root child is never touched by any splice + // (it's a root-level child and a splice target is never the root — see + // `splicedTree`'s root guard — so it always survives untouched, no + // special-case re-appending needed here the way a full rescan needs + // `appendHiddenSpaceNodeIfNeeded`). + // + // Unlike a prune, a splice can introduce brand-new nodes (the freshly + // rescanned subtree), which start out `.caution`/no-duplicate-group from + // `FileTreeBuilder` — same as any fresh scan — so safety tagging and + // duplicate detection both re-run over the whole tree, exactly as they + // do after `scan(url:)`. Safety tagging is awaited synchronously before + // anything else touches `newTree.records`: it and `DuplicateDetector` + // both mutate that array in place, so — same reasoning as `scan(url:)` — + // they can't be allowed to run concurrently with each other. + private func applySplicedTree(_ newTree: FileTree, from oldTree: FileTree) async { + let selectedPath = selectedNode.map { oldTree.path(of: $0.index) } + let drillPaths = drillStack.map { oldTree.path(of: $0.index) } + + await Task.detached(priority: .userInitiated) { + Self.tagSafetyLevels(tree: newTree) + }.value + + self.tree = newTree + + if let selectedPath, let idx = Self.findIndex(forPath: selectedPath, in: newTree) { + selectedNode = FileNode(tree: newTree, index: idx) + } else { + selectedNode = nil + } + + var newDrillStack: [FileNode] = [] + for path in drillPaths { + guard let idx = Self.findIndex(forPath: path, in: newTree) else { break } + newDrillStack.append(FileNode(tree: newTree, index: idx)) + } + drillStack = newDrillStack + + let rootNode = FileNode(tree: newTree, index: newTree.rootIndex) + let map = ExtensionColorMap(root: rootNode) + colorMap = map + + extensionTask?.cancel() + extensionTask = Task.detached(priority: .userInitiated) { [weak self] in + let summaries = Self.buildExtensionSummaries(tree: newTree, map: map) + guard !Task.isCancelled else { return } + await MainActor.run { self?.extensionSummaries = summaries } + } + + duplicateTask?.cancel() + duplicateTask = Task.detached(priority: .utility) { [weak self] in + let detector = DuplicateDetector() + await detector.detect(in: newTree) + guard !Task.isCancelled else { return } + let groups = Self.buildDuplicateGroups(tree: newTree) + guard !Task.isCancelled else { return } + await MainActor.run { + self?.duplicatesReady = true + self?.duplicateGroups = groups + } + } + + isComputingLayout = true + await recomputeLayout() } // Walk the tree by path components to find the FSNode for a given path. diff --git a/Tests/IncrementalRefreshTests.swift b/Tests/IncrementalRefreshTests.swift new file mode 100644 index 0000000..1552d71 --- /dev/null +++ b/Tests/IncrementalRefreshTests.swift @@ -0,0 +1,379 @@ +import XCTest +@testable import MacDirStat + +// Coverage for the Phase 4d incremental splice refresh: `FileTree.replacingSubtree(at:with:)` +// (the pure tree-topology algorithm) and `ScanViewModel.splicedTree(afterChangeAt:in:)` (the +// per-directory-rescan-and-splice orchestration that `handleFileSystemChanges` now uses instead +// of a full rescan on every FSEvents notification). +final class IncrementalRefreshTests: XCTestCase { + + // MARK: - FileTree.replacingSubtree: pure algorithm + + // Builds the same small fixture FileTreePruneTests uses: + // root (/scan) size 900 + // ├── big.bin (500) + // └── sub (dir) size 400 + // ├── a.txt (300) + // └── b.txt (100) + private func makeFixtureTree() -> FileTree { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big = FSNode(url: URL(fileURLWithPath: "/scan/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: root) + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 400, fileExtension: "", parent: root) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 300, fileExtension: "txt", parent: sub) + let b = FSNode(url: URL(fileURLWithPath: "/scan/sub/b.txt"), name: "b.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: sub) + sub.children = [a, b] + root.children = [big, sub] + root.size = big.size + sub.size + return FileTreeBuilder.build(from: root, rootPath: "/scan") + } + + private func node(named name: String, in tree: FileTree) -> FileNode { + for i in 0..= 0 && start + cnt <= tree.childIndices.count, "child span out of range at \(i)", file: file, line: line) + var previousSize: Int64? + for offset in 0..= 0 && child < count, "child index \(child) out of range", file: file, line: line) + XCTAssertEqual(tree.parentIndex[child], i, "child \(child)'s parentIndex must point back to \(i)", file: file, line: line) + let size = tree.records[child].size + if let previousSize { + XCTAssertGreaterThanOrEqual(previousSize, size, "children must stay sorted size-desc", file: file, line: line) + } + previousSize = size + } + } + XCTAssertEqual(tree.parentIndex[tree.rootIndex], -1, "root must have no parent", file: file, line: line) + } + + // A "freshly rescanned sub" replacement: a.txt shrinks 300->200, b.txt is + // gone, c.txt (50) is new -> new sub totals 250 (was 400). + private func makeShrunkSubtree() -> FileTree { + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 200, fileExtension: "txt", parent: sub) + let c = FSNode(url: URL(fileURLWithPath: "/scan/sub/c.txt"), name: "c.txt", isDirectory: false, size: 50, fileExtension: "txt", parent: sub) + sub.children = [a, c] + sub.size = 250 + return FileTreeBuilder.build(from: sub, rootPath: "/scan/sub") + } + + // A "freshly rescanned sub" replacement that grew past big.bin's 500, + // to exercise the ancestor re-sort path. + private func makeGrownSubtree() -> FileTree { + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big2 = FSNode(url: URL(fileURLWithPath: "/scan/sub/big2.bin"), name: "big2.bin", isDirectory: false, size: 600, fileExtension: "bin", parent: sub) + sub.children = [big2] + sub.size = 600 + return FileTreeBuilder.build(from: sub, rootPath: "/scan/sub") + } + + func test_replacingSubtree_shrinks_ancestor_sizes_by_exact_delta() { + let tree = makeFixtureTree() + let subNode = node(named: "sub", in: tree) + let replacement = makeShrunkSubtree() + + let spliced = tree.replacingSubtree(at: subNode.index, with: replacement) + + let newRoot = FileNode(tree: spliced, index: spliced.rootIndex) + XCTAssertEqual(newRoot.size, 900 - 400 + 250, "root size must reflect exactly the size delta of the replaced subtree") + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(newSub.size, 250) + assertValidTopology(spliced) + } + + func test_replacingSubtree_replaces_node_set_under_target() { + let tree = makeFixtureTree() + let subNode = node(named: "sub", in: tree) + let replacement = makeShrunkSubtree() + + let spliced = tree.replacingSubtree(at: subNode.index, with: replacement) + + let newRoot = FileNode(tree: spliced, index: spliced.rootIndex) + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(Set(newSub.children.map(\.name)), Set(["a.txt", "c.txt"]), "b.txt must be gone, c.txt must be present") + XCTAssertNil(newSub.children.first { $0.name == "b.txt" }) + let newA = newSub.children.first { $0.name == "a.txt" }! + XCTAssertEqual(newA.size, 200, "a.txt's size must reflect the fresh rescan, not the stale one") + } + + func test_replacingSubtree_preserves_untouched_sibling() { + let tree = makeFixtureTree() + let subNode = node(named: "sub", in: tree) + let replacement = makeShrunkSubtree() + + let spliced = tree.replacingSubtree(at: subNode.index, with: replacement) + + let newRoot = FileNode(tree: spliced, index: spliced.rootIndex) + let bigStill = newRoot.children.first { $0.name == "big.bin" } + XCTAssertNotNil(bigStill) + XCTAssertEqual(bigStill?.size, 500, "an untouched sibling subtree must be completely unaffected by the splice") + } + + func test_replacingSubtree_path_reconstruction_correct_for_spliced_and_survivors() { + let tree = makeFixtureTree() + let subNode = node(named: "sub", in: tree) + let replacement = makeShrunkSubtree() + + let spliced = tree.replacingSubtree(at: subNode.index, with: replacement) + + let newRoot = FileNode(tree: spliced, index: spliced.rootIndex) + XCTAssertEqual(newRoot.url.path, "/scan") + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(newSub.url.path, "/scan/sub") + let newC = newSub.children.first { $0.name == "c.txt" }! + XCTAssertEqual(newC.url.path, "/scan/sub/c.txt", "a brand-new spliced-in node must still reconstruct its path correctly") + let bigStill = newRoot.children.first { $0.name == "big.bin" }! + XCTAssertEqual(bigStill.url.path, "/scan/big.bin", "a surviving node's path must still be correct after the splice") + } + + func test_replacingSubtree_resorts_ancestor_chain_when_target_grows_past_a_sibling() { + let tree = makeFixtureTree() + let subNode = node(named: "sub", in: tree) + let replacement = makeGrownSubtree() // 600, now bigger than big.bin's 500 + + let spliced = tree.replacingSubtree(at: subNode.index, with: replacement) + + let newRoot = FileNode(tree: spliced, index: spliced.rootIndex) + XCTAssertEqual(newRoot.children.map(\.name), ["sub", "big.bin"], "sub must now sort ahead of big.bin") + assertValidTopology(spliced) + } + + func test_replacingSubtree_leaves_original_tree_untouched() { + let tree = makeFixtureTree() + let subNode = node(named: "sub", in: tree) + let replacement = makeShrunkSubtree() + let originalRootSize = tree.records[tree.rootIndex].size + let originalCount = tree.records.count + + _ = tree.replacingSubtree(at: subNode.index, with: replacement) + + XCTAssertEqual(tree.records[tree.rootIndex].size, originalRootSize, "the original tree must not mutate") + XCTAssertEqual(tree.records.count, originalCount) + } + + func test_replacingSubtree_at_root_returns_self_unchanged() { + let tree = makeFixtureTree() + let replacement = makeShrunkSubtree() + + let result = tree.replacingSubtree(at: tree.rootIndex, with: replacement) + + XCTAssertTrue(result === tree, "replacing the root must be a no-op, returning the same instance") + } + + func test_replacingSubtree_carries_over_duplicate_group_and_safety_on_untouched_survivors() { + let tree = makeFixtureTree() + let bigNode = node(named: "big.bin", in: tree) + let groupID = UUID() + tree.setDuplicateGroupID(groupID, at: bigNode.index) + tree.setSafety(.safe, at: bigNode.index) + let subNode = node(named: "sub", in: tree) + let replacement = makeShrunkSubtree() + + let spliced = tree.replacingSubtree(at: subNode.index, with: replacement) + + let splicedBig = node(named: "big.bin", in: spliced) + XCTAssertEqual(splicedBig.duplicateGroupID, groupID) + XCTAssertEqual(splicedBig.safetyLevel, .safe) + } + + // MARK: - ScanViewModel.splicedTree: fallback cases + + func test_splicedTree_root_change_falls_back_to_nil() { + let tree = makeFixtureTree() + + XCTAssertNil(ScanViewModel.splicedTree(afterChangeAt: "/scan", in: tree)) + } + + func test_splicedTree_root_change_with_trailing_slash_falls_back_to_nil() { + let tree = makeFixtureTree() + + XCTAssertNil(ScanViewModel.splicedTree(afterChangeAt: "/scan/", in: tree)) + } + + func test_splicedTree_unresolvable_path_falls_back_to_nil() { + let tree = makeFixtureTree() + + XCTAssertNil(ScanViewModel.splicedTree(afterChangeAt: "/scan/does/not/exist", in: tree)) + } + + func test_splicedTree_path_outside_tree_falls_back_to_nil() { + let tree = makeFixtureTree() + + XCTAssertNil(ScanViewModel.splicedTree(afterChangeAt: "/somewhere/else", in: tree)) + } + + func test_splicedTree_autosummarized_target_falls_back_to_nil() { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let bigDir = FSNode(url: URL(fileURLWithPath: "/scan/node_modules"), name: "node_modules", isDirectory: true, size: 12_345, fileExtension: "", parent: root) + bigDir.isAutoSummarized = true + bigDir.descendantFileCount = 9_999 + root.children = [bigDir] + root.size = bigDir.size + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + + XCTAssertNil(ScanViewModel.splicedTree(afterChangeAt: "/scan/node_modules", in: tree), "a summarized node has no children to splice into") + } + + func test_splicedTree_path_inside_autosummarized_node_falls_back_to_nil() { + // Same fixture as above, but the "changed" path is one level deeper + // than the summarized node itself — never materialized in the tree + // at all, so the path-component walk simply can't resolve it. + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let bigDir = FSNode(url: URL(fileURLWithPath: "/scan/node_modules"), name: "node_modules", isDirectory: true, size: 12_345, fileExtension: "", parent: root) + bigDir.isAutoSummarized = true + root.children = [bigDir] + root.size = bigDir.size + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + + XCTAssertNil(ScanViewModel.splicedTree(afterChangeAt: "/scan/node_modules/some-package", in: tree)) + } + + // MARK: - Key correctness check: splice result == full fresh rescan + + @MainActor + private func waitUntil(timeout: TimeInterval = 5, _ condition: () -> Bool) async { + let deadline = Date().addingTimeInterval(timeout) + while !condition(), Date() < deadline { + try? await Task.sleep(nanoseconds: 10_000_000) + } + } + + private struct NodeSnapshot: Hashable { + let relativePath: String + let isDirectory: Bool + let size: Int64 + } + + // Flattens every descendant of `node` (node included, as "") into a + // set keyed by its path relative to `node`'s own tree root — order- and + // index-independent, so it's safe to compare across two entirely + // separate `FileTree` instances built from independent scans. + private func snapshot(_ root: FileNode) -> Set { + var result: Set = [] + let rootPath = root.tree.rootPath + func walk(_ n: FileNode) { + let full = n.url.path + let relative = full == rootPath ? "" : String(full.dropFirst(rootPath.count)) + result.insert(NodeSnapshot(relativePath: relative, isDirectory: n.isDirectory, size: n.size)) + for child in n.children { walk(child) } + } + walk(root) + return result + } + + @MainActor + private func scanAndWait(_ url: URL) async -> ScanViewModel { + let prior = UserDefaults.standard.object(forKey: "realtimeMonitoring") as? Bool + UserDefaults.standard.set(false, forKey: "realtimeMonitoring") + defer { + if let prior { UserDefaults.standard.set(prior, forKey: "realtimeMonitoring") } + else { UserDefaults.standard.removeObject(forKey: "realtimeMonitoring") } + } + let vm = ScanViewModel() + vm.updateLayoutSize(CGSize(width: 400, height: 400)) + vm.scan(url: url) + await waitUntil { !vm.isScanning && !vm.isComputingLayout } + return vm + } + + // The key correctness test the plan calls for: build a real tree from a + // real temp directory (via the actual FileScanner, exactly like a real + // scan), mutate a subdirectory on disk (add a file, remove a file, + // resize a file, add a nested directory), splice just that directory via + // `ScanViewModel.splicedTree`, and compare the result against an + // entirely independent, from-scratch full rescan of the same now-mutated + // directory tree (a second real `FileScanner` run via a second + // `ScanViewModel`). The two must describe exactly the same node set and + // sizes — a splice is only a cheaper way to arrive at the same state a + // full rescan would. + func test_splice_result_matches_full_fresh_rescan() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + try Data(repeating: 1, count: 8192).write(to: tmp.appendingPathComponent("keep.bin")) + let sub = tmp.appendingPathComponent("sub") + try FileManager.default.createDirectory(at: sub, withIntermediateDirectories: true) + try Data(repeating: 2, count: 4096).write(to: sub.appendingPathComponent("a.txt")) + let toRemove = sub.appendingPathComponent("b.txt") + try Data(repeating: 3, count: 2048).write(to: toRemove) + let toResize = sub.appendingPathComponent("c.txt") + try Data(repeating: 4, count: 1024).write(to: toResize) + + let before = await scanAndWait(tmp) + guard let beforeTree = await before.tree else { return XCTFail("initial scan should populate a tree") } + guard let subIndex = (0.. Date: Fri, 24 Jul 2026 22:06:36 +0300 Subject: [PATCH 23/41] fix(scanner): make summary walk async, removing thread-blocking bridge summarizeSubtree kept a synchronous signature and bridged into its nested worker pool by handing it to a detached Task and blocking the caller on a DispatchSemaphore. The caller runs on a Swift-concurrency cooperative thread and the nested pool needs threads from that same pool, so enough simultaneous summaries could block every cooperative thread waiting on work that had no thread left to run on. The blast radius was the whole scan, not one directory: a wedged summary keeps the main WorkQueue's inFlight count above zero, so isFinished never trips and every other scan worker spins forever. The poll loop also never returned on cancellation, so Stop Scan could not recover it. Make summarizeSubtree async and await the task group directly: no thread is ever blocked and cancellation propagates through structured concurrency. Drops the now-unused SummaryResultBox. Adds a regression test with 24 concurrent summarizing siblings (3x the worker cap). --- Sources/Scanner/AtomicDirectorySummary.swift | 88 ++++++-------------- Sources/Scanner/FileScanner.swift | 6 +- Tests/AutoSummaryTests.swift | 73 ++++++++++++++++ 3 files changed, 103 insertions(+), 64 deletions(-) diff --git a/Sources/Scanner/AtomicDirectorySummary.swift b/Sources/Scanner/AtomicDirectorySummary.swift index 97967b3..e9a14f2 100644 --- a/Sources/Scanner/AtomicDirectorySummary.swift +++ b/Sources/Scanner/AtomicDirectorySummary.swift @@ -139,26 +139,6 @@ private final class SummaryAccumulator: @unchecked Sendable { } } -// Tiny lock-guarded box used only to carry the first error thrown by the -// worker pool back across the sync/async bridge in `summarizeSubtree`. -private final class SummaryResultBox: @unchecked Sendable { - private var lock = os_unfair_lock() - private var storedError: Error? - - func setErrorIfAbsent(_ error: Error) { - os_unfair_lock_lock(&lock) - if storedError == nil { storedError = error } - os_unfair_lock_unlock(&lock) - } - - var error: Error? { - os_unfair_lock_lock(&lock) - let result = storedError - os_unfair_lock_unlock(&lock) - return result - } -} - // Parallel walk of the subtree rooted at a directory already deemed a // summarization candidate. Applies the exact same scan semantics as the main // traversal (symlink/hidden/exclusion/mount-point skips, hardlink + @@ -182,25 +162,28 @@ private final class SummaryResultBox: @unchecked Sendable { // hazard to worry about; the only shared mutable state is the queue and the // accumulator, both lock-guarded. // -// Concurrency bridge: this function's signature stays synchronous (`throws`, -// not `async`) so its one call site in FileScanner.swift is unchanged - it -// already runs on one of the main scan's worker threads, itself inside an -// async Task. To drive `withThrowingTaskGroup` from here, it hands the pool -// off to a detached Task and blocks this thread on a semaphore, polling with -// a short timeout so it can keep re-checking the ORIGINAL caller's -// cancellation via the passed-in `cancel` closure (`Task.checkCancellation` -// is dynamic-scoped to whatever Task is executing at the call site, so -// checking it here - still on the calling worker's own Task - reflects the -// real scan's cancellation; the detached pool Task is a separate Task, so -// checking cancellation from inside it would never see the outer scan being -// cancelled unless we explicitly forward it, which is what the polling loop -// below does via `poolTask.cancel()`). +// Concurrency: this function is `async` and drives its nested worker pool +// with a plain `withThrowingTaskGroup`, awaited directly by the calling scan +// worker. It deliberately does NOT block a thread to bridge sync->async. +// +// An earlier version kept a synchronous signature and bridged by handing the +// pool to a detached Task while blocking the caller's thread on a +// DispatchSemaphore. That was a forward-progress hazard: the caller runs on a +// Swift-concurrency cooperative thread, and the nested pool needs threads +// from that same pool to run. With enough sibling directories summarizing at +// once (a monorepo with several node_modules trees), every cooperative thread +// could end up blocked waiting for work that has no thread left to run on. +// Worse, a wedged summary keeps the main scan's `inFlight` counter above zero +// forever, so `WorkQueue.isFinished` never trips and EVERY other scan worker +// spins too - one stuck summary hangs the whole scan, and cancellation could +// not recover it. Awaiting the group directly removes the hazard entirely: +// no thread is ever blocked, and cancellation propagates through structured +// concurrency for free. // -// This runs nested inside one main-scan worker's task; a nested worker pool -// is fine here. Oversubscription (main pool x summary pool) is bounded (both -// pools cap at 8 workers) and self-limiting: only directories big enough to -// trip auto-summarization spin up a nested pool at all, and macOS's -// thread-pool scheduling handles the resulting oversubscription gracefully. +// Oversubscription (main pool x summary pool) remains bounded - both pools +// cap at 8 workers, and only directories big enough to trip auto-summarization +// spin up a nested pool at all - but it is now merely a scheduling concern, +// not a correctness one. func summarizeSubtree( rootEntries: [BulkDirEntry], rootPath: String, @@ -208,7 +191,7 @@ func summarizeSubtree( config: ScanConfig, visited: VisitedSet, cancel: @Sendable @escaping () throws -> Void -) throws -> (allocatedSize: Int64, fileCount: Int) { +) async throws -> (allocatedSize: Int64, fileCount: Int) { var totalAllocated: Int64 = 0 var fileCount = 0 var seedDirs: [String] = [] @@ -248,34 +231,17 @@ func summarizeSubtree( let queue = SummaryDirQueue(seed: seedDirs) let accumulator = SummaryAccumulator(allocatedSize: totalAllocated, fileCount: fileCount) - let resultBox = SummaryResultBox() let workerCount = min(max(2, ProcessInfo.processInfo.activeProcessorCount / 2), 8) - let semaphore = DispatchSemaphore(value: 0) - let poolTask = Task.detached(priority: .userInitiated) { - do { - try await withThrowingTaskGroup(of: Void.self) { group in - for _ in 0..> the 8-worker cap on either pool, so many summaries + // are guaranteed to be in flight at the same time. Each candidate is + // named `node_modules` so it summarizes via the deterministic + // named-layout shortcut rather than depending on the filesystem's + // allocated-size rounding to satisfy the average-size heuristic. + let siblingCount = 24 + let filesPerSibling = 12 + var expectedFiles = 0 + + for s in 0.. Int { + (node.isAutoSummarized ? 1 : 0) + node.children.reduce(0) { $0 + countSummarized($1) } + } + private func findNode(_ root: FileNode, path: [String]) -> FileNode? { var current = root for name in path { From 14303ac50c887dae268cb7229a101948b02d7f4b Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 22:10:43 +0300 Subject: [PATCH 24/41] fix(compare): report file<->directory type changes instead of dropping them A path present in both trees is skipped by the added/removed loops, and the grew/shrank loop only compares file-to-file, so a node that changed kind fell through every branch and never appeared in the diff. Worse, children of a path that became a directory were reported as plain 'added' rows, because the collapse rule only asked whether the parent path existed before, not whether it existed as a directory. Add a .replaced kind for same-path type changes (reported regardless of size, since neither size branch would fire when the sizes happen to match), and tighten both collapse rules to require the parent to have existed AS A DIRECTORY, so the replaced node's own row implies its contents. --- Sources/Model/ScanComparison.swift | 50 ++++++++++++++- Sources/Views/Comparison/ComparisonView.swift | 8 ++- Tests/ScanComparisonTests.swift | 63 +++++++++++++++++++ 3 files changed, 118 insertions(+), 3 deletions(-) diff --git a/Sources/Model/ScanComparison.swift b/Sources/Model/ScanComparison.swift index 8303eca..03e2e13 100644 --- a/Sources/Model/ScanComparison.swift +++ b/Sources/Model/ScanComparison.swift @@ -10,6 +10,11 @@ public struct ScanChange: Identifiable, Hashable, Sendable { case removed case grew case shrank + // Same path, different kind of thing: a file was replaced by a + // directory or vice versa. Reported as its own row because neither + // "added"/"removed" (the path exists on both sides) nor + // "grew"/"shrank" (which compare like with like) describes it. + case replaced } public let relativePath: String @@ -55,7 +60,12 @@ public enum ScanComparison { // "added" row, exactly like Radix's diff-row suppression. for (relativePath, afterNodeIndex) in afterIndex { guard beforeIndex[relativePath] == nil else { continue } - if let parentPath = parentRelativePath(of: relativePath), beforeIndex[parentPath] == nil { + // The parent counts as pre-existing only if it was there AND was + // already a directory. If it was a *file* that has since become a + // directory, everything now inside it is new content implied by + // that node's own "replaced" row, so suppress it here. + if let parentPath = parentRelativePath(of: relativePath), + !existsAsDirectory(parentPath, in: beforeIndex, of: before) { continue } let record = after.records[afterNodeIndex] @@ -75,7 +85,11 @@ public enum ScanComparison { // reported. for (relativePath, beforeNodeIndex) in beforeIndex { guard afterIndex[relativePath] == nil else { continue } - if let parentPath = parentRelativePath(of: relativePath), afterIndex[parentPath] == nil { + // Mirror of the added-side rule: if the parent is now a file where + // it used to be a directory, this node's disappearance is implied + // by the parent's "replaced" row. + if let parentPath = parentRelativePath(of: relativePath), + !existsAsDirectory(parentPath, in: afterIndex, of: after) { continue } let record = before.records[beforeNodeIndex] @@ -100,6 +114,24 @@ public enum ScanComparison { guard let afterNodeIndex = afterIndex[relativePath] else { continue } let beforeRecord = before.records[beforeNodeIndex] let afterRecord = after.records[afterNodeIndex] + + // Type change (file <-> directory) at the same path. Reported + // regardless of whether the size happens to match, since the node + // is fundamentally a different thing now — and without this the + // pair would fall through both guards below and vanish from the + // report entirely. + if beforeRecord.isDirectory != afterRecord.isDirectory { + changes.append(ScanChange( + relativePath: relativePath, + name: afterRecord.name, + isDirectory: afterRecord.isDirectory, + kind: .replaced, + beforeSize: beforeRecord.size, + afterSize: afterRecord.size + )) + continue + } + guard !beforeRecord.isDirectory, !afterRecord.isDirectory else { continue } guard beforeRecord.size != afterRecord.size else { continue } changes.append(ScanChange( @@ -148,4 +180,18 @@ public enum ScanComparison { } return String(relativePath[.. Bool { + guard let nodeIndex = index[relativePath] else { return false } + return tree.records[nodeIndex].isDirectory + } } diff --git a/Sources/Views/Comparison/ComparisonView.swift b/Sources/Views/Comparison/ComparisonView.swift index 78bedb7..39dcd3f 100644 --- a/Sources/Views/Comparison/ComparisonView.swift +++ b/Sources/Views/Comparison/ComparisonView.swift @@ -83,6 +83,7 @@ struct ComparisonView: View { let removed = changes.filter { $0.kind == .removed }.count let grew = changes.filter { $0.kind == .grew }.count let shrank = changes.filter { $0.kind == .shrank }.count + let replaced = changes.filter { $0.kind == .replaced }.count let netDelta = changes.reduce(Int64(0)) { $0 + $1.delta } return HStack(spacing: 14) { @@ -90,6 +91,9 @@ struct ComparisonView: View { summaryPill(count: removed, label: "removed", color: .red, icon: "minus.circle") summaryPill(count: grew, label: "grew", color: .orange, icon: "arrow.up.circle") summaryPill(count: shrank, label: "shrank", color: .teal, icon: "arrow.down.circle") + if replaced > 0 { + summaryPill(count: replaced, label: "replaced", color: .purple, icon: "arrow.triangle.swap") + } Spacer() @@ -141,6 +145,7 @@ private struct ChangeRow: View { case .removed: return "minus.circle.fill" case .grew: return "arrow.up.circle.fill" case .shrank: return "arrow.down.circle.fill" + case .replaced: return "arrow.triangle.swap" } } @@ -150,6 +155,7 @@ private struct ChangeRow: View { case .removed: return .red case .grew: return .orange case .shrank: return .teal + case .replaced: return .purple } } @@ -192,7 +198,7 @@ private struct ChangeRow: View { Text(ByteFormatter.string(from: change.beforeSize)) .font(.system(size: 11, design: .monospaced)) .foregroundStyle(.secondary) - case .grew, .shrank: + case .grew, .shrank, .replaced: Text("\(ByteFormatter.string(from: change.beforeSize)) → \(ByteFormatter.string(from: change.afterSize))") .font(.system(size: 11, design: .monospaced)) .foregroundStyle(.secondary) diff --git a/Tests/ScanComparisonTests.swift b/Tests/ScanComparisonTests.swift index a98a6df..5c5260c 100644 --- a/Tests/ScanComparisonTests.swift +++ b/Tests/ScanComparisonTests.swift @@ -125,4 +125,67 @@ final class ScanComparisonTests: XCTestCase { XCTAssertEqual(changes[0].kind, .added) XCTAssertEqual(changes[0].afterSize, 300) } + + // MARK: - Type changes at the same path (file <-> directory) + // + // A path present in BOTH trees is skipped by the added/removed loops, and + // the grew/shrank loop only compares file-to-file. A node that changed + // kind therefore used to fall through every branch and vanish from the + // report entirely, while its new children were reported as plain "added" + // rows (their parent path technically existed before, as a file). + + private func singleNodeTree(isDirectory: Bool, size: Int64, withChild child: (name: String, size: Int64)?) -> FileTree { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let a = FSNode(url: URL(fileURLWithPath: "/scan/A"), name: "A", isDirectory: isDirectory, size: size, fileExtension: isDirectory ? "" : "bin", parent: root) + if let child { + let inner = FSNode(url: URL(fileURLWithPath: "/scan/A/\(child.name)"), name: child.name, isDirectory: false, size: child.size, fileExtension: "txt", parent: a) + a.children = [inner] + } + root.children = [a] + root.size = a.size + return FileTreeBuilder.build(from: root, rootPath: "/scan") + } + + func test_file_becoming_directory_is_reported_once_and_children_suppressed() { + let before = singleNodeTree(isDirectory: false, size: 10, withChild: nil) + let after = singleNodeTree(isDirectory: true, size: 2_000, withChild: ("inner.txt", 2_000)) + + let changes = ScanComparison.compare(before: before, after: after) + + XCTAssertEqual(changes.count, 1, "expected exactly one row for the replaced node, got: \(changes.map { "\($0.relativePath):\($0.kind)" })") + XCTAssertEqual(changes[0].relativePath, "A") + XCTAssertEqual(changes[0].kind, .replaced) + XCTAssertTrue(changes[0].isDirectory) + XCTAssertEqual(changes[0].beforeSize, 10) + XCTAssertEqual(changes[0].afterSize, 2_000) + XCTAssertEqual(changes[0].delta, 1_990) + } + + func test_directory_becoming_file_is_reported_once_and_old_children_suppressed() { + let before = singleNodeTree(isDirectory: true, size: 100, withChild: ("inner.txt", 100)) + let after = singleNodeTree(isDirectory: false, size: 500, withChild: nil) + + let changes = ScanComparison.compare(before: before, after: after) + + XCTAssertEqual(changes.count, 1, "expected exactly one row for the replaced node, got: \(changes.map { "\($0.relativePath):\($0.kind)" })") + XCTAssertEqual(changes[0].relativePath, "A") + XCTAssertEqual(changes[0].kind, .replaced) + XCTAssertFalse(changes[0].isDirectory) + XCTAssertEqual(changes[0].beforeSize, 100) + XCTAssertEqual(changes[0].afterSize, 500) + } + + // A type change with an identical size still matters: neither size-based + // branch would fire, so this specifically guards the "reported regardless + // of size" rule. + func test_type_change_with_identical_size_is_still_reported() { + let before = singleNodeTree(isDirectory: false, size: 100, withChild: nil) + let after = singleNodeTree(isDirectory: true, size: 100, withChild: ("inner.txt", 100)) + + let changes = ScanComparison.compare(before: before, after: after) + + XCTAssertEqual(changes.count, 1) + XCTAssertEqual(changes[0].kind, .replaced) + XCTAssertEqual(changes[0].delta, 0) + } } From a2783e24687768be14dc711c8afbf9b7f2f642fa Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 22:23:27 +0300 Subject: [PATCH 25/41] fix(cleanup): promote surviving hardlink twin when trashing a size carrier --- Sources/Model/FileTree.swift | 68 +++++++++++++++++++ Tests/FileTreePruneTests.swift | 118 +++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) diff --git a/Sources/Model/FileTree.swift b/Sources/Model/FileTree.swift index 879f506..e0ba05a 100644 --- a/Sources/Model/FileTree.swift +++ b/Sources/Model/FileTree.swift @@ -153,6 +153,19 @@ public final class FileTree: @unchecked Sendable { // resulting old->new index map. This is O(n) total rather than O(n·k) // for k removed subtrees, while remaining just as simple to reason about // as folding would be (a fold is also correct here — just slower). + // + // Hardlink-aware (first-seen-wins survivor promotion): the scanner's + // convention is that a hardlinked inode's allocated size is carried by + // exactly one node (the "size carrier"; see `bulkAllocatedSize` / + // `VisitedSet` in FileScanner.swift), while every other node sharing its + // `hardLinkRef` sits at size 0. If a removed subtree contains a size + // carrier, and a node OUTSIDE the removed set shares its `hardLinkRef` + // and is still at size 0, that surviving twin is promoted to carry the + // size instead — otherwise the inode's bytes would simply vanish from + // the tree (the disk usage didn't change just because one of the + // inode's N links got removed). If no such survivor exists, every link + // to that inode is gone, and the existing subtract-only behavior below + // is already correct. public func removingSubtrees(at indices: [Int]) -> FileTree { let count = records.count @@ -196,6 +209,36 @@ public final class FileTree: @unchecked Sendable { newParentIndex[newIdx] = oldParent >= 0 ? oldToNew[oldParent] : -1 } + // Hardlink survivor promotion, computed against the ORIGINAL tree + // before any deltas are applied. One pass over survivors builds + // ref -> first zero-size surviving index (not O(n·k) per carrier), + // then one pass over removed nodes finds carriers and looks each up. + var survivorTwinByRef: [HardLinkRef: Int] = [:] + for old in 0..() + var promotions: [(survivorOld: Int, size: Int64)] = [] + for old in 0.. 0 else { continue } + // Defensive: first-seen-wins means at most one carrier per ref + // should ever exist, but never promote the same ref twice. + guard promotedRefs.insert(ref).inserted else { continue } + guard let survivorOld = survivorTwinByRef[ref] else { continue } // no survivor -> no promotion + promotions.append((survivorOld, records[old].size)) + } + for (survivorOld, size) in promotions { + newRecords[oldToNew[survivorOld]].size = size + var ancestorOld = parentIndex[survivorOld] + while ancestorOld >= 0 { + newRecords[oldToNew[ancestorOld]].size += size + ancestorOld = parentIndex[ancestorOld] + } + } + // Subtract each removed subtree's root size from every one of its // ancestors. Only process seeds whose immediate parent is NOT itself // removed — if the parent is removed too, this seed is a descendant @@ -241,6 +284,31 @@ public final class FileTree: @unchecked Sendable { newChildCount[newIdx] = kept } + // Re-sort every ancestor span a promotion could have disturbed: the + // promoted twin's own parent (one member jumped from 0 to `size`) + // and every ancestor above it up to root (same members, but one now + // has a different size) — same reasoning as `replacingSubtree`'s own + // ancestor re-sort below. Deduped across promotions since several + // independent hardlink pairs can share ancestors (e.g. root). + if !promotions.isEmpty { + var ancestorNewIndices = Set() + for (survivorOld, _) in promotions { + var a = parentIndex[survivorOld] + while a >= 0 { + ancestorNewIndices.insert(oldToNew[a]) + a = parentIndex[a] + } + } + for newIdx in ancestorNewIndices { + let start = newChildStart[newIdx] + let cnt = newChildCount[newIdx] + guard cnt > 1 else { continue } + var slice = Array(newChildIndices[start..<(start + cnt)]) + slice.sort { newRecords[$0].size > newRecords[$1].size } + newChildIndices.replaceSubrange(start..<(start + cnt), with: slice) + } + } + return FileTree( records: newRecords, parentIndex: newParentIndex, diff --git a/Tests/FileTreePruneTests.swift b/Tests/FileTreePruneTests.swift index 814b24f..e412c06 100644 --- a/Tests/FileTreePruneTests.swift +++ b/Tests/FileTreePruneTests.swift @@ -233,6 +233,124 @@ final class FileTreePruneTests: XCTestCase { XCTAssertEqual(newRoot.size, 900 - 300, "duplicate seeds in the list must not subtract twice") assertValidTopology(pruned) } + + // MARK: - removingSubtrees: hardlink survivor promotion (BUG 1) + + // A single hardlinked pair split across two sibling directories: + // root (/scan) size 500 + // ├── A (dir) size 500 + // │ └── big.bin (500, ref R, carrier) + // └── B (dir) size 0 + // └── twin.bin (0, ref R) + private func makeHardlinkPairFixture() -> (root: FSNode, dirA: FSNode, big: FSNode, dirB: FSNode, twin: FSNode) { + let ref = HardLinkRef(dev: 1, ino: 42) + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let dirA = FSNode(url: URL(fileURLWithPath: "/scan/A"), name: "A", isDirectory: true, size: 500, fileExtension: "", parent: root) + let big = FSNode(url: URL(fileURLWithPath: "/scan/A/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: dirA) + big.hardLinkRef = ref + dirA.children = [big] + let dirB = FSNode(url: URL(fileURLWithPath: "/scan/B"), name: "B", isDirectory: true, size: 0, fileExtension: "", parent: root) + let twin = FSNode(url: URL(fileURLWithPath: "/scan/B/twin.bin"), name: "twin.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirB) + twin.hardLinkRef = ref + dirB.children = [twin] + root.children = [dirA, dirB] + root.size = dirA.size + dirB.size + return (root, dirA, big, dirB, twin) + } + + func test_removingSubtree_trashing_size_carrier_promotes_surviving_twin_root_total_unchanged() { + let (root, _, big, _, _) = makeHardlinkPairFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let bigNode = node(named: "big.bin", in: tree) + let originalRootSize = tree.records[tree.rootIndex].size // 500 + + let pruned = tree.removingSubtree(at: bigNode.index) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.size, originalRootSize, "root total must be unchanged — the disk usage didn't change, only which path carries it") + let newDirA = newRoot.children.first { $0.name == "A" }! + XCTAssertEqual(newDirA.size, 0, "carrier's own ancestor (A) must still lose the size") + let newDirB = newRoot.children.first { $0.name == "B" }! + XCTAssertEqual(newDirB.size, 500, "twin's ancestor (B) must gain exactly the promoted size") + let newTwin = newDirB.children.first { $0.name == "twin.bin" }! + XCTAssertEqual(newTwin.size, 500, "surviving twin must be promoted to the carrier's full size") + assertValidTopology(pruned) + } + + func test_removingSubtree_trashing_zero_size_twin_does_not_promote_and_leaves_root_unchanged() { + let (root, _, big, _, twin) = makeHardlinkPairFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let twinNode = node(named: "twin.bin", in: tree) + let originalRootSize = tree.records[tree.rootIndex].size // 500 + _ = big; _ = twin + + let pruned = tree.removingSubtree(at: twinNode.index) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.size, originalRootSize, "subtracting a 0-size twin must leave the root total unchanged") + let newDirA = newRoot.children.first { $0.name == "A" }! + XCTAssertEqual(newDirA.size, 500, "the surviving carrier must be untouched — no promotion should have happened") + let newBig = newDirA.children.first { $0.name == "big.bin" }! + XCTAssertEqual(newBig.size, 500) + assertValidTopology(pruned) + } + + func test_removingSubtrees_both_hardlink_links_removed_together_drops_size_exactly_once() { + let (root, _, big, _, twin) = makeHardlinkPairFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let bigNode = node(named: "big.bin", in: tree) + let twinNode = node(named: "twin.bin", in: tree) + _ = big; _ = twin + + let pruned = tree.removingSubtrees(at: [bigNode.index, twinNode.index]) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.size, 0, "with no surviving link, the inode's size must drop out exactly once (500 - 500 - 0)") + assertValidTopology(pruned) + } + + func test_removingSubtrees_two_independent_hardlink_pairs_each_promote_their_own_twin() { + let refR1 = HardLinkRef(dev: 1, ino: 100) + let refR2 = HardLinkRef(dev: 1, ino: 200) + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + + let dirA1 = FSNode(url: URL(fileURLWithPath: "/scan/A1"), name: "A1", isDirectory: true, size: 500, fileExtension: "", parent: root) + let bigA = FSNode(url: URL(fileURLWithPath: "/scan/A1/bigA.bin"), name: "bigA.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: dirA1) + bigA.hardLinkRef = refR1 + dirA1.children = [bigA] + + let dirA2 = FSNode(url: URL(fileURLWithPath: "/scan/A2"), name: "A2", isDirectory: true, size: 0, fileExtension: "", parent: root) + let twinA = FSNode(url: URL(fileURLWithPath: "/scan/A2/twinA.bin"), name: "twinA.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirA2) + twinA.hardLinkRef = refR1 + dirA2.children = [twinA] + + let dirB1 = FSNode(url: URL(fileURLWithPath: "/scan/B1"), name: "B1", isDirectory: true, size: 300, fileExtension: "", parent: root) + let bigB = FSNode(url: URL(fileURLWithPath: "/scan/B1/bigB.bin"), name: "bigB.bin", isDirectory: false, size: 300, fileExtension: "bin", parent: dirB1) + bigB.hardLinkRef = refR2 + dirB1.children = [bigB] + + let dirB2 = FSNode(url: URL(fileURLWithPath: "/scan/B2"), name: "B2", isDirectory: true, size: 0, fileExtension: "", parent: root) + let twinB = FSNode(url: URL(fileURLWithPath: "/scan/B2/twinB.bin"), name: "twinB.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirB2) + twinB.hardLinkRef = refR2 + dirB2.children = [twinB] + + root.children = [dirA1, dirA2, dirB1, dirB2] + root.size = 500 + 0 + 300 + 0 // 800 + + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let bigANode = node(named: "bigA.bin", in: tree) + let bigBNode = node(named: "bigB.bin", in: tree) + + let pruned = tree.removingSubtrees(at: [bigANode.index, bigBNode.index]) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.size, 800, "root total must be unchanged: each pair's bytes just moved to its own surviving twin") + let newTwinA = newRoot.children.first { $0.name == "A2" }!.children.first { $0.name == "twinA.bin" }! + XCTAssertEqual(newTwinA.size, 500, "twinA must be promoted with its own pair's size") + let newTwinB = newRoot.children.first { $0.name == "B2" }!.children.first { $0.name == "twinB.bin" }! + XCTAssertEqual(newTwinB.size, 300, "twinB must be promoted with its own pair's size, independently of twinA") + assertValidTopology(pruned) + } } // MARK: - ScanViewModel-level integration From 52a663be7c0fbfbe187633092a3250d720dfd710 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 22:23:32 +0300 Subject: [PATCH 26/41] fix(refresh): keep hardlink dedup across the splice boundary --- Sources/ViewModels/ScanViewModel.swift | 50 ++++++++++++- Tests/IncrementalRefreshTests.swift | 99 ++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/Sources/ViewModels/ScanViewModel.swift b/Sources/ViewModels/ScanViewModel.swift index 65c6cab..27a3f50 100644 --- a/Sources/ViewModels/ScanViewModel.swift +++ b/Sources/ViewModels/ScanViewModel.swift @@ -402,7 +402,26 @@ public final class ScanViewModel: ObservableObject { let node = FileNode(tree: tree, index: index) let showHiddenFiles = UserDefaults.standard.bool(forKey: "showHiddenFiles") let excludedNames = parseExcludedNames() - var seenRefs = Set() + + // Cross-tree hardlink dedup (BUG 2 fix): `scanSubtree`'s own + // `treeRoot` parameter — meant to catch a hardlink whose twin lives + // outside the directory being rescanned via `firstNode(withRef:in:)` + // — only makes sense against a live FSNode tree, which no longer + // exists on this flat-store splice path, so it stays `nil` below and + // that check can never fire. Left alone, that means only hardlinks + // *within* this one rescanned directory would get deduped (via the + // fresh, otherwise-empty `seenRefs` below), and any twin living + // outside the spliced subtree would double-count. Pre-seeding + // `seenRefs` before the call replaces that dead check: it contains + // every `hardLinkRef` that appears anywhere in `tree` OUTSIDE the + // subtree being replaced, but ONLY when that outside occurrence is + // itself the size carrier (size > 0). That asymmetry matters — if + // the carrier instead lives INSIDE the subtree being replaced (i.e. + // the outside twin is the 0-size loser), seeding on the twin's mere + // presence would make the rescan zero its own copy too, and the + // inode's bytes would vanish from the tree entirely (both copies at + // 0) instead of correctly moving to whichever copy is now first-seen. + var seenRefs = Self.hardLinkRefsOutsideSubtree(rootedAt: index, in: tree) let freshNode = scanSubtree( url: node.url, parent: nil, @@ -418,6 +437,35 @@ public final class ScanViewModel: ObservableObject { return tree.replacingSubtree(at: index, with: subtree) } + // BUG 2 fix support for `splicedTree`: marks every node inside the + // subtree rooted at `subtreeRootIndex` via the same iterative DFS + // `FileTree.removingSubtrees`/`replacingSubtree` use, then collects every + // `hardLinkRef` whose occurrence OUTSIDE that subtree is the size + // carrier (`size > 0`). See the comment at the `splicedTree` call site + // for why the `size > 0` condition (rather than mere presence) matters. + private nonisolated static func hardLinkRefsOutsideSubtree(rootedAt subtreeRootIndex: Int, in tree: FileTree) -> Set { + let count = tree.records.count + var inside = [Bool](repeating: false, count: count) + var stack = [subtreeRootIndex] + while let i = stack.popLast() { + if inside[i] { continue } + inside[i] = true + let start = tree.childStart[i] + let cnt = tree.childCount[i] + for offset in 0..() + for i in 0.. 0 { + refs.insert(ref) + } + } + return refs + } + // See the last bullet of `splicedTree`'s doc comment above. private nonisolated static func containsUnsummarizedGeneratedDirectory(_ node: FSNode) -> Bool { for child in node.children where child.isDirectory { diff --git a/Tests/IncrementalRefreshTests.swift b/Tests/IncrementalRefreshTests.swift index 1552d71..44f685f 100644 --- a/Tests/IncrementalRefreshTests.swift +++ b/Tests/IncrementalRefreshTests.swift @@ -376,4 +376,103 @@ final class IncrementalRefreshTests: XCTestCase { } XCTAssertEqual(Set(newSub.children.map(\.name)), Set(["a.txt", "new.bin"]), "the splice must reflect the on-disk addition") } + + // MARK: - BUG 2: splice must keep hardlink dedup across the splice boundary + + // Builds a real temp tree with one hardlinked pair split across two + // sibling directories (`dirX/big.bin` <-> `dirY/link.bin`), the exact + // shape of the plan's BUG 2 scenario. Returns the inode's `HardLinkRef` + // so the tests below can find out, after the initial scan, which side + // the scanner's first-seen-wins picked as the carrier (deterministic per + // run, but not something the test should hard-code). + private func makeHardlinkAcrossDirsFixture() throws -> (tmp: URL, ref: HardLinkRef) { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + let dirX = tmp.appendingPathComponent("dirX") + let dirY = tmp.appendingPathComponent("dirY") + try FileManager.default.createDirectory(at: dirX, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: dirY, withIntermediateDirectories: true) + + let bigPath = dirX.appendingPathComponent("big.bin") + try Data(repeating: 9, count: 262_144).write(to: bigPath) + try FileManager.default.linkItem(at: bigPath, to: dirY.appendingPathComponent("link.bin")) + // An unrelated file in each directory so a splice of either side has + // something else in it besides the hardlink half. + try Data(repeating: 1, count: 128).write(to: dirX.appendingPathComponent("other.txt")) + try Data(repeating: 2, count: 128).write(to: dirY.appendingPathComponent("other.txt")) + + var st = stat() + XCTAssertEqual(lstat(bigPath.path, &st), 0) + let ref = HardLinkRef(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) + return (tmp, ref) + } + + // Finds the node sharing `ref` whose size does (or doesn't, per + // `wantCarrier`) match the first-seen-wins carrier convention, then + // climbs to its direct child-of-root directory name — "the directory + // holding the carrier/twin" from the plan's scenario. + private func topLevelDirName(carryingRef wantCarrier: Bool, ref: HardLinkRef, tree: FileTree) -> String? { + for i in 0.. 0) == wantCarrier else { continue } + var current = i + while tree.parentIndex[current] != tree.rootIndex { + let p = tree.parentIndex[current] + guard p >= 0 else { return nil } + current = p + } + return tree.records[current].name + } + return nil + } + + // Splicing the directory holding the NON-carrier (0-size) twin must not + // re-materialize the full size a second time: the carrier lives outside + // the spliced subtree, so it must be pre-seeded into `seenRefs` and the + // rescanned copy must come back at 0, exactly as it was. + func test_splice_of_directory_holding_hardlink_loser_matches_full_rescan_total() async throws { + let (tmp, ref) = try makeHardlinkAcrossDirsFixture() + defer { try? FileManager.default.removeItem(at: tmp) } + + let before = await scanAndWait(tmp) + guard let beforeTree = await before.tree else { return XCTFail("initial scan should populate a tree") } + guard let loserDirName = topLevelDirName(carryingRef: false, ref: ref, tree: beforeTree) else { + return XCTFail("expected to find the hardlink's 0-size twin in the initial scan") + } + + guard let spliced = ScanViewModel.splicedTree(afterChangeAt: tmp.appendingPathComponent(loserDirName).path, in: beforeTree) else { + return XCTFail("splice should succeed for a plain, non-summarized subdirectory") + } + + let after = await scanAndWait(tmp) + guard let afterTree = await after.tree else { return XCTFail("second full rescan should populate a tree") } + + let splicedRoot = FileNode(tree: spliced, index: spliced.rootIndex) + let afterRoot = FileNode(tree: afterTree, index: afterTree.rootIndex) + XCTAssertEqual(splicedRoot.size, afterRoot.size, "splicing the directory holding the hardlink LOSER must not double-count the carrier living outside it") + } + + // The mirrored case: splicing the directory holding the CARRIER must let + // the rescan re-take the full size for that (still 0-seeded-locally) + // ref, while the untouched outside 0-size twin correctly stays at 0. + func test_splice_of_directory_holding_hardlink_carrier_matches_full_rescan_total() async throws { + let (tmp, ref) = try makeHardlinkAcrossDirsFixture() + defer { try? FileManager.default.removeItem(at: tmp) } + + let before = await scanAndWait(tmp) + guard let beforeTree = await before.tree else { return XCTFail("initial scan should populate a tree") } + guard let carrierDirName = topLevelDirName(carryingRef: true, ref: ref, tree: beforeTree) else { + return XCTFail("expected to find the hardlink carrier in the initial scan") + } + + guard let spliced = ScanViewModel.splicedTree(afterChangeAt: tmp.appendingPathComponent(carrierDirName).path, in: beforeTree) else { + return XCTFail("splice should succeed for a plain, non-summarized subdirectory") + } + + let after = await scanAndWait(tmp) + guard let afterTree = await after.tree else { return XCTFail("second full rescan should populate a tree") } + + let splicedRoot = FileNode(tree: spliced, index: spliced.rootIndex) + let afterRoot = FileNode(tree: afterTree, index: afterTree.rootIndex) + XCTAssertEqual(splicedRoot.size, afterRoot.size, "splicing the directory holding the hardlink CARRIER must still match a full fresh rescan's total") + } } From 15604851add747e99451112a5f77faee19a1eb0d Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 22:35:18 +0300 Subject: [PATCH 27/41] test(scanner): cover summary-walk cancellation with a test that has teeth The end-to-end scan-then-cancel test added alongside the async fix looked like it covered cancellation during summarization, but mutation testing showed it passes even with EVERY cancellation check inside the summary path deleted: the cancel lands during the initial traversal, before any summary walk has begun. Replace it with a direct call to summarizeSubtree driven by a cancel closure that throws partway through, which fails under that same mutation, and keep the end-to-end case under a name that reflects what it actually verifies (a cancelled scan with summarization enabled never emits .completed). --- Tests/AutoSummaryTests.swift | 122 +++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/Tests/AutoSummaryTests.swift b/Tests/AutoSummaryTests.swift index 0ccdcb9..53dae17 100644 --- a/Tests/AutoSummaryTests.swift +++ b/Tests/AutoSummaryTests.swift @@ -128,6 +128,128 @@ final class AutoSummaryTests: XCTestCase { (node.isAutoSummarized ? 1 : 0) + node.children.reduce(0) { $0 + countSummarized($1) } } + // The other half of the forward-progress bug: with the old blocking + // bridge, the poll loop only forwarded cancellation to the detached pool + // and kept spinning, so a wedged summary could not be recovered by + // "Stop Scan" at all. + // + // Drives `summarizeSubtree` DIRECTLY with a cancel closure that throws + // partway through, rather than cancelling a whole scan. An end-to-end + // scan-then-cancel test looks like it covers this but does not: the + // cancellation almost always lands during the initial traversal, before + // any summary walk has begun, so the test passes even with every + // cancellation check inside the summary path deleted (verified by + // mutation). Calling the walk directly is deterministic and actually + // fails if the walk stops honouring `cancel`. + func test_summary_walk_propagates_cancellation() async throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + + // Enough nested directories that the walk makes many cancel() calls. + for d in 0..<40 { + let deep = root.appendingPathComponent("m\(d)", isDirectory: true) + try FileManager.default.createDirectory(at: deep, withIntermediateDirectories: true) + for f in 0..<10 { + try Data(repeating: 1, count: 32).write(to: deep.appendingPathComponent("f\(f).dat")) + } + } + + var st = stat() + XCTAssertEqual(lstat(root.path, &st), 0) + let rootDev = UInt64(bitPattern: Int64(st.st_dev)) + + let fd = open(root.path, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW) + XCTAssertGreaterThanOrEqual(fd, 0) + defer { close(fd) } + let rootEntries = try listDirectoryEntries(path: root.path, fd: fd, forceFallback: false) + + struct StopWalking: Error {} + // Let a few directories through, then refuse to continue. + let callCount = Counter() + let config = ScanConfig( + excludedNames: [], + showHiddenFiles: false, + forceFallbackEnum: false, + autoSummarizeEnabled: true + ) + + do { + _ = try await summarizeSubtree( + rootEntries: rootEntries, + rootPath: root.path, + rootDev: rootDev, + config: config, + visited: VisitedSet() + ) { + if callCount.increment() > 5 { throw StopWalking() } + } + XCTFail("summarizeSubtree must propagate the cancel closure's error, not swallow it") + } catch is StopWalking { + // Expected: the walk asked, was told to stop, and unwound. + } + } + + // Thread-safe call counter: the summary pool invokes `cancel` from + // several workers at once. + private final class Counter: @unchecked Sendable { + private var lock = os_unfair_lock() + private var value = 0 + func increment() -> Int { + os_unfair_lock_lock(&lock) + value += 1 + let result = value + os_unfair_lock_unlock(&lock) + return result + } + } + + // Complements the direct test above: a full scan with summarization + // enabled must still cancel cleanly and never emit `.completed`. + func test_cancelled_scan_with_summarization_never_completes() async throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + + // Enough summarizable siblings, each with real depth and file count, + // that a full uncancelled scan takes appreciable time. + for s in 0..<12 { + let holder = root.appendingPathComponent("pkg\(s)", isDirectory: true) + let target = holder.appendingPathComponent("node_modules", isDirectory: true) + for d in 0..<12 { + let deep = target.appendingPathComponent("m\(d)", isDirectory: true) + try FileManager.default.createDirectory(at: deep, withIntermediateDirectories: true) + for f in 0..<25 { + try Data(repeating: 9, count: 64).write(to: deep.appendingPathComponent("f\(f).dat")) + } + } + } + + let elapsed: Double = await withExcludedFolderNames(".git,DerivedData,.Trash") { + await withAutoSummarize(true) { + let scanner = FileScanner() + var sawCompleted = false + let stream = await scanner.scan(url: root) + let start = DispatchTime.now() + let consumeTask = Task { + for await progress in stream { + if case .completed = progress { sawCompleted = true } + } + } + await scanner.cancel() + _ = await consumeTask.value + let seconds = Double(DispatchTime.now().uptimeNanoseconds - start.uptimeNanoseconds) / 1_000_000_000 + + XCTAssertFalse(sawCompleted, "a scan cancelled during summarization must not emit .completed") + return seconds + } + } + + // Generous bound: the point is "it unwinds", not a precise deadline. + // Against a wedged pool this would never return at all. + XCTAssertLessThan(elapsed, 10.0, "cancellation during a summary walk should tear down promptly") + } + private func findNode(_ root: FileNode, path: [String]) -> FileNode? { var current = root for name in path { From 550ce90535fc738b821cd2c414fea6d0ccee39e5 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 22:51:05 +0300 Subject: [PATCH 28/41] fix(refresh): promote surviving hardlink twin when a carrier vanishes externally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FSEvents-driven splices only rescan the changed subtree, so when an external process (Finder, rm, a build tool — not this app's own Trash action) deletes the link that carried a hardlinked inode's allocated size, the splice's plain size delta just subtracts the vanished bytes and any zero-size twin elsewhere in the tree stays at 0, permanently under-reporting real disk usage until a full rescan. Adds FileTree.promotingSurvivingTwin(ref:size:), the live-refresh twin of removingSubtrees' existing Move-to-Trash survivor promotion, and wires ScanViewModel.splicedTree to diff the old and new subtrees' carried hardlink refs and promote a surviving twin for any ref that's now orphaned. --- Sources/Model/FileTree.swift | 61 +++++++++++ Sources/ViewModels/ScanViewModel.swift | 64 +++++++++++- Tests/IncrementalRefreshTests.swift | 137 +++++++++++++++++++++++++ 3 files changed, 261 insertions(+), 1 deletion(-) diff --git a/Sources/Model/FileTree.swift b/Sources/Model/FileTree.swift index e0ba05a..9746859 100644 --- a/Sources/Model/FileTree.swift +++ b/Sources/Model/FileTree.swift @@ -466,4 +466,65 @@ public final class FileTree: @unchecked Sendable { rootPath: rootPath ) } + + // MARK: - Splice-time hardlink survivor promotion (FIX 1, refresh-side twin of removingSubtrees' promotion) + + // The live-refresh counterpart to `removingSubtrees`' hardlink survivor + // promotion above, but simpler: it doesn't need to find the orphaned + // ref/size pairs itself (the caller — `ScanViewModel.splicedTree` — + // already worked out which `hardLinkRef`s the OLD (stale) subtree + // carried but the freshly-rescanned NEW subtree doesn't, by diffing the + // two subtrees' own carried-ref sets), it just needs to do the + // promotion: find a surviving zero-size twin for `ref` ANYWHERE in this + // tree (the search doesn't need to exclude the freshly-spliced subtree — + // if that subtree happens to itself contain the zero-size twin, that's + // still a perfectly valid — arguably the most natural — promotion + // target), set it to `size`, and bubble that size up its ancestor chain, + // re-sorting every span the jump from 0 to `size` could have disturbed — + // same reasoning as `removingSubtrees`' own promotion resort above. + // + // Returns `self` unchanged if no zero-size twin for `ref` exists + // anywhere — defensive; the caller only ever calls this for a ref it + // already knows was carried by the subtree it just replaced, but a + // missing survivor (every link to the inode is now gone) is not an + // error, just nothing to promote. + public func promotingSurvivingTwin(ref: HardLinkRef, size: Int64) -> FileTree { + guard let twinIndex = records.firstIndex(where: { $0.hardLinkRef == ref && $0.size == 0 }) else { + return self + } + + var newRecords = records + newRecords[twinIndex].size = size + + // The twin's own parent span (one member jumped from 0 to `size`) + // and every ancestor above it up to root (same members, but one now + // has a different size). + var resortTargets: [Int] = [] + var ancestor = parentIndex[twinIndex] + while ancestor >= 0 { + newRecords[ancestor].size += size + resortTargets.append(ancestor) + ancestor = parentIndex[ancestor] + } + + var newChildIndices = childIndices + for newIdx in resortTargets { + let start = childStart[newIdx] + let cnt = childCount[newIdx] + guard cnt > 1 else { continue } + var slice = Array(newChildIndices[start..<(start + cnt)]) + slice.sort { newRecords[$0].size > newRecords[$1].size } + newChildIndices.replaceSubrange(start..<(start + cnt), with: slice) + } + + return FileTree( + records: newRecords, + parentIndex: parentIndex, + childStart: childStart, + childCount: childCount, + childIndices: newChildIndices, + rootIndex: rootIndex, + rootPath: rootPath + ) + } } diff --git a/Sources/ViewModels/ScanViewModel.swift b/Sources/ViewModels/ScanViewModel.swift index 27a3f50..e43bf7c 100644 --- a/Sources/ViewModels/ScanViewModel.swift +++ b/Sources/ViewModels/ScanViewModel.swift @@ -422,6 +422,21 @@ public final class ScanViewModel: ObservableObject { // inode's bytes would vanish from the tree entirely (both copies at // 0) instead of correctly moving to whichever copy is now first-seen. var seenRefs = Self.hardLinkRefsOutsideSubtree(rootedAt: index, in: tree) + + // FIX 1 setup: capture which `hardLinkRef`s the OLD (stale) subtree + // itself carried (size > 0), against the ORIGINAL tree, before it's + // replaced below. This is the only place that information survives — + // once `replacingSubtree` swaps the stale nodes out, there's no way + // to tell "this ref isn't carried by the new subtree" apart from + // "this ref was never carried by anything in this subtree at all". + // See the comment at the promotion call site below for why this + // matters: an external deletion (Finder/`rm`/a build tool — NOT this + // app's own Trash action, which `FileTree.removingSubtrees` already + // covers) of the carrier link inside this subtree leaves the inode + // still fully allocated via a twin elsewhere, and nothing else in + // this function would ever notice. + let oldCarriedRefs = Self.carriedHardLinkRefs(insideSubtreeRootedAt: index, in: tree) + let freshNode = scanSubtree( url: node.url, parent: nil, @@ -434,7 +449,54 @@ public final class ScanViewModel: ObservableObject { guard freshNode.name != "node_modules", !containsUnsummarizedGeneratedDirectory(freshNode) else { return nil } let subtree = FileTreeBuilder.build(from: freshNode, rootPath: node.url.path) - return tree.replacingSubtree(at: index, with: subtree) + var result = tree.replacingSubtree(at: index, with: subtree) + + // FIX 1: promote a surviving twin for every ref the OLD subtree + // carried but the freshly-rescanned NEW subtree does not — the + // on-disk link that carried the inode's bytes vanished for a reason + // this splice can't otherwise account for (an external delete, not + // this app's own Trash action). Without this, the bytes would simply + // drop out of the tree until the next full rescan, even though the + // inode is still fully allocated via a twin outside this subtree (or + // even inside it — see `promotingSurvivingTwin`'s doc comment for why + // a plain whole-tree search for the twin is correct either way). + // Multiple independent orphaned refs (two unrelated hardlink pairs + // both losing their carrier in the same splice) each promote their + // own twin, one call per ref. + if !oldCarriedRefs.isEmpty { + var newCarriedRefs = Set() + for record in subtree.records where record.hardLinkRef != nil && record.size > 0 { + newCarriedRefs.insert(record.hardLinkRef!) + } + for (ref, size) in oldCarriedRefs where !newCarriedRefs.contains(ref) { + result = result.promotingSurvivingTwin(ref: ref, size: size) + } + } + + return result + } + + // FIX 1 support for `splicedTree`: collects `hardLinkRef -> size` for + // every node INSIDE the subtree rooted at `subtreeRootIndex` that is + // itself the size carrier (`size > 0`) for its ref — the set of refs an + // external deletion inside this subtree could orphan. Mirrors + // `hardLinkRefsOutsideSubtree`'s traversal but walks IN rather than + // computing the complement, and needs the size (not just the ref) so the + // caller can promote a survivor to the exact right amount. + private nonisolated static func carriedHardLinkRefs(insideSubtreeRootedAt subtreeRootIndex: Int, in tree: FileTree) -> [HardLinkRef: Int64] { + var result: [HardLinkRef: Int64] = [:] + var stack = [subtreeRootIndex] + while let i = stack.popLast() { + if let ref = tree.records[i].hardLinkRef, tree.records[i].size > 0 { + result[ref] = tree.records[i].size + } + let start = tree.childStart[i] + let cnt = tree.childCount[i] + for offset in 0.. dirY/firstLink.bin + let firstBigPath = dirX.appendingPathComponent("firstBig.bin") + try Data(repeating: 9, count: 262_144).write(to: firstBigPath) + try FileManager.default.linkItem(at: firstBigPath, to: dirY.appendingPathComponent("firstLink.bin")) + var firstSt = stat() + XCTAssertEqual(lstat(firstBigPath.path, &firstSt), 0) + let firstRef = HardLinkRef(dev: UInt64(bitPattern: Int64(firstSt.st_dev)), ino: UInt64(firstSt.st_ino)) + + // Pair 2: dirX/secondBig.bin <-> dirY/secondLink.bin — a completely + // independent inode, unrelated to pair 1. + let secondBigPath = dirX.appendingPathComponent("secondBig.bin") + try Data(repeating: 3, count: 131_072).write(to: secondBigPath) + try FileManager.default.linkItem(at: secondBigPath, to: dirY.appendingPathComponent("secondLink.bin")) + var secondSt = stat() + XCTAssertEqual(lstat(secondBigPath.path, &secondSt), 0) + let secondRef = HardLinkRef(dev: UInt64(bitPattern: Int64(secondSt.st_dev)), ino: UInt64(secondSt.st_ino)) + + let before = await scanAndWait(tmp) + guard let beforeTree = await before.tree else { return XCTFail("initial scan should populate a tree") } + guard let firstCarrierDirName = topLevelDirName(carryingRef: true, ref: firstRef, tree: beforeTree), + let secondCarrierDirName = topLevelDirName(carryingRef: true, ref: secondRef, tree: beforeTree), + firstCarrierDirName == secondCarrierDirName + else { return XCTFail("expected both pairs to share the same first-seen-wins carrier directory") } + let carrierDirName = firstCarrierDirName + let twinDirName = carrierDirName == "dirX" ? "dirY" : "dirX" + let carrierDir = tmp.appendingPathComponent(carrierDirName) + + // Delete BOTH carrier links out of the shared carrier directory. + // Both inodes are still fully allocated via their respective twins + // in the other directory. + for url in try FileManager.default.contentsOfDirectory(at: carrierDir, includingPropertiesForKeys: nil) { + var st = stat() + guard lstat(url.path, &st) == 0 else { continue } + let ref = HardLinkRef(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) + if ref == firstRef || ref == secondRef { + try FileManager.default.removeItem(at: url) + } + } + + guard let spliced = ScanViewModel.splicedTree(afterChangeAt: carrierDir.path, in: beforeTree) else { + return XCTFail("splice should succeed for a plain, non-summarized subdirectory") + } + + let after = await scanAndWait(tmp) + guard let afterTree = await after.tree else { return XCTFail("rescan should populate a tree") } + + let splicedRoot = FileNode(tree: spliced, index: spliced.rootIndex) + let afterRoot = FileNode(tree: afterTree, index: afterTree.rootIndex) + XCTAssertEqual( + splicedRoot.size, + afterRoot.size, + "deleting two independent hardlinks' size carriers from the same directory must promote both surviving twins" + ) + + // Confirm both promotions actually happened (not just a total that + // happens to match by coincidence): the untouched sibling directory's + // two remaining links must now each carry their own pair's full size. + guard let splicedTwinDir = splicedRoot.children.first(where: { $0.name == twinDirName }) else { + return XCTFail("the twin directory must survive the splice untouched (it wasn't the spliced one)") + } + let firstTwin = splicedTwinDir.children.first { $0.hardLinkRef == firstRef } + let secondTwin = splicedTwinDir.children.first { $0.hardLinkRef == secondRef } + XCTAssertEqual(firstTwin?.size, 262_144, "the first pair's surviving twin must be promoted to its own pair's full size") + XCTAssertEqual(secondTwin?.size, 131_072, "the second pair's surviving twin must be promoted to its own pair's full size, independently of the first pair") + } } From b97fbf88be1a927800bfc03052f3d7487751a9c6 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Fri, 24 Jul 2026 22:52:34 +0300 Subject: [PATCH 29/41] fix(model): re-sort the removed node's ancestor chain after subtree removal removingSubtrees re-sorted spans along the promoted twin's ancestor chain, but never along the plain-removed node's own ancestor chain. An ancestor whose size merely shrank (potentially straight to 0, if it was itself the hardlink carrier) could end up out of order within its own parent's span, violating the size-desc invariant FileNode.children promises even though the underlying gap predates hardlink handling entirely. Unifies the promotion and subtraction ancestor-resort sets into one deduped pass, and extracts the shared span-resort logic (also reused by replacingSubtree and promotingSurvivingTwin) into a single resortSpans helper. --- Sources/Model/FileTree.swift | 83 ++++++++++++++++++---------------- Tests/FileTreePruneTests.swift | 72 +++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 40 deletions(-) diff --git a/Sources/Model/FileTree.swift b/Sources/Model/FileTree.swift index 9746859..4c23e87 100644 --- a/Sources/Model/FileTree.swift +++ b/Sources/Model/FileTree.swift @@ -230,11 +230,21 @@ public final class FileTree: @unchecked Sendable { guard let survivorOld = survivorTwinByRef[ref] else { continue } // no survivor -> no promotion promotions.append((survivorOld, records[old].size)) } + // Every span a size change below could disturb, collected as we go + // and deduped (a promotion and a plain subtraction can easily share + // ancestors, e.g. root) so each span is only ever re-sorted once at + // the end, after `newChildIndices` exists to sort. + var resortTargets = Set() + for (survivorOld, size) in promotions { newRecords[oldToNew[survivorOld]].size = size + // The promoted twin's own parent span (one member jumped from 0 + // to `size`) and every ancestor above it up to root — same + // members up there, but one of them now has a different size. var ancestorOld = parentIndex[survivorOld] while ancestorOld >= 0 { newRecords[oldToNew[ancestorOld]].size += size + resortTargets.insert(oldToNew[ancestorOld]) ancestorOld = parentIndex[ancestorOld] } } @@ -246,6 +256,17 @@ public final class FileTree: @unchecked Sendable { // already folded into the ancestor chain when that higher seed was // processed, so subtracting again here would double-count. Also // de-dupe in case the same index appears more than once in `indices`. + // + // FIX 2: every ancestor visited here also goes into `resortTargets`. + // The removed seed's immediate parent just shrank (possibly straight + // to 0), which can leave it out of order within ITS OWN parent's + // span — a pre-existing gap in this plain-subtraction path (nothing + // to do with hardlinks) that survivor promotion just makes easy to + // trigger, since a directory's size can drop straight to 0 while an + // untouched sibling stays put. Including the shrunk node's own span + // too is harmless (it's already correctly ordered by the child-span + // rebuild below), but keeping it in the same set as the ancestors + // above it is what actually fixes the sibling ordering. var processedRoots = Set() for seed in indices { guard seed != rootIndex, seed >= 0, seed < count, removed[seed] else { continue } @@ -257,6 +278,7 @@ public final class FileTree: @unchecked Sendable { var ancestorOld = parent while ancestorOld >= 0 { newRecords[oldToNew[ancestorOld]].size -= removedSize + resortTargets.insert(oldToNew[ancestorOld]) ancestorOld = parentIndex[ancestorOld] } } @@ -284,30 +306,7 @@ public final class FileTree: @unchecked Sendable { newChildCount[newIdx] = kept } - // Re-sort every ancestor span a promotion could have disturbed: the - // promoted twin's own parent (one member jumped from 0 to `size`) - // and every ancestor above it up to root (same members, but one now - // has a different size) — same reasoning as `replacingSubtree`'s own - // ancestor re-sort below. Deduped across promotions since several - // independent hardlink pairs can share ancestors (e.g. root). - if !promotions.isEmpty { - var ancestorNewIndices = Set() - for (survivorOld, _) in promotions { - var a = parentIndex[survivorOld] - while a >= 0 { - ancestorNewIndices.insert(oldToNew[a]) - a = parentIndex[a] - } - } - for newIdx in ancestorNewIndices { - let start = newChildStart[newIdx] - let cnt = newChildCount[newIdx] - guard cnt > 1 else { continue } - var slice = Array(newChildIndices[start..<(start + cnt)]) - slice.sort { newRecords[$0].size > newRecords[$1].size } - newChildIndices.replaceSubrange(start..<(start + cnt), with: slice) - } - } + Self.resortSpans(resortTargets, records: newRecords, childStart: newChildStart, childCount: newChildCount, into: &newChildIndices) return FileTree( records: newRecords, @@ -320,6 +319,24 @@ public final class FileTree: @unchecked Sendable { ) } + // Shared ancestor-resort helper used by `removingSubtrees`, + // `replacingSubtree`, and `promotingSurvivingTwin`: re-sorts the child + // span at each index in `targets` by size descending, in place. All + // three only ever need to re-sort a handful of ancestor spans a size + // change could have disturbed, never the whole tree, so this stays + // proportional to tree depth × number of affected chains rather than + // total node count. + private static func resortSpans(_ targets: S, records: [FileNodeRecord], childStart: [Int], childCount: [Int], into childIndices: inout [Int]) where S.Element == Int { + for newIdx in targets { + let start = childStart[newIdx] + let cnt = childCount[newIdx] + guard cnt > 1 else { continue } + var slice = Array(childIndices[start..<(start + cnt)]) + slice.sort { records[$0].size > records[$1].size } + childIndices.replaceSubrange(start..<(start + cnt), with: slice) + } + } + // MARK: - Splice (incremental live-refresh, replacing the full-rescan interim) // Returns a NEW tree where the subtree rooted at `index` is replaced @@ -447,14 +464,7 @@ public final class FileTree: @unchecked Sendable { ancestorNewIndices.append(oldToNew[a]) a = parentIndex[a] } - for newIdx in ancestorNewIndices { - let start = newChildStart[newIdx] - let cnt = newChildCount[newIdx] - guard cnt > 1 else { continue } - var slice = Array(newChildIndices[start..<(start + cnt)]) - slice.sort { newRecords[$0].size > newRecords[$1].size } - newChildIndices.replaceSubrange(start..<(start + cnt), with: slice) - } + Self.resortSpans(ancestorNewIndices, records: newRecords, childStart: newChildStart, childCount: newChildCount, into: &newChildIndices) return FileTree( records: newRecords, @@ -508,14 +518,7 @@ public final class FileTree: @unchecked Sendable { } var newChildIndices = childIndices - for newIdx in resortTargets { - let start = childStart[newIdx] - let cnt = childCount[newIdx] - guard cnt > 1 else { continue } - var slice = Array(newChildIndices[start..<(start + cnt)]) - slice.sort { newRecords[$0].size > newRecords[$1].size } - newChildIndices.replaceSubrange(start..<(start + cnt), with: slice) - } + Self.resortSpans(resortTargets, records: newRecords, childStart: childStart, childCount: childCount, into: &newChildIndices) return FileTree( records: newRecords, diff --git a/Tests/FileTreePruneTests.swift b/Tests/FileTreePruneTests.swift index e412c06..7d56a78 100644 --- a/Tests/FileTreePruneTests.swift +++ b/Tests/FileTreePruneTests.swift @@ -351,6 +351,78 @@ final class FileTreePruneTests: XCTestCase { XCTAssertEqual(newTwinB.size, 300, "twinB must be promoted with its own pair's size, independently of twinA") assertValidTopology(pruned) } + + // MARK: - removingSubtrees: removed node's own ancestor chain must stay sorted (BUG 2 / FIX 2) + + // The plan's exact repro: + // root + // ├─ P size 900 + // │ ├─ A (dir, size 500) + // │ │ └─ big.bin (500, ref R, carrier) + // │ └─ C (dir, size 400, untouched) + // │ └─ other.bin (400, unrelated) + // └─ Q (dir, size 0) + // └─ twin.bin (0, ref R) + // `P.children` starts `[A(500), C(400)]`. Removing `big.bin` promotes + // twin.bin to 500 (A's own promotion-resort logic already covers Q's and + // root's spans), but A itself drops straight to 0 — nothing in the old + // code re-sorted P's span to reflect that, leaving `P.children == + // [A(0), C(400)]`, which violates the size-desc invariant even though + // this has nothing to do with hardlinks (a plain size *shrink* would + // trigger the exact same gap; the hardlink promotion path just makes a + // 500->0 drop easy to construct in one step). + private func makeAncestorResortFixture() -> (root: FSNode, dirP: FSNode, dirA: FSNode, big: FSNode, dirC: FSNode, other: FSNode, dirQ: FSNode, twin: FSNode) { + let ref = HardLinkRef(dev: 1, ino: 77) + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + + let dirP = FSNode(url: URL(fileURLWithPath: "/scan/P"), name: "P", isDirectory: true, size: 900, fileExtension: "", parent: root) + let dirA = FSNode(url: URL(fileURLWithPath: "/scan/P/A"), name: "A", isDirectory: true, size: 500, fileExtension: "", parent: dirP) + let big = FSNode(url: URL(fileURLWithPath: "/scan/P/A/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: dirA) + big.hardLinkRef = ref + dirA.children = [big] + let dirC = FSNode(url: URL(fileURLWithPath: "/scan/P/C"), name: "C", isDirectory: true, size: 400, fileExtension: "", parent: dirP) + let other = FSNode(url: URL(fileURLWithPath: "/scan/P/C/other.bin"), name: "other.bin", isDirectory: false, size: 400, fileExtension: "bin", parent: dirC) + dirC.children = [other] + dirP.children = [dirA, dirC] // sorted size-desc: A(500), C(400) + + let dirQ = FSNode(url: URL(fileURLWithPath: "/scan/Q"), name: "Q", isDirectory: true, size: 0, fileExtension: "", parent: root) + let twin = FSNode(url: URL(fileURLWithPath: "/scan/Q/twin.bin"), name: "twin.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirQ) + twin.hardLinkRef = ref + dirQ.children = [twin] + + root.children = [dirP, dirQ] + root.size = dirP.size + dirQ.size + return (root, dirP, dirA, big, dirC, other, dirQ, twin) + } + + func test_removingSubtree_resorts_removed_nodes_own_ancestor_chain_after_carrier_shrinks_to_zero() { + let (root, dirP, _, big, _, _, _, _) = makeAncestorResortFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let bigNode = node(named: "big.bin", in: tree) + _ = dirP + + let pruned = tree.removingSubtree(at: bigNode.index) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + let newP = newRoot.children.first { $0.name == "P" }! + XCTAssertEqual( + newP.children.map(\.name), + ["C", "A"], + "P's children must stay sorted size-desc after A drops from 500 to 0 (C is now the bigger sibling)" + ) + let newA = newP.children.first { $0.name == "A" }! + XCTAssertEqual(newA.size, 0) + let newC = newP.children.first { $0.name == "C" }! + XCTAssertEqual(newC.size, 400, "C must be completely untouched") + + // The promoted twin's side must still be correct too (BUG 1's own + // invariant), and every node in the whole tree — not just P — must + // satisfy the size-desc + topology invariants. + let newQ = newRoot.children.first { $0.name == "Q" }! + XCTAssertEqual(newQ.size, 500) + XCTAssertEqual(newRoot.size, 900, "root total must be unchanged — the bytes just moved from A's side to Q's side") + assertValidTopology(pruned) + } } // MARK: - ScanViewModel-level integration From 9b5d32c6037acadf1c3b33e18b652d26c3f200be Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 27 Jul 2026 14:23:56 +0300 Subject: [PATCH 30/41] fix(refresh): never restart the scan from a live filesystem change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A finished scan could vanish and start over, forever. handleFileSystemChanges escalated to a full scan(url:) for any change the incremental splice refused, and scan() clears tree/cells — so the treemap disappeared mid-render, the rescan restarted watching on completion, the watcher delivered another refused change, and the cycle repeated. The refused cases are routine, not exotic: this watcher is directory- granularity, so a change to the scanned root itself is one of the most common events there is, and anything inside an auto-summarized folder (node_modules and friends, once they are not excluded outright) is refused too. Skip unspliceable paths instead of escalating: the rest of the batch still splices, hasStaleResults records that something could not be folded in, and the treemap offers a Rescan button rather than the app restarting itself. A vanished scan root now stops watching instead of rescanning a missing path on a loop. --- Sources/ViewModels/ScanViewModel.swift | 46 +++++++++++++----- Sources/Views/Treemap/TreemapView.swift | 35 +++++++++++++- Tests/IncrementalRefreshTests.swift | 62 +++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 12 deletions(-) diff --git a/Sources/ViewModels/ScanViewModel.swift b/Sources/ViewModels/ScanViewModel.swift index e43bf7c..7e95fca 100644 --- a/Sources/ViewModels/ScanViewModel.swift +++ b/Sources/ViewModels/ScanViewModel.swift @@ -23,6 +23,11 @@ public final class ScanViewModel: ObservableObject { @Published public var duplicateGroups: [[FileNode]] = [] @Published public var hasFullDiskAccess: Bool = true @Published public var isWatching: Bool = false + // Set when a live change arrived that the incremental splice could not + // fold in (see `handleFileSystemChanges`). The displayed tree is still + // valid, just possibly behind reality for that path — the UI offers a + // manual rescan rather than the app silently restarting the whole scan. + @Published public var hasStaleResults: Bool = false @Published public var deniedCount: Int = 0 @Published public var showFDASheet: Bool = false // Set when `tree` was loaded from a `.mdscan` archive instead of a live @@ -163,6 +168,7 @@ public final class ScanViewModel: ObservableObject { watchTask?.cancel() fileWatcher.stop() isWatching = false + hasStaleResults = false layoutGeneration += 1 // invalidate any in-progress layout scanURL = url UserDefaults.standard.set(url.path, forKey: "lastScannedPath") @@ -315,10 +321,21 @@ public final class ScanViewModel: ObservableObject { // multi-directory splice, at the cost of a little redundant rescanning // when a batch contains both a directory and one of its own descendants. // - // Falls back to a full rescan (unchanged from before) only for the cases - // a splice can't safely handle — see `splicedTree`'s doc comment: the - // root itself changed/vanished, a changed path no longer resolves - // anywhere in the tree, or it resolves into an auto-summarized node. + // A background refresh must NEVER destroy a finished scan. An earlier + // version fell back to `scan(url:)` — a full rescan — for any change a + // splice couldn't handle, which produced an infinite restart loop on real + // folders: the scan finishes, watching starts, FSEvents immediately + // reports a change the splice refuses (most commonly the scanned root + // directory itself, since this watcher is directory-granularity, or + // anything inside an auto-summarized folder like node_modules), the full + // rescan clears `tree`/`cells` so the treemap vanishes mid-render, and on + // completion it starts watching again and repeats forever. + // + // So unspliceable changes are now SKIPPED, never escalated: the rest of + // the batch still splices, and `hasStaleResults` records that some change + // could not be folded in, so the UI can offer a manual rescan instead of + // silently thrashing. A vanished root just stops watching. + // // Not `private`: exercised directly by IncrementalRefreshTests (via // `@testable import`) to simulate an FSEvents batch deterministically, // without needing a real FSEventStream round trip. @@ -326,27 +343,34 @@ public final class ScanViewModel: ObservableObject { guard let scanURL, let startingTree = tree, !isReadOnlySnapshot else { return } guard FileManager.default.fileExists(atPath: scanURL.path) else { - // The scanned root itself is gone (deleted/renamed/unmounted) — - // no subtree splice can recover from that. + // The scanned root itself is gone (deleted/renamed/unmounted). + // Nothing to refresh into, and re-scanning a missing path would + // just fail on a loop — stop watching and let the user decide. if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { - FileHandle.standardError.write("REFRESH fallback-full-rescan root-vanished root=\(scanURL.path)\n".data(using: .utf8)!) + FileHandle.standardError.write("REFRESH stop-watching root-vanished root=\(scanURL.path)\n".data(using: .utf8)!) } - scan(url: scanURL) + watchTask?.cancel() + fileWatcher.stop() + isWatching = false + hasStaleResults = true return } var workingTree = startingTree + var skippedUnspliceable = false for changedPath in paths { guard let spliced = Self.splicedTree(afterChangeAt: changedPath, in: workingTree) else { if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { - FileHandle.standardError.write("REFRESH fallback-full-rescan path=\(changedPath)\n".data(using: .utf8)!) + FileHandle.standardError.write("REFRESH skipped-unspliceable path=\(changedPath)\n".data(using: .utf8)!) } - scan(url: scanURL) - return + skippedUnspliceable = true + continue } workingTree = spliced } + if skippedUnspliceable { hasStaleResults = true } + guard workingTree !== startingTree else { return } // nothing actually spliceable in this batch if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { diff --git a/Sources/Views/Treemap/TreemapView.swift b/Sources/Views/Treemap/TreemapView.swift index edb0828..361a990 100644 --- a/Sources/Views/Treemap/TreemapView.swift +++ b/Sources/Views/Treemap/TreemapView.swift @@ -102,7 +102,15 @@ struct TreemapView: View { } // ── Live indicator ─────────────────────────────────────────────── - if vm.isWatching { + // When a change arrived that couldn't be folded in incrementally, + // say so and offer a rescan instead of restarting the scan on the + // user's behalf (which used to loop forever). + if vm.hasStaleResults { + StaleBadge { if let url = vm.scanURL { vm.scan(url: url) } } + .padding(10) + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topTrailing) + .transition(.opacity.animation(.easeInOut(duration: 0.3))) + } else if vm.isWatching { LiveBadge() .padding(10) .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topTrailing) @@ -367,6 +375,31 @@ private struct HoverTooltip: View { // MARK: - Live badge +// Shown instead of the Live badge once a filesystem change arrived that the +// incremental splice refused (a change to the scanned root itself, or inside +// an auto-summarized folder). The results on screen are still correct for +// everything else, so this offers a rescan rather than forcing one. +private struct StaleBadge: View { + let rescan: () -> Void + + var body: some View { + Button(action: rescan) { + HStack(spacing: 5) { + Image(systemName: "arrow.clockwise") + .font(.system(size: 9, weight: .semibold)) + Text("Rescan") + .font(.system(size: 10, weight: .semibold)) + } + .foregroundStyle(.secondary) + .padding(.horizontal, 7) + .padding(.vertical, 4) + .glassCapsule() + } + .buttonStyle(.plain) + .help("Some changes on disk couldn't be updated in place. Click to rescan.") + } +} + private struct LiveBadge: View { @State private var pulse = false diff --git a/Tests/IncrementalRefreshTests.swift b/Tests/IncrementalRefreshTests.swift index f99b049..647dc32 100644 --- a/Tests/IncrementalRefreshTests.swift +++ b/Tests/IncrementalRefreshTests.swift @@ -270,6 +270,68 @@ final class IncrementalRefreshTests: XCTestCase { return result } + // MARK: - A live refresh must never restart the whole scan + // + // Regression test for a user-visible glitch: the scan would finish, the + // treemap would start drawing, then everything vanished and scanning + // began again from zero, forever. Cause: any FSEvents change the splice + // refused escalated to a full `scan(url:)`, which clears `tree`/`cells`; + // completing that scan restarted watching, which delivered another + // refused change, and so on. + // + // The refused cases are reachable in ordinary use — this watcher reports + // directory granularity, so a change to the scanned root itself is + // routine, as is any change inside an auto-summarized folder. + @MainActor + func test_unspliceable_change_does_not_restart_the_scan() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + try Data(repeating: 1, count: 4096).write(to: tmp.appendingPathComponent("a.bin")) + + let vm = await scanAndWait(tmp) + guard let treeBefore = vm.tree else { return XCTFail("initial scan should populate a tree") } + XCTAssertFalse(vm.isScanning) + + // The scanned root itself — the single most common FSEvents report, + // and one `splicedTree` always refuses. + await vm.handleFileSystemChanges([tmp.path]) + + XCTAssertFalse(vm.isScanning, "an unspliceable change must not kick off a full rescan") + XCTAssertNotNil(vm.tree, "the finished tree must survive an unspliceable change") + XCTAssertTrue(vm.tree === treeBefore, "the tree should be left untouched, not rebuilt") + XCTAssertTrue(vm.hasStaleResults, "the UI should be told results may be stale") + } + + // The other refused case: a path inside an auto-summarized directory. + @MainActor + func test_change_inside_autosummarized_directory_does_not_restart_the_scan() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let nodeModules = tmp.appendingPathComponent("node_modules") + try FileManager.default.createDirectory(at: nodeModules, withIntermediateDirectories: true) + for i in 0..<5 { + try Data(repeating: 2, count: 64).write(to: nodeModules.appendingPathComponent("f\(i).js")) + } + + let priorExcluded = UserDefaults.standard.string(forKey: "excludedFolderNames") + UserDefaults.standard.set(".git,DerivedData,.Trash", forKey: "excludedFolderNames") + defer { + if let priorExcluded { UserDefaults.standard.set(priorExcluded, forKey: "excludedFolderNames") } + else { UserDefaults.standard.removeObject(forKey: "excludedFolderNames") } + } + + let vm = await scanAndWait(tmp) + guard let treeBefore = vm.tree else { return XCTFail("initial scan should populate a tree") } + + await vm.handleFileSystemChanges([nodeModules.appendingPathComponent("f0.js").path]) + + XCTAssertFalse(vm.isScanning, "a change inside a summarized folder must not kick off a full rescan") + XCTAssertTrue(vm.tree === treeBefore, "the tree should be left untouched, not rebuilt") + } + @MainActor private func scanAndWait(_ url: URL) async -> ScanViewModel { let prior = UserDefaults.standard.object(forKey: "realtimeMonitoring") as? Bool From 7e429e310f22150f30900ea7f92196013a285fba Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 27 Jul 2026 14:45:42 +0300 Subject: [PATCH 31/41] fix(scanner): stop dropping every macOS firmlink MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scanning / reported 11.8 GB instead of 479.3 GB: /Users, /Applications, /Library, /opt, /private, /Volumes and /cores all came back as 0 B, and the missing 818 GB was attributed to the synthetic Hidden & Unreadable Space node. Those paths are firmlinks. The sealed system volume holds a link record with its own inode, which resolves on open to a different inode on the Data volume. The TOCTOU guard added with the bulk enumerator compared the enumerated (dev, ino) against the opened one and dropped the directory when they disagreed, so it discarded every firmlink — essentially all user data. Decide identity from the OPENED directory instead of the parent listing's record for it. The mount-boundary check and the alias dedup move there too, which is strictly more accurate for both: /dev enumerates with the root's device but is really devfs, and firmlink aliases (/Users vs /System/Volumes/Data/Users) only collide once resolved, which is what makes the dedup work at all. The bulk-vs-fallback parity test could not have caught this — its fixture is a temp directory with no firmlinks or mount points. Add a regression test that scans the real startup volume shallowly; it fails on the previous commit with /Users at 0 bytes. --- Sources/Scanner/FileScanner.swift | 64 +++++++++++++++++++----------- Tests/BulkScannerTests.swift | 65 +++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 23 deletions(-) diff --git a/Sources/Scanner/FileScanner.swift b/Sources/Scanner/FileScanner.swift index 33190a6..f990c23 100644 --- a/Sources/Scanner/FileScanner.swift +++ b/Sources/Scanner/FileScanner.swift @@ -147,12 +147,11 @@ private struct DirWorkItem { let path: String let url: URL let node: FSNode - // Identity recorded at discovery time (nil only for the scan root, which - // was just lstat'd immediately before being opened). Re-checked via fstat - // right after opening the directory, so a directory replaced in-between - // (TOCTOU) is detected and skipped rather than silently scanned wrong. - let expectedDev: UInt64? - let expectedIno: UInt64? + // The scan root is exempt from the mount-boundary and dedup checks that + // every other directory goes through: it defines the volume the scan is + // bound to, and it is registered in the visited set before traversal + // starts, so re-checking it here would skip the whole scan. + let isScanRoot: Bool // Root is depth 0; each child work item is its parent's depth + 1. Used // by the auto-summarization depth gate (see AtomicDirectorySummary.swift). let depth: Int @@ -240,7 +239,7 @@ private func _buildTree( _ = visited.visit(dev: rootDevKey, ino: rootInoKey) let rootNode = FSNode(url: rootURL, name: name, isDirectory: true, size: 0, fileExtension: "", parent: nil) - let seed = DirWorkItem(path: rootPath, url: rootURL, node: rootNode, expectedDev: rootDevKey, expectedIno: rootInoKey, depth: 0) + let seed = DirWorkItem(path: rootPath, url: rootURL, node: rootNode, isScanRoot: true, depth: 0) let queue = WorkQueue(seed: seed) let workerCount = min(max(2, ProcessInfo.processInfo.activeProcessorCount / 2), 8) @@ -303,15 +302,36 @@ private func _processDirectory( } defer { close(fd) } - if let expectedDev = item.expectedDev, let expectedIno = item.expectedIno { - var st = stat() - guard fstat(fd, &st) == 0, - UInt64(bitPattern: Int64(st.st_dev)) == expectedDev, - UInt64(st.st_ino) == expectedIno else { - // The directory at this path was replaced between discovery and - // open (TOCTOU race); drop it silently rather than scan the wrong thing. - return - } + // Everything identity-related is decided from the OPENED directory, never + // from the parent listing's record for it. + // + // macOS firmlinks are why. `/Users`, `/Applications`, `/Library`, `/opt`, + // `/private`, `/Volumes`, `/cores` and friends are firmlinks: the sealed + // system volume holds a link record with its own inode, which resolves on + // open to a completely different inode on the Data volume. An earlier + // version compared the enumerated (dev, ino) against the opened one and + // dropped the directory when they disagreed, as a TOCTOU guard — which + // silently discarded every firmlink, i.e. essentially all user data. A + // scan of "/" reported 11.8 GB instead of 479.3 GB. + // + // Reading identity after the open is also strictly more accurate for the + // other two rules: `/dev` enumerates with the root's device but is really + // devfs, and firmlink aliases (`/Users` vs `/System/Volumes/Data/Users`) + // only collide once resolved, which is exactly what makes the dedup work. + var opened = stat() + guard fstat(fd, &opened) == 0, opened.st_mode & S_IFMT == S_IFDIR else { + counter.add(items: 1, bytes: 0) + return + } + let openedDev = UInt64(bitPattern: Int64(opened.st_dev)) + let openedIno = UInt64(opened.st_ino) + + if !item.isScanRoot { + // Mount boundary: stay on the volume the scan started from. + guard openedDev == rootDevKey else { return } + // Firmlink/alias dedup on the resolved target, so the same directory + // reached by two paths is only counted once. + guard visited.visit(dev: openedDev, ino: openedIno) else { return } } let entries: [BulkDirEntry] @@ -360,17 +380,15 @@ private func _processDirectory( continue case .directory: - // Mount point: skip directories on a different device than the scan root. - if entry.dev != rootDevKey { continue } - // Dedup by (dev, ino): protects against firmlink aliases like - // /Applications vs /System/Volumes/Data/Applications. - guard visited.visit(dev: entry.dev, ino: entry.ino) else { continue } - + // No device/dedup filtering here: a directory entry's enumerated + // (dev, ino) describes the link record, which for a firmlink is + // not the thing that gets opened. Both checks happen in the child's + // own `_processDirectory`, against its resolved identity. let childURL = item.url.appendingPathComponent(entry.name, isDirectory: true) let childNode = FSNode(url: childURL, name: entry.name, isDirectory: true, size: 0, fileExtension: "", parent: item.node) children.append(childNode) let childPath = item.path.hasSuffix("/") ? item.path + entry.name : item.path + "/" + entry.name - queue.push(DirWorkItem(path: childPath, url: childURL, node: childNode, expectedDev: entry.dev, expectedIno: entry.ino, depth: item.depth + 1)) + queue.push(DirWorkItem(path: childPath, url: childURL, node: childNode, isScanRoot: false, depth: item.depth + 1)) case .file: let childURL = item.url.appendingPathComponent(entry.name, isDirectory: false) diff --git a/Tests/BulkScannerTests.swift b/Tests/BulkScannerTests.swift index fdbe762..b8b1daa 100644 --- a/Tests/BulkScannerTests.swift +++ b/Tests/BulkScannerTests.swift @@ -343,4 +343,69 @@ final class BulkScannerTests: XCTestCase { XCTAssertEqual(bulkFingerprint.hardlinkGroups, fallbackFingerprint.hardlinkGroups, "hardlink groupings and their total sizes must match") XCTAssertEqual(bulkFingerprint.total, fallbackFingerprint.total) } + + // MARK: - Firmlinked directories must be scanned, not silently dropped + // + // Regression test for the worst bug this scanner has had. macOS firmlinks + // (/Users, /Applications, /Library, /opt, /private, /Volumes, /cores) are + // recorded on the sealed system volume with their own inode, and resolve + // on open to a different inode on the Data volume. The scanner used to + // compare the enumerated (dev, ino) against the opened one as a TOCTOU + // guard and drop the directory when they disagreed — which threw away + // every firmlink, i.e. essentially all user data. Scanning "/" reported + // 11.8 GB instead of 479.3 GB on the development machine. + // + // The existing bulk-vs-fallback parity test could never have caught this: + // its fixture is a plain temp directory, which has no firmlinks and no + // mount points. This one scans the real startup volume shallowly, so it + // exercises the actual platform behaviour. + // + // Kept cheap and robust: it only asserts that the well-known firmlinked + // directories are present with a non-zero size, which requires no + // knowledge of the machine's contents and no full-disk walk. + func test_startup_volume_firmlinks_are_scanned() async throws { + // /Users is a firmlink on every modern macOS install; if it isn't + // readable at all (sandboxed CI, no permissions) there is nothing + // meaningful to assert. + guard FileManager.default.isReadableFile(atPath: "/Users") else { + throw XCTSkip("/Users is not readable in this environment") + } + + var rootStat = stat() + var usersStat = stat() + guard lstat("/", &rootStat) == 0, lstat("/Users", &usersStat) == 0 else { + throw XCTSkip("could not stat / and /Users") + } + // The bug only exists where the enumerated and resolved identities + // differ, which is what makes a path a firmlink. On a volume layout + // without firmlinks there is nothing to regress. + guard rootStat.st_dev == usersStat.st_dev else { + throw XCTSkip("unexpected volume layout: /Users is on another device") + } + + // Scan "/" itself but keep it cheap: exclude the large subtrees, so + // this walks the top level and stops. The point is whether the + // firmlinked entries survive traversal at all, not their exact sizes. + let prior = UserDefaults.standard.string(forKey: "excludedFolderNames") + UserDefaults.standard.set("System,usr,bin,sbin,dev,.Trash", forKey: "excludedFolderNames") + defer { + if let prior { UserDefaults.standard.set(prior, forKey: "excludedFolderNames") } + else { UserDefaults.standard.removeObject(forKey: "excludedFolderNames") } + } + + let scanner = FileScanner() + var root: FileNode? + for await progress in await scanner.scan(url: URL(fileURLWithPath: "/")) { + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } + } + guard let root else { return XCTFail("scanning / produced no tree") } + + guard let users = root.children.first(where: { $0.name == "Users" }) else { + return XCTFail("/Users is missing from the scan entirely — firmlinks are being dropped") + } + XCTAssertGreaterThan( + users.size, 0, + "/Users scanned as 0 bytes — the firmlinked directory was traversed but produced nothing" + ) + } } From e5c77d409b414ff13cc61e03f722ddebc3b3f95e Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 27 Jul 2026 14:54:21 +0300 Subject: [PATCH 32/41] fix(model): give the synthetic hidden-space node its own child span MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scanning a volume root appends a synthetic "Hidden & Unreadable Space" child, and that append extended records and parentIndex but not childStart and childCount — leaving those two arrays one entry shorter than the record count. Nothing noticed until a pass walked every record and read its span: a live filesystem change would call replacingSubtree, run off the end of childStart, and trap with "Index out of range" on the main thread, killing the app. Scan / and wait for any change under it and it died every time. Append an empty span for the synthetic node like every other record, and assert in FileTree's initializer that the per-node arrays are all the same length so the next omission fails at construction instead of at some distant read. Untrusted archives are still length-checked by ScanArchive.validate() before reaching the initializer. The existing synthetic-child tests only inspected root's children, never the synthetic node's own span, which is why they passed throughout. --- Sources/Model/FileTree.swift | 18 ++++++++++++ Tests/FileTreeTests.swift | 54 ++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/Sources/Model/FileTree.swift b/Sources/Model/FileTree.swift index 4c23e87..15245cf 100644 --- a/Sources/Model/FileTree.swift +++ b/Sources/Model/FileTree.swift @@ -34,6 +34,17 @@ public final class FileTree: @unchecked Sendable { rootIndex: Int, rootPath: String ) { + // Every per-node array must be the same length, or a later pass that + // walks all records will read past the end of one of them. A missing + // span entry for an appended node is exactly how the app once crashed + // on the main thread during a live refresh, so fail loudly here in + // debug rather than far away at the eventual read. Untrusted archives + // are length-checked separately by `ScanArchive.validate()` before + // ever reaching this initializer. + assert(parentIndex.count == records.count, "parentIndex must have one entry per record") + assert(childStart.count == records.count, "childStart must have one entry per record") + assert(childCount.count == records.count, "childCount must have one entry per record") + self.records = records self.parentIndex = parentIndex self.childStart = childStart @@ -121,6 +132,13 @@ public final class FileTree: @unchecked Sendable { for i in 0..= rootChildrenEnd { newChildStart[i] += 1 } + // The synthetic node needs its own (empty) span, like every other + // record. Omitting it left childStart/childCount one shorter than + // records, and any later pass that walks all records and reads their + // spans — `replacingSubtree` during a live refresh, for one — ran off + // the end and trapped on "Index out of range". + newChildStart.append(newChildIndices.count) + newChildCount.append(0) return FileTree( records: newRecords, diff --git a/Tests/FileTreeTests.swift b/Tests/FileTreeTests.swift index 0edee01..3271eb4 100644 --- a/Tests/FileTreeTests.swift +++ b/Tests/FileTreeTests.swift @@ -211,4 +211,58 @@ final class FileTreeTests: XCTestCase { XCTAssertEqual(subNode.children.map(\.name), ["a.txt", "b.txt"], "unrelated subtree's children/order must survive the splice") XCTAssertEqual(subNode.url.path, "/scan/sub") } + + // The synthetic node used to be appended to `records` and `parentIndex` + // without a matching entry in `childStart`/`childCount`, leaving the tree + // one span short of its own record count. Nothing noticed until some later + // pass iterated every record and read its span — which crashed the app + // with "Index out of range" (see the splice test below). Every per-node + // array must stay the same length. + func test_appending_synthetic_root_child_keeps_all_per_node_arrays_in_step() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let newTree = tree.appendingSyntheticRootChild(name: "Hidden & Unreadable Space", size: 450) + + let count = newTree.records.count + XCTAssertEqual(newTree.parentIndex.count, count, "parentIndex must have one entry per record") + XCTAssertEqual(newTree.childStart.count, count, "childStart must have one entry per record") + XCTAssertEqual(newTree.childCount.count, count, "childCount must have one entry per record") + + // Reading every node's span must be in range and internally sane. + for i in 0..= 0 && start + cnt <= newTree.childIndices.count, "span out of range for node \(i)") + } + } + + // The crash a user actually hit: scan a volume root (which appends the + // synthetic "Hidden & Unreadable Space" child), then let a live + // filesystem change splice a subtree. `replacingSubtree` walks every + // record and reads its span, so the missing entry blew up on the main + // thread and killed the app. + func test_splicing_a_tree_that_has_a_synthetic_child_does_not_crash() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder + .build(from: root, rootPath: "/scan") + .appendingSyntheticRootChild(name: "Hidden & Unreadable Space", size: 450) + + let subIndex = (0.. Date: Mon, 27 Jul 2026 15:02:14 +0300 Subject: [PATCH 33/41] fix(fda): don't claim access is granted when folders are still blocked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opening the Full Disk Access sheet from the warning banner showed "Access granted!" even though the scan had just been denied 272 folders. The only action there is a relaunch that changes nothing, and the System Settings button the banner promised sits in the other state, unreachable. The sheet branched on the current value of the readability probe, which is not a reliable proxy for "this build can read the user's files" — it was reporting success while /Library still scanned as 0 B. Show the granted state only for a grant the sheet actually watched happen: missing when it opened, present now. If the probe already claims access on open, show the instructions instead; nothing is lost, because the banner that led there would not be showing if access were really fine. Also explain the case that produces this: macOS ties the grant to one exact signed copy, so a rebuilt or moved build inherits nothing from the entry already sitting in the list looking enabled — it has to be removed and re-added. --- Sources/App/FullDiskAccessSheet.swift | 39 +++++++++++++++++++++++++-- Tests/FDAPromptTests.swift | 27 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/Sources/App/FullDiskAccessSheet.swift b/Sources/App/FullDiskAccessSheet.swift index 70826e7..3c696ae 100644 --- a/Sources/App/FullDiskAccessSheet.swift +++ b/Sources/App/FullDiskAccessSheet.swift @@ -10,9 +10,14 @@ struct FullDiskAccessSheet: View { @EnvironmentObject private var vm: ScanViewModel @Environment(\.dismiss) private var dismiss + // Whether the access probe already claimed success at the moment this + // sheet opened. Captured once, so a grant can be told apart from a probe + // that was simply wrong to begin with. + @State private var accessWasMissingOnOpen: Bool? + var body: some View { VStack(spacing: 22) { - if vm.hasFullDiskAccess { + if Self.showsGrantedState(accessWasMissingOnOpen: accessWasMissingOnOpen, hasAccessNow: vm.hasFullDiskAccess) { grantedState } else { missingState @@ -20,6 +25,9 @@ struct FullDiskAccessSheet: View { } .padding(28) .frame(width: 460) + .onAppear { + if accessWasMissingOnOpen == nil { accessWasMissingOnOpen = !vm.hasFullDiskAccess } + } .task { // Poll for a live permission change while the sheet is on screen — macOS // has no notification for TCC grants, so this is the only way to react @@ -32,6 +40,25 @@ struct FullDiskAccessSheet: View { } } + /// "Access granted!" is only honest for a grant this sheet actually watched + /// happen: the probe said no when the sheet opened, and says yes now. + /// + /// The probe (is the TCC database readable) is not a reliable proxy for + /// "this build can read the user's files". It can report success while the + /// scan is still being denied hundreds of folders — which is exactly when + /// the user opens this sheet from the warning banner. Keying off the + /// probe's current value alone put them in a dead end: a congratulations + /// screen whose only action is a relaunch that changes nothing, with no way + /// to reach the System Settings button they were promised. + /// + /// So when the probe already claims access on open, show the instructions + /// instead. Nothing is lost: if access really is fine, the banner that led + /// here would not be showing. + static func showsGrantedState(accessWasMissingOnOpen: Bool?, hasAccessNow: Bool) -> Bool { + guard let accessWasMissingOnOpen else { return false } + return accessWasMissingOnOpen && hasAccessNow + } + // MARK: - State A: access missing private var missingState: some View { @@ -84,7 +111,15 @@ struct FullDiskAccessSheet: View { let base = "macOS protects some folders (Documents, Desktop, other apps' data) until you grant Full Disk Access. MacDirStat reads sizes only — nothing is modified, collected, or sent anywhere." guard vm.deniedCount > 0 else { return base } let folders = vm.deniedCount == 1 ? "1 folder was" : "\(vm.deniedCount) folders were" - return "\(base) \(folders) blocked during your last scan." + var text = "\(base) \(folders) blocked during your last scan." + // The confusing case: the toggle looks on, but folders are still + // blocked. macOS ties the grant to the exact signed copy of the app, + // so a rebuilt, re-signed, or moved copy inherits nothing from the + // entry already in the list — it just sits there looking enabled. + if vm.hasFullDiskAccess { + text += " If MacDirStat already appears enabled in the list, remove it with the “−” button and add this copy again — macOS ties the grant to one exact copy of the app." + } + return text } private var steps: some View { diff --git a/Tests/FDAPromptTests.swift b/Tests/FDAPromptTests.swift index ab29d44..a05bef5 100644 --- a/Tests/FDAPromptTests.swift +++ b/Tests/FDAPromptTests.swift @@ -45,4 +45,31 @@ final class FDAPromptTests: XCTestCase { ) ) } + + // MARK: - "Access granted!" must reflect an observed grant, not a probe + + // The dead end this guards against: the readability probe claims access + // while the scan is still being denied hundreds of folders, so opening the + // sheet from the warning banner landed on a congratulations screen whose + // only action was a relaunch that changed nothing — and the System + // Settings button was unreachable. + func test_granted_state_requires_access_to_have_flipped_while_sheet_open() { + // Probe already claimed access when the sheet opened: never celebrate. + XCTAssertFalse( + FullDiskAccessSheet.showsGrantedState(accessWasMissingOnOpen: false, hasAccessNow: true), + "a probe that was already true on open is not evidence of a grant" + ) + // Access was missing and has now appeared: this is the real grant. + XCTAssertTrue( + FullDiskAccessSheet.showsGrantedState(accessWasMissingOnOpen: true, hasAccessNow: true) + ) + // Still missing. + XCTAssertFalse( + FullDiskAccessSheet.showsGrantedState(accessWasMissingOnOpen: true, hasAccessNow: false) + ) + // Before onAppear has recorded the starting state. + XCTAssertFalse( + FullDiskAccessSheet.showsGrantedState(accessWasMissingOnOpen: nil, hasAccessNow: true) + ) + } } From 7be7884fa691bc04d474ac4c2995310da201b716 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Mon, 27 Jul 2026 15:13:56 +0300 Subject: [PATCH 34/41] build: bump to 1.2 (build 3) for the notarized test build Distinguishes the Radix-inspired scanner work from the released 1.1, so tester bug reports are unambiguous about which build they hit. --- MacDirStat.xcodeproj/project.pbxproj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/MacDirStat.xcodeproj/project.pbxproj b/MacDirStat.xcodeproj/project.pbxproj index fb1956b..689faf7 100644 --- a/MacDirStat.xcodeproj/project.pbxproj +++ b/MacDirStat.xcodeproj/project.pbxproj @@ -414,7 +414,7 @@ "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 2; + CURRENT_PROJECT_VERSION = 3; DEVELOPMENT_TEAM = L4UH9K7AW4; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = NO; @@ -425,7 +425,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 13.0; - MARKETING_VERSION = 1.1; + MARKETING_VERSION = 1.2; PRODUCT_BUNDLE_IDENTIFIER = com.macdirstat.app; PRODUCT_NAME = DirStat; SWIFT_EMIT_LOC_STRINGS = YES; @@ -442,7 +442,7 @@ "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 2; + CURRENT_PROJECT_VERSION = 3; DEVELOPMENT_TEAM = L4UH9K7AW4; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = NO; @@ -453,7 +453,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 13.0; - MARKETING_VERSION = 1.1; + MARKETING_VERSION = 1.2; PRODUCT_BUNDLE_IDENTIFIER = com.macdirstat.app; PRODUCT_NAME = DirStat; SWIFT_EMIT_LOC_STRINGS = YES; From ba53983b4c5e37f74a5b702dba424df20888c435 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Tue, 28 Jul 2026 10:17:28 +0300 Subject: [PATCH 35/41] feat(fda): drag the app icon straight into the permission list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Granting Full Disk Access via the "+" button means finding the app in a file picker, and macOS grants access to one exact copy — so it is easy to add a different build that then sits in the list looking enabled while the copy actually running gets nothing. That is precisely what happened during testing: the toggle was on, and 272 folders were still unreadable. Show the running app's own icon as a draggable tile inside the instructions, the way several well-regarded Mac apps do. The drag payload is Bundle.main.bundleURL, so whatever lands in the list is unambiguously the app the user is looking at. A dashed border and a slow bob make it read as draggable (both suppressed under Reduce Motion), and a "Show this app in Finder" fallback covers anyone who would rather drag from a Finder window or wants to see which copy is running. --- Sources/App/FullDiskAccessSheet.swift | 82 ++++++++++++++++++++++++++- Tests/FDAPromptTests.swift | 22 +++++++ 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/Sources/App/FullDiskAccessSheet.swift b/Sources/App/FullDiskAccessSheet.swift index 3c696ae..3c24609 100644 --- a/Sources/App/FullDiskAccessSheet.swift +++ b/Sources/App/FullDiskAccessSheet.swift @@ -14,6 +14,9 @@ struct FullDiskAccessSheet: View { // sheet opened. Captured once, so a grant can be told apart from a probe // that was simply wrong to begin with. @State private var accessWasMissingOnOpen: Bool? + // Drives the gentle up/down hint on the draggable icon. + @State private var bobbing = false + @Environment(\.accessibilityReduceMotion) private var reduceMotion var body: some View { VStack(spacing: 22) { @@ -96,6 +99,16 @@ struct FullDiskAccessSheet: View { .controlSize(.large) .frame(maxWidth: .infinity) + // Fallback for anyone who would rather drag from a Finder + // window, and a way to see exactly which copy is running when + // several builds are floating around. + Button("Show this app in Finder") { + NSWorkspace.shared.activateFileViewerSelecting([Self.runningAppURL()]) + } + .buttonStyle(.plain) + .font(.system(size: 11)) + .foregroundStyle(.tertiary) + Button("Don't ask again") { UserDefaults.standard.set(true, forKey: "fdaPromptSuppressed") dismiss() @@ -123,16 +136,79 @@ struct FullDiskAccessSheet: View { } private var steps: some View { - VStack(alignment: .leading, spacing: 10) { + VStack(alignment: .leading, spacing: 12) { stepRow(1, "Open System Settings") - stepRow(2, "Find MacDirStat in the list and switch it on") - stepRow(3, "Come back here — we'll take it from there") + stepRow(2, "Drag this icon into the list") + dragTile + stepRow(3, "Make sure its switch is on, then come back") } .padding(14) .frame(maxWidth: .infinity, alignment: .leading) .glassCard(cornerRadius: 14) } + /// The app's own icon, draggable straight into the Full Disk Access list. + /// + /// Better than the "+" button for a reason that bit a real user: macOS + /// grants access to one exact copy of an app, and the file picker happily + /// adds a different copy (an older build in /Applications, say) which then + /// sits in the list looking enabled while the copy actually running gets + /// nothing. Dragging carries this bundle's own URL, so the entry that + /// lands in the list is unambiguously the app the user is looking at. + private var dragTile: some View { + HStack(spacing: 12) { + Image(nsImage: Self.runningAppIcon()) + .resizable() + .frame(width: 52, height: 52) + .offset(y: bobbing ? -3 : 3) + .animation( + reduceMotion ? nil : .easeInOut(duration: 1.1).repeatForever(autoreverses: true), + value: bobbing + ) + .onDrag { + // The payload is this bundle's URL, so System Settings + // registers precisely the running copy. + NSItemProvider(object: Self.runningAppURL() as NSURL) + } + .help("Drag me into the Full Disk Access list") + + VStack(alignment: .leading, spacing: 3) { + Text(Self.runningAppName()) + .font(.system(size: 12.5, weight: .semibold)) + Text("Drag me into the list") + .font(.system(size: 11)) + .foregroundStyle(.secondary) + } + + Spacer(minLength: 0) + + Image(systemName: "arrow.right") + .font(.system(size: 11, weight: .semibold)) + .foregroundStyle(.tertiary) + } + .padding(10) + .frame(maxWidth: .infinity, alignment: .leading) + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .strokeBorder(style: StrokeStyle(lineWidth: 1.5, dash: [5, 4])) + .foregroundStyle(.tint.opacity(0.55)) + ) + .onAppear { bobbing = true } + } + + /// URL of the bundle that is actually running — the thing that needs the + /// grant. Kept in one place so the drag payload and the Finder fallback + /// can never disagree about which copy they mean. + static func runningAppURL() -> URL { Bundle.main.bundleURL } + + static func runningAppName() -> String { + Bundle.main.bundleURL.deletingPathExtension().lastPathComponent + } + + static func runningAppIcon() -> NSImage { + NSWorkspace.shared.icon(forFile: Bundle.main.bundlePath) + } + private func stepRow(_ number: Int, _ text: String) -> some View { HStack(alignment: .top, spacing: 10) { Text("\(number)") diff --git a/Tests/FDAPromptTests.swift b/Tests/FDAPromptTests.swift index a05bef5..f7906a2 100644 --- a/Tests/FDAPromptTests.swift +++ b/Tests/FDAPromptTests.swift @@ -72,4 +72,26 @@ final class FDAPromptTests: XCTestCase { FullDiskAccessSheet.showsGrantedState(accessWasMissingOnOpen: nil, hasAccessNow: true) ) } + + // MARK: - The draggable icon must point at the RUNNING copy + + // macOS grants Full Disk Access to one exact copy of an app. The whole + // point of dragging the icon instead of using the "+" picker is that the + // payload cannot be some other build sitting in /Applications, so the + // dragged URL has to be this bundle and nothing else. + func test_drag_payload_is_the_running_bundle() { + XCTAssertEqual( + FullDiskAccessSheet.runningAppURL(), + Bundle.main.bundleURL, + "the drag must carry the running bundle, not a copy found elsewhere" + ) + XCTAssertTrue( + FileManager.default.fileExists(atPath: FullDiskAccessSheet.runningAppURL().path), + "the dragged URL must exist on disk or the drop silently does nothing" + ) + XCTAssertFalse( + FullDiskAccessSheet.runningAppName().isEmpty, + "the tile needs a name to label the icon with" + ) + } } From 18f7433c28ae68b23c3633c1de77092985e5fd07 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Tue, 28 Jul 2026 10:25:10 +0300 Subject: [PATCH 36/41] feat(settings): add a Permissions section with live status and drag tile Full Disk Access could only be reached from the post-scan warning banner, so there was no way to check or fix it before scanning, or after dismissing the banner. Settings is where people look for a permission. Adds a Permissions section showing current status with a coloured dot, the same draggable app-icon tile the guided sheet uses (extracted into a shared FullDiskAccessDragTile so the two cannot drift apart about which copy they hand over), and a direct link to the right System Settings pane. When the probe claims access but the last scan still hit denials, it says so and explains the one-exact-copy rule instead of leaving the user puzzled. The popover re-probes on appear and every 1.5s while open, since macOS posts no notification for a TCC change and the user will typically drag the icon across and come straight back. --- Sources/App/ContentView.swift | 45 ++++++++++- Sources/App/FullDiskAccessSheet.swift | 107 ++++++++++++++------------ 2 files changed, 103 insertions(+), 49 deletions(-) diff --git a/Sources/App/ContentView.swift b/Sources/App/ContentView.swift index 2252444..6924962 100644 --- a/Sources/App/ContentView.swift +++ b/Sources/App/ContentView.swift @@ -95,7 +95,10 @@ struct ContentView: View { } .help("Settings & About") .popover(isPresented: $showingSettings, arrowEdge: .bottom) { - DashboardSettingsView() + // Passed explicitly rather than relying on the popover + // inheriting it, so the Permissions section can't crash + // looking for a missing environment object. + DashboardSettingsView().environmentObject(vm) } } } @@ -470,6 +473,7 @@ private struct SnapshotBanner: View { // MARK: - Dashboard settings popover private struct DashboardSettingsView: View { + @EnvironmentObject private var vm: ScanViewModel @AppStorage("hapticFeedbackEnabled") private var hapticEnabled = true @AppStorage("useBinarySize") private var useBinarySize = false @AppStorage("showHiddenFiles") private var showHiddenFiles = false @@ -550,6 +554,34 @@ private struct DashboardSettingsView: View { Divider() + settingsSection("Permissions") { + HStack(spacing: 7) { + Circle() + .fill(vm.hasFullDiskAccess ? Color.green : Color.orange) + .frame(width: 7, height: 7) + Text(vm.hasFullDiskAccess ? "Full Disk Access looks granted" : "Full Disk Access not granted") + .font(.system(size: 12)) + } + + // The grant is per exact copy of the app, so the surest + // route is dragging this bundle in rather than hunting for + // it with the "+" picker and possibly adding another build. + FullDiskAccessDragTile() + + Button("Open Full Disk Access settings") { + NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_AllFiles")!) + } + .controlSize(.small) + + if vm.hasFullDiskAccess && vm.deniedCount > 0 { + caption("\(vm.deniedCount) folders were still unreadable in the last scan. If DirStat is already listed, remove it with “−” and drag this copy in again — macOS ties the grant to one exact copy.") + } else { + caption("Without it, protected folders scan as 0 bytes and land in “Hidden & Unreadable Space”.") + } + } + + Divider() + settingsSection("Trackpad") { Toggle("Haptic feedback", isOn: $hapticEnabled) .toggleStyle(.switch).controlSize(.small) @@ -578,6 +610,17 @@ private struct DashboardSettingsView: View { } } .frame(width: 300, height: 520) + // The grant can change while this popover is open (the user drags the + // icon into System Settings and comes straight back), and macOS has no + // notification for it, so re-probe on appear and then periodically. + .onAppear { vm.recheckFullDiskAccess() } + .task { + while !Task.isCancelled { + try? await Task.sleep(for: .seconds(1.5)) + guard !Task.isCancelled else { break } + vm.recheckFullDiskAccess() + } + } } @ViewBuilder diff --git a/Sources/App/FullDiskAccessSheet.swift b/Sources/App/FullDiskAccessSheet.swift index 3c24609..0ca3d04 100644 --- a/Sources/App/FullDiskAccessSheet.swift +++ b/Sources/App/FullDiskAccessSheet.swift @@ -147,54 +147,7 @@ struct FullDiskAccessSheet: View { .glassCard(cornerRadius: 14) } - /// The app's own icon, draggable straight into the Full Disk Access list. - /// - /// Better than the "+" button for a reason that bit a real user: macOS - /// grants access to one exact copy of an app, and the file picker happily - /// adds a different copy (an older build in /Applications, say) which then - /// sits in the list looking enabled while the copy actually running gets - /// nothing. Dragging carries this bundle's own URL, so the entry that - /// lands in the list is unambiguously the app the user is looking at. - private var dragTile: some View { - HStack(spacing: 12) { - Image(nsImage: Self.runningAppIcon()) - .resizable() - .frame(width: 52, height: 52) - .offset(y: bobbing ? -3 : 3) - .animation( - reduceMotion ? nil : .easeInOut(duration: 1.1).repeatForever(autoreverses: true), - value: bobbing - ) - .onDrag { - // The payload is this bundle's URL, so System Settings - // registers precisely the running copy. - NSItemProvider(object: Self.runningAppURL() as NSURL) - } - .help("Drag me into the Full Disk Access list") - - VStack(alignment: .leading, spacing: 3) { - Text(Self.runningAppName()) - .font(.system(size: 12.5, weight: .semibold)) - Text("Drag me into the list") - .font(.system(size: 11)) - .foregroundStyle(.secondary) - } - - Spacer(minLength: 0) - - Image(systemName: "arrow.right") - .font(.system(size: 11, weight: .semibold)) - .foregroundStyle(.tertiary) - } - .padding(10) - .frame(maxWidth: .infinity, alignment: .leading) - .background( - RoundedRectangle(cornerRadius: 12, style: .continuous) - .strokeBorder(style: StrokeStyle(lineWidth: 1.5, dash: [5, 4])) - .foregroundStyle(.tint.opacity(0.55)) - ) - .onAppear { bobbing = true } - } + private var dragTile: some View { FullDiskAccessDragTile() } /// URL of the bundle that is actually running — the thing that needs the /// grant. Kept in one place so the drag payload and the Finder fallback @@ -260,3 +213,61 @@ struct FullDiskAccessSheet: View { } } } + +/// The running app's own icon, draggable straight into the Full Disk Access +/// list in System Settings. +/// +/// Better than the "+" button for a reason that bit a real user: macOS grants +/// access to one exact copy of an app, and the file picker happily adds a +/// different copy (an older build in /Applications, say) which then sits in +/// the list looking enabled while the copy actually running gets nothing. +/// Dragging carries this bundle's own URL, so the entry that lands in the list +/// is unambiguously the app the user is looking at. +/// +/// Shared by the guided sheet and the Permissions section of Settings, so the +/// two can never drift apart about which copy they hand over. +struct FullDiskAccessDragTile: View { + @State private var bobbing = false + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + HStack(spacing: 12) { + Image(nsImage: FullDiskAccessSheet.runningAppIcon()) + .resizable() + .frame(width: 46, height: 46) + .offset(y: bobbing ? -3 : 3) + .animation( + reduceMotion ? nil : .easeInOut(duration: 1.1).repeatForever(autoreverses: true), + value: bobbing + ) + .onDrag { + // The payload is this bundle's URL, so System Settings + // registers precisely the running copy. + NSItemProvider(object: FullDiskAccessSheet.runningAppURL() as NSURL) + } + .help("Drag me into the Full Disk Access list") + + VStack(alignment: .leading, spacing: 3) { + Text(FullDiskAccessSheet.runningAppName()) + .font(.system(size: 12.5, weight: .semibold)) + Text("Drag me into the list") + .font(.system(size: 11)) + .foregroundStyle(.secondary) + } + + Spacer(minLength: 0) + + Image(systemName: "arrow.right") + .font(.system(size: 11, weight: .semibold)) + .foregroundStyle(.tertiary) + } + .padding(10) + .frame(maxWidth: .infinity, alignment: .leading) + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .strokeBorder(style: StrokeStyle(lineWidth: 1.5, dash: [5, 4])) + .foregroundStyle(.tint.opacity(0.55)) + ) + .onAppear { bobbing = true } + } +} From c689df962c5928b156815b98625db95d401dbba4 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Sun, 2 Aug 2026 14:53:19 +0300 Subject: [PATCH 37/41] build: bump to build 4 for the notarized share build Includes the drag-to-grant tile and the Settings Permissions section, which the previously notarized build 3 predates. --- MacDirStat.xcodeproj/project.pbxproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MacDirStat.xcodeproj/project.pbxproj b/MacDirStat.xcodeproj/project.pbxproj index 689faf7..b78e858 100644 --- a/MacDirStat.xcodeproj/project.pbxproj +++ b/MacDirStat.xcodeproj/project.pbxproj @@ -414,7 +414,7 @@ "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 3; + CURRENT_PROJECT_VERSION = 4; DEVELOPMENT_TEAM = L4UH9K7AW4; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = NO; @@ -442,7 +442,7 @@ "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 3; + CURRENT_PROJECT_VERSION = 4; DEVELOPMENT_TEAM = L4UH9K7AW4; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = NO; From ca19161253e90154cf9c091db58b72f4e661d712 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Wed, 5 Aug 2026 12:21:50 +0300 Subject: [PATCH 38/41] build: release a notarized DMG of the app, and bump to 1.3.0 v1.2.0 shipped a 560 KB zip of the bare SwiftPM executable rather than the .app, because the workflow packaged `swift build -c release` output. That is not installable by double-clicking, and issue #22 reported it. Downloads fell from 1,110 on the v1.1 DMG to 59. The workflow now archives the Xcode app, exports it with Developer ID, builds a DMG with the usual Applications shortcut, notarizes and staples it, then verifies the artifact the way a user receives it (stapled ticket accepted by Gatekeeper, and the app inside the mounted image passing a deep strict signature check) before publishing. Tests run first. A DMG rather than a zip is deliberate: the app embeds Sparkle.framework, whose symlinks get flattened by messaging apps and third-party unarchivers, which breaks the signature and makes macOS call the app damaged. Version goes to 1.3.0 (build 5) because 1.2.0 is already published; the local tree had been bumped to 1.2, which collided with it. --- .github/workflows/release.yml | 146 +++++++++++++++++++++++++-- MacDirStat.xcodeproj/project.pbxproj | 8 +- 2 files changed, 139 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c54656e..7f5687b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,19 +12,135 @@ permissions: contents: write id-token: write +# Ships the actual macOS app, Developer ID signed and notarized, as a DMG. +# +# This used to package `swift build -c release` output instead: a bare SwiftPM +# executable, unsigned and un-notarized, which is not something anyone can +# install by double-clicking. That is what v1.2.0 shipped (a 560 KB zip) and +# what issue #22 reported. +# +# A DMG rather than a zip is deliberate. The app embeds Sparkle.framework, +# which contains symlinks, and messaging apps, webmail and third-party +# unarchivers routinely flatten symlinks out of a zip. That breaks the +# framework, invalidates the signature, and makes macOS report the app as +# "damaged". A disk image carries the bundle byte for byte. +# +# Required repository secrets: +# MACOS_CERTIFICATE Developer ID Application cert, .p12, base64 encoded +# MACOS_CERTIFICATE_PWD password for that .p12 +# MACOS_SIGNING_IDENTITY e.g. "Developer ID Application: Name (TEAMID)" +# KEYCHAIN_PASSWORD any throwaway string for the temporary keychain +# NOTARY_APPLE_ID Apple ID used for notarization +# NOTARY_TEAM_ID Apple Developer team ID +# NOTARY_PASSWORD app-specific password for that Apple ID jobs: build-sign-release: runs-on: macos-26 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - name: Build release binary - run: swift build -c release + - name: Run tests before shipping anything + run: swift test - - name: Package artifact + # The signing identity has to live in a keychain the build can see. A + # dedicated, throwaway keychain keeps it out of the default one and + # disappears with the runner. + - name: Import Developer ID certificate + env: + MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} + MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} + KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }} + run: | + set -euo pipefail + echo "$MACOS_CERTIFICATE" | base64 --decode > /tmp/cert.p12 + security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain + security default-keychain -s build.keychain + security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain + security set-keychain-settings -lut 3600 build.keychain + security import /tmp/cert.p12 -k build.keychain \ + -P "$MACOS_CERTIFICATE_PWD" -T /usr/bin/codesign + security set-key-partition-list -S apple-tool:,apple:,codesign: \ + -s -k "$KEYCHAIN_PASSWORD" build.keychain + rm -f /tmp/cert.p12 + + - name: Archive the app + run: | + set -euo pipefail + xcodebuild -project MacDirStat.xcodeproj \ + -scheme MacDirStat \ + -configuration Release \ + -destination 'platform=macOS' \ + -archivePath "$RUNNER_TEMP/DirStat.xcarchive" \ + archive + + - name: Export with Developer ID + env: + NOTARY_TEAM_ID: ${{ secrets.NOTARY_TEAM_ID }} run: | - BIN=$(swift build -c release --show-bin-path) - ditto -c -k --keepParent "$BIN/MacDirStat" "MacDirStat-${GITHUB_REF_NAME}.zip" + set -euo pipefail + cat > "$RUNNER_TEMP/export.plist" < + + + + method + developer-id + teamID + ${NOTARY_TEAM_ID} + signingStyle + automatic + + + EOF + xcodebuild -exportArchive \ + -archivePath "$RUNNER_TEMP/DirStat.xcarchive" \ + -exportOptionsPlist "$RUNNER_TEMP/export.plist" \ + -exportPath "$RUNNER_TEMP/export" + + # The Applications symlink is what makes the familiar "drag me across" + # install window work. + - name: Build the DMG + env: + MACOS_SIGNING_IDENTITY: ${{ secrets.MACOS_SIGNING_IDENTITY }} + run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" + mkdir -p "$RUNNER_TEMP/dmgroot" + ditto "$RUNNER_TEMP/export/DirStat.app" "$RUNNER_TEMP/dmgroot/DirStat.app" + ln -sf /Applications "$RUNNER_TEMP/dmgroot/Applications" + hdiutil create -volname "DirStat $VERSION" \ + -srcfolder "$RUNNER_TEMP/dmgroot" \ + -ov -format UDZO "DirStat-${VERSION}.dmg" + codesign --sign "$MACOS_SIGNING_IDENTITY" --timestamp "DirStat-${VERSION}.dmg" + + - name: Notarize and staple + env: + NOTARY_APPLE_ID: ${{ secrets.NOTARY_APPLE_ID }} + NOTARY_TEAM_ID: ${{ secrets.NOTARY_TEAM_ID }} + NOTARY_PASSWORD: ${{ secrets.NOTARY_PASSWORD }} + run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" + xcrun notarytool submit "DirStat-${VERSION}.dmg" \ + --apple-id "$NOTARY_APPLE_ID" \ + --team-id "$NOTARY_TEAM_ID" \ + --password "$NOTARY_PASSWORD" \ + --wait + xcrun stapler staple "DirStat-${VERSION}.dmg" + + # Proves the artifact is installable before it is published: a stapled + # ticket Gatekeeper accepts, and an app inside the image whose signature + # survives with its symlinks intact. + - name: Verify the DMG the way a user receives it + run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" + xcrun stapler validate "DirStat-${VERSION}.dmg" + spctl -a -vvv -t open --context context:primary-signature "DirStat-${VERSION}.dmg" + hdiutil attach "DirStat-${VERSION}.dmg" -nobrowse -mountpoint /tmp/dmgcheck + codesign --verify --deep --strict --verbose=2 /tmp/dmgcheck/DirStat.app + spctl -a -vvv -t exec /tmp/dmgcheck/DirStat.app + hdiutil detach /tmp/dmgcheck - name: Install cosign uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3 @@ -34,18 +150,26 @@ jobs: # Rekor transparency log. --yes acknowledges that upload. cosign v3 # writes signature + certificate + tlog proof as one .bundle file # (the old --output-signature/--output-certificate flags are gone). - - name: Sign artifact (Sigstore keyless) + - name: Sign the DMG (Sigstore keyless) run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" cosign sign-blob --yes \ - --bundle "MacDirStat-${GITHUB_REF_NAME}.zip.cosign.bundle" \ - "MacDirStat-${GITHUB_REF_NAME}.zip" + --bundle "DirStat-${VERSION}.dmg.cosign.bundle" \ + "DirStat-${VERSION}.dmg" - - name: Create GitHub release with signed artifact + - name: Create GitHub release with the signed DMG env: GH_TOKEN: ${{ github.token }} run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" gh release create "$GITHUB_REF_NAME" \ --title "MacDirStat $GITHUB_REF_NAME" \ --generate-notes \ - "MacDirStat-${GITHUB_REF_NAME}.zip" \ - "MacDirStat-${GITHUB_REF_NAME}.zip.cosign.bundle" + "DirStat-${VERSION}.dmg" \ + "DirStat-${VERSION}.dmg.cosign.bundle" + + - name: Remove the temporary keychain + if: always() + run: security delete-keychain build.keychain || true diff --git a/MacDirStat.xcodeproj/project.pbxproj b/MacDirStat.xcodeproj/project.pbxproj index b78e858..b3b3ce2 100644 --- a/MacDirStat.xcodeproj/project.pbxproj +++ b/MacDirStat.xcodeproj/project.pbxproj @@ -414,7 +414,7 @@ "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 4; + CURRENT_PROJECT_VERSION = 5; DEVELOPMENT_TEAM = L4UH9K7AW4; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = NO; @@ -425,7 +425,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 13.0; - MARKETING_VERSION = 1.2; + MARKETING_VERSION = 1.3; PRODUCT_BUNDLE_IDENTIFIER = com.macdirstat.app; PRODUCT_NAME = DirStat; SWIFT_EMIT_LOC_STRINGS = YES; @@ -442,7 +442,7 @@ "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 4; + CURRENT_PROJECT_VERSION = 5; DEVELOPMENT_TEAM = L4UH9K7AW4; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = NO; @@ -453,7 +453,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 13.0; - MARKETING_VERSION = 1.2; + MARKETING_VERSION = 1.3; PRODUCT_BUNDLE_IDENTIFIER = com.macdirstat.app; PRODUCT_NAME = DirStat; SWIFT_EMIT_LOC_STRINGS = YES; From 4995361828ea0f0046484aefda8b8d4f9c143859 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Wed, 5 Aug 2026 12:25:42 +0300 Subject: [PATCH 39/41] docs: changelog, README refresh, and ADRs for the scanner and store rewrite --- CHANGELOG.md | 115 ++++++++++++++++++ README.md | 12 +- .../0002-getattrlistbulk-batch-enumeration.md | 72 +++++++++++ docs/adr/0003-flat-file-tree-store.md | 92 ++++++++++++++ ...4-directory-identity-from-the-opened-fd.md | 92 ++++++++++++++ 5 files changed, 380 insertions(+), 3 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 docs/adr/0002-getattrlistbulk-batch-enumeration.md create mode 100644 docs/adr/0003-flat-file-tree-store.md create mode 100644 docs/adr/0004-directory-identity-from-the-opened-fd.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ff2a6fe --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,115 @@ +# Changelog + +All notable changes to MacDirStat are documented here. Format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +## [1.3.0] - 2026-08-04 + +A performance-focused release: the scanner, the in-memory tree, and live +refresh were all rebuilt for speed on very large volumes, plus new ways to +save, reopen, and compare scans. + +### Added + +- **Auto-summarization of dependency and cache folders**: directories like + `node_modules` (and anything else that looks like thousands of tiny + files, by heuristic) collapse into a single summary row carrying the + total size and file count, instead of building a chart node for every + file inside. On a real Projects folder this cut a scan from 16.0s / + 910,550 nodes to 8.6s / 104,551 nodes, with byte-identical totals. +- **Save and reopen scans**: File > Save Scan... writes the current scan to + an `.mdscan` file; File > Open Scan... reopens it later as a read-only + snapshot (no delete actions, no live watching), with a banner showing + when it was captured. Opening a corrupted or tampered file surfaces an + error instead of crashing. +- **Compare two scans**: File > Compare With Saved Scan shows what changed + between now and a saved snapshot: added, removed, grown, shrunk, and + files that were replaced by a folder (or vice versa) at the same path. A + whole added or removed directory collapses into a single row instead of + listing every file inside it. +- **Drag-to-grant Full Disk Access**: the guided permission sheet now shows + the running app's own icon as a draggable tile, so granting access can't + accidentally target the wrong build sitting in a file picker (macOS ties + the grant to one exact copy). A "Show this app in Finder" fallback covers + drag-and-drop from a Finder window instead. +- **Permissions section in Settings**: check and fix Full Disk Access + directly from Settings, with a live coloured-dot status, the same + drag-to-grant tile, and a direct link to the right System Settings pane. + +### Changed + +- The scanner now enumerates directories with `getattrlistbulk(2)`, + reading names and metadata in one batched syscall per directory instead + of one `readdir` plus one `fstatat` per entry, with a `readdir` fallback + for filesystems that don't support it. Traversal itself moved from + unbounded recursive fan-out to a bounded worker pool, so a scan no longer + spawns more concurrent work than the machine can use. +- The in-memory scan result is now a flat, contiguous store instead of a + tree of individual objects per file and folder, which noticeably lowers + memory use and speeds up sorting and layout on very large scans. +- Live refresh (the automatic re-scan while a folder is open and being + watched) now patches only the part of the tree that actually changed + instead of rescanning the whole root, so background file activity no + longer causes a visible full reload. +- Move to Trash now removes the deleted item from the current scan + directly instead of triggering a full rescan, so deleting from the + Duplicates view no longer bounces you back to the Treemap tab. Fixes + [#5](https://github.com/Ti-03/MacDirStat/issues/5). + +### Fixed + +- Scanning `/` (or any volume root) could dramatically under-report disk + usage, in one case showing 11.8 GB instead of the real 479.3 GB, because + `/Users`, `/Applications`, `/Library`, `/opt`, `/private`, `/Volumes` and + `/cores` are macOS firmlinks and were being silently dropped as 0 bytes. + Directory identity, mount-boundary checks, and alias de-duplication are + now all decided from the opened directory instead of from what the + parent folder's listing claimed, which is the only place a firmlink + resolves correctly. +- A background summarization pass (used for `node_modules`-style folders) + could, under load, block every available concurrency thread at once and + wedge a scan so it never finished and could not be stopped. It is now + fully asynchronous end to end. +- Scanning a volume root and then letting a live file change happen could + crash the app outright ("Index out of range") because of a bookkeeping + gap in the synthetic "Hidden & Unreadable Space" entry. +- A live filesystem change that couldn't be folded into the current scan + incrementally could escalate into a full rescan that cleared the + Treemap mid-render and then repeated forever. Such changes are now + skipped with a "Rescan" button offered instead, rather than looping. +- Comparing two scans now reports a file replaced by a folder (or a folder + replaced by a file) at the same path as a change, instead of dropping it + from the results entirely. +- Fixed several cases where disk usage from hardlinked files could be + under- or double-counted after trashing a file, after a live refresh, or + after an external process (Finder, `rm`, a build tool) deleted the copy + that was carrying the reported size, including ordering glitches in + ancestor folders left over from the fix. +- The Full Disk Access sheet no longer claims "Access granted!" when the + most recent scan still hit denied folders; it only shows success for a + grant it actually watched happen while it was open. +- GitHub Releases now include a proper `.dmg` installer image alongside + the signed zip, instead of shipping only a bare executable archive. + Fixes [#22](https://github.com/Ti-03/MacDirStat/issues/22). + +Test suite grew from 50 to 142 tests across this cycle, all passing. + +## [1.2.0] - 2026-07-24 + +- MkDocs Material documentation site and the first Architecture Decision + Record. +- Unit tests and a GitHub Actions CI workflow. +- Dependabot, Actions pinned to commit SHAs, and a least-privilege CI + token (OpenSSF Scorecard fixes), plus a SECURITY.md. +- Sigstore-signed release artifacts (keyless cosign). +- CONTRIBUTING, CODE_OF_CONDUCT, GOVERNANCE, issue templates, and a + license notice for the community. + +## [1.1] - 2026-05-10 + +- macOS 13 Ventura and later are now supported (previously macOS 26 only). +- The Liquid Glass UI gracefully falls back on older macOS versions. + +## [1.0] - 2026-05-04 + +- First public release. diff --git a/README.md b/README.md index 7509fb5..c832181 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,11 @@ MacDirStat scans any folder and turns your filesystem into an interactive sunbur - **File type breakdown** — top file types with a searchable list of all types - **Move to Trash** — right-click any arc or row to trash it, chart refreshes instantly - **CSV export** — dump the full scan as a spreadsheet -- **Settings** — toggle haptic feedback on/off +- **Save and reopen scans** — write a completed scan to an `.mdscan` file and reopen it later as a read-only snapshot (no delete actions, no live watching) +- **Compare two scans** — diff the current scan against a saved one: added, removed, grown, and shrunk items, with a whole added or removed folder collapsing into one row (File > Compare With Saved Scan) +- **Auto-summarization** — dependency and cache folders (node_modules and the like) collapse into a single row with their total size and file count, instead of a node per file +- **Full Disk Access, made unambiguous** — drag the app's own icon straight into the permission list, since macOS grants access to one exact copy and a file picker makes it easy to add the wrong one +- **Settings** — appearance (units, color scheme, default tab), file scanning (hidden files, excluded folders), behaviour (auto-scan on launch, real-time monitoring), a Permissions section with live Full Disk Access status, and haptic feedback ## Screenshots @@ -84,11 +88,13 @@ log. ## Tech -Pure Swift + SwiftUI — no Electron, no web views, no dependencies. +Pure Swift + SwiftUI — no Electron, no web views; Sparkle for auto-updates is the only dependency. | Layer | What | |---|---| -| Scanner | POSIX `opendir`/`fstatat` with async task groups — parallel, cancellable | +| Scanner | `getattrlistbulk(2)` batch enumeration (name + metadata per syscall), with a `readdir` fallback and a bounded worker pool draining an iterative work queue — parallel, cancellable | +| Store | Flat struct-of-arrays `FileTree` — contiguous records, no per-node heap object, paths reconstructed on demand | +| Auto-summarization | Dependency/cache folders (`node_modules` and similar) collapse into one summary node with a deep size and file count, instead of a node per file | | Layout | Custom sunburst partition algorithm (band-width from view size) | | Renderer | SwiftUI `Canvas` — draws 1,000+ arcs at 30 fps | | Haptics | `NSHapticFeedbackManager` — intensity scales with file size | diff --git a/docs/adr/0002-getattrlistbulk-batch-enumeration.md b/docs/adr/0002-getattrlistbulk-batch-enumeration.md new file mode 100644 index 0000000..cd74c3a --- /dev/null +++ b/docs/adr/0002-getattrlistbulk-batch-enumeration.md @@ -0,0 +1,72 @@ +# 0002. Enumerate directories with getattrlistbulk, not per-entry stat + +## Status + +Accepted, 2026-07-24. + +## Context + +The scanner's previous directory walk did what most POSIX code does: open a +directory, call `readdir` to get one entry's name, then call `fstatat` on +that name to get its metadata (size, type, inode). That is two syscalls per +entry, and on a directory with thousands of files (a `node_modules`, a +Photos library, a build output folder) that means thousands of syscalls +just to enumerate one directory, before any of the actual scanning work +happens. + +Traversal itself compounded the problem: each directory spawned a new +recursive task for every subdirectory it found, with no cap. On a wide, +shallow tree (again, `node_modules` is the canonical case) this could fan +out into far more concurrent tasks than the machine has cores to run them +on, all contending for the same thread pool. + +macOS offers `getattrlistbulk(2)`, a single syscall that returns a batch of +directory entries with their names and requested attributes (size, type, +inode, flags) together, amortizing the syscall cost across the whole batch +instead of paying it per entry. Not every filesystem or mount supports it, +though (some network and legacy filesystems don't), so it cannot be the +only path. + +## Decision + +Replace the `readdir`/`fstatat` pair with `getattrlistbulk(2)` as the +primary enumeration path in `BulkDirectoryEnumerator`, requesting name and +all previously-`fstatat`'d metadata in one call per batch. Keep a +`readdir`-based fallback path for volumes that reject `getattrlistbulk` +(detected by the syscall itself failing, not by pre-checking filesystem +type), so no volume becomes unscannable. + +Change traversal from unbounded recursive task fan-out to an iterative work +queue drained by a bounded worker pool sized +`min(max(2, cores/2), 8)`: enough parallelism to keep disk and CPU busy, +capped so a wide directory tree cannot spawn more concurrent work than the +machine can usefully run. + +All existing scan semantics (allocated vs. logical size, symlink/hidden/ +exclusion skips, mount-point and hardlink dedup, access-denied surfacing) +are preserved identically in both the bulk and fallback paths, and a +fingerprint test scans the same fixture tree once with bulk enumeration and +once with the fallback forced, asserting the resulting trees are identical +byte-for-byte in every field that matters. The public `FileScanner`/ +`FSNode`/`ScanProgress` API is unchanged, so this is purely an internal +rewrite. + +Measured on `/Applications`: 1.63s with `getattrlistbulk` vs. 2.02s with +the `readdir` fallback, on the same machine and directory. + +## Consequences + +- Two enumeration code paths now exist and must be kept in sync; the + parity test is the thing that makes that safe; anyone changing what a + scan captures (a new attribute, a new skip rule) must update both paths + or the parity test will (correctly) fail. +- The bounded worker pool means a scan of a very wide, shallow tree no + longer transiently spikes to thousands of concurrent tasks, at a small + cost in wall-clock time on such trees compared to fully unbounded + fan-out; this trade was accepted because unbounded fan-out was also + the root cause of thread-pool exhaustion bugs fixed later on this branch + (see the async summary-walk fix). +- `getattrlistbulk` returning inode/device information here is what later + interacted badly with macOS firmlinks (see ADR 0004): the enumerator's + own report of an entry's identity cannot be trusted for firmlinked + directories, only what you get back from actually opening them. diff --git a/docs/adr/0003-flat-file-tree-store.md b/docs/adr/0003-flat-file-tree-store.md new file mode 100644 index 0000000..6411012 --- /dev/null +++ b/docs/adr/0003-flat-file-tree-store.md @@ -0,0 +1,92 @@ +# 0003. Replace the FSNode class tree with a flat struct-of-arrays store + +## Status + +Accepted, 2026-07-24. + +## Context + +The app's in-memory representation of a scan was a tree of `FSNode` class +instances: one heap-allocated object per file and folder, each holding a +`URL`, a `UUID`, a size, a type, an array of child references, and a weak +parent reference. That is a natural way to model a filesystem tree, and it +made the early implementation straightforward, but it does not scale well: + +- Every file and folder in a scan is a separate heap allocation and a + separate reference-counted object, which is slow to allocate in bulk and + slow for ARC to tear down when a scan is discarded or replaced. +- Sorting children by size, walking the tree for layout, or searching by + path all chase pointers through scattered heap objects instead of + scanning contiguous memory, which is unfriendly to the CPU cache on + scans with hundreds of thousands of nodes. +- A `URL` and a `UUID` per node is more retained state per node than a + treemap actually needs to render and interact with a chart: identity for + UI purposes doesn't require a real `URL`, and the path can always be + rebuilt from parent links when it's actually needed (opening in Finder, + computing a delete target). + +On the auto-summarization work (ADR-adjacent, not its own ADR) this became +a hard blocker: summarizing away a folder full of tiny files back into +individual `FSNode`s only for them to be discarded again the moment the +summary collapses them was pure waste, both in time and peak memory. + +## Decision + +Replace `FSNode` as the model layer's tree representation with `FileTree`, +a flat struct-of-arrays store: a `FileNodeRecord` per node holds only what +the UI and business logic actually need (name, size, type, safety level, +hardlink ref), stored in a single contiguous `records` array. Parent/child +relationships are represented as an index into that array (a +`UInt32`-style parent index per record) plus a `childStart`/`childCount` +span describing a contiguous run of children, rather than child arrays or +object references. `FileNode` is a lightweight handle (effectively an +index plus a reference to the owning `FileTree`) that call sites use in +place of the old class reference; paths are reconstructed on demand by +walking parent indices up to the root, not stored per node. + +`FSNode` is kept, deliberately, as the scanner's internal build type: the +scanner still constructs a tree incrementally while walking the +filesystem, where a class tree with real parent/child object references is +the natural shape for that job (nodes get created, reparented, and +summarized away before the scan finishes). Once a scan completes, it is +converted once into a `FileTree`, which is what the rest of the app (view +model, layout, duplicates, compare, archive) operates on from then on. +Everything downstream of the scanner only ever sees `FileTree`. + +The split that makes later live-refresh and hardlink work tractable is +that a `FileTree`'s topology (the parent indices and child spans) is +treated as effectively immutable once built, and reshaped only through a +small number of whole-operation functions (`replacingSubtree`, +`removingSubtrees`, `promotingSurvivingTwin`) that rebuild the affected +spans and ancestor sizes in one pass, rather than through ad hoc mutation +of individual records. Per-record fields that do change in place, size +above all, are mutated directly in the `records` array without touching +topology at all. + +## Consequences + +- Real numbers from the migration: `swift build`/`swift test` stayed fully + green throughout (the migration touched essentially every test file), + and the store change alone, independent of auto-summarization, is what + made scans of hundreds of thousands of nodes practical to hold in memory + and sort/lay out responsively. +- Every consumer of the tree (layout, duplicates, compare, archive, + live-refresh) works in terms of `FileNode` handles and `FileTree` + queries, not object references; code that assumes it can hold a stable + reference to "a node" across a topology-mutating operation is wrong by + construction; handles must be re-resolved after any `replacingSubtree`/ + `removingSubtrees` call. +- The immutable-topology/mutable-fields split is what several later fixes + on this branch depend on being respected: the synthetic "Hidden & + Unreadable Space" node bug (records/parentIndex extended but childStart/ + childCount left one entry short) and the ancestor re-sort gap after + hardlink removal were both violations of this split slipping through + because a new code path mutated part of the arrays without going through + the shared span-rebuilding helpers. `FileTree`'s initializer now asserts + all per-node arrays are the same length specifically to catch the first + kind of gap at construction time instead of at some distant read. +- `.mdscan` archives serialize `FileNodeRecord`/`HardLinkRef`/ + `SafetyLevel` directly (they're `Codable`), which is only straightforward + because they are plain value types with no object graph to break out of; + this would have been considerably more awkward to do safely with the old + `FSNode` class tree. diff --git a/docs/adr/0004-directory-identity-from-the-opened-fd.md b/docs/adr/0004-directory-identity-from-the-opened-fd.md new file mode 100644 index 0000000..803f16f --- /dev/null +++ b/docs/adr/0004-directory-identity-from-the-opened-fd.md @@ -0,0 +1,92 @@ +# 0004. Decide directory identity, mount boundaries, and dedup from the opened directory, never the parent listing + +## Status + +Accepted, 2026-07-27. + +## Context + +The bulk enumerator (ADR 0002) added a TOCTOU (time-of-check-to-time-of-use) +guard: when a parent directory's listing reports an entry's `(dev, ino)`, +and the scanner later opens that entry to recurse into it, the guard +compared the opened directory's `(dev, ino)` against what the listing had +claimed and treated a mismatch as evidence that the entry changed out from +under the scan (swapped for a symlink, replaced, etc.), and dropped it. +That is a reasonable-sounding safety check, and it shipped as part of the +same commit that introduced batch enumeration. + +It broke nearly the entire scan. Running MacDirStat on `/` reported +**11.8 GB used** instead of the real **479.3 GB** on the same machine. +`/Users`, `/Applications`, `/Library`, `/opt`, `/private`, `/Volumes`, and +`/cores` all came back as 0 bytes, and the missing ~818 GB was silently +folded into the synthetic "Hidden & Unreadable Space" node, so nothing +about the failure looked like an error, it just looked like a very +restricted disk. Since Users and Applications carry essentially all of a +typical Mac's non-system data, this was not an edge case: it was +functionally "the scanner doesn't work." + +The cause: every one of those paths is a macOS firmlink. Since Catalina, +the boot volume is split into a read-only system volume and a writable +data volume, joined by firmlinks, kernel-level directory aliases that +present as ordinary directories in one namespace but resolve to a +different location, and a different inode, on the other volume. A parent +listing's `getattrlistbulk`/`readdir` entry for `/Users` reports the +firmlink's own identity; opening `/Users` and asking the open file +descriptor for its identity reports the Data volume's identity underneath. +Those two `(dev, ino)` pairs are supposed to disagree, that disagreement is +what a firmlink *is*, not evidence of a race. The TOCTOU guard, applied at +the listing level, could not tell a real swapped-directory attack apart +from completely ordinary firmlink resolution, and rejected both. + +The existing bulk-vs-fallback parity test (ADR 0002) could not have caught +this: its fixture is a temporary directory with no firmlinks or mount +points, so both enumeration paths agreed with each other and both were +equally wrong about the real world. + +## Decision + +Directory identity, the mount-boundary check (is this entry on a different +volume than its parent, and should the scan cross it), and alias +deduplication (has this physical location already been visited under a +different path, e.g. `/Users` vs. `/System/Volumes/Data/Users`) are now all +decided from the *opened* directory, never from the parent listing's report +of it. Concretely: open the directory first, ask the open descriptor for +its `(dev, ino)`, and make every identity-sensitive decision from that +value. The listing is used only to know what names exist and to get a +cheap first-pass size/type hint; it is never treated as authoritative about +identity. + +This is not just a bug fix but a strictly more correct model for two +things that were already true before firmlinks entered the picture: + +- `/dev` enumerates with the root filesystem's device number, but is + actually `devfs`; only the opened identity reveals that. +- Firmlink aliases only collide (resolve to the same underlying location) + *after* opening; comparing pre-open identities can never detect the + aliasing that alias-dedup exists to catch in the first place. + +A regression test scans the real startup volume shallowly and asserts +`/Users` (or the equivalent well-known firmlink on the test machine) is +reported with nonzero size; it fails against the pre-fix commit with +`/Users` at 0 bytes, and passes after. + +## Consequences + +- Any future scanner change that adds a new identity-sensitive decision + (a new dedup rule, a new mount-crossing rule, a new "have I seen this + before" check) must key it off the identity obtained after `open()`, + not off anything read from a parent directory's listing. This is the + single most important invariant in the scanner: violating it silently + drops firmlinked data with no error, no crash, and a plausible-looking + (merely wrong) total. +- There is deliberately no TOCTOU guard left at the listing level anymore. + A real swap-after-list race is not defended against by comparing + listing-time identity to open-time identity, because that comparison + cannot distinguish an attack from a firmlink. If TOCTOU hardening is + wanted again, it needs a different signal than pre-open vs. post-open + identity. +- Whoever touches this code next should re-run the shallow `/` scan + regression test (not just the synthetic-fixture parity test) before + trusting any change to enumeration, identity, or dedup logic; a synthetic + temp-directory fixture will never exercise firmlinks and cannot catch + this class of bug. From a2068b5b4caece088666014a4d290d4af3d26b36 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Wed, 5 Aug 2026 12:28:02 +0300 Subject: [PATCH 40/41] docs: drop em dashes from the lines this branch added to the README --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c832181..7762b70 100644 --- a/README.md +++ b/README.md @@ -37,11 +37,11 @@ MacDirStat scans any folder and turns your filesystem into an interactive sunbur - **File type breakdown** — top file types with a searchable list of all types - **Move to Trash** — right-click any arc or row to trash it, chart refreshes instantly - **CSV export** — dump the full scan as a spreadsheet -- **Save and reopen scans** — write a completed scan to an `.mdscan` file and reopen it later as a read-only snapshot (no delete actions, no live watching) -- **Compare two scans** — diff the current scan against a saved one: added, removed, grown, and shrunk items, with a whole added or removed folder collapsing into one row (File > Compare With Saved Scan) -- **Auto-summarization** — dependency and cache folders (node_modules and the like) collapse into a single row with their total size and file count, instead of a node per file -- **Full Disk Access, made unambiguous** — drag the app's own icon straight into the permission list, since macOS grants access to one exact copy and a file picker makes it easy to add the wrong one -- **Settings** — appearance (units, color scheme, default tab), file scanning (hidden files, excluded folders), behaviour (auto-scan on launch, real-time monitoring), a Permissions section with live Full Disk Access status, and haptic feedback +- **Save and reopen scans**: write a completed scan to an `.mdscan` file and reopen it later as a read-only snapshot (no delete actions, no live watching) +- **Compare two scans**: diff the current scan against a saved one: added, removed, grown, and shrunk items, with a whole added or removed folder collapsing into one row (File > Compare With Saved Scan) +- **Auto-summarization**: dependency and cache folders (node_modules and the like) collapse into a single row with their total size and file count, instead of a node per file +- **Full Disk Access, made unambiguous**: drag the app's own icon straight into the permission list, since macOS grants access to one exact copy and a file picker makes it easy to add the wrong one +- **Settings**: appearance (units, color scheme, default tab), file scanning (hidden files, excluded folders), behaviour (auto-scan on launch, real-time monitoring), a Permissions section with live Full Disk Access status, and haptic feedback ## Screenshots @@ -88,12 +88,12 @@ log. ## Tech -Pure Swift + SwiftUI — no Electron, no web views; Sparkle for auto-updates is the only dependency. +Pure Swift + SwiftUI. No Electron, no web views; Sparkle for auto-updates is the only dependency. | Layer | What | |---|---| -| Scanner | `getattrlistbulk(2)` batch enumeration (name + metadata per syscall), with a `readdir` fallback and a bounded worker pool draining an iterative work queue — parallel, cancellable | -| Store | Flat struct-of-arrays `FileTree` — contiguous records, no per-node heap object, paths reconstructed on demand | +| Scanner | `getattrlistbulk(2)` batch enumeration (name + metadata per syscall), with a `readdir` fallback and a bounded worker pool draining an iterative work queue, parallel and cancellable | +| Store | Flat struct-of-arrays `FileTree`: contiguous records, no per-node heap object, paths reconstructed on demand | | Auto-summarization | Dependency/cache folders (`node_modules` and similar) collapse into one summary node with a deep size and file count, instead of a node per file | | Layout | Custom sunburst partition algorithm (band-width from view size) | | Renderer | SwiftUI `Canvas` — draws 1,000+ arcs at 30 fps | From 7cb2fa2c1ec3e14d677f58d79be50cac70b314f0 Mon Sep 17 00:00:00 2001 From: Ti-03 Date: Wed, 5 Aug 2026 12:54:31 +0300 Subject: [PATCH 41/41] ci(release): skip signing cleanly when no credentials are configured The Developer ID private key is deliberately kept on the maintainer's machine rather than uploaded to repository secrets, so releases are built and notarized locally and published by hand. Without a guard, tagging would start the release job and fail partway through, which is close to the failure that produced the unusable v1.2.0 asset. The job now probes for the signing secret first: present, it builds, notarizes, verifies and publishes as before; absent, it runs the tests, leaves a notice explaining that the DMG is published manually, and stops. --- .github/workflows/release.yml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7f5687b..705edbd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,10 +42,30 @@ jobs: - name: Run tests before shipping anything run: swift test + # Signing and notarization need credentials that intentionally may not + # be present: the Developer ID private key is kept on the maintainer's + # machine rather than uploaded here, and releases are then built and + # published by hand. Without this probe a tag would start the job and + # fail partway, which is the failure mode that produced the unusable + # v1.2.0 asset. When the secrets are absent the job runs the tests and + # stops cleanly instead. + - name: Can this runner sign? + id: caps + env: + MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} + run: | + if [ -n "$MACOS_CERTIFICATE" ]; then + echo "can_sign=true" >> "$GITHUB_OUTPUT" + else + echo "can_sign=false" >> "$GITHUB_OUTPUT" + echo "::notice::No signing secrets configured. Tests ran; publish the notarized DMG manually. See the secret list at the top of this workflow to automate it." + fi + # The signing identity has to live in a keychain the build can see. A # dedicated, throwaway keychain keeps it out of the default one and # disappears with the runner. - name: Import Developer ID certificate + if: steps.caps.outputs.can_sign == 'true' env: MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} @@ -64,6 +84,7 @@ jobs: rm -f /tmp/cert.p12 - name: Archive the app + if: steps.caps.outputs.can_sign == 'true' run: | set -euo pipefail xcodebuild -project MacDirStat.xcodeproj \ @@ -74,6 +95,7 @@ jobs: archive - name: Export with Developer ID + if: steps.caps.outputs.can_sign == 'true' env: NOTARY_TEAM_ID: ${{ secrets.NOTARY_TEAM_ID }} run: | @@ -100,6 +122,7 @@ jobs: # The Applications symlink is what makes the familiar "drag me across" # install window work. - name: Build the DMG + if: steps.caps.outputs.can_sign == 'true' env: MACOS_SIGNING_IDENTITY: ${{ secrets.MACOS_SIGNING_IDENTITY }} run: | @@ -114,6 +137,7 @@ jobs: codesign --sign "$MACOS_SIGNING_IDENTITY" --timestamp "DirStat-${VERSION}.dmg" - name: Notarize and staple + if: steps.caps.outputs.can_sign == 'true' env: NOTARY_APPLE_ID: ${{ secrets.NOTARY_APPLE_ID }} NOTARY_TEAM_ID: ${{ secrets.NOTARY_TEAM_ID }} @@ -132,6 +156,7 @@ jobs: # ticket Gatekeeper accepts, and an app inside the image whose signature # survives with its symlinks intact. - name: Verify the DMG the way a user receives it + if: steps.caps.outputs.can_sign == 'true' run: | set -euo pipefail VERSION="${GITHUB_REF_NAME#v}" @@ -143,6 +168,7 @@ jobs: hdiutil detach /tmp/dmgcheck - name: Install cosign + if: steps.caps.outputs.can_sign == 'true' uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3 # Keyless signing: cosign exchanges the job's OIDC token for a @@ -151,6 +177,7 @@ jobs: # writes signature + certificate + tlog proof as one .bundle file # (the old --output-signature/--output-certificate flags are gone). - name: Sign the DMG (Sigstore keyless) + if: steps.caps.outputs.can_sign == 'true' run: | set -euo pipefail VERSION="${GITHUB_REF_NAME#v}" @@ -159,6 +186,7 @@ jobs: "DirStat-${VERSION}.dmg" - name: Create GitHub release with the signed DMG + if: steps.caps.outputs.can_sign == 'true' env: GH_TOKEN: ${{ github.token }} run: | @@ -171,5 +199,5 @@ jobs: "DirStat-${VERSION}.dmg.cosign.bundle" - name: Remove the temporary keychain - if: always() + if: always() && steps.caps.outputs.can_sign == 'true' run: security delete-keychain build.keychain || true