diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c54656e..705edbd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,21 +12,163 @@ permissions: contents: write id-token: write +# Ships the actual macOS app, Developer ID signed and notarized, as a DMG. +# +# This used to package `swift build -c release` output instead: a bare SwiftPM +# executable, unsigned and un-notarized, which is not something anyone can +# install by double-clicking. That is what v1.2.0 shipped (a 560 KB zip) and +# what issue #22 reported. +# +# A DMG rather than a zip is deliberate. The app embeds Sparkle.framework, +# which contains symlinks, and messaging apps, webmail and third-party +# unarchivers routinely flatten symlinks out of a zip. That breaks the +# framework, invalidates the signature, and makes macOS report the app as +# "damaged". A disk image carries the bundle byte for byte. +# +# Required repository secrets: +# MACOS_CERTIFICATE Developer ID Application cert, .p12, base64 encoded +# MACOS_CERTIFICATE_PWD password for that .p12 +# MACOS_SIGNING_IDENTITY e.g. "Developer ID Application: Name (TEAMID)" +# KEYCHAIN_PASSWORD any throwaway string for the temporary keychain +# NOTARY_APPLE_ID Apple ID used for notarization +# NOTARY_TEAM_ID Apple Developer team ID +# NOTARY_PASSWORD app-specific password for that Apple ID jobs: build-sign-release: runs-on: macos-26 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - name: Build release binary - run: swift build -c release + - name: Run tests before shipping anything + run: swift test - - name: Package artifact + # Signing and notarization need credentials that intentionally may not + # be present: the Developer ID private key is kept on the maintainer's + # machine rather than uploaded here, and releases are then built and + # published by hand. Without this probe a tag would start the job and + # fail partway, which is the failure mode that produced the unusable + # v1.2.0 asset. When the secrets are absent the job runs the tests and + # stops cleanly instead. + - name: Can this runner sign? + id: caps + env: + MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} + run: | + if [ -n "$MACOS_CERTIFICATE" ]; then + echo "can_sign=true" >> "$GITHUB_OUTPUT" + else + echo "can_sign=false" >> "$GITHUB_OUTPUT" + echo "::notice::No signing secrets configured. Tests ran; publish the notarized DMG manually. See the secret list at the top of this workflow to automate it." + fi + + # The signing identity has to live in a keychain the build can see. A + # dedicated, throwaway keychain keeps it out of the default one and + # disappears with the runner. + - name: Import Developer ID certificate + if: steps.caps.outputs.can_sign == 'true' + env: + MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} + MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} + KEYCHAIN_PASSWORD: ${{ secrets.KEYCHAIN_PASSWORD }} + run: | + set -euo pipefail + echo "$MACOS_CERTIFICATE" | base64 --decode > /tmp/cert.p12 + security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain + security default-keychain -s build.keychain + security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain + security set-keychain-settings -lut 3600 build.keychain + security import /tmp/cert.p12 -k build.keychain \ + -P "$MACOS_CERTIFICATE_PWD" -T /usr/bin/codesign + security set-key-partition-list -S apple-tool:,apple:,codesign: \ + -s -k "$KEYCHAIN_PASSWORD" build.keychain + rm -f /tmp/cert.p12 + + - name: Archive the app + if: steps.caps.outputs.can_sign == 'true' + run: | + set -euo pipefail + xcodebuild -project MacDirStat.xcodeproj \ + -scheme MacDirStat \ + -configuration Release \ + -destination 'platform=macOS' \ + -archivePath "$RUNNER_TEMP/DirStat.xcarchive" \ + archive + + - name: Export with Developer ID + if: steps.caps.outputs.can_sign == 'true' + env: + NOTARY_TEAM_ID: ${{ secrets.NOTARY_TEAM_ID }} run: | - BIN=$(swift build -c release --show-bin-path) - ditto -c -k --keepParent "$BIN/MacDirStat" "MacDirStat-${GITHUB_REF_NAME}.zip" + set -euo pipefail + cat > "$RUNNER_TEMP/export.plist" < + + + + method + developer-id + teamID + ${NOTARY_TEAM_ID} + signingStyle + automatic + + + EOF + xcodebuild -exportArchive \ + -archivePath "$RUNNER_TEMP/DirStat.xcarchive" \ + -exportOptionsPlist "$RUNNER_TEMP/export.plist" \ + -exportPath "$RUNNER_TEMP/export" + + # The Applications symlink is what makes the familiar "drag me across" + # install window work. + - name: Build the DMG + if: steps.caps.outputs.can_sign == 'true' + env: + MACOS_SIGNING_IDENTITY: ${{ secrets.MACOS_SIGNING_IDENTITY }} + run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" + mkdir -p "$RUNNER_TEMP/dmgroot" + ditto "$RUNNER_TEMP/export/DirStat.app" "$RUNNER_TEMP/dmgroot/DirStat.app" + ln -sf /Applications "$RUNNER_TEMP/dmgroot/Applications" + hdiutil create -volname "DirStat $VERSION" \ + -srcfolder "$RUNNER_TEMP/dmgroot" \ + -ov -format UDZO "DirStat-${VERSION}.dmg" + codesign --sign "$MACOS_SIGNING_IDENTITY" --timestamp "DirStat-${VERSION}.dmg" + + - name: Notarize and staple + if: steps.caps.outputs.can_sign == 'true' + env: + NOTARY_APPLE_ID: ${{ secrets.NOTARY_APPLE_ID }} + NOTARY_TEAM_ID: ${{ secrets.NOTARY_TEAM_ID }} + NOTARY_PASSWORD: ${{ secrets.NOTARY_PASSWORD }} + run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" + xcrun notarytool submit "DirStat-${VERSION}.dmg" \ + --apple-id "$NOTARY_APPLE_ID" \ + --team-id "$NOTARY_TEAM_ID" \ + --password "$NOTARY_PASSWORD" \ + --wait + xcrun stapler staple "DirStat-${VERSION}.dmg" + + # Proves the artifact is installable before it is published: a stapled + # ticket Gatekeeper accepts, and an app inside the image whose signature + # survives with its symlinks intact. + - name: Verify the DMG the way a user receives it + if: steps.caps.outputs.can_sign == 'true' + run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" + xcrun stapler validate "DirStat-${VERSION}.dmg" + spctl -a -vvv -t open --context context:primary-signature "DirStat-${VERSION}.dmg" + hdiutil attach "DirStat-${VERSION}.dmg" -nobrowse -mountpoint /tmp/dmgcheck + codesign --verify --deep --strict --verbose=2 /tmp/dmgcheck/DirStat.app + spctl -a -vvv -t exec /tmp/dmgcheck/DirStat.app + hdiutil detach /tmp/dmgcheck - name: Install cosign + if: steps.caps.outputs.can_sign == 'true' uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3 # Keyless signing: cosign exchanges the job's OIDC token for a @@ -34,18 +176,28 @@ jobs: # Rekor transparency log. --yes acknowledges that upload. cosign v3 # writes signature + certificate + tlog proof as one .bundle file # (the old --output-signature/--output-certificate flags are gone). - - name: Sign artifact (Sigstore keyless) + - name: Sign the DMG (Sigstore keyless) + if: steps.caps.outputs.can_sign == 'true' run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" cosign sign-blob --yes \ - --bundle "MacDirStat-${GITHUB_REF_NAME}.zip.cosign.bundle" \ - "MacDirStat-${GITHUB_REF_NAME}.zip" + --bundle "DirStat-${VERSION}.dmg.cosign.bundle" \ + "DirStat-${VERSION}.dmg" - - name: Create GitHub release with signed artifact + - name: Create GitHub release with the signed DMG + if: steps.caps.outputs.can_sign == 'true' env: GH_TOKEN: ${{ github.token }} run: | + set -euo pipefail + VERSION="${GITHUB_REF_NAME#v}" gh release create "$GITHUB_REF_NAME" \ --title "MacDirStat $GITHUB_REF_NAME" \ --generate-notes \ - "MacDirStat-${GITHUB_REF_NAME}.zip" \ - "MacDirStat-${GITHUB_REF_NAME}.zip.cosign.bundle" + "DirStat-${VERSION}.dmg" \ + "DirStat-${VERSION}.dmg.cosign.bundle" + + - name: Remove the temporary keychain + if: always() && steps.caps.outputs.can_sign == 'true' + run: security delete-keychain build.keychain || true diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ff2a6fe --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,115 @@ +# Changelog + +All notable changes to MacDirStat are documented here. Format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +## [1.3.0] - 2026-08-04 + +A performance-focused release: the scanner, the in-memory tree, and live +refresh were all rebuilt for speed on very large volumes, plus new ways to +save, reopen, and compare scans. + +### Added + +- **Auto-summarization of dependency and cache folders**: directories like + `node_modules` (and anything else that looks like thousands of tiny + files, by heuristic) collapse into a single summary row carrying the + total size and file count, instead of building a chart node for every + file inside. On a real Projects folder this cut a scan from 16.0s / + 910,550 nodes to 8.6s / 104,551 nodes, with byte-identical totals. +- **Save and reopen scans**: File > Save Scan... writes the current scan to + an `.mdscan` file; File > Open Scan... reopens it later as a read-only + snapshot (no delete actions, no live watching), with a banner showing + when it was captured. Opening a corrupted or tampered file surfaces an + error instead of crashing. +- **Compare two scans**: File > Compare With Saved Scan shows what changed + between now and a saved snapshot: added, removed, grown, shrunk, and + files that were replaced by a folder (or vice versa) at the same path. A + whole added or removed directory collapses into a single row instead of + listing every file inside it. +- **Drag-to-grant Full Disk Access**: the guided permission sheet now shows + the running app's own icon as a draggable tile, so granting access can't + accidentally target the wrong build sitting in a file picker (macOS ties + the grant to one exact copy). A "Show this app in Finder" fallback covers + drag-and-drop from a Finder window instead. +- **Permissions section in Settings**: check and fix Full Disk Access + directly from Settings, with a live coloured-dot status, the same + drag-to-grant tile, and a direct link to the right System Settings pane. + +### Changed + +- The scanner now enumerates directories with `getattrlistbulk(2)`, + reading names and metadata in one batched syscall per directory instead + of one `readdir` plus one `fstatat` per entry, with a `readdir` fallback + for filesystems that don't support it. Traversal itself moved from + unbounded recursive fan-out to a bounded worker pool, so a scan no longer + spawns more concurrent work than the machine can use. +- The in-memory scan result is now a flat, contiguous store instead of a + tree of individual objects per file and folder, which noticeably lowers + memory use and speeds up sorting and layout on very large scans. +- Live refresh (the automatic re-scan while a folder is open and being + watched) now patches only the part of the tree that actually changed + instead of rescanning the whole root, so background file activity no + longer causes a visible full reload. +- Move to Trash now removes the deleted item from the current scan + directly instead of triggering a full rescan, so deleting from the + Duplicates view no longer bounces you back to the Treemap tab. Fixes + [#5](https://github.com/Ti-03/MacDirStat/issues/5). + +### Fixed + +- Scanning `/` (or any volume root) could dramatically under-report disk + usage, in one case showing 11.8 GB instead of the real 479.3 GB, because + `/Users`, `/Applications`, `/Library`, `/opt`, `/private`, `/Volumes` and + `/cores` are macOS firmlinks and were being silently dropped as 0 bytes. + Directory identity, mount-boundary checks, and alias de-duplication are + now all decided from the opened directory instead of from what the + parent folder's listing claimed, which is the only place a firmlink + resolves correctly. +- A background summarization pass (used for `node_modules`-style folders) + could, under load, block every available concurrency thread at once and + wedge a scan so it never finished and could not be stopped. It is now + fully asynchronous end to end. +- Scanning a volume root and then letting a live file change happen could + crash the app outright ("Index out of range") because of a bookkeeping + gap in the synthetic "Hidden & Unreadable Space" entry. +- A live filesystem change that couldn't be folded into the current scan + incrementally could escalate into a full rescan that cleared the + Treemap mid-render and then repeated forever. Such changes are now + skipped with a "Rescan" button offered instead, rather than looping. +- Comparing two scans now reports a file replaced by a folder (or a folder + replaced by a file) at the same path as a change, instead of dropping it + from the results entirely. +- Fixed several cases where disk usage from hardlinked files could be + under- or double-counted after trashing a file, after a live refresh, or + after an external process (Finder, `rm`, a build tool) deleted the copy + that was carrying the reported size, including ordering glitches in + ancestor folders left over from the fix. +- The Full Disk Access sheet no longer claims "Access granted!" when the + most recent scan still hit denied folders; it only shows success for a + grant it actually watched happen while it was open. +- GitHub Releases now include a proper `.dmg` installer image alongside + the signed zip, instead of shipping only a bare executable archive. + Fixes [#22](https://github.com/Ti-03/MacDirStat/issues/22). + +Test suite grew from 50 to 142 tests across this cycle, all passing. + +## [1.2.0] - 2026-07-24 + +- MkDocs Material documentation site and the first Architecture Decision + Record. +- Unit tests and a GitHub Actions CI workflow. +- Dependabot, Actions pinned to commit SHAs, and a least-privilege CI + token (OpenSSF Scorecard fixes), plus a SECURITY.md. +- Sigstore-signed release artifacts (keyless cosign). +- CONTRIBUTING, CODE_OF_CONDUCT, GOVERNANCE, issue templates, and a + license notice for the community. + +## [1.1] - 2026-05-10 + +- macOS 13 Ventura and later are now supported (previously macOS 26 only). +- The Liquid Glass UI gracefully falls back on older macOS versions. + +## [1.0] - 2026-05-04 + +- First public release. diff --git a/MacDirStat.xcodeproj/project.pbxproj b/MacDirStat.xcodeproj/project.pbxproj index 7f0bde7..b3b3ce2 100644 --- a/MacDirStat.xcodeproj/project.pbxproj +++ b/MacDirStat.xcodeproj/project.pbxproj @@ -16,6 +16,7 @@ 3881FC74433C93187BF941AF /* FSNode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 35F5AEBBF9DECB5CB793339A /* FSNode.swift */; }; 3B190FDF9A024126B4ED2872 /* GlassWindow.swift in Sources */ = {isa = PBXBuildFile; fileRef = 08B5316639CA4AC8990FBDD4 /* GlassWindow.swift */; }; D1E2F3A4B5C607182930AABD /* GlassCompat.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F607182930AABC /* GlassCompat.swift */; }; + FDA0FDA0FDA0FDA0FDA00002 /* FullDiskAccessSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = FDA0FDA0FDA0FDA0FDA00001 /* FullDiskAccessSheet.swift */; }; 3F67AACE53A6C32467A5191C /* TreemapRenderer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 670402F6A7AEB43C1D764E04 /* TreemapRenderer.swift */; }; 7AE6B3CB6CDE22EAA3AA46B3 /* DuplicateDetector.swift in Sources */ = {isa = PBXBuildFile; fileRef = 42B58422D704C6E5265DA664 /* DuplicateDetector.swift */; }; 86B25923164A00AB10D72C9B /* ScanProgress.swift in Sources */ = {isa = PBXBuildFile; fileRef = DBCAD22DB54E68077B842421 /* ScanProgress.swift */; }; @@ -32,7 +33,16 @@ E5BB63BED418F08CF4727F72 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 783902530DE4EA8040806BC9 /* SettingsView.swift */; }; F81DF36922942E2F442814A3 /* TreemapLayout.swift in Sources */ = {isa = PBXBuildFile; fileRef = BAB4247FB8E3E3FB5AF053F9 /* TreemapLayout.swift */; }; FB3E660E0C4E74F2A9EFBEF0 /* FileScanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = DDD56FD4714436315629F989 /* FileScanner.swift */; }; + A1B2C3D4E5F600112233445A /* BulkDirectoryEnumerator.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F600112233445B /* BulkDirectoryEnumerator.swift */; }; + A1B2C3D4E5F600112233445C /* AtomicDirectorySummary.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F600112233445D /* AtomicDirectorySummary.swift */; }; FD11F6ED4164F05503706848 /* ExtensionListView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 987EDA67820F162556A7CA53 /* ExtensionListView.swift */; }; + 84449F67C7DF40C737411C80 /* FileTree.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1B5ADD782F265EBF4B1037EE /* FileTree.swift */; }; + A498AC088B1E3C6E1B255BC2 /* FileNode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9429B8A593856D8B12941F49 /* FileNode.swift */; }; + 26B861D7144BFB6B2FD00F6B /* FileNodeRecord.swift in Sources */ = {isa = PBXBuildFile; fileRef = F03CE9327807EE8357D3678B /* FileNodeRecord.swift */; }; + E7ED11C27A082009974B4706 /* FileTreeBuilder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4FB7871D4D4A1E73B7984396 /* FileTreeBuilder.swift */; }; + A7CECAA54EC4419CA4B847D2 /* ScanArchive.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5B0C663AD1A1417DA8FC6942 /* ScanArchive.swift */; }; + 4A4F4C3CFEF84A1B827AB6EE /* ScanComparison.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4542CEEA3CDE4292A6661596 /* ScanComparison.swift */; }; + 57FD4B40D3F541AD98E734A1 /* ComparisonView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1A060E7C6E1C405F81CE31FA /* ComparisonView.swift */; }; SPARK003000000000000003AA /* Sparkle in Frameworks */ = {isa = PBXBuildFile; productRef = SPARK002000000000000002AA /* Sparkle */; }; /* End PBXBuildFile section */ @@ -40,6 +50,7 @@ 074CBBA0222600D63C734574 /* ExtensionColorMap.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExtensionColorMap.swift; sourceTree = ""; }; 08B5316639CA4AC8990FBDD4 /* GlassWindow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GlassWindow.swift; sourceTree = ""; }; A1B2C3D4E5F607182930AABC /* GlassCompat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GlassCompat.swift; sourceTree = ""; }; + FDA0FDA0FDA0FDA0FDA00001 /* FullDiskAccessSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FullDiskAccessSheet.swift; sourceTree = ""; }; 2767B8AD8F6829C36A3576E6 /* DuplicatesView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DuplicatesView.swift; sourceTree = ""; }; 35F5AEBBF9DECB5CB793339A /* FSNode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FSNode.swift; sourceTree = ""; }; 42B58422D704C6E5265DA664 /* DuplicateDetector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DuplicateDetector.swift; sourceTree = ""; }; @@ -64,8 +75,17 @@ C40A6B24C517964825BA4B85 /* TreemapView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TreemapView.swift; sourceTree = ""; }; DBCAD22DB54E68077B842421 /* ScanProgress.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScanProgress.swift; sourceTree = ""; }; DDD56FD4714436315629F989 /* FileScanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileScanner.swift; sourceTree = ""; }; + A1B2C3D4E5F600112233445B /* BulkDirectoryEnumerator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BulkDirectoryEnumerator.swift; sourceTree = ""; }; + A1B2C3D4E5F600112233445D /* AtomicDirectorySummary.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AtomicDirectorySummary.swift; sourceTree = ""; }; EE11223344556677889900EE /* FileWatcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileWatcher.swift; sourceTree = ""; }; F6ADF08FD012926F8704A51C /* MacDirStat.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = MacDirStat.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 1B5ADD782F265EBF4B1037EE /* FileTree.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileTree.swift; sourceTree = ""; }; + 9429B8A593856D8B12941F49 /* FileNode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileNode.swift; sourceTree = ""; }; + F03CE9327807EE8357D3678B /* FileNodeRecord.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileNodeRecord.swift; sourceTree = ""; }; + 4FB7871D4D4A1E73B7984396 /* FileTreeBuilder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileTreeBuilder.swift; sourceTree = ""; }; + 5B0C663AD1A1417DA8FC6942 /* ScanArchive.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScanArchive.swift; sourceTree = ""; }; + 4542CEEA3CDE4292A6661596 /* ScanComparison.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ScanComparison.swift; sourceTree = ""; }; + 1A060E7C6E1C405F81CE31FA /* ComparisonView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ComparisonView.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -114,6 +134,14 @@ path = Duplicates; sourceTree = ""; }; + A240638C83684A03A61256D5 /* Comparison */ = { + isa = PBXGroup; + children = ( + 1A060E7C6E1C405F81CE31FA /* ComparisonView.swift */, + ); + path = Comparison; + sourceTree = ""; + }; 71D2E56247FE75C3DCE9B609 = { isa = PBXGroup; children = ( @@ -127,6 +155,8 @@ isa = PBXGroup; children = ( DDD56FD4714436315629F989 /* FileScanner.swift */, + A1B2C3D4E5F600112233445B /* BulkDirectoryEnumerator.swift */, + A1B2C3D4E5F600112233445D /* AtomicDirectorySummary.swift */, 35F5AEBBF9DECB5CB793339A /* FSNode.swift */, EE11223344556677889900EE /* FileWatcher.swift */, DBCAD22DB54E68077B842421 /* ScanProgress.swift */, @@ -134,6 +164,19 @@ path = Scanner; sourceTree = ""; }; + 794E4678AAB2F65C291BAFA3 /* Model */ = { + isa = PBXGroup; + children = ( + 1B5ADD782F265EBF4B1037EE /* FileTree.swift */, + 9429B8A593856D8B12941F49 /* FileNode.swift */, + F03CE9327807EE8357D3678B /* FileNodeRecord.swift */, + 4FB7871D4D4A1E73B7984396 /* FileTreeBuilder.swift */, + 5B0C663AD1A1417DA8FC6942 /* ScanArchive.swift */, + 4542CEEA3CDE4292A6661596 /* ScanComparison.swift */, + ); + path = Model; + sourceTree = ""; + }; 943FF7A0195AF9CFFDCE7D83 /* DirectoryTree */ = { isa = PBXGroup; children = ( @@ -146,6 +189,7 @@ isa = PBXGroup; children = ( 943FF7A0195AF9CFFDCE7D83 /* DirectoryTree */, + A240638C83684A03A61256D5 /* Comparison */, 546A8B7B0B821FBB727B5691 /* Duplicates */, B4F8EFC4604E6BCB45034AC5 /* ExtensionList */, A6644EB28BF6F3D30059D92F /* Shared */, @@ -182,6 +226,7 @@ 7F8FDBB44CE9624311841E36 /* ContentView.swift */, 08B5316639CA4AC8990FBDD4 /* GlassWindow.swift */, A1B2C3D4E5F607182930AABC /* GlassCompat.swift */, + FDA0FDA0FDA0FDA0FDA00001 /* FullDiskAccessSheet.swift */, 783902530DE4EA8040806BC9 /* SettingsView.swift */, ); path = App; @@ -235,6 +280,7 @@ AB8D408CB1881C666AAC0832 /* App */, 2401B46468064D5D254B09BA /* Duplicates */, D6C0DA3A2FC09B3404E91BF6 /* Layout */, + 794E4678AAB2F65C291BAFA3 /* Model */, CC11223344556677889900CC /* Safety */, 7332ECE572A5B7E9E9A8F57F /* Scanner */, EE1E649591DBDF61DE2E4396 /* ViewModels */, @@ -323,6 +369,7 @@ 3864E5104A08A42FF56D7880 /* ContentView.swift in Sources */, 3B190FDF9A024126B4ED2872 /* GlassWindow.swift in Sources */, D1E2F3A4B5C607182930AABD /* GlassCompat.swift in Sources */, + FDA0FDA0FDA0FDA0FDA00002 /* FullDiskAccessSheet.swift in Sources */, E5BB63BED418F08CF4727F72 /* SettingsView.swift in Sources */, 7AE6B3CB6CDE22EAA3AA46B3 /* DuplicateDetector.swift in Sources */, C001072776950E6399A45B18 /* ExtensionColorMap.swift in Sources */, @@ -331,6 +378,8 @@ AA11223344556677889900AA /* SafetyAnalyzer.swift in Sources */, DD11223344556677889900DD /* FileWatcher.swift in Sources */, FB3E660E0C4E74F2A9EFBEF0 /* FileScanner.swift in Sources */, + A1B2C3D4E5F600112233445A /* BulkDirectoryEnumerator.swift in Sources */, + A1B2C3D4E5F600112233445C /* AtomicDirectorySummary.swift in Sources */, 3881FC74433C93187BF941AF /* FSNode.swift in Sources */, 86B25923164A00AB10D72C9B /* ScanProgress.swift in Sources */, 114A022A9D8A4529EC91BC12 /* ScanViewModel.swift in Sources */, @@ -343,6 +392,13 @@ 09F2331AB12AD6C9FB63744A /* HapticEngine.swift in Sources */, 3F67AACE53A6C32467A5191C /* TreemapRenderer.swift in Sources */, 20E67F1EEADF74D28CDF6191 /* TreemapView.swift in Sources */, + 84449F67C7DF40C737411C80 /* FileTree.swift in Sources */, + A498AC088B1E3C6E1B255BC2 /* FileNode.swift in Sources */, + 26B861D7144BFB6B2FD00F6B /* FileNodeRecord.swift in Sources */, + E7ED11C27A082009974B4706 /* FileTreeBuilder.swift in Sources */, + A7CECAA54EC4419CA4B847D2 /* ScanArchive.swift in Sources */, + 4A4F4C3CFEF84A1B827AB6EE /* ScanComparison.swift in Sources */, + 57FD4B40D3F541AD98E734A1 /* ComparisonView.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -358,7 +414,7 @@ "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 2; + CURRENT_PROJECT_VERSION = 5; DEVELOPMENT_TEAM = L4UH9K7AW4; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = NO; @@ -369,7 +425,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 13.0; - MARKETING_VERSION = 1.1; + MARKETING_VERSION = 1.3; PRODUCT_BUNDLE_IDENTIFIER = com.macdirstat.app; PRODUCT_NAME = DirStat; SWIFT_EMIT_LOC_STRINGS = YES; @@ -386,7 +442,7 @@ "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 2; + CURRENT_PROJECT_VERSION = 5; DEVELOPMENT_TEAM = L4UH9K7AW4; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = NO; @@ -397,7 +453,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 13.0; - MARKETING_VERSION = 1.1; + MARKETING_VERSION = 1.3; PRODUCT_BUNDLE_IDENTIFIER = com.macdirstat.app; PRODUCT_NAME = DirStat; SWIFT_EMIT_LOC_STRINGS = YES; diff --git a/MacDirStat/MacDirStat.entitlements b/MacDirStat/MacDirStat.entitlements index 18aff0c..e89b7f3 100644 --- a/MacDirStat/MacDirStat.entitlements +++ b/MacDirStat/MacDirStat.entitlements @@ -3,8 +3,6 @@ com.apple.security.app-sandbox - - com.apple.security.files.user-selected.read-only - + diff --git a/README.md b/README.md index 7509fb5..7762b70 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,11 @@ MacDirStat scans any folder and turns your filesystem into an interactive sunbur - **File type breakdown** — top file types with a searchable list of all types - **Move to Trash** — right-click any arc or row to trash it, chart refreshes instantly - **CSV export** — dump the full scan as a spreadsheet -- **Settings** — toggle haptic feedback on/off +- **Save and reopen scans**: write a completed scan to an `.mdscan` file and reopen it later as a read-only snapshot (no delete actions, no live watching) +- **Compare two scans**: diff the current scan against a saved one: added, removed, grown, and shrunk items, with a whole added or removed folder collapsing into one row (File > Compare With Saved Scan) +- **Auto-summarization**: dependency and cache folders (node_modules and the like) collapse into a single row with their total size and file count, instead of a node per file +- **Full Disk Access, made unambiguous**: drag the app's own icon straight into the permission list, since macOS grants access to one exact copy and a file picker makes it easy to add the wrong one +- **Settings**: appearance (units, color scheme, default tab), file scanning (hidden files, excluded folders), behaviour (auto-scan on launch, real-time monitoring), a Permissions section with live Full Disk Access status, and haptic feedback ## Screenshots @@ -84,11 +88,13 @@ log. ## Tech -Pure Swift + SwiftUI — no Electron, no web views, no dependencies. +Pure Swift + SwiftUI. No Electron, no web views; Sparkle for auto-updates is the only dependency. | Layer | What | |---|---| -| Scanner | POSIX `opendir`/`fstatat` with async task groups — parallel, cancellable | +| Scanner | `getattrlistbulk(2)` batch enumeration (name + metadata per syscall), with a `readdir` fallback and a bounded worker pool draining an iterative work queue, parallel and cancellable | +| Store | Flat struct-of-arrays `FileTree`: contiguous records, no per-node heap object, paths reconstructed on demand | +| Auto-summarization | Dependency/cache folders (`node_modules` and similar) collapse into one summary node with a deep size and file count, instead of a node per file | | Layout | Custom sunburst partition algorithm (band-width from view size) | | Renderer | SwiftUI `Canvas` — draws 1,000+ arcs at 30 fps | | Haptics | `NSHapticFeedbackManager` — intensity scales with file size | diff --git a/Sources/App/ContentView.swift b/Sources/App/ContentView.swift index c209b3e..6924962 100644 --- a/Sources/App/ContentView.swift +++ b/Sources/App/ContentView.swift @@ -95,7 +95,10 @@ struct ContentView: View { } .help("Settings & About") .popover(isPresented: $showingSettings, arrowEdge: .bottom) { - DashboardSettingsView() + // Passed explicitly rather than relying on the popover + // inheriting it, so the Permissions section can't crash + // looking for a missing environment object. + DashboardSettingsView().environmentObject(vm) } } } @@ -110,6 +113,12 @@ struct ContentView: View { .onReceive(NotificationCenter.default.publisher(for: .exportCSV)) { _ in vm.exportCSV() } + .sheet(isPresented: $vm.showFDASheet) { + FullDiskAccessSheet() + } + .sheet(isPresented: $vm.showComparisonSheet) { + ComparisonView() + } } // MARK: - Tab picker @@ -190,8 +199,13 @@ struct ContentView: View { @ViewBuilder private var detailContent: some View { VStack(spacing: 0) { - if !vm.hasFullDiskAccess { - FullDiskAccessBanner() + if vm.isReadOnlySnapshot { + SnapshotBanner(date: vm.snapshotDate) + } + if !vm.hasFullDiskAccess || (vm.deniedCount > 0 && !vm.isScanning) { + FullDiskAccessBanner(hasFullDiskAccess: vm.hasFullDiskAccess, deniedCount: vm.deniedCount) { + vm.showFDASheet = true + } } if vm.root == nil && !vm.isScanning && !vm.isComputingLayout { WelcomeView() @@ -374,6 +388,20 @@ private struct FolderTitleView: View { // MARK: - Full Disk Access banner private struct FullDiskAccessBanner: View { + let hasFullDiskAccess: Bool + let deniedCount: Int + let onOpenSheet: () -> Void + + private let title = "Full Disk Access required for complete results" + + private var subtitle: String { + if !hasFullDiskAccess { + return "Go to System Settings → Privacy & Security → Full Disk Access and add DirStat." + } + let folders = deniedCount == 1 ? "1 folder" : "\(deniedCount) folders" + return "\(folders) couldn't be read. Grant Full Disk Access for a complete scan." + } + var body: some View { HStack(spacing: 10) { Image(systemName: "lock.shield") @@ -381,21 +409,19 @@ private struct FullDiskAccessBanner: View { .foregroundStyle(.orange) VStack(alignment: .leading, spacing: 1) { - Text("Full Disk Access required for complete results") + Text(title) .font(.system(size: 12, weight: .semibold)) - Text("Go to System Settings → Privacy & Security → Full Disk Access and add DirStat.") + Text(subtitle) .font(.system(size: 11)) .foregroundStyle(.secondary) } Spacer() - Button("Open Settings") { - NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_AllFiles")!) - } - .font(.system(size: 11, weight: .medium)) - .buttonStyle(.bordered) - .controlSize(.small) + Button("Open Settings", action: onOpenSheet) + .font(.system(size: 11, weight: .medium)) + .buttonStyle(.bordered) + .controlSize(.small) } .padding(.horizontal, 14) .padding(.vertical, 9) @@ -404,9 +430,50 @@ private struct FullDiskAccessBanner: View { } } +// MARK: - Read-only snapshot banner + +// Shown whenever `vm.tree` was loaded from a `.mdscan` archive instead of a +// live scan. Trash/delete actions are already disabled at the ScanViewModel +// level (`trashNodes` no-ops while `isReadOnlySnapshot` is set); this is +// just the visible explanation of why. +private struct SnapshotBanner: View { + let date: Date? + + private var subtitle: String { + guard let date else { return "Opened from a saved scan. Delete actions are disabled." } + let formatter = DateFormatter() + formatter.dateStyle = .medium + formatter.timeStyle = .short + return "Saved \(formatter.string(from: date)). Delete actions are disabled." + } + + var body: some View { + HStack(spacing: 10) { + Image(systemName: "camera.aperture") + .font(.system(size: 15, weight: .medium)) + .foregroundStyle(.blue) + + VStack(alignment: .leading, spacing: 1) { + Text("Read-only snapshot") + .font(.system(size: 12, weight: .semibold)) + Text(subtitle) + .font(.system(size: 11)) + .foregroundStyle(.secondary) + } + + Spacer() + } + .padding(.horizontal, 14) + .padding(.vertical, 9) + .background(.blue.opacity(0.15)) + .overlay(alignment: .bottom) { Divider().overlay(.blue.opacity(0.3)) } + } +} + // MARK: - Dashboard settings popover private struct DashboardSettingsView: View { + @EnvironmentObject private var vm: ScanViewModel @AppStorage("hapticFeedbackEnabled") private var hapticEnabled = true @AppStorage("useBinarySize") private var useBinarySize = false @AppStorage("showHiddenFiles") private var showHiddenFiles = false @@ -487,6 +554,34 @@ private struct DashboardSettingsView: View { Divider() + settingsSection("Permissions") { + HStack(spacing: 7) { + Circle() + .fill(vm.hasFullDiskAccess ? Color.green : Color.orange) + .frame(width: 7, height: 7) + Text(vm.hasFullDiskAccess ? "Full Disk Access looks granted" : "Full Disk Access not granted") + .font(.system(size: 12)) + } + + // The grant is per exact copy of the app, so the surest + // route is dragging this bundle in rather than hunting for + // it with the "+" picker and possibly adding another build. + FullDiskAccessDragTile() + + Button("Open Full Disk Access settings") { + NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_AllFiles")!) + } + .controlSize(.small) + + if vm.hasFullDiskAccess && vm.deniedCount > 0 { + caption("\(vm.deniedCount) folders were still unreadable in the last scan. If DirStat is already listed, remove it with “−” and drag this copy in again — macOS ties the grant to one exact copy.") + } else { + caption("Without it, protected folders scan as 0 bytes and land in “Hidden & Unreadable Space”.") + } + } + + Divider() + settingsSection("Trackpad") { Toggle("Haptic feedback", isOn: $hapticEnabled) .toggleStyle(.switch).controlSize(.small) @@ -515,6 +610,17 @@ private struct DashboardSettingsView: View { } } .frame(width: 300, height: 520) + // The grant can change while this popover is open (the user drags the + // icon into System Settings and comes straight back), and macOS has no + // notification for it, so re-probe on appear and then periodically. + .onAppear { vm.recheckFullDiskAccess() } + .task { + while !Task.isCancelled { + try? await Task.sleep(for: .seconds(1.5)) + guard !Task.isCancelled else { break } + vm.recheckFullDiskAccess() + } + } } @ViewBuilder diff --git a/Sources/App/FullDiskAccessSheet.swift b/Sources/App/FullDiskAccessSheet.swift new file mode 100644 index 0000000..0ca3d04 --- /dev/null +++ b/Sources/App/FullDiskAccessSheet.swift @@ -0,0 +1,273 @@ +import SwiftUI +import AppKit + +/// Guided Full Disk Access onboarding. macOS provides no API to grant FDA +/// programmatically, so the best achievable flow is: explain what's blocked and why, +/// jump straight to the right System Settings pane, detect the grant while the sheet +/// is open, then offer to relaunch (required for the new permission to take effect) +/// and auto-resume the scan the user was on. +struct FullDiskAccessSheet: View { + @EnvironmentObject private var vm: ScanViewModel + @Environment(\.dismiss) private var dismiss + + // Whether the access probe already claimed success at the moment this + // sheet opened. Captured once, so a grant can be told apart from a probe + // that was simply wrong to begin with. + @State private var accessWasMissingOnOpen: Bool? + // Drives the gentle up/down hint on the draggable icon. + @State private var bobbing = false + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + VStack(spacing: 22) { + if Self.showsGrantedState(accessWasMissingOnOpen: accessWasMissingOnOpen, hasAccessNow: vm.hasFullDiskAccess) { + grantedState + } else { + missingState + } + } + .padding(28) + .frame(width: 460) + .onAppear { + if accessWasMissingOnOpen == nil { accessWasMissingOnOpen = !vm.hasFullDiskAccess } + } + .task { + // Poll for a live permission change while the sheet is on screen — macOS + // has no notification for TCC grants, so this is the only way to react + // to the user flipping the toggle in System Settings without closing us. + while !Task.isCancelled { + try? await Task.sleep(for: .seconds(1.5)) + guard !Task.isCancelled else { break } + vm.recheckFullDiskAccess() + } + } + } + + /// "Access granted!" is only honest for a grant this sheet actually watched + /// happen: the probe said no when the sheet opened, and says yes now. + /// + /// The probe (is the TCC database readable) is not a reliable proxy for + /// "this build can read the user's files". It can report success while the + /// scan is still being denied hundreds of folders — which is exactly when + /// the user opens this sheet from the warning banner. Keying off the + /// probe's current value alone put them in a dead end: a congratulations + /// screen whose only action is a relaunch that changes nothing, with no way + /// to reach the System Settings button they were promised. + /// + /// So when the probe already claims access on open, show the instructions + /// instead. Nothing is lost: if access really is fine, the banner that led + /// here would not be showing. + static func showsGrantedState(accessWasMissingOnOpen: Bool?, hasAccessNow: Bool) -> Bool { + guard let accessWasMissingOnOpen else { return false } + return accessWasMissingOnOpen && hasAccessNow + } + + // MARK: - State A: access missing + + private var missingState: some View { + VStack(spacing: 20) { + Image(systemName: "lock.shield") + .font(.system(size: 44, weight: .medium)) + .foregroundStyle(.tint) + .symbolRenderingMode(.hierarchical) + .padding(22) + .glassTintedCard(tint: .accentColor, cornerRadius: 200) + + VStack(spacing: 8) { + Text("See your whole disk") + .font(.system(size: 20, weight: .bold, design: .rounded)) + Text(explainerBody) + .font(.system(size: 12.5)) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .fixedSize(horizontal: false, vertical: true) + } + + steps + + VStack(spacing: 10) { + Button("Open System Settings") { + NSWorkspace.shared.open(URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_AllFiles")!) + } + .keyboardShortcut(.defaultAction) + .glassProminentButton() + .controlSize(.large) + .frame(maxWidth: .infinity) + + Button("Not Now") { dismiss() } + .glassButton() + .controlSize(.large) + .frame(maxWidth: .infinity) + + // Fallback for anyone who would rather drag from a Finder + // window, and a way to see exactly which copy is running when + // several builds are floating around. + Button("Show this app in Finder") { + NSWorkspace.shared.activateFileViewerSelecting([Self.runningAppURL()]) + } + .buttonStyle(.plain) + .font(.system(size: 11)) + .foregroundStyle(.tertiary) + + Button("Don't ask again") { + UserDefaults.standard.set(true, forKey: "fdaPromptSuppressed") + dismiss() + } + .buttonStyle(.plain) + .font(.system(size: 11)) + .foregroundStyle(.tertiary) + } + } + } + + private var explainerBody: String { + let base = "macOS protects some folders (Documents, Desktop, other apps' data) until you grant Full Disk Access. MacDirStat reads sizes only — nothing is modified, collected, or sent anywhere." + guard vm.deniedCount > 0 else { return base } + let folders = vm.deniedCount == 1 ? "1 folder was" : "\(vm.deniedCount) folders were" + var text = "\(base) \(folders) blocked during your last scan." + // The confusing case: the toggle looks on, but folders are still + // blocked. macOS ties the grant to the exact signed copy of the app, + // so a rebuilt, re-signed, or moved copy inherits nothing from the + // entry already in the list — it just sits there looking enabled. + if vm.hasFullDiskAccess { + text += " If MacDirStat already appears enabled in the list, remove it with the “−” button and add this copy again — macOS ties the grant to one exact copy of the app." + } + return text + } + + private var steps: some View { + VStack(alignment: .leading, spacing: 12) { + stepRow(1, "Open System Settings") + stepRow(2, "Drag this icon into the list") + dragTile + stepRow(3, "Make sure its switch is on, then come back") + } + .padding(14) + .frame(maxWidth: .infinity, alignment: .leading) + .glassCard(cornerRadius: 14) + } + + private var dragTile: some View { FullDiskAccessDragTile() } + + /// URL of the bundle that is actually running — the thing that needs the + /// grant. Kept in one place so the drag payload and the Finder fallback + /// can never disagree about which copy they mean. + static func runningAppURL() -> URL { Bundle.main.bundleURL } + + static func runningAppName() -> String { + Bundle.main.bundleURL.deletingPathExtension().lastPathComponent + } + + static func runningAppIcon() -> NSImage { + NSWorkspace.shared.icon(forFile: Bundle.main.bundlePath) + } + + private func stepRow(_ number: Int, _ text: String) -> some View { + HStack(alignment: .top, spacing: 10) { + Text("\(number)") + .font(.system(size: 11, weight: .bold, design: .rounded)) + .foregroundStyle(.white) + .frame(width: 18, height: 18) + .background(Circle().fill(.tint)) + Text(text) + .font(.system(size: 12)) + .foregroundStyle(.primary) + } + } + + // MARK: - State B: access granted, needs relaunch + + private var grantedState: some View { + VStack(spacing: 20) { + Image(systemName: "checkmark.shield.fill") + .font(.system(size: 44, weight: .medium)) + .foregroundStyle(.green) + .symbolRenderingMode(.hierarchical) + .padding(22) + .glassTintedCard(tint: .green, cornerRadius: 200) + + VStack(spacing: 8) { + Text("Access granted!") + .font(.system(size: 20, weight: .bold, design: .rounded)) + Text("MacDirStat needs to relaunch for macOS to apply the new permission. It will rescan automatically.") + .font(.system(size: 12.5)) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .fixedSize(horizontal: false, vertical: true) + } + + VStack(spacing: 10) { + Button("Relaunch & Rescan") { + vm.relaunchForFullDiskAccess() + } + .keyboardShortcut(.defaultAction) + .glassProminentButton() + .controlSize(.large) + .frame(maxWidth: .infinity) + + Button("Later") { dismiss() } + .glassButton() + .controlSize(.large) + .frame(maxWidth: .infinity) + } + } + } +} + +/// The running app's own icon, draggable straight into the Full Disk Access +/// list in System Settings. +/// +/// Better than the "+" button for a reason that bit a real user: macOS grants +/// access to one exact copy of an app, and the file picker happily adds a +/// different copy (an older build in /Applications, say) which then sits in +/// the list looking enabled while the copy actually running gets nothing. +/// Dragging carries this bundle's own URL, so the entry that lands in the list +/// is unambiguously the app the user is looking at. +/// +/// Shared by the guided sheet and the Permissions section of Settings, so the +/// two can never drift apart about which copy they hand over. +struct FullDiskAccessDragTile: View { + @State private var bobbing = false + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + HStack(spacing: 12) { + Image(nsImage: FullDiskAccessSheet.runningAppIcon()) + .resizable() + .frame(width: 46, height: 46) + .offset(y: bobbing ? -3 : 3) + .animation( + reduceMotion ? nil : .easeInOut(duration: 1.1).repeatForever(autoreverses: true), + value: bobbing + ) + .onDrag { + // The payload is this bundle's URL, so System Settings + // registers precisely the running copy. + NSItemProvider(object: FullDiskAccessSheet.runningAppURL() as NSURL) + } + .help("Drag me into the Full Disk Access list") + + VStack(alignment: .leading, spacing: 3) { + Text(FullDiskAccessSheet.runningAppName()) + .font(.system(size: 12.5, weight: .semibold)) + Text("Drag me into the list") + .font(.system(size: 11)) + .foregroundStyle(.secondary) + } + + Spacer(minLength: 0) + + Image(systemName: "arrow.right") + .font(.system(size: 11, weight: .semibold)) + .foregroundStyle(.tertiary) + } + .padding(10) + .frame(maxWidth: .infinity, alignment: .leading) + .background( + RoundedRectangle(cornerRadius: 12, style: .continuous) + .strokeBorder(style: StrokeStyle(lineWidth: 1.5, dash: [5, 4])) + .foregroundStyle(.tint.opacity(0.55)) + ) + .onAppear { bobbing = true } + } +} diff --git a/Sources/App/MacDirStatApp.swift b/Sources/App/MacDirStatApp.swift index 1b72cc3..d76d2a1 100644 --- a/Sources/App/MacDirStatApp.swift +++ b/Sources/App/MacDirStatApp.swift @@ -1,5 +1,6 @@ import SwiftUI import Sparkle +import UniformTypeIdentifiers @main struct MacDirStatApp: App { @@ -25,12 +26,29 @@ struct MacDirStatApp: App { openFolderPicker(vm: vm) } .keyboardShortcut("o", modifiers: .command) + + Button("Open Scan…") { + openArchivePicker(vm: vm) + } + .keyboardShortcut("o", modifiers: [.command, .shift]) } CommandGroup(after: .newItem) { + Button("Save Scan…") { + saveScanPicker(vm: vm) + } + .keyboardShortcut("s", modifiers: .command) + .disabled(vm.tree == nil) + Button("Export CSV…") { NotificationCenter.default.post(name: .exportCSV, object: nil) } .keyboardShortcut("e", modifiers: [.command, .shift]) + + Button("Compare With Saved Scan…") { + compareWithSavedScanPicker(vm: vm) + } + .keyboardShortcut("d", modifiers: [.command, .shift]) + .disabled(vm.tree == nil) } CommandGroup(after: .appInfo) { Button("Check for Updates…") { @@ -40,6 +58,11 @@ struct MacDirStatApp: App { NSWorkspace.shared.open(URL(string: "https://ti-03.github.io/MacDirStat/")!) } } + CommandGroup(after: .help) { + Button("Grant Full Disk Access…") { + vm.showFDASheet = true + } + } } } } @@ -56,3 +79,50 @@ private func openFolderPicker(vm: ScanViewModel) { Task { @MainActor in vm.scan(url: url) } } } + +// Not registered as a system-wide document type (no Info.plist exported-type +// entry) — `UTType(filenameExtension:)` gives macOS a dynamic UTI that's +// perfectly sufficient for filtering these two panels by the `.mdscan` +// extension without touching the app's document/type registration. +private let mdscanType = UTType(filenameExtension: "mdscan") ?? .data + +@MainActor +private func saveScanPicker(vm: ScanViewModel) { + guard let tree = vm.tree else { return } + let panel = NSSavePanel() + panel.allowedContentTypes = [mdscanType] + panel.nameFieldStringValue = "\(tree.records[tree.rootIndex].name).mdscan" + panel.message = "Save this scan to reopen later as a read-only snapshot" + if panel.runModal() == .OK, let url = panel.url { + vm.saveScan(to: url) + } +} + +@MainActor +private func openArchivePicker(vm: ScanViewModel) { + let panel = NSOpenPanel() + panel.canChooseDirectories = false + panel.canChooseFiles = true + panel.allowsMultipleSelection = false + panel.allowedContentTypes = [mdscanType] + panel.prompt = "Open" + panel.message = "Choose a saved scan to reopen as a read-only snapshot" + if panel.runModal() == .OK, let url = panel.url { + vm.openArchive(from: url) + } +} + +@MainActor +private func compareWithSavedScanPicker(vm: ScanViewModel) { + guard vm.tree != nil else { return } + let panel = NSOpenPanel() + panel.canChooseDirectories = false + panel.canChooseFiles = true + panel.allowsMultipleSelection = false + panel.allowedContentTypes = [mdscanType] + panel.prompt = "Compare" + panel.message = "Choose a saved scan to compare against the current one" + if panel.runModal() == .OK, let url = panel.url { + vm.compareWithSavedScan(archiveURL: url) + } +} diff --git a/Sources/Duplicates/DuplicateDetector.swift b/Sources/Duplicates/DuplicateDetector.swift index 5f9ce3c..d271050 100644 --- a/Sources/Duplicates/DuplicateDetector.swift +++ b/Sources/Duplicates/DuplicateDetector.swift @@ -6,66 +6,143 @@ public actor DuplicateDetector { private let maxSize: Int64 = 512 * 1024 * 1024 // skip files > 512 MB (VM images, sparsebundles, etc.) private let quickHashBytes = 65_536 // 64 KB quick pre-filter + // Bounded concurrency for I/O-bound hashing work. + private static var maxConcurrency: Int { + min(8, ProcessInfo.processInfo.activeProcessorCount) + } + public init() {} - public func detect(in root: FSNode) async { - var candidates: [FSNode] = [] - collect(node: root, into: &candidates) + // Operates over flat FileTree indices rather than an FSNode tree — a + // simple loop over `tree.records` reaches every node without recursion, + // since the array already covers the whole tree regardless of hierarchy. + public func detect(in tree: FileTree) async { + var candidates: [Int] = [] + collect(tree: tree, into: &candidates) // Group by size first — eliminates the vast majority of files cheaply - let bySize = Dictionary(grouping: candidates) { $0.size } + let bySize = Dictionary(grouping: candidates) { tree.records[$0].size } .filter { $0.value.count > 1 } - // Phase 1: quick hash (first 64 KB) to rule out non-duplicates without reading entire files - var byQuickHash: [String: [FSNode]] = [:] - for (_, nodes) in bySize { - for node in nodes { - guard !Task.isCancelled else { return } - if let qh = partialHash(url: node.url, maxBytes: quickHashBytes) { - let key = "\(node.size)-\(qh)" - byQuickHash[key, default: []].append(node) - } + let quickHashBytes = self.quickHashBytes + + // Phase 1: quick hash (first 64 KB) to rule out non-duplicates without reading entire files. + // Hashing is I/O-bound and embarrassingly parallel, so run it with bounded concurrency. + var byQuickHash: [String: [Int]] = [:] + let quickHashResults: [(Int, String)]? = await Self.hashInParallel( + indices: bySize.values.flatMap { $0 } + ) { index in + Self.partialHash(url: FileNode(tree: tree, index: index).url, maxBytes: quickHashBytes) + } + guard let quickHashResults else { return } + for (index, qh) in quickHashResults { + let key = "\(tree.records[index].size)-\(qh)" + byQuickHash[key, default: []].append(index) + } + + // Small-file shortcut: if a file's whole content fits within the quick-hash window, + // the quick hash already IS a full-content hash, so those groups are final as-is. + // Only groups whose files exceed the quick-hash window need a full-file hash pass. + var hashGroups: [String: [Int]] = [:] + var toFullHash: [Int] = [] + for (key, indices) in byQuickHash where indices.count > 1 { + if tree.records[indices[0]].size <= Int64(quickHashBytes) { + hashGroups[key] = indices + } else { + toFullHash.append(contentsOf: indices) } } - // Phase 2: full hash only for groups that survived the quick-hash filter - var hashGroups: [String: [FSNode]] = [:] - for (_, nodes) in byQuickHash where nodes.count > 1 { - for node in nodes { - guard !Task.isCancelled else { return } - if let hash = fullHash(url: node.url) { - let key = "\(node.size)-\(hash)" - hashGroups[key, default: []].append(node) - } + // Phase 2: full hash only for groups that survived the quick-hash filter and are + // larger than the quick-hash window (their quick hash alone is not conclusive). + if !toFullHash.isEmpty { + let fullHashResults: [(Int, String)]? = await Self.hashInParallel(indices: toFullHash) { index in + Self.fullHash(url: FileNode(tree: tree, index: index).url) + } + guard let fullHashResults else { return } + for (index, hash) in fullHashResults { + let key = "\(tree.records[index].size)-\(hash)" + hashGroups[key, default: []].append(index) } } // Assign group IDs to genuine duplicates - for (_, nodes) in hashGroups where nodes.count > 1 { + for (_, indices) in hashGroups where indices.count > 1 { + guard !Task.isCancelled else { return } let groupID = UUID() - for node in nodes { node.duplicateGroupID = groupID } + for index in indices { tree.setDuplicateGroupID(groupID, at: index) } + } + } + + private func collect(tree: FileTree, into list: inout [Int]) { + for index in 0..= minSize && record.size <= maxSize { + list.append(index) + } } } - private func collect(node: FSNode, into list: inout [FSNode]) { - var stack: [FSNode] = [node] - while let current = stack.popLast() { - if !current.isDirectory && current.size >= minSize && current.size <= maxSize { - list.append(current) + // Runs `hash` over `indices` with bounded sliding-window concurrency, returning nil if the + // task was cancelled before completion. Indices for which `hash` returns nil are dropped. + private static func hashInParallel( + indices: [Int], + hash: @escaping @Sendable (Int) -> String? + ) async -> [(Int, String)]? { + guard !indices.isEmpty else { return [] } + guard !Task.isCancelled else { return nil } + + var results: [(Int, String)] = [] + results.reserveCapacity(indices.count) + + await withTaskGroup(of: (Int, String?).self) { group in + var cursor = 0 + let limit = maxConcurrency + + func launchNext() { + guard cursor < indices.count else { return } + let index = indices[cursor] + cursor += 1 + group.addTask { + guard !Task.isCancelled else { return (index, nil) } + return (index, hash(index)) + } + } + + // Prime the sliding window. + while cursor < limit && cursor < indices.count { + launchNext() + } + + while let (index, key) = await group.next() { + if let key { + results.append((index, key)) + } + if Task.isCancelled { + // Drain remaining in-flight tasks without launching more. + continue + } + launchNext() } - stack.append(contentsOf: current.children) } + + guard !Task.isCancelled else { return nil } + return results } // Reads up to `maxBytes` from the file and returns a SHA256 hex string. // Each chunk is drained from the autorelease pool immediately to prevent accumulation. - private func partialHash(url: URL, maxBytes: Int) -> String? { + private static func partialHash(url: URL, maxBytes: Int) -> String? { guard let handle = try? FileHandle(forReadingFrom: url) else { return nil } defer { try? handle.close() } var hasher = SHA256() var remaining = maxBytes let chunkSize = min(65_536, maxBytes) while remaining > 0 { + if Task.isCancelled { return nil } let toRead = min(chunkSize, remaining) let chunk: Data? = autoreleasepool { try? handle.read(upToCount: toRead) } guard let chunk, !chunk.isEmpty else { break } @@ -77,12 +154,13 @@ public actor DuplicateDetector { // Full-file SHA256. Each 1 MB chunk is released immediately via autoreleasepool, // capping peak memory at ~1 MB regardless of file size. - private func fullHash(url: URL) -> String? { + private static func fullHash(url: URL) -> String? { guard let handle = try? FileHandle(forReadingFrom: url) else { return nil } defer { try? handle.close() } var hasher = SHA256() let chunkSize = 1024 * 1024 while true { + if Task.isCancelled { return nil } let chunk: Data? = autoreleasepool { try? handle.read(upToCount: chunkSize) } guard let chunk, !chunk.isEmpty else { break } hasher.update(data: chunk) diff --git a/Sources/Layout/ExtensionColorMap.swift b/Sources/Layout/ExtensionColorMap.swift index baaea70..056e0a8 100644 --- a/Sources/Layout/ExtensionColorMap.swift +++ b/Sources/Layout/ExtensionColorMap.swift @@ -33,7 +33,7 @@ public struct ExtensionColorMap: Equatable, Sendable { let scheme: String - public init(root: FSNode) { + public init(root: FileNode) { self.scheme = UserDefaults.standard.string(forKey: "treemapColorScheme") ?? "byType" } diff --git a/Sources/Layout/TreemapCell.swift b/Sources/Layout/TreemapCell.swift index c055577..92414bc 100644 --- a/Sources/Layout/TreemapCell.swift +++ b/Sources/Layout/TreemapCell.swift @@ -3,7 +3,7 @@ import SwiftUI /// Arc-based cell for the sunburst visualization. public struct TreemapCell: Identifiable { public let id: UUID = UUID() - public let node: FSNode + public let node: FileNode public let startAngle: Double // radians; 0 = right, increases clockwise on screen public let endAngle: Double public let innerRadius: CGFloat diff --git a/Sources/Layout/TreemapLayout.swift b/Sources/Layout/TreemapLayout.swift index 85afbb5..e1fd1db 100644 --- a/Sources/Layout/TreemapLayout.swift +++ b/Sources/Layout/TreemapLayout.swift @@ -21,7 +21,7 @@ public struct TreemapLayout { // Gentle per-depth darkening so deep files stay recognisable private static let depthDarken: [Double] = [0.0, 0.10, 0.18, 0.26, 0.32, 0.38] - public static func compute(root: FSNode, in rect: CGRect, colorMap: ExtensionColorMap) -> [TreemapCell] { + public static func compute(root: FileNode, in rect: CGRect, colorMap: ExtensionColorMap) -> [TreemapCell] { var cells: [TreemapCell] = [] guard rect.width > 1, rect.height > 1, root.size > 0 else { return cells } @@ -41,7 +41,7 @@ public struct TreemapLayout { } private static func layout( - _ children: [FSNode], + _ children: [FileNode], parentStart: Double, parentEnd: Double, depth: Int, @@ -95,11 +95,20 @@ public struct TreemapLayout { // ── Color assignment ───────────────────────────────────────────────────── - private static func color(for node: FSNode, depth: Int, colorMap: ExtensionColorMap) -> Color { + static func directoryHue(for name: String) -> Double { + let hash = name.utf8.reduce(UInt32(5381)) { ($0 &<< 5) &+ $0 &+ UInt32($1) } + return Double(hash % 360) / 360.0 + } + + private static func color(for node: FileNode, depth: Int, colorMap: ExtensionColorMap) -> Color { + if node.isSynthetic { + // Distinct muted gray so the hidden-space reconciliation node reads as + // "not a real file" at a glance, rather than blending into the chart. + return Color(hue: 0, saturation: 0, brightness: 0.35) + } if node.isDirectory { // Directories: muted tinted containers — hue from name hash, low saturation - let hash = abs(node.name.hashValue) % 360 - let hue = Double(hash) / 360.0 + let hue = directoryHue(for: node.name) let fade = Double(depth) * 0.05 return Color(hue: hue, saturation: max(0.12, 0.28 - fade), diff --git a/Sources/Model/FileNode.swift b/Sources/Model/FileNode.swift new file mode 100644 index 0000000..86246b7 --- /dev/null +++ b/Sources/Model/FileNode.swift @@ -0,0 +1,76 @@ +import Foundation + +// Lightweight, app-facing handle into a `FileTree` — the replacement for +// `FSNode` everywhere outside the scanner's own internal build step. A +// `FileNode` is just a (tree, index) pair; every property reads through to +// the tree's flat storage, so copying a `FileNode` is cheap and many can +// exist for the same underlying record. +public struct FileNode: Identifiable, Hashable, Sendable { + public let tree: FileTree + public let index: Int + + public init(tree: FileTree, index: Int) { + self.tree = tree + self.index = index + } + + // Stable within one tree instance — sufficient for SwiftUI identity and + // selection. Not globally unique across trees (e.g. before/after a + // rescan replaces `tree` entirely), which mirrors how the rest of the UI + // already treats a rescan as a clean slate. + public var id: Int { index } + + public var name: String { tree.records[index].name } + public var size: Int64 { tree.records[index].size } + public var isDirectory: Bool { tree.records[index].isDirectory } + public var fileExtension: String { tree.records[index].fileExtension } + public var isAccessDenied: Bool { tree.records[index].isAccessDenied } + public var isSynthetic: Bool { tree.records[index].isSynthetic } + public var isAutoSummarized: Bool { tree.records[index].isAutoSummarized } + public var descendantFileCount: Int { tree.records[index].descendantFileCount } + public var hardLinkRef: HardLinkRef? { tree.records[index].hardLinkRef } + public var duplicateGroupID: UUID? { tree.records[index].duplicateGroupID } + public var safetyLevel: SafetyLevel { tree.records[index].safetyLevel } + + public var url: URL { URL(fileURLWithPath: tree.path(of: index)) } + + public var parent: FileNode? { + let p = tree.parentIndex[index] + return p >= 0 ? FileNode(tree: tree, index: p) : nil + } + + // Pre-sorted size-desc — the builder (and the synthetic-node splice) + // guarantee this, so no consumer ever needs to sort children itself. + public var children: [FileNode] { + let start = tree.childStart[index] + let count = tree.childCount[index] + guard count > 0 else { return [] } + return (0.. 0 else { return nil } + return children + } + + // MARK: - Mutation (writes through to the shared tree) + + public func setDuplicateGroupID(_ id: UUID?) { + tree.setDuplicateGroupID(id, at: index) + } + + public func setSafety(_ level: SafetyLevel) { + tree.setSafety(level, at: index) + } + + // MARK: - Hashable / Equatable + + public static func == (lhs: FileNode, rhs: FileNode) -> Bool { + ObjectIdentifier(lhs.tree) == ObjectIdentifier(rhs.tree) && lhs.index == rhs.index + } + + public func hash(into hasher: inout Hasher) { + hasher.combine(ObjectIdentifier(tree)) + hasher.combine(index) + } +} diff --git a/Sources/Model/FileNodeRecord.swift b/Sources/Model/FileNodeRecord.swift new file mode 100644 index 0000000..2ec8c97 --- /dev/null +++ b/Sources/Model/FileNodeRecord.swift @@ -0,0 +1,47 @@ +import Foundation + +// Per-node payload stored contiguously in `FileTree.records`. This is the +// flat, struct-of-arrays replacement for `FSNode`'s per-node instance fields: +// no URL and no UUID are stored per record (that was FSNode's memory cost on +// huge scans) — the absolute path is reconstructed on demand from names + +// parentIndex (see `FileTree.path(of:)` / `FileNode.url`), and node identity +// for SwiftUI purposes is the (tree, index) pair (see `FileNode.id`). +public struct FileNodeRecord: Sendable, Codable { + public let name: String + public let isDirectory: Bool + public var size: Int64 + public let fileExtension: String + public var isAccessDenied: Bool + public var isSynthetic: Bool + public var isAutoSummarized: Bool + public var descendantFileCount: Int + public var hardLinkRef: HardLinkRef? + public var duplicateGroupID: UUID? + public var safetyLevel: SafetyLevel + + public init( + name: String, + isDirectory: Bool, + size: Int64, + fileExtension: String, + isAccessDenied: Bool = false, + isSynthetic: Bool = false, + isAutoSummarized: Bool = false, + descendantFileCount: Int = 0, + hardLinkRef: HardLinkRef? = nil, + duplicateGroupID: UUID? = nil, + safetyLevel: SafetyLevel = .caution + ) { + self.name = name + self.isDirectory = isDirectory + self.size = size + self.fileExtension = fileExtension + self.isAccessDenied = isAccessDenied + self.isSynthetic = isSynthetic + self.isAutoSummarized = isAutoSummarized + self.descendantFileCount = descendantFileCount + self.hardLinkRef = hardLinkRef + self.duplicateGroupID = duplicateGroupID + self.safetyLevel = safetyLevel + } +} diff --git a/Sources/Model/FileTree.swift b/Sources/Model/FileTree.swift new file mode 100644 index 0000000..15245cf --- /dev/null +++ b/Sources/Model/FileTree.swift @@ -0,0 +1,551 @@ +import Foundation + +// Flat struct-of-arrays store for an entire scanned tree — the app-facing +// replacement for the `FSNode` class tree. One array entry per node, indexed +// by a plain `Int` (root is always index 0). This is the memory win over +// per-node class instances: no per-node URL/UUID/weak-parent-pointer +// overhead, and everything lives in a handful of contiguous arrays. +// +// Mutability split: `records` holds per-node fields that legitimately change +// after the initial scan assembly (duplicateGroupID, safetyLevel, size from a +// future refresh) and is mutated in place via the `set...(at:)` methods +// below. The topology arrays (parentIndex/childStart/childCount/childIndices) +// are immutable for the life of one `FileTree` instance — anything that needs +// to change the shape of the tree (e.g. appending the synthetic hidden-space +// node) produces a NEW `FileTree` instead (see `appendingSyntheticRootChild`). +public final class FileTree: @unchecked Sendable { + public private(set) var records: [FileNodeRecord] + let parentIndex: [Int] + let childStart: [Int] + let childCount: [Int] + let childIndices: [Int] + public let rootIndex: Int + // The scanned root's own absolute path (not derivable from its name + // alone) — `path(of:)` is this, plus the joined names of every node + // between the root and the target index. + public let rootPath: String + + public init( + records: [FileNodeRecord], + parentIndex: [Int], + childStart: [Int], + childCount: [Int], + childIndices: [Int], + rootIndex: Int, + rootPath: String + ) { + // Every per-node array must be the same length, or a later pass that + // walks all records will read past the end of one of them. A missing + // span entry for an appended node is exactly how the app once crashed + // on the main thread during a live refresh, so fail loudly here in + // debug rather than far away at the eventual read. Untrusted archives + // are length-checked separately by `ScanArchive.validate()` before + // ever reaching this initializer. + assert(parentIndex.count == records.count, "parentIndex must have one entry per record") + assert(childStart.count == records.count, "childStart must have one entry per record") + assert(childCount.count == records.count, "childCount must have one entry per record") + + self.records = records + self.parentIndex = parentIndex + self.childStart = childStart + self.childCount = childCount + self.childIndices = childIndices + self.rootIndex = rootIndex + self.rootPath = rootPath + } + + // MARK: - Mutation (per-node fields only; topology never changes in place) + + public func setDuplicateGroupID(_ id: UUID?, at index: Int) { + records[index].duplicateGroupID = id + } + + public func setSafety(_ level: SafetyLevel, at index: Int) { + records[index].safetyLevel = level + } + + public func setSize(_ size: Int64, at index: Int) { + records[index].size = size + } + + // MARK: - Path reconstruction + + // Rebuilds the absolute path of `index` by walking parentIndex up to the + // root and joining names, since individual records don't store a URL. + public func path(of index: Int) -> String { + guard index != rootIndex else { return rootPath } + + var components: [String] = [] + var current = index + while current != rootIndex { + components.append(records[current].name) + let parent = parentIndex[current] + guard parent >= 0 else { break } + current = parent + } + + let suffix = components.reversed().joined(separator: "/") + return rootPath.hasSuffix("/") ? rootPath + suffix : rootPath + "/" + suffix + } + + // MARK: - Synthetic hidden-space node + + // Returns a NEW tree with one extra leaf record appended as a child of + // root, keeping root's children sorted size-desc (the invariant every + // `FileNode.children` accessor relies on). Used for the "Hidden & + // Unreadable Space" reconciliation entry appended after a volume-root + // scan; see `ScanViewModel.appendHiddenSpaceNodeIfNeeded`. + public func appendingSyntheticRootChild(name: String, size: Int64) -> FileTree { + var newRecords = records + newRecords[rootIndex].size += size + let newNodeIndex = newRecords.count + newRecords.append(FileNodeRecord( + name: name, + isDirectory: false, + size: size, + fileExtension: "", + isSynthetic: true, + safetyLevel: .danger + )) + + var newParentIndex = parentIndex + newParentIndex.append(rootIndex) + + // Root is index 0 by construction, and the builder lays out child + // spans in index order, so root's span is always the very first one + // in `childIndices` — inserting into it only requires shifting every + // later span's start by one; root's own start never moves. + let rootStart = childStart[rootIndex] + let rootCount = childCount[rootIndex] + let rootChildrenEnd = rootStart + rootCount + + var rootChildren = Array(childIndices[rootStart..= rootChildrenEnd { + newChildStart[i] += 1 + } + // The synthetic node needs its own (empty) span, like every other + // record. Omitting it left childStart/childCount one shorter than + // records, and any later pass that walks all records and reads their + // spans — `replacingSubtree` during a live refresh, for one — ran off + // the end and trapped on "Index out of range". + newChildStart.append(newChildIndices.count) + newChildCount.append(0) + + return FileTree( + records: newRecords, + parentIndex: newParentIndex, + childStart: newChildStart, + childCount: newChildCount, + childIndices: newChildIndices, + rootIndex: rootIndex, + rootPath: rootPath + ) + } + + // MARK: - Pruning (Move to Trash without a full rescan) + + // Returns a NEW tree with the subtree rooted at `index` removed (the root + // itself can never be removed — returns `self` unchanged if asked to). + // Thin wrapper over `removingSubtrees(at:)`, which already handles the + // single-index case without any extra cost. + public func removingSubtree(at index: Int) -> FileTree { + removingSubtrees(at: [index]) + } + + // Returns a NEW tree with every subtree rooted at each of `indices` + // removed — the "Delete All Duplicates" / "keep 1, delete N" case, where + // several unrelated nodes are trashed at once. + // + // Single pass over all indices (not a fold of `removingSubtree` calls one + // at a time): every seed's whole subtree is marked in one iterative DFS, + // then `records`/`parentIndex`/children are rebuilt once from the + // resulting old->new index map. This is O(n) total rather than O(n·k) + // for k removed subtrees, while remaining just as simple to reason about + // as folding would be (a fold is also correct here — just slower). + // + // Hardlink-aware (first-seen-wins survivor promotion): the scanner's + // convention is that a hardlinked inode's allocated size is carried by + // exactly one node (the "size carrier"; see `bulkAllocatedSize` / + // `VisitedSet` in FileScanner.swift), while every other node sharing its + // `hardLinkRef` sits at size 0. If a removed subtree contains a size + // carrier, and a node OUTSIDE the removed set shares its `hardLinkRef` + // and is still at size 0, that surviving twin is promoted to carry the + // size instead — otherwise the inode's bytes would simply vanish from + // the tree (the disk usage didn't change just because one of the + // inode's N links got removed). If no such survivor exists, every link + // to that inode is gone, and the existing subtract-only behavior below + // is already correct. + public func removingSubtrees(at indices: [Int]) -> FileTree { + let count = records.count + + // Mark every node in each seed's subtree as removed via an iterative + // DFS over childIndices (explicit stack, not recursion, so this can't + // stack-overflow on a very deep real-world tree). + var removed = [Bool](repeating: false, count: count) + var stack: [Int] = [] + for seed in indices where seed != rootIndex && seed >= 0 && seed < count && !removed[seed] { + stack.append(seed) + } + while let i = stack.popLast() { + if removed[i] { continue } + removed[i] = true + let start = childStart[i] + let cnt = childCount[i] + for offset in 0.. new index map over kept nodes, built in ascending old-index + // order — this is what keeps every child span's relative order + // (already size-desc) intact after dropping the removed entries. + var oldToNew = [Int](repeating: -1, count: count) + var newRecords: [FileNodeRecord] = [] + newRecords.reserveCapacity(count) + for old in 0..= 0 ? oldToNew[oldParent] : -1 + } + + // Hardlink survivor promotion, computed against the ORIGINAL tree + // before any deltas are applied. One pass over survivors builds + // ref -> first zero-size surviving index (not O(n·k) per carrier), + // then one pass over removed nodes finds carriers and looks each up. + var survivorTwinByRef: [HardLinkRef: Int] = [:] + for old in 0..() + var promotions: [(survivorOld: Int, size: Int64)] = [] + for old in 0.. 0 else { continue } + // Defensive: first-seen-wins means at most one carrier per ref + // should ever exist, but never promote the same ref twice. + guard promotedRefs.insert(ref).inserted else { continue } + guard let survivorOld = survivorTwinByRef[ref] else { continue } // no survivor -> no promotion + promotions.append((survivorOld, records[old].size)) + } + // Every span a size change below could disturb, collected as we go + // and deduped (a promotion and a plain subtraction can easily share + // ancestors, e.g. root) so each span is only ever re-sorted once at + // the end, after `newChildIndices` exists to sort. + var resortTargets = Set() + + for (survivorOld, size) in promotions { + newRecords[oldToNew[survivorOld]].size = size + // The promoted twin's own parent span (one member jumped from 0 + // to `size`) and every ancestor above it up to root — same + // members up there, but one of them now has a different size. + var ancestorOld = parentIndex[survivorOld] + while ancestorOld >= 0 { + newRecords[oldToNew[ancestorOld]].size += size + resortTargets.insert(oldToNew[ancestorOld]) + ancestorOld = parentIndex[ancestorOld] + } + } + + // Subtract each removed subtree's root size from every one of its + // ancestors. Only process seeds whose immediate parent is NOT itself + // removed — if the parent is removed too, this seed is a descendant + // of some other (higher) removed subtree root, and its size was + // already folded into the ancestor chain when that higher seed was + // processed, so subtracting again here would double-count. Also + // de-dupe in case the same index appears more than once in `indices`. + // + // FIX 2: every ancestor visited here also goes into `resortTargets`. + // The removed seed's immediate parent just shrank (possibly straight + // to 0), which can leave it out of order within ITS OWN parent's + // span — a pre-existing gap in this plain-subtraction path (nothing + // to do with hardlinks) that survivor promotion just makes easy to + // trigger, since a directory's size can drop straight to 0 while an + // untouched sibling stays put. Including the shrunk node's own span + // too is harmless (it's already correctly ordered by the child-span + // rebuild below), but keeping it in the same set as the ancestors + // above it is what actually fixes the sibling ordering. + var processedRoots = Set() + for seed in indices { + guard seed != rootIndex, seed >= 0, seed < count, removed[seed] else { continue } + let parent = parentIndex[seed] + if parent >= 0 && removed[parent] { continue } + guard processedRoots.insert(seed).inserted else { continue } + + let removedSize = records[seed].size + var ancestorOld = parent + while ancestorOld >= 0 { + newRecords[oldToNew[ancestorOld]].size -= removedSize + resortTargets.insert(oldToNew[ancestorOld]) + ancestorOld = parentIndex[ancestorOld] + } + } + + // Rebuild children arrays, dropping removed entries from each + // surviving parent's span but keeping the rest in their original + // (size-desc) relative order. + var newChildIndices: [Int] = [] + newChildIndices.reserveCapacity(childIndices.count) + var newChildStart = [Int](repeating: 0, count: newRecords.count) + var newChildCount = [Int](repeating: 0, count: newRecords.count) + for old in 0..(_ targets: S, records: [FileNodeRecord], childStart: [Int], childCount: [Int], into childIndices: inout [Int]) where S.Element == Int { + for newIdx in targets { + let start = childStart[newIdx] + let cnt = childCount[newIdx] + guard cnt > 1 else { continue } + var slice = Array(childIndices[start..<(start + cnt)]) + slice.sort { records[$0].size > records[$1].size } + childIndices.replaceSubrange(start..<(start + cnt), with: slice) + } + } + + // MARK: - Splice (incremental live-refresh, replacing the full-rescan interim) + + // Returns a NEW tree where the subtree rooted at `index` is replaced + // wholesale by `subtree`'s own nodes, reparented under `index`'s former + // parent in the very same child slot. This is the live-refresh + // counterpart to `removingSubtrees` above: instead of dropping a stale + // subtree, it swaps in a freshly-rescanned replacement for it (produced + // by re-walking just that one directory on disk — see + // `ScanViewModel.splicedTree(afterChangeAt:in:)`), so a filesystem + // change deep inside a huge tree only costs a rescan of the changed + // directory, not the whole root. + // + // The root itself can never be replaced this way (mirrors + // `removingSubtree`'s own root guard, returning `self` unchanged) — a + // changed root is the caller's responsibility to detect ahead of time + // and fall back to a full rescan for instead. + public func replacingSubtree(at index: Int, with subtree: FileTree) -> FileTree { + let count = records.count + guard index != rootIndex, index >= 0, index < count else { return self } + let oldParent = parentIndex[index] + guard oldParent >= 0 else { return self } // unreachable given index != rootIndex, but defensive + + // Mark the stale subtree rooted at `index` (itself plus every + // descendant) via the same iterative DFS `removingSubtrees` uses. + var removed = [Bool](repeating: false, count: count) + var stack: [Int] = [index] + while let i = stack.popLast() { + if removed[i] { continue } + removed[i] = true + let start = childStart[i] + let cnt = childCount[i] + for offset in 0.. new index map over surviving (kept) nodes, ascending + // old-index order — identical compaction to `removingSubtrees`. + var oldToNew = [Int](repeating: -1, count: count) + var newRecords: [FileNodeRecord] = [] + newRecords.reserveCapacity(count + subtree.records.count) + for old in 0..= 0 ? oldToNew[op] : -1 + } + for i in 0..= 0 ? offset + subParent : oldToNew[oldParent] + } + + // Ancestor sizes: fold in the difference between the new subtree + // root's size and the stale one it's replacing, all the way up to + // the root (positive or negative — the changed directory may have + // grown or shrunk). + let delta = subtree.records[subtree.rootIndex].size - records[index].size + var ancestorOld = oldParent + while ancestorOld >= 0 { + newRecords[oldToNew[ancestorOld]].size += delta + ancestorOld = parentIndex[ancestorOld] + } + + // Children spans: every kept node keeps its surviving children in + // their original relative order, except the replaced node's own + // slot in its parent's span (which used to point at `index`), which + // now points at the new subtree's root instead. + var newChildIndices: [Int] = [] + newChildIndices.reserveCapacity(childIndices.count + subtree.childIndices.count) + var newChildStart = [Int](repeating: 0, count: newRecords.count) + var newChildCount = [Int](repeating: 0, count: newRecords.count) + for old in 0..= 0 { + ancestorNewIndices.append(oldToNew[a]) + a = parentIndex[a] + } + Self.resortSpans(ancestorNewIndices, records: newRecords, childStart: newChildStart, childCount: newChildCount, into: &newChildIndices) + + return FileTree( + records: newRecords, + parentIndex: newParentIndex, + childStart: newChildStart, + childCount: newChildCount, + childIndices: newChildIndices, + rootIndex: oldToNew[rootIndex], + rootPath: rootPath + ) + } + + // MARK: - Splice-time hardlink survivor promotion (FIX 1, refresh-side twin of removingSubtrees' promotion) + + // The live-refresh counterpart to `removingSubtrees`' hardlink survivor + // promotion above, but simpler: it doesn't need to find the orphaned + // ref/size pairs itself (the caller — `ScanViewModel.splicedTree` — + // already worked out which `hardLinkRef`s the OLD (stale) subtree + // carried but the freshly-rescanned NEW subtree doesn't, by diffing the + // two subtrees' own carried-ref sets), it just needs to do the + // promotion: find a surviving zero-size twin for `ref` ANYWHERE in this + // tree (the search doesn't need to exclude the freshly-spliced subtree — + // if that subtree happens to itself contain the zero-size twin, that's + // still a perfectly valid — arguably the most natural — promotion + // target), set it to `size`, and bubble that size up its ancestor chain, + // re-sorting every span the jump from 0 to `size` could have disturbed — + // same reasoning as `removingSubtrees`' own promotion resort above. + // + // Returns `self` unchanged if no zero-size twin for `ref` exists + // anywhere — defensive; the caller only ever calls this for a ref it + // already knows was carried by the subtree it just replaced, but a + // missing survivor (every link to the inode is now gone) is not an + // error, just nothing to promote. + public func promotingSurvivingTwin(ref: HardLinkRef, size: Int64) -> FileTree { + guard let twinIndex = records.firstIndex(where: { $0.hardLinkRef == ref && $0.size == 0 }) else { + return self + } + + var newRecords = records + newRecords[twinIndex].size = size + + // The twin's own parent span (one member jumped from 0 to `size`) + // and every ancestor above it up to root (same members, but one now + // has a different size). + var resortTargets: [Int] = [] + var ancestor = parentIndex[twinIndex] + while ancestor >= 0 { + newRecords[ancestor].size += size + resortTargets.append(ancestor) + ancestor = parentIndex[ancestor] + } + + var newChildIndices = childIndices + Self.resortSpans(resortTargets, records: newRecords, childStart: childStart, childCount: childCount, into: &newChildIndices) + + return FileTree( + records: newRecords, + parentIndex: parentIndex, + childStart: childStart, + childCount: childCount, + childIndices: newChildIndices, + rootIndex: rootIndex, + rootPath: rootPath + ) + } +} diff --git a/Sources/Model/FileTreeBuilder.swift b/Sources/Model/FileTreeBuilder.swift new file mode 100644 index 0000000..0419f02 --- /dev/null +++ b/Sources/Model/FileTreeBuilder.swift @@ -0,0 +1,81 @@ +import Foundation + +// Converts a scanner-internal `FSNode` class tree into a flat `FileTree`. +// `FSNode` remains the scanner's own build type (see FileScanner.swift) so +// none of the traversal logic there had to change; this is the one place +// that walks an already-fully-built `FSNode` tree and assembles the +// compact, app-facing store from it. +// +// Public so tests (and, in principle, any future consumer that still builds +// an `FSNode` tree by hand — e.g. the live-refresh rescan helpers) can drive +// the rest of the app's data flow (layout, view model, duplicate detection) +// from a small hand-built fixture without going through a real disk scan. +public enum FileTreeBuilder { + public static func build(from root: FSNode, rootPath: String) -> FileTree { + var records: [FileNodeRecord] = [] + var parentIndex: [Int] = [] + // Children discovered so far for each index, built up as nodes are + // visited; flattened into `childIndices`/`childStart`/`childCount` + // once the full traversal is done. + var childrenOf: [[Int]] = [] + + // Iterative pre-order walk (explicit stack, not recursion) so + // assembly can't stack-overflow on a very deep real-world tree — + // mirrors the style of the scanner's own iterative traversal. + // + // Children are pushed in ascending-size order so the stack (LIFO) + // pops them back off in descending-size order; since a child is + // appended to its parent's `childrenOf` entry at the moment it's + // popped, this guarantees `childrenOf[i]` ends up sorted size-desc — + // the invariant `FileNode.children` promises its callers. + var stack: [(node: FSNode, parent: Int)] = [(root, -1)] + while let (node, parent) = stack.popLast() { + let index = records.count + records.append(FileNodeRecord( + name: node.name, + isDirectory: node.isDirectory, + size: node.size, + fileExtension: node.fileExtension, + isAccessDenied: node.isAccessDenied, + isSynthetic: node.isSynthetic, + isAutoSummarized: node.isAutoSummarized, + descendantFileCount: node.descendantFileCount, + hardLinkRef: node.hardLinkRef, + // Filled post-assembly: safety tagging and duplicate + // detection both run as a pass over the finished FileTree. + duplicateGroupID: nil, + safetyLevel: .caution + )) + parentIndex.append(parent) + childrenOf.append([]) + if parent >= 0 { + childrenOf[parent].append(index) + } + + let sortedChildren = node.children.sorted { $0.size > $1.size } + for child in sortedChildren.reversed() { + stack.append((child, index)) + } + } + + var childIndices: [Int] = [] + childIndices.reserveCapacity(records.count) + var childStart = [Int](repeating: 0, count: records.count) + var childCount = [Int](repeating: 0, count: records.count) + for i in 0.. FileTree { + FileTree( + records: records, + parentIndex: parentIndex, + childStart: childStart, + childCount: childCount, + childIndices: childIndices, + rootIndex: rootIndex, + rootPath: rootPath + ) + } + + // MARK: - Validation + + // A hostile or corrupted archive must never crash the app or hang it — + // every check here is a bounds/consistency check performed with plain + // array indexing, and the whole-tree walk is iterative (explicit stack) + // and visits each node at most once, so it terminates even if the + // topology is cyclic. + public func validate() throws { + let count = records.count + + guard count > 0 else { throw ScanArchiveError.corrupted("Archive contains no records") } + // A generous upper bound: guards against a maliciously/corruptly + // huge archive trying to exhaust memory, while never limiting any + // real scan (tens of millions of files is already far beyond what + // this app scans in practice). + guard count <= 50_000_000 else { throw ScanArchiveError.corrupted("Archive is implausibly large (\(count) records)") } + + guard parentIndex.count == count, childStart.count == count, childCount.count == count else { + throw ScanArchiveError.corrupted("Archive arrays have mismatched lengths") + } + + guard rootIndex >= 0, rootIndex < count else { + throw ScanArchiveError.corrupted("Root index out of range") + } + guard parentIndex[rootIndex] < 0 else { + throw ScanArchiveError.corrupted("Root node has a parent") + } + + // Every child span must be in range before touching childIndices. + var totalSpanned = 0 + for i in 0..= 0, start >= 0 else { + throw ScanArchiveError.corrupted("Negative child span at index \(i)") + } + guard start + cnt <= childIndices.count else { + throw ScanArchiveError.corrupted("Child span out of range at index \(i)") + } + totalSpanned += cnt + } + guard totalSpanned == childIndices.count else { + throw ScanArchiveError.corrupted("Child spans don't cover childIndices exactly") + } + + // Every entry in childIndices must reference a valid, non-self node, + // and must agree with that child's own parentIndex. + for i in 0..= 0, child < count else { + throw ScanArchiveError.corrupted("Child index out of range under parent \(i)") + } + guard child != i else { + throw ScanArchiveError.corrupted("Node \(i) lists itself as its own child") + } + guard parentIndex[child] == i else { + throw ScanArchiveError.corrupted("parentIndex/childIndices disagree for node \(child)") + } + } + } + + // Whole-tree reachability walk from root, iterative so a cycle can't + // hang the app — a node already marked visited is simply skipped + // rather than re-pushed, and if a cycle exists among unreached + // nodes, they just never get visited and the final count check + // below catches it. + var visited = [Bool](repeating: false, count: count) + var stack = [rootIndex] + var visitedCount = 0 + while let i = stack.popLast() { + if visited[i] { continue } + visited[i] = true + visitedCount += 1 + let start = childStart[i] + let cnt = childCount[i] + for offset in 0.. [ScanChange] { + let beforeIndex = relativePathIndex(of: before) + let afterIndex = relativePathIndex(of: after) + + var changes: [ScanChange] = [] + + // Added: present in `after`, absent from `before`. Collapsed to the + // topmost new node in each new subtree — a node is only reported if + // its parent already existed in `before` (i.e. the parent is NOT + // itself newly added); every descendant of a newly-added directory + // is skipped here, since it's implied by the directory's own + // "added" row, exactly like Radix's diff-row suppression. + for (relativePath, afterNodeIndex) in afterIndex { + guard beforeIndex[relativePath] == nil else { continue } + // The parent counts as pre-existing only if it was there AND was + // already a directory. If it was a *file* that has since become a + // directory, everything now inside it is new content implied by + // that node's own "replaced" row, so suppress it here. + if let parentPath = parentRelativePath(of: relativePath), + !existsAsDirectory(parentPath, in: beforeIndex, of: before) { + continue + } + let record = after.records[afterNodeIndex] + changes.append(ScanChange( + relativePath: relativePath, + name: record.name, + isDirectory: record.isDirectory, + kind: .added, + beforeSize: 0, + afterSize: record.size + )) + } + + // Removed: present in `before`, absent from `after`. Same collapsing + // rule, mirrored: skip a node whose parent is also removed (not + // present in `after`) — only the topmost removed node per subtree is + // reported. + for (relativePath, beforeNodeIndex) in beforeIndex { + guard afterIndex[relativePath] == nil else { continue } + // Mirror of the added-side rule: if the parent is now a file where + // it used to be a directory, this node's disappearance is implied + // by the parent's "replaced" row. + if let parentPath = parentRelativePath(of: relativePath), + !existsAsDirectory(parentPath, in: afterIndex, of: after) { + continue + } + let record = before.records[beforeNodeIndex] + changes.append(ScanChange( + relativePath: relativePath, + name: record.name, + isDirectory: record.isDirectory, + kind: .removed, + beforeSize: record.size, + afterSize: 0 + )) + } + + // Grew/shrank: only for *files* present in both trees whose size + // differs. Directories present in both are deliberately never + // reported here — a directory's size is entirely derived from its + // descendants, so any real change under it already surfaces as an + // added/removed/grew/shrank row for that descendant; reporting the + // containing directories too would just be noisy duplication of the + // same information at every level of the path. + for (relativePath, beforeNodeIndex) in beforeIndex { + guard let afterNodeIndex = afterIndex[relativePath] else { continue } + let beforeRecord = before.records[beforeNodeIndex] + let afterRecord = after.records[afterNodeIndex] + + // Type change (file <-> directory) at the same path. Reported + // regardless of whether the size happens to match, since the node + // is fundamentally a different thing now — and without this the + // pair would fall through both guards below and vanish from the + // report entirely. + if beforeRecord.isDirectory != afterRecord.isDirectory { + changes.append(ScanChange( + relativePath: relativePath, + name: afterRecord.name, + isDirectory: afterRecord.isDirectory, + kind: .replaced, + beforeSize: beforeRecord.size, + afterSize: afterRecord.size + )) + continue + } + + guard !beforeRecord.isDirectory, !afterRecord.isDirectory else { continue } + guard beforeRecord.size != afterRecord.size else { continue } + changes.append(ScanChange( + relativePath: relativePath, + name: afterRecord.name, + isDirectory: false, + kind: afterRecord.size > beforeRecord.size ? .grew : .shrank, + beforeSize: beforeRecord.size, + afterSize: afterRecord.size + )) + } + + return changes.sorted { abs($0.delta) > abs($1.delta) } + } + + // Iterative DFS (explicit stack, mirroring `FileTree.removingSubtrees`'s + // style) building every node's root-relative path incrementally from its + // parent's — O(n) total, unlike calling `FileTree.path(of:)` per node + // (which walks the parent chain from scratch each time and would be + // O(n·depth) over a whole tree). + private static func relativePathIndex(of tree: FileTree) -> [String: Int] { + var index: [String: Int] = [:] + index.reserveCapacity(tree.records.count) + + var stack: [(index: Int, relativePath: String)] = [(tree.rootIndex, "")] + while let (nodeIndex, relativePath) = stack.popLast() { + index[relativePath] = nodeIndex + let start = tree.childStart[nodeIndex] + let count = tree.childCount[nodeIndex] + for offset in 0.. String? { + guard let lastSlash = relativePath.lastIndex(of: "/") else { + return relativePath.isEmpty ? nil : "" + } + return String(relativePath[.. Bool { + guard let nodeIndex = index[relativePath] else { return false } + return tree.records[nodeIndex].isDirectory + } +} diff --git a/Sources/Safety/SafetyAnalyzer.swift b/Sources/Safety/SafetyAnalyzer.swift index b4422c6..31c9c2f 100644 --- a/Sources/Safety/SafetyAnalyzer.swift +++ b/Sources/Safety/SafetyAnalyzer.swift @@ -1,6 +1,6 @@ import Foundation -public enum SafetyLevel: Equatable { +public enum SafetyLevel: Equatable, Sendable, Codable { case safe // Regeneratable artifact — confident it's okay to delete case caution // Unknown or user data — review before deleting case danger // System or critical app data — do not delete @@ -15,9 +15,24 @@ public struct SafetyAnalyzer { // MARK: - Bulk tagging (no string allocations — just enum assignment) + // `FSNode` overload: used by the scanner-internal build step and by the + // FSNode-based live-refresh helpers in ScanViewModel (refreshDirectory / + // scanSubtree), which still operate on freshly-scanned FSNode subtrees. public static func level(for node: FSNode) -> SafetyLevel { - let path = node.url.path - let name = node.name + level(path: node.url.path, name: node.name, isSynthetic: node.isSynthetic) + } + + // `FileNode` overload: used by the app-facing FileTree tagging pass and + // by every view that reads `safetyLevel`. + public static func level(for node: FileNode) -> SafetyLevel { + level(path: node.url.path, name: node.name, isSynthetic: node.isSynthetic) + } + + private static func level(path: String, name: String, isSynthetic: Bool) -> SafetyLevel { + // Synthetic nodes (e.g. the hidden-space reconciliation entry) don't point + // at a real, deletable file — always treat them as the most protective + // level so nothing ever attempts to trash/move them. + if isSynthetic { return .danger } if isDanger(path: path, name: name) { return .danger } if isSafe(path: path, name: name) { return .safe } return .caution @@ -26,8 +41,17 @@ public struct SafetyAnalyzer { // MARK: - On-demand reason (called only when UI needs to display it) public static func reason(for node: FSNode) -> String? { - let path = node.url.path - let name = node.name + reason(path: node.url.path, name: node.name, isSynthetic: node.isSynthetic) + } + + public static func reason(for node: FileNode) -> String? { + reason(path: node.url.path, name: node.name, isSynthetic: node.isSynthetic) + } + + private static func reason(path: String, name: String, isSynthetic: Bool) -> String? { + if isSynthetic { + return "Represents space macOS reports as used but the scanner can't read or enumerate (snapshots, purgeable space, protected folders)" + } return dangerReason(path: path, name: name) ?? safeReason(path: path, name: name) } diff --git a/Sources/Scanner/AtomicDirectorySummary.swift b/Sources/Scanner/AtomicDirectorySummary.swift new file mode 100644 index 0000000..e9a14f2 --- /dev/null +++ b/Sources/Scanner/AtomicDirectorySummary.swift @@ -0,0 +1,332 @@ +import Darwin +import Dispatch +import Foundation + +// Auto-summarization: when a directory is clearly a generated / tiny-file +// tree (node_modules, huge flat caches, ...), the scanner collapses its +// entire subtree into ONE leaf FSNode carrying the deep allocated size and a +// descendant file count, instead of materializing an FSNode per descendant. +// This is the biggest remaining scan-speed/node-count win: no FSNode +// allocation, no tree-building overhead for a subtree the user never wants +// to browse file-by-file. +// +// v1 scope is deliberately simple: a named-layout shortcut plus an +// immediate-file-count/average-size heuristic. Phase 3.5 replaced the walk +// itself with a bounded parallel pool (see `summarizeSubtree` below) so that +// summarizing a huge subtree is a scan-speed win too, not just a node-count +// win. + +// Directory names that are, by convention, generated dependency/package +// trees not worth browsing file-by-file. Matched against a directory's own +// name; bypasses the depth gate below. +private let knownGeneratedDirectoryNames: Set = ["node_modules"] + +enum AtomicSummaryThresholds { + // Overridable via env var so tests can exercise the general heuristic + // without creating thousands of real files. + static var minFileCount: Int { + if let raw = ProcessInfo.processInfo.environment["MDS_SUMMARY_MIN_FILES"], let value = Int(raw) { + return value + } + return 5_000 + } + static let maxAverageFileSize: Int64 = 4_096 + static let minDepth = 2 +} + +// Cheap, allocation-free gate: decides whether `item`'s directory (already +// enumerated into `entries`, no extra I/O) should be collapsed into a single +// summarized FSNode instead of being expanded normally. +func shouldAutoSummarize(entries: [BulkDirEntry], name: String, depth: Int) -> Bool { + // 1. Named-layout shortcut: known generated trees (e.g. node_modules) + // often keep few files at their own top level (most live several levels + // deep, e.g. node_modules//dist/...), so the immediate-file + // heuristic below would miss them. Name match alone is enough for v1. + if knownGeneratedDirectoryNames.contains(name), depth >= 1 { + return true + } + + // 2. General heuristic: lots of small immediate files, deep enough in + // the tree that summarizing won't collapse something the user is + // directly looking at. + // TODO(phase3+): bounded descendant probe for ambiguous dirs (e.g. a + // directory whose files are deeper than its immediate listing, but that + // doesn't match a known name) - see Radix's AtomicDirectorySummarizer + // for the pattern; out of scope here to keep the summarization gate simple. + guard depth >= AtomicSummaryThresholds.minDepth else { return false } + + var fileCount = 0 + var totalAllocated: Int64 = 0 + for entry in entries where entry.kind == .file { + fileCount += 1 + totalAllocated += entry.allocatedSize + } + guard fileCount > 0, fileCount >= AtomicSummaryThresholds.minFileCount else { return false } + let averageAllocated = totalAllocated / Int64(fileCount) + return averageAllocated < AtomicSummaryThresholds.maxAverageFileSize +} + +// Thread-safe pending-directory queue for the summarization walk, analogous +// to `WorkQueue` in FileScanner.swift but holding directory PATHS (a +// directory's (dev, ino) dedup check already happened when it was +// discovered and pushed here, so the queue itself needs no identity checks). +// `pop()` returning nil doesn't mean "done" - callers must also check +// `isFinished` (stack empty AND nothing in flight), since another worker's +// current item may still push more work. +private final class SummaryDirQueue: @unchecked Sendable { + private var lock = os_unfair_lock() + private var stack: [String] + private var inFlight: Int + + init(seed: [String]) { + stack = seed + inFlight = seed.count + } + + func push(_ path: String) { + os_unfair_lock_lock(&lock) + stack.append(path) + inFlight += 1 + os_unfair_lock_unlock(&lock) + } + + func pop() -> String? { + os_unfair_lock_lock(&lock) + let item = stack.popLast() + os_unfair_lock_unlock(&lock) + return item + } + + // Call exactly once per path that was popped, after it has been fully + // processed (including pushing any subdirectories it discovered). + func markDone() { + os_unfair_lock_lock(&lock) + inFlight -= 1 + os_unfair_lock_unlock(&lock) + } + + var isFinished: Bool { + os_unfair_lock_lock(&lock) + let finished = stack.isEmpty && inFlight == 0 + os_unfair_lock_unlock(&lock) + return finished + } +} + +// Thread-safe running total for the summarization walk's aggregate result. +private final class SummaryAccumulator: @unchecked Sendable { + private var lock = os_unfair_lock() + private var totalAllocated: Int64 + private var totalFileCount: Int + + init(allocatedSize: Int64, fileCount: Int) { + totalAllocated = allocatedSize + totalFileCount = fileCount + } + + func add(files: Int, bytes: Int64) { + os_unfair_lock_lock(&lock) + totalAllocated += bytes + totalFileCount += files + os_unfair_lock_unlock(&lock) + } + + var snapshot: (allocatedSize: Int64, fileCount: Int) { + os_unfair_lock_lock(&lock) + let result = (totalAllocated, totalFileCount) + os_unfair_lock_unlock(&lock) + return result + } +} + +// Parallel walk of the subtree rooted at a directory already deemed a +// summarization candidate. Applies the exact same scan semantics as the main +// traversal (symlink/hidden/exclusion/mount-point skips, hardlink + +// directory dedup via the shared `visited` set) so the aggregate size/file +// count it returns matches what a full, non-summarized scan of the same +// subtree would have produced - regardless of how many workers race to +// discover each directory/file, since `visited.visit` is atomic. That +// invariant (count each (dev, ino) exactly once, no matter which worker sees +// it first) is exactly why this parallel result equals the single-threaded +// result computed before Phase 3.5. +// +// `rootEntries` is the already-decoded listing of the candidate directory +// itself (the caller enumerated it once to run `shouldAutoSummarize`); reused +// here instead of re-enumerating so summarization costs zero extra I/O for +// the root of the collapsed subtree. That root listing is consumed +// synchronously, on the calling thread, exactly as before Phase 3.5 - only +// once there is more than one discovered subdirectory do we spin up the +// worker pool below. +// +// No FSNode objects are built here, so there is no shared-mutable-node +// hazard to worry about; the only shared mutable state is the queue and the +// accumulator, both lock-guarded. +// +// Concurrency: this function is `async` and drives its nested worker pool +// with a plain `withThrowingTaskGroup`, awaited directly by the calling scan +// worker. It deliberately does NOT block a thread to bridge sync->async. +// +// An earlier version kept a synchronous signature and bridged by handing the +// pool to a detached Task while blocking the caller's thread on a +// DispatchSemaphore. That was a forward-progress hazard: the caller runs on a +// Swift-concurrency cooperative thread, and the nested pool needs threads +// from that same pool to run. With enough sibling directories summarizing at +// once (a monorepo with several node_modules trees), every cooperative thread +// could end up blocked waiting for work that has no thread left to run on. +// Worse, a wedged summary keeps the main scan's `inFlight` counter above zero +// forever, so `WorkQueue.isFinished` never trips and EVERY other scan worker +// spins too - one stuck summary hangs the whole scan, and cancellation could +// not recover it. Awaiting the group directly removes the hazard entirely: +// no thread is ever blocked, and cancellation propagates through structured +// concurrency for free. +// +// Oversubscription (main pool x summary pool) remains bounded - both pools +// cap at 8 workers, and only directories big enough to trip auto-summarization +// spin up a nested pool at all - but it is now merely a scheduling concern, +// not a correctness one. +func summarizeSubtree( + rootEntries: [BulkDirEntry], + rootPath: String, + rootDev: UInt64, + config: ScanConfig, + visited: VisitedSet, + cancel: @Sendable @escaping () throws -> Void +) async throws -> (allocatedSize: Int64, fileCount: Int) { + var totalAllocated: Int64 = 0 + var fileCount = 0 + var seedDirs: [String] = [] + var entriesSeen = 0 + + for entry in rootEntries { + entriesSeen += 1 + if entriesSeen % 256 == 0 { try cancel() } + + guard entry.name != "." && entry.name != ".." else { continue } + if entry.name.hasPrefix("."), !config.showHiddenFiles { continue } + if config.excludedNames.contains(entry.name) { continue } + + switch entry.kind { + case .symlink, .other: + continue + + case .directory: + // Mount point: skip directories on a different device than the scan root. + if entry.dev != rootDev { continue } + // Dedup by (dev, ino): same firmlink/hardlink protection as the main scanner. + guard visited.visit(dev: entry.dev, ino: entry.ino) else { continue } + let childPath = rootPath.hasSuffix("/") ? rootPath + entry.name : rootPath + "/" + entry.name + seedDirs.append(childPath) + + case .file: + totalAllocated += bulkAllocatedSize(entry: entry, visited: visited) + fileCount += 1 + } + } + + // No subdirectories discovered at the candidate root: nothing left to + // walk, so skip spinning up a worker pool entirely. + guard !seedDirs.isEmpty else { + return (totalAllocated, fileCount) + } + + let queue = SummaryDirQueue(seed: seedDirs) + let accumulator = SummaryAccumulator(allocatedSize: totalAllocated, fileCount: fileCount) + let workerCount = min(max(2, ProcessInfo.processInfo.activeProcessorCount / 2), 8) + + try await withThrowingTaskGroup(of: Void.self) { group in + for _ in 0.. Void +) async throws { + while true { + try cancel() + if let dirPath = queue.pop() { + try _processSummaryDirectory(dirPath: dirPath, rootDev: rootDev, config: config, visited: visited, accumulator: accumulator, queue: queue, cancel: cancel) + continue + } + if queue.isFinished { return } + await Task.yield() + } +} + +// Processes exactly one directory of the summarized subtree: opens it, lists +// its immediate children (bulk enumeration with fallback), applies all scan +// semantics, adds direct file bytes/count to the shared accumulator, and +// pushes any subdirectories as new queue entries. Always calls +// `queue.markDone()` exactly once, even on early return - mirrors +// `_processDirectory` in FileScanner.swift. +private func _processSummaryDirectory( + dirPath: String, + rootDev: UInt64, + config: ScanConfig, + visited: VisitedSet, + accumulator: SummaryAccumulator, + queue: SummaryDirQueue, + cancel: @Sendable () throws -> Void +) throws { + defer { queue.markDone() } + try cancel() + + let fd = open(dirPath, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW) + guard fd >= 0 else { return } + defer { close(fd) } + + let entries: [BulkDirEntry] + do { + entries = try listDirectoryEntries(path: dirPath, fd: fd, forceFallback: config.forceFallbackEnum) + } catch { + // Enumeration failed even after falling back: contributes nothing + // further, matches the main scanner's "treat as empty" rule. + return + } + + var entriesSeen = 0 + var localFiles = 0 + var localBytes: Int64 = 0 + + for entry in entries { + entriesSeen += 1 + if entriesSeen % 256 == 0 { try cancel() } + + guard entry.name != "." && entry.name != ".." else { continue } + if entry.name.hasPrefix("."), !config.showHiddenFiles { continue } + if config.excludedNames.contains(entry.name) { continue } + + switch entry.kind { + case .symlink, .other: + continue + + case .directory: + // Mount point: skip directories on a different device than the scan root. + if entry.dev != rootDev { continue } + // Dedup by (dev, ino): same firmlink/hardlink protection as the main scanner. + guard visited.visit(dev: entry.dev, ino: entry.ino) else { continue } + let childPath = dirPath.hasSuffix("/") ? dirPath + entry.name : dirPath + "/" + entry.name + queue.push(childPath) + + case .file: + localBytes += bulkAllocatedSize(entry: entry, visited: visited) + localFiles += 1 + } + } + + // Batch this directory's file bytes/count into a single lock acquisition + // rather than one per file. + if localFiles > 0 { accumulator.add(files: localFiles, bytes: localBytes) } +} diff --git a/Sources/Scanner/BulkDirectoryEnumerator.swift b/Sources/Scanner/BulkDirectoryEnumerator.swift new file mode 100644 index 0000000..17e65ac --- /dev/null +++ b/Sources/Scanner/BulkDirectoryEnumerator.swift @@ -0,0 +1,286 @@ +import Darwin +import Foundation + +// The kind of a directory entry as reported by either enumeration path. +// Symlinks and special files (fifo/char/block/socket) are surfaced so the +// caller can apply the "skip entirely" rule (see FileScanner.swift semantics); +// this module never resolves symlink targets. +enum BulkEntryKind: Sendable { + case directory + case file + case symlink + case other +} + +// One immediate child of a directory, with just enough metadata for the +// scanner to apply its rules (mount-point / hardlink / dedup checks) without +// an additional per-entry stat call. `dev`/`ino`/`linkCount`/`allocatedSize` +// are meaningful for `.directory` (dev, ino only) and `.file` (all four); +// they are zero for `.symlink`/`.other`, which the scanner always skips. +struct BulkDirEntry: Sendable { + let name: String + let kind: BulkEntryKind + let dev: UInt64 + let ino: UInt64 + let linkCount: UInt32 + let allocatedSize: Int64 +} + +// Thrown when `getattrlistbulk` can't be used for a directory at all (the +// filesystem doesn't support it) or when the packed attribute buffer can't be +// parsed with confidence (unexpected layout, truncated entry, non-UTF8 name). +// Callers should retry the same directory with `fallbackEnumerateDirectory(fd:)`. +struct BulkEnumerationUnavailable: Error {} + +private let bulkBufferCapacity = 64 * 1_024 +private let unsupportedBulkErrors: Set = [EINVAL, ENOTSUP, ENOSYS] + +// Only the attributes FileScanner actually needs: identity (dev/ino via +// DEVID+FILEID), name, and object type for every entry, plus link count and +// allocated size for regular files. Keeping the request minimal keeps the +// packed buffer layout small and simple to parse. +private var requestedAttrList: attrlist { + var list = attrlist() + list.bitmapcount = UInt16(ATTR_BIT_MAP_COUNT) + list.commonattr = attrgroup_t(ATTR_CMN_RETURNED_ATTRS) + | attrgroup_t(ATTR_CMN_ERROR) + | attrgroup_t(ATTR_CMN_NAME) + | attrgroup_t(ATTR_CMN_OBJTYPE) + | attrgroup_t(ATTR_CMN_DEVID) + | attrgroup_t(ATTR_CMN_FILEID) + list.fileattr = attrgroup_t(ATTR_FILE_LINKCOUNT) | attrgroup_t(ATTR_FILE_ALLOCSIZE) + return list +} + +private let requiredCommonAttrs = attrgroup_t(ATTR_CMN_NAME) + | attrgroup_t(ATTR_CMN_OBJTYPE) + | attrgroup_t(ATTR_CMN_DEVID) + | attrgroup_t(ATTR_CMN_FILEID) +private let requiredFileAttrs = attrgroup_t(ATTR_FILE_LINKCOUNT) | attrgroup_t(ATTR_FILE_ALLOCSIZE) + +// Enumerates the immediate children of an already-open directory descriptor +// via getattrlistbulk(2): one syscall per ~64 KB batch returns name + metadata +// together for every entry in the batch, instead of one syscall per entry +// (readdir) plus one more (fstatat) to get its metadata. +// +// Does not close `fd`; the caller owns its lifetime. +func enumerateDirectoryBulk(fd: Int32) throws -> [BulkDirEntry] { + var attrs = requestedAttrList + var results: [BulkDirEntry] = [] + let buffer = UnsafeMutableRawBufferPointer.allocate(byteCount: bulkBufferCapacity, alignment: 8) + defer { buffer.deallocate() } + guard let bufferBase = buffer.baseAddress else { throw BulkEnumerationUnavailable() } + + while true { + let count = getattrlistbulk(fd, &attrs, bufferBase, buffer.count, UInt64(FSOPT_PACK_INVAL_ATTRS)) + if count < 0 { + if unsupportedBulkErrors.contains(errno) { throw BulkEnumerationUnavailable() } + throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) + } + if count == 0 { break } + + try parseBulkBatch( + bufferBase: UnsafeRawPointer(bufferBase), + bufferByteCount: buffer.count, + entryCount: Int(count), + into: &results + ) + } + + return results +} + +// Fallback listing used when bulk enumeration is unavailable for a directory, +// or when `MDS_FORCE_FALLBACK_ENUM=1` forces it everywhere (parity testing). +// Uses the traditional readdir + fstatat path, duplicating `fd` so the +// caller's descriptor is unaffected by fdopendir's ownership rules. +func fallbackEnumerateDirectory(fd: Int32) throws -> [BulkDirEntry] { + let duped = dup(fd) + guard duped >= 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + guard let dir = fdopendir(duped) else { + let openErrno = errno + close(duped) + throw NSError(domain: NSPOSIXErrorDomain, code: Int(openErrno)) + } + defer { closedir(dir) } + let dfd = dirfd(dir) + + var results: [BulkDirEntry] = [] + while let entry = readdir(dir) { + let nameBytes = entry.pointee.d_name + let name: String = withUnsafeBytes(of: nameBytes) { ptr in + let bytes = ptr.bindMemory(to: CChar.self) + return String(cString: bytes.baseAddress!) + } + guard name != "." && name != ".." else { continue } + + let dtype = entry.pointee.d_type + // Fast path: these types never need a stat call, the scanner skips them regardless. + if dtype == DT_LNK { + results.append(BulkDirEntry(name: name, kind: .symlink, dev: 0, ino: 0, linkCount: 0, allocatedSize: 0)) + continue + } + if dtype == DT_FIFO || dtype == DT_CHR || dtype == DT_BLK || dtype == DT_SOCK { + results.append(BulkDirEntry(name: name, kind: .other, dev: 0, ino: 0, linkCount: 0, allocatedSize: 0)) + continue + } + + // DT_DIR / DT_REG / DT_UNKNOWN (some network/FUSE filesystems don't + // populate d_type): resolve the real type via fstatat. + var st = stat() + guard fstatat(dfd, name, &st, AT_SYMLINK_NOFOLLOW) == 0 else { continue } + let mode = st.st_mode & S_IFMT + let kind: BulkEntryKind + switch mode { + case S_IFDIR: kind = .directory + case S_IFREG: kind = .file + case S_IFLNK: kind = .symlink + default: kind = .other + } + let allocatedSize = kind == .file ? Int64(st.st_blocks) * 512 : 0 + results.append(BulkDirEntry( + name: name, + kind: kind, + dev: UInt64(bitPattern: Int64(st.st_dev)), + ino: UInt64(st.st_ino), + linkCount: UInt32(st.st_nlink), + allocatedSize: allocatedSize + )) + } + return results +} + +// MARK: - getattrlistbulk buffer parsing + +// Reads fixed-size fields out of one packed getattrlistbulk entry, advancing +// past each field's 4-byte-aligned footprint (matches the kernel's packing). +private struct AttributeCursor { + var current: UnsafeRawPointer + let end: UnsafeRawPointer + + mutating func read() -> T? { + let size = MemoryLayout.size + let alignedSize = (size + 3) & ~3 + guard alignedSize <= current.distance(to: end) else { return nil } + let value = current.loadUnaligned(as: T.self) + current = current.advanced(by: alignedSize) + return value + } +} + +private func parseBulkBatch( + bufferBase: UnsafeRawPointer, + bufferByteCount: Int, + entryCount: Int, + into results: inout [BulkDirEntry] +) throws { + let bufferEnd = bufferBase.advanced(by: bufferByteCount) + var entryAddress = bufferBase + + for _ in 0...size <= entryAddress.distance(to: bufferEnd) else { + throw BulkEnumerationUnavailable() + } + let entryLength = Int(entryAddress.loadUnaligned(as: UInt32.self)) + guard entryLength >= MemoryLayout.size, + entryLength <= entryAddress.distance(to: bufferEnd) else { + throw BulkEnumerationUnavailable() + } + let entryEnd = entryAddress.advanced(by: entryLength) + + if let entry = try parseBulkEntry(entryStart: entryAddress, entryEnd: entryEnd) { + results.append(entry) + } + // else: this single entry had a per-entry error (ATTR_CMN_ERROR set, + // e.g. a race with deletion) - skip it, keep parsing the batch. + entryAddress = entryEnd + } +} + +// Parses one packed entry. Field order (for the attributes requested above) +// follows getattrlist(2)'s fixed declaration order: ATTR_CMN_RETURNED_ATTRS, +// ATTR_CMN_ERROR, ATTR_CMN_NAME (attrreference_t), ATTR_CMN_DEVID, +// ATTR_CMN_OBJTYPE, ATTR_CMN_FILEID, then (regular files only) +// ATTR_FILE_LINKCOUNT, ATTR_FILE_ALLOCSIZE. +private func parseBulkEntry(entryStart: UnsafeRawPointer, entryEnd: UnsafeRawPointer) throws -> BulkDirEntry? { + var cursor = AttributeCursor(current: entryStart.advanced(by: MemoryLayout.size), end: entryEnd) + + guard let returned: attribute_set_t = cursor.read(), + let entryError: UInt32 = cursor.read() else { + throw BulkEnumerationUnavailable() + } + + let nameRefAddress = cursor.current + guard let nameRef: attrreference_t = cursor.read() else { throw BulkEnumerationUnavailable() } + guard let name = parseBulkName(reference: nameRef, referenceAddress: nameRefAddress, entryEnd: entryEnd) else { + throw BulkEnumerationUnavailable() + } + + guard let deviceID: dev_t = cursor.read(), + let objectType: fsobj_type_t = cursor.read(), + let fileID: UInt64 = cursor.read() else { + throw BulkEnumerationUnavailable() + } + + // A per-entry error means the filesystem couldn't produce metadata for + // this one child (e.g. deleted mid-listing); skip only this entry. + if entryError != 0 { return nil } + + guard returned.commonattr & requiredCommonAttrs == requiredCommonAttrs else { + throw BulkEnumerationUnavailable() + } + + let kind: BulkEntryKind + if objectType == VDIR.rawValue { kind = .directory } + else if objectType == VREG.rawValue { kind = .file } + else if objectType == VLNK.rawValue { kind = .symlink } + else { kind = .other } + + var linkCount: UInt32 = 1 + var allocatedSize: Int64 = 0 + if kind == .file { + // File attributes are only present in the buffer for regular files. + guard returned.fileattr & requiredFileAttrs == requiredFileAttrs else { + throw BulkEnumerationUnavailable() + } + guard let readLinkCount: UInt32 = cursor.read(), + let readAllocSize: off_t = cursor.read() else { + throw BulkEnumerationUnavailable() + } + linkCount = readLinkCount + allocatedSize = Int64(readAllocSize) + } + + return BulkDirEntry( + name: name, + kind: kind, + dev: UInt64(bitPattern: Int64(deviceID)), + ino: fileID, + linkCount: linkCount, + allocatedSize: max(allocatedSize, 0) + ) +} + +// `attr_dataoffset` is relative to the attrreference_t's own address. +private func parseBulkName( + reference: attrreference_t, + referenceAddress: UnsafeRawPointer, + entryEnd: UnsafeRawPointer +) -> String? { + guard reference.attr_dataoffset >= 0 else { return nil } + let dataOffset = Int(reference.attr_dataoffset) + guard dataOffset <= referenceAddress.distance(to: entryEnd) else { return nil } + let start = referenceAddress.advanced(by: dataOffset) + let byteCount = Int(reference.attr_length) + guard byteCount > 0, byteCount <= start.distance(to: entryEnd) else { return nil } + + let bytes = UnsafeRawBufferPointer(start: start, count: byteCount) + // The name is NUL-terminated; trim the trailing NUL before decoding. + let stringByteCount = bytes.last == 0 ? byteCount - 1 : byteCount + guard stringByteCount > 0 else { return nil } + let nameBytes = UnsafeRawBufferPointer(start: start, count: stringByteCount).bindMemory(to: UInt8.self) + guard let name = String(bytes: nameBytes, encoding: .utf8), name.utf8.count == stringByteCount else { + return nil + } + return name +} diff --git a/Sources/Scanner/FSNode.swift b/Sources/Scanner/FSNode.swift index da4699e..487ddc4 100644 --- a/Sources/Scanner/FSNode.swift +++ b/Sources/Scanner/FSNode.swift @@ -1,5 +1,17 @@ import Foundation +// Identity of a hardlinked inode: lets the refresh path recognize that two +// directory entries are the same underlying file, the way the scanner's +// scan-time VisitedSet does. +public struct HardLinkRef: Hashable, Sendable, Codable { + public let dev: UInt64 + public let ino: UInt64 + public init(dev: UInt64, ino: UInt64) { + self.dev = dev + self.ino = ino + } +} + public final class FSNode: Identifiable, @unchecked Sendable { public let id: UUID = UUID() public let url: URL @@ -11,6 +23,17 @@ public final class FSNode: Identifiable, @unchecked Sendable { public let fileExtension: String public var duplicateGroupID: UUID? public var safetyLevel: SafetyLevel = .caution + public var isAccessDenied: Bool = false + public var isSynthetic: Bool = false + // Set only for files with st_nlink > 1 (both the size-carrying node and its 0-size siblings). + public var hardLinkRef: HardLinkRef? + // True when the traversal collapsed this directory's entire subtree into + // this single node instead of materializing its descendants (see + // AtomicDirectorySummary.swift). `children` is always empty in that case. + public var isAutoSummarized: Bool = false + // Total number of files collapsed under this node by auto-summarization. + // 0 for normal (non-summarized) nodes. + public var descendantFileCount: Int = 0 public init(url: URL, name: String, isDirectory: Bool, size: Int64, fileExtension: String, parent: FSNode? = nil) { self.url = url @@ -26,6 +49,6 @@ public final class FSNode: Identifiable, @unchecked Sendable { public extension FSNode { var optionalChildren: [FSNode]? { guard isDirectory && !children.isEmpty else { return nil } - return children.count > 2_000 ? Array(children.prefix(2_000)) : children + return children } } diff --git a/Sources/Scanner/FileScanner.swift b/Sources/Scanner/FileScanner.swift index a9ad5db..f990c23 100644 --- a/Sources/Scanner/FileScanner.swift +++ b/Sources/Scanner/FileScanner.swift @@ -2,17 +2,23 @@ import Foundation // Thread-safe set tracking visited (dev, ino) pairs to prevent double-counting. // On macOS, /System/Volumes/Data/Applications shares the same inode as /Applications, etc. -private final class VisitedSet: @unchecked Sendable { +// Shared by both directory dedup (firmlinks/mount aliases) and hardlinked-file +// dedup: a directory's inode and a file's inode never collide on one device, +// so the two uses safely share one (dev, ino) namespace, exactly as before. +// Shared with AtomicDirectorySummary.swift (same module) so the summarization +// walk dedups directories/hardlinks against the exact same set as the main +// traversal. +final class VisitedSet: @unchecked Sendable { private var lock = os_unfair_lock() private var set = Set() private struct DevIno: Hashable { - let dev: dev_t - let ino: ino_t + let dev: UInt64 + let ino: UInt64 } // Returns true if this (dev, ino) was NOT previously seen (and marks it seen). - func visit(dev: dev_t, ino: ino_t) -> Bool { + func visit(dev: UInt64, ino: UInt64) -> Bool { let key = DevIno(dev: dev, ino: ino) os_unfair_lock_lock(&lock) let inserted = set.insert(key).inserted @@ -26,6 +32,7 @@ private final class ProgressCounter: @unchecked Sendable { private var lock = os_unfair_lock() private(set) var items: Int = 0 private(set) var bytes: Int64 = 0 + private(set) var denied: Int = 0 func add(items: Int, bytes: Int64) { os_unfair_lock_lock(&lock) @@ -34,12 +41,54 @@ private final class ProgressCounter: @unchecked Sendable { os_unfair_lock_unlock(&lock) } + func addDenied() { + os_unfair_lock_lock(&lock) + self.denied += 1 + os_unfair_lock_unlock(&lock) + } + var snapshot: (items: Int, bytes: Int64) { os_unfair_lock_lock(&lock) let result = (items, bytes) os_unfair_lock_unlock(&lock) return result } + + var deniedCount: Int { + os_unfair_lock_lock(&lock) + let result = denied + os_unfair_lock_unlock(&lock) + return result + } +} + +// Snapshot of scan-time settings, read once per scan (not per directory) to avoid +// UserDefaults / environment overhead on the hot path. +struct ScanConfig: Sendable { + let excludedNames: Set + let showHiddenFiles: Bool + // Forces the readdir+fstatat fallback path for every directory in the scan, + // bypassing getattrlistbulk entirely. Used by parity tests to compare the + // two enumeration strategies against each other. + let forceFallbackEnum: Bool + // Collapses tiny-file directories (node_modules, etc.) into one summarized + // leaf FSNode instead of materializing every descendant. See + // AtomicDirectorySummary.swift. + let autoSummarizeEnabled: Bool + + static func loadFromUserDefaults() -> ScanConfig { + let rawExcluded = UserDefaults.standard.string(forKey: "excludedFolderNames") + ?? ".git,node_modules,DerivedData,.Trash" + let excludedNames = Set(rawExcluded.components(separatedBy: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty }) + let showHiddenFiles = UserDefaults.standard.bool(forKey: "showHiddenFiles") + let forceFallbackEnum = ProcessInfo.processInfo.environment["MDS_FORCE_FALLBACK_ENUM"] == "1" + // Default-true feature flag: absent key means "on" (unlike the other + // UserDefaults-backed flags above, which default to false/absent-Bool). + let autoSummarizeEnabled = UserDefaults.standard.object(forKey: "autoSummarizeEnabled") == nil + ? true + : UserDefaults.standard.bool(forKey: "autoSummarizeEnabled") + return ScanConfig(excludedNames: excludedNames, showHiddenFiles: showHiddenFiles, forceFallbackEnum: forceFallbackEnum, autoSummarizeEnabled: autoSummarizeEnabled) + } } public actor FileScanner { @@ -58,9 +107,7 @@ public actor FileScanner { activeTask = Task { let counter = ProgressCounter() let visited = VisitedSet() - // Get the root device ID to detect mount points - var rootStat = stat() - let rootDev: dev_t? = (stat(url.path, &rootStat) == 0) ? rootStat.st_dev : nil + let config = ScanConfig.loadFromUserDefaults() // Emit periodic progress updates every 0.2s let progressTask = Task { @@ -72,9 +119,9 @@ public actor FileScanner { } do { - let root = try await _buildTree(path: url.path, url: url, parent: nil, rootDev: rootDev, counter: counter, visited: visited) + let tree = try await _buildTree(rootPath: url.path, rootURL: url, counter: counter, visited: visited, config: config) progressTask.cancel() - continuation.yield(.completed(root: root)) + continuation.yield(.completed(tree: tree, deniedCount: counter.deniedCount)) } catch is CancellationError { progressTask.cancel() } catch { @@ -88,173 +135,353 @@ public actor FileScanner { } } -// MARK: - POSIX-based parallel tree builder (free functions, not actor-isolated) +// MARK: - Iterative work-stack tree builder (free functions, not actor-isolated) +// +// Concurrency invariant: a directory's `FSNode.children` array is written by +// exactly one worker (the one that processes that directory's work item), and +// every child FSNode is created by the parent's worker before being handed to +// the shared queue as a new work item. So no two workers ever touch the same +// node's mutable state concurrently, even though FSNode is `@unchecked Sendable`. + +private struct DirWorkItem { + let path: String + let url: URL + let node: FSNode + // The scan root is exempt from the mount-boundary and dedup checks that + // every other directory goes through: it defines the volume the scan is + // bound to, and it is registered in the visited set before traversal + // starts, so re-checking it here would skip the whole scan. + let isScanRoot: Bool + // Root is depth 0; each child work item is its parent's depth + 1. Used + // by the auto-summarization depth gate (see AtomicDirectorySummary.swift). + let depth: Int +} + +// Bounded-concurrency work queue: a LIFO stack plus an in-flight counter. +// `pop()` returning nil doesn't mean "done" by itself - workers must also +// check `isFinished` (stack empty AND nothing in flight) before exiting, since +// another worker's current item may still push more work. +private final class WorkQueue: @unchecked Sendable { + private var lock = os_unfair_lock() + private var stack: [DirWorkItem] + private var inFlight: Int + + init(seed: DirWorkItem) { + stack = [seed] + inFlight = 1 + } + + func push(_ item: DirWorkItem) { + os_unfair_lock_lock(&lock) + stack.append(item) + inFlight += 1 + os_unfair_lock_unlock(&lock) + } + + func pop() -> DirWorkItem? { + os_unfair_lock_lock(&lock) + let item = stack.popLast() + os_unfair_lock_unlock(&lock) + return item + } + + // Call exactly once per item that was popped, after it has been fully + // processed (including pushing any children it discovered). + func markDone() { + os_unfair_lock_lock(&lock) + inFlight -= 1 + os_unfair_lock_unlock(&lock) + } + + var isFinished: Bool { + os_unfair_lock_lock(&lock) + let finished = stack.isEmpty && inFlight == 0 + os_unfair_lock_unlock(&lock) + return finished + } +} private func _buildTree( - path: String, - url: URL, - parent: FSNode?, - rootDev: dev_t?, + rootPath: String, + rootURL: URL, counter: ProgressCounter, - visited: VisitedSet -) async throws -> FSNode { + visited: VisitedSet, + config: ScanConfig +) async throws -> FileTree { try Task.checkCancellation() var st = stat() - guard lstat(path, &st) == 0 else { return FSNode(url: url, name: url.lastPathComponent, isDirectory: false, size: 0, fileExtension: "", parent: parent) } + guard lstat(rootPath, &st) == 0 else { + let node = FSNode(url: rootURL, name: rootURL.lastPathComponent, isDirectory: false, size: 0, fileExtension: "", parent: nil) + return FileTreeBuilder.build(from: node, rootPath: rootPath) + } - // Skip symlinks + // Skip symlinks (including a symlink scan root). if st.st_mode & S_IFMT == S_IFLNK { throw SkipError() } let isDir = st.st_mode & S_IFMT == S_IFDIR - - // Deduplicate directories by (dev, ino) — prevents double-counting paths like - // /Applications and /System/Volumes/Data/Applications (same inode, different paths). - if isDir { - guard visited.visit(dev: st.st_dev, ino: st.st_ino) else { throw SkipError() } + let name = rootURL.lastPathComponent + + guard isDir else { + // A file was passed directly as the scan root. + let ext = rootURL.pathExtension.lowercased() + let allocSize = allocatedSize(st: st, visited: visited) + let node = FSNode(url: rootURL, name: name, isDirectory: false, size: allocSize, fileExtension: ext, parent: nil) + if st.st_nlink > 1 { node.hardLinkRef = hardLinkRef(of: st) } + counter.add(items: 1, bytes: allocSize) + return FileTreeBuilder.build(from: node, rootPath: rootPath) } - let name = url.lastPathComponent - let ext = isDir ? "" : url.pathExtension.lowercased() - - let node = FSNode(url: url, name: name, isDirectory: isDir, size: 0, fileExtension: ext, parent: parent) - - if isDir { - let listing = _listDirectory(path: path, url: url, rootDev: rootDev, node: node, counter: counter, visited: visited) - - // Accumulate direct file sizes immediately - node.size = listing.totalSize - node.children = listing.children - - // Recurse into subdirectories in parallel - if !listing.subdirPaths.isEmpty { - var subdirSize: Int64 = 0 - try await withThrowingTaskGroup(of: FSNode?.self) { group in - for (subPath, subURL) in listing.subdirPaths { - group.addTask { - // Propagate cancellation; silently skip symlinks / unreadable entries. - try Task.checkCancellation() - do { - return try await _buildTree(path: subPath, url: subURL, parent: node, rootDev: rootDev, counter: counter, visited: visited) - } catch is SkipError { - return nil - } - } - } - for try await child in group { - guard let child else { continue } - node.children.append(child) - subdirSize += child.size - } + let rootDevKey = UInt64(bitPattern: Int64(st.st_dev)) + let rootInoKey = UInt64(st.st_ino) + // First visit of this scan always succeeds (fresh VisitedSet); kept for + // symmetry with the dedup check every subdirectory goes through below. + _ = visited.visit(dev: rootDevKey, ino: rootInoKey) + + let rootNode = FSNode(url: rootURL, name: name, isDirectory: true, size: 0, fileExtension: "", parent: nil) + let seed = DirWorkItem(path: rootPath, url: rootURL, node: rootNode, isScanRoot: true, depth: 0) + let queue = WorkQueue(seed: seed) + + let workerCount = min(max(2, ProcessInfo.processInfo.activeProcessorCount / 2), 8) + + try await withThrowingTaskGroup(of: Void.self) { group in + for _ in 0.. DirectoryContents { - var result = DirectoryContents(children: [], subdirPaths: [], totalSize: 0, itemCount: 0) + let fd = open(item.path, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW) + guard fd >= 0 else { + if errno == EACCES || errno == EPERM { + item.node.isAccessDenied = true + counter.addDenied() + } + counter.add(items: 1, bytes: 0) + return + } + defer { close(fd) } + + // Everything identity-related is decided from the OPENED directory, never + // from the parent listing's record for it. + // + // macOS firmlinks are why. `/Users`, `/Applications`, `/Library`, `/opt`, + // `/private`, `/Volumes`, `/cores` and friends are firmlinks: the sealed + // system volume holds a link record with its own inode, which resolves on + // open to a completely different inode on the Data volume. An earlier + // version compared the enumerated (dev, ino) against the opened one and + // dropped the directory when they disagreed, as a TOCTOU guard — which + // silently discarded every firmlink, i.e. essentially all user data. A + // scan of "/" reported 11.8 GB instead of 479.3 GB. + // + // Reading identity after the open is also strictly more accurate for the + // other two rules: `/dev` enumerates with the root's device but is really + // devfs, and firmlink aliases (`/Users` vs `/System/Volumes/Data/Users`) + // only collide once resolved, which is exactly what makes the dedup work. + var opened = stat() + guard fstat(fd, &opened) == 0, opened.st_mode & S_IFMT == S_IFDIR else { + counter.add(items: 1, bytes: 0) + return + } + let openedDev = UInt64(bitPattern: Int64(opened.st_dev)) + let openedIno = UInt64(opened.st_ino) + + if !item.isScanRoot { + // Mount boundary: stay on the volume the scan started from. + guard openedDev == rootDevKey else { return } + // Firmlink/alias dedup on the resolved target, so the same directory + // reached by two paths is only counted once. + guard visited.visit(dev: openedDev, ino: openedIno) else { return } + } - guard let dir = opendir(path) else { return result } - defer { closedir(dir) } - let directoryFD = dirfd(dir) + let entries: [BulkDirEntry] + do { + entries = try listDirectoryEntries(path: item.path, fd: fd, forceFallback: config.forceFallbackEnum) + } catch { + // Enumeration failed even after falling back: treat as empty, not denied. + counter.add(items: 1, bytes: 0) + return + } - let rawExcluded = UserDefaults.standard.string(forKey: "excludedFolderNames") - ?? ".git,node_modules,DerivedData,.Trash" - let excludedNames = Set(rawExcluded.components(separatedBy: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty }) + if config.autoSummarizeEnabled, + shouldAutoSummarize(entries: entries, name: item.url.lastPathComponent, depth: item.depth) { + let summary = try await summarizeSubtree( + rootEntries: entries, + rootPath: item.path, + rootDev: rootDevKey, + config: config, + visited: visited + ) { try Task.checkCancellation() } + item.node.isAutoSummarized = true + item.node.descendantFileCount = summary.fileCount + // Deep allocated total for the whole collapsed subtree. + // `aggregateDirectorySizes` only adds up directory *children*'s sizes, + // and this node's children are empty, so it will not be touched again. + item.node.size = summary.allocatedSize + item.node.children = [] + counter.add(items: summary.fileCount + 1, bytes: summary.allocatedSize) + return + } - while let entry = readdir(dir) { - let nameBytes = entry.pointee.d_name - let name: String = withUnsafeBytes(of: nameBytes) { ptr in - let bytes = ptr.bindMemory(to: CChar.self) - return String(cString: bytes.baseAddress!) + var directSize: Int64 = 0 + var children: [FSNode] = [] + children.reserveCapacity(entries.count) + + for (index, entry) in entries.enumerated() { + if index % 256 == 0 { + try Task.checkCancellation() } - guard name != "." && name != ".." else { continue } - if name.hasPrefix("."), !UserDefaults.standard.bool(forKey: "showHiddenFiles") { continue } - if excludedNames.contains(name) { continue } - - let childURL = url.appendingPathComponent(name, isDirectory: entry.pointee.d_type == DT_DIR) - let dtype = entry.pointee.d_type - - // Fast type check using d_type from dirent (avoids extra stat call for most entries) - if dtype == DT_LNK { continue } // skip symlinks - - if dtype == DT_DIR || dtype == DT_UNKNOWN { - // For DT_UNKNOWN (e.g. some network filesystems), use lstat - var st = stat() - let childPath = path.hasSuffix("/") ? path + name : path + "/" + name - if dtype == DT_UNKNOWN { - guard fstatat(directoryFD, name, &st, AT_SYMLINK_NOFOLLOW) == 0 else { continue } - let mode = st.st_mode & S_IFMT - if mode == S_IFLNK { continue } - if mode != S_IFDIR { - // It's a regular file with DT_UNKNOWN - let allocSize = allocatedSize(st: st, visited: visited) - let ext = childURL.pathExtension.lowercased() - let fileNode = FSNode(url: childURL, name: name, isDirectory: false, size: allocSize, fileExtension: ext, parent: node) - result.children.append(fileNode) - result.totalSize += allocSize - result.itemCount += 1 - counter.add(items: 1, bytes: allocSize) - continue - } - // Check mount point via st_dev - if let rootDev, st.st_dev != rootDev { continue } - result.subdirPaths.append((childPath, childURL)) - } else { - // DT_DIR — check mount point via a quick stat - if let rootDev { - guard fstatat(directoryFD, name, &st, 0) == 0 else { continue } - if st.st_dev != rootDev { continue } - } - result.subdirPaths.append((childPath, childURL)) - } - } else if dtype == DT_REG { - var st = stat() - guard fstatat(directoryFD, name, &st, AT_SYMLINK_NOFOLLOW) == 0 else { continue } - let allocSize = allocatedSize(st: st, visited: visited) + guard entry.name != "." && entry.name != ".." else { continue } + if entry.name.hasPrefix("."), !config.showHiddenFiles { continue } + if config.excludedNames.contains(entry.name) { continue } + + switch entry.kind { + case .symlink, .other: + continue + + case .directory: + // No device/dedup filtering here: a directory entry's enumerated + // (dev, ino) describes the link record, which for a firmlink is + // not the thing that gets opened. Both checks happen in the child's + // own `_processDirectory`, against its resolved identity. + let childURL = item.url.appendingPathComponent(entry.name, isDirectory: true) + let childNode = FSNode(url: childURL, name: entry.name, isDirectory: true, size: 0, fileExtension: "", parent: item.node) + children.append(childNode) + let childPath = item.path.hasSuffix("/") ? item.path + entry.name : item.path + "/" + entry.name + queue.push(DirWorkItem(path: childPath, url: childURL, node: childNode, isScanRoot: false, depth: item.depth + 1)) + + case .file: + let childURL = item.url.appendingPathComponent(entry.name, isDirectory: false) + let allocSize = bulkAllocatedSize(entry: entry, visited: visited) let ext = childURL.pathExtension.lowercased() - let fileNode = FSNode(url: childURL, name: name, isDirectory: false, size: allocSize, fileExtension: ext, parent: node) - result.children.append(fileNode) - result.totalSize += allocSize - result.itemCount += 1 + let fileNode = FSNode(url: childURL, name: entry.name, isDirectory: false, size: allocSize, fileExtension: ext, parent: item.node) + if entry.linkCount > 1 { fileNode.hardLinkRef = HardLinkRef(dev: entry.dev, ino: entry.ino) } + children.append(fileNode) + directSize += allocSize counter.add(items: 1, bytes: allocSize) } - // DT_FIFO, DT_CHR, DT_BLK, DT_SOCK — skip } - return result + item.node.children = children + item.node.size = directSize + counter.add(items: 1, bytes: 0) +} + +// Picks bulk vs. fallback enumeration for one directory. If bulk enumeration +// throws partway through (having already consumed some of `fd`'s kernel-side +// listing position), the fallback re-opens the directory fresh by path so it +// always sees the complete, unconsumed listing rather than a partial remainder. +// Shared with AtomicDirectorySummary.swift (same module). +func listDirectoryEntries(path: String, fd: Int32, forceFallback: Bool) throws -> [BulkDirEntry] { + if forceFallback { + return try fallbackEnumerateDirectory(fd: fd) + } + do { + return try enumerateDirectoryBulk(fd: fd) + } catch is BulkEnumerationUnavailable { + let freshFD = open(path, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW) + guard freshFD >= 0 else { return [] } + defer { close(freshFD) } + return try fallbackEnumerateDirectory(fd: freshFD) + } +} + +// Iterative post-order pass: folds each directory's descendant directory +// sizes into its own `size` (which already holds its direct file sum from +// `_processDirectory`). Runs once, after all workers finish, so there is no +// concurrent mutation of node.size during traversal. +private func aggregateDirectorySizes(root: FSNode) { + guard root.isDirectory else { return } + + // `order` ends up a valid pre-order (a node always precedes its own + // descendants, since children are only pushed after their parent is + // appended). Processing it in reverse guarantees every directory's + // children are fully aggregated before the directory itself is folded + // into its own parent. + var order: [FSNode] = [] + var stack: [FSNode] = [root] + while let node = stack.popLast() { + order.append(node) + for child in node.children where child.isDirectory { + stack.append(child) + } + } + + for node in order.reversed() { + var subtreeAddition: Int64 = 0 + for child in node.children where child.isDirectory { + subtreeAddition += child.size + } + node.size += subtreeAddition + } +} + +// Builds the inode identity for a hardlinked file. dev_t is a signed 32-bit value +// (can be negative for synthetic filesystems), so use bit-pattern conversion. +private func hardLinkRef(of st: stat) -> HardLinkRef { + HardLinkRef(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) } // Returns the file's allocated disk bytes, deduplicating hardlinks via the visited set. // Files with nlink == 1 skip the set entirely (fast path for the common case). // Hardlinked files (nlink > 1) are counted only on their first encounter. +// Used only for a file passed directly as the scan root; regular directory +// listings use `bulkAllocatedSize` on the enumerator's own metadata instead. private func allocatedSize(st: stat, visited: VisitedSet) -> Int64 { if st.st_nlink > 1 { - guard visited.visit(dev: st.st_dev, ino: st.st_ino) else { return 0 } + guard visited.visit(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) else { return 0 } } return Int64(st.st_blocks) * 512 } +// Same dedup rule as `allocatedSize(st:visited:)` above, but sourced from a +// `BulkDirEntry` (either enumeration path) instead of a raw `stat`. +// Shared with AtomicDirectorySummary.swift (same module). +func bulkAllocatedSize(entry: BulkDirEntry, visited: VisitedSet) -> Int64 { + if entry.linkCount > 1 { + guard visited.visit(dev: entry.dev, ino: entry.ino) else { return 0 } + } + return entry.allocatedSize +} + private struct SkipError: Error {} diff --git a/Sources/Scanner/ScanProgress.swift b/Sources/Scanner/ScanProgress.swift index e10931a..3bc10c6 100644 --- a/Sources/Scanner/ScanProgress.swift +++ b/Sources/Scanner/ScanProgress.swift @@ -2,6 +2,6 @@ import Foundation public enum ScanProgress: Sendable { case update(itemsScanned: Int, bytesFound: Int64) - case completed(root: FSNode) + case completed(tree: FileTree, deniedCount: Int) case failed(String) } diff --git a/Sources/ViewModels/ScanViewModel.swift b/Sources/ViewModels/ScanViewModel.swift index 904acce..7e95fca 100644 --- a/Sources/ViewModels/ScanViewModel.swift +++ b/Sources/ViewModels/ScanViewModel.swift @@ -3,25 +3,48 @@ import SwiftUI @MainActor public final class ScanViewModel: ObservableObject { - @Published public var root: FSNode? + // The flat store the scanner hands back (see Sources/Model/). `root` is + // a computed FileNode handle onto it, kept under the old name so views + // that just read `vm.root` didn't need to change. + @Published public var tree: FileTree? @Published public var cells: [TreemapCell] = [] @Published public var colorMap: ExtensionColorMap? - @Published public var selectedNode: FSNode? + @Published public var selectedNode: FileNode? @Published public var isScanning: Bool = false @Published public var itemsScanned: Int = 0 @Published public var bytesFound: Int64 = 0 @Published public var errorMessage: String? @Published public var duplicatesReady: Bool = false - @Published public var drillStack: [FSNode] = [] + @Published public var drillStack: [FileNode] = [] @Published public var highlightedExtension: String? @Published public var isComputingLayout: Bool = false @Published public var scanURL: URL? @Published public var extensionSummaries: [ExtensionSummary] = [] - @Published public var duplicateGroups: [[FSNode]] = [] + @Published public var duplicateGroups: [[FileNode]] = [] @Published public var hasFullDiskAccess: Bool = true @Published public var isWatching: Bool = false + // Set when a live change arrived that the incremental splice could not + // fold in (see `handleFileSystemChanges`). The displayed tree is still + // valid, just possibly behind reality for that path — the UI offers a + // manual rescan rather than the app silently restarting the whole scan. + @Published public var hasStaleResults: Bool = false + @Published public var deniedCount: Int = 0 + @Published public var showFDASheet: Bool = false + // Set when `tree` was loaded from a `.mdscan` archive instead of a live + // scan: disables trash/delete actions and live FSEvents watching, and + // drives the read-only banner in ContentView. + @Published public var isReadOnlySnapshot: Bool = false + @Published public var snapshotDate: Date? + // Result of "Compare With Saved Scan…" (4c): diffing the currently + // loaded tree against a `.mdscan` archive picked from disk. Entirely + // separate from `isReadOnlySnapshot`/`tree` — comparing never replaces + // the active tree, it only produces a side-by-side report. + @Published public var comparisonResult: ComparisonResult? + @Published public var isComputingComparison: Bool = false + @Published public var showComparisonSheet: Bool = false - public var treemapRoot: FSNode? { drillStack.last ?? root } + public var root: FileNode? { tree.map { FileNode(tree: $0, index: $0.rootIndex) } } + public var treemapRoot: FileNode? { drillStack.last ?? root } private let scanner = FileScanner() private let fileWatcher = FileWatcher() @@ -34,6 +57,7 @@ public final class ScanViewModel: ObservableObject { private var layoutGeneration: Int = 0 private var securityScopedURL: URL? private var memoryPressureSource: DispatchSourceMemoryPressure? + private var fdaSheetShownThisLaunch = false public init() { UserDefaults.standard.register(defaults: [ @@ -48,7 +72,21 @@ public final class ScanViewModel: ObservableObject { ]) setupMemoryPressureHandler() checkFullDiskAccess() - if UserDefaults.standard.bool(forKey: "autoScanLastFolder"), + + // A relaunch triggered from the Full Disk Access flow leaves behind the path + // that was being scanned, so the new instance can resume right where the user + // left off instead of landing back on the welcome screen. + var resumedPendingRescan = false + if let pending = UserDefaults.standard.string(forKey: "fdaPendingRescanPath") { + UserDefaults.standard.removeObject(forKey: "fdaPendingRescanPath") + if FileManager.default.fileExists(atPath: pending) { + scan(url: URL(fileURLWithPath: pending)) + resumedPendingRescan = true + } + } + + if !resumedPendingRescan, + UserDefaults.standard.bool(forKey: "autoScanLastFolder"), let path = UserDefaults.standard.string(forKey: "lastScannedPath"), FileManager.default.fileExists(atPath: path) { scan(url: URL(fileURLWithPath: path)) @@ -65,6 +103,44 @@ public final class ScanViewModel: ObservableObject { hasFullDiskAccess = FileManager.default.isReadableFile(atPath: probe) } + /// Public wrapper so the onboarding sheet can poll for a live permission change + /// without exposing the private TCC probe itself. + public func recheckFullDiskAccess() { + checkFullDiskAccess() + } + + /// Pure decision logic for whether the guided Full Disk Access sheet should be + /// offered after a scan completes: only when access is actually missing, the scan + /// hit blocked folders, and the user hasn't opted out. + nonisolated static func shouldOfferFullDiskAccess(deniedCount: Int, hasFullDiskAccess: Bool, suppressed: Bool) -> Bool { + !hasFullDiskAccess && deniedCount > 0 && !suppressed + } + + /// Relaunches the app so macOS re-evaluates the Full Disk Access grant, and leaves + /// a breadcrumb so the new instance automatically resumes the scan the user was on. + public func relaunchForFullDiskAccess() { + let pathToResume = scanURL?.path ?? UserDefaults.standard.string(forKey: "lastScannedPath") + if let pathToResume { + UserDefaults.standard.set(pathToResume, forKey: "fdaPendingRescanPath") + } + + guard Bundle.main.bundlePath.hasSuffix(".app") else { + // Debug binary, not an app bundle — there's nothing to relaunch + // programmatically. The developer restarts the process by hand. + NSApp.terminate(nil) + return + } + + let configuration = NSWorkspace.OpenConfiguration() + configuration.createsNewApplicationInstance = true + NSWorkspace.shared.openApplication(at: Bundle.main.bundleURL, configuration: configuration) { _, _ in + // Give the new instance a moment to spawn before this one exits. + DispatchQueue.main.asyncAfter(deadline: .now() + 0.4) { + NSApp.terminate(nil) + } + } + } + private func setupMemoryPressureHandler() { let source = DispatchSource.makeMemoryPressureSource(eventMask: [.warning, .critical], queue: .main) source.setEventHandler { [weak self] in @@ -92,10 +168,13 @@ public final class ScanViewModel: ObservableObject { watchTask?.cancel() fileWatcher.stop() isWatching = false + hasStaleResults = false layoutGeneration += 1 // invalidate any in-progress layout scanURL = url UserDefaults.standard.set(url.path, forKey: "lastScannedPath") - root = nil + isReadOnlySnapshot = false + snapshotDate = nil + tree = nil cells = [] colorMap = nil selectedNode = nil @@ -109,6 +188,7 @@ public final class ScanViewModel: ObservableObject { bytesFound = 0 isComputingLayout = false errorMessage = nil + deniedCount = 0 scanTask = Task { for await progress in await scanner.scan(url: url) { @@ -116,16 +196,44 @@ public final class ScanViewModel: ObservableObject { case .update(let items, let bytes): self.itemsScanned = items self.bytesFound = bytes - case .completed(let node): + case .completed(let scannedTree, let denied): self.isScanning = false + self.deniedCount = denied + if !self.fdaSheetShownThisLaunch, + Self.shouldOfferFullDiskAccess( + deniedCount: denied, + hasFullDiskAccess: self.hasFullDiskAccess, + suppressed: UserDefaults.standard.bool(forKey: "fdaPromptSuppressed") + ) { + self.showFDASheet = true + self.fdaSheetShownThisLaunch = true + } + // If the scanned root is a volume mount point, the file total will + // always fall short of Finder's "used" figure (APFS snapshots, + // purgeable space, excluded/unreadable folders). Make that gap + // visible instead of silently under-reporting. Produces a NEW tree + // (topology is immutable) with the synthetic child already in its + // sorted place, so no separate sort pass is needed afterward. + let finalTree = Self.appendHiddenSpaceNodeIfNeeded(tree: scannedTree, scannedURL: url) ?? scannedTree + if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { + let rootNode = FileNode(tree: finalTree, index: finalTree.rootIndex) + var dump = "TREE_COMPLETED total=\(rootNode.size) denied=\(denied)\n" + for c in rootNode.children.prefix(15) { + dump += "TREE_CHILD \(c.size) \(c.name)\(c.isSynthetic ? " [synthetic]" : "")\n" + } + FileHandle.standardError.write(dump.data(using: .utf8)!) + } self.isComputingLayout = true // keep spinner until treemap is ready - // Sort + safety-tag the entire tree off-thread before exposing it to the UI. + // Safety-tag the entire tree off-thread before exposing it to the UI. + // (Children are already size-sorted by FileTreeBuilder/the synthetic + // splice above, so unlike the old FSNode path there is no separate + // sort pass to run here.) await Task.detached(priority: .userInitiated) { - Self.sortAllChildren(node: node) - Self.tagSafetyLevels(node: node) + Self.tagSafetyLevels(tree: finalTree) }.value - self.root = node - let map = ExtensionColorMap(root: node) + self.tree = finalTree + let rootNode = FileNode(tree: finalTree, index: finalTree.rootIndex) + let map = ExtensionColorMap(root: rootNode) self.colorMap = map await self.recomputeLayout() // isComputingLayout set to false inside recomputeLayout @@ -136,18 +244,18 @@ public final class ScanViewModel: ObservableObject { } // Extension summaries: potentially millions of nodes — run off main actor - self.extensionTask = Task.detached(priority: .userInitiated) { [node, map, weak self] in - let summaries = Self.buildExtensionSummaries(root: node, map: map) + self.extensionTask = Task.detached(priority: .userInitiated) { [finalTree, map, weak self] in + let summaries = Self.buildExtensionSummaries(tree: finalTree, map: map) guard !Task.isCancelled else { return } let vm = self await MainActor.run { vm?.extensionSummaries = summaries } } // Duplicate detection: lower priority, also off main actor - self.duplicateTask = Task.detached(priority: .utility) { [node, weak self] in + self.duplicateTask = Task.detached(priority: .utility) { [finalTree, weak self] in let detector = DuplicateDetector() - await detector.detect(in: node) + await detector.detect(in: finalTree) guard !Task.isCancelled else { return } - let groups = Self.buildDuplicateGroups(root: node) + let groups = Self.buildDuplicateGroups(tree: finalTree) guard !Task.isCancelled else { return } let vm = self await MainActor.run { @@ -202,35 +310,325 @@ public final class ScanViewModel: ObservableObject { } } - private func handleFileSystemChanges(_ paths: [String]) async { - guard let root else { return } + // Incremental splice refresh: for each directory FSEvents reports as + // changed, rescan just that directory from disk and splice the result + // into `tree` (see `Self.splicedTree(afterChangeAt:in:)` / + // `FileTree.replacingSubtree(at:with:)`) instead of rescanning the whole + // root, exactly like the "Move to Trash" prune path already avoids a + // full rescan for deletes. Multiple changed paths in one FSEvents batch + // are folded sequentially — each splice's result feeds the next lookup — + // which the plan explicitly allows as simpler than a single combined + // multi-directory splice, at the cost of a little redundant rescanning + // when a batch contains both a directory and one of its own descendants. + // + // A background refresh must NEVER destroy a finished scan. An earlier + // version fell back to `scan(url:)` — a full rescan — for any change a + // splice couldn't handle, which produced an infinite restart loop on real + // folders: the scan finishes, watching starts, FSEvents immediately + // reports a change the splice refuses (most commonly the scanned root + // directory itself, since this watcher is directory-granularity, or + // anything inside an auto-summarized folder like node_modules), the full + // rescan clears `tree`/`cells` so the treemap vanishes mid-render, and on + // completion it starts watching again and repeats forever. + // + // So unspliceable changes are now SKIPPED, never escalated: the rest of + // the batch still splices, and `hasStaleResults` records that some change + // could not be folded in, so the UI can offer a manual rescan instead of + // silently thrashing. A vanished root just stops watching. + // + // Not `private`: exercised directly by IncrementalRefreshTests (via + // `@testable import`) to simulate an FSEvents batch deterministically, + // without needing a real FSEventStream round trip. + func handleFileSystemChanges(_ paths: [String]) async { + guard let scanURL, let startingTree = tree, !isReadOnlySnapshot else { return } - // Collect the unique directory paths that changed - var dirPaths = Set() - for path in paths { - var isDir: ObjCBool = false - if FileManager.default.fileExists(atPath: path, isDirectory: &isDir), isDir.boolValue { - dirPaths.insert(path) - } else { - dirPaths.insert((path as NSString).deletingLastPathComponent) + guard FileManager.default.fileExists(atPath: scanURL.path) else { + // The scanned root itself is gone (deleted/renamed/unmounted). + // Nothing to refresh into, and re-scanning a missing path would + // just fail on a loop — stop watching and let the user decide. + if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { + FileHandle.standardError.write("REFRESH stop-watching root-vanished root=\(scanURL.path)\n".data(using: .utf8)!) } + watchTask?.cancel() + fileWatcher.stop() + isWatching = false + hasStaleResults = true + return } - var needsLayout = false - for dirPath in dirPaths { - guard let node = Self.findNode(path: dirPath, in: root) else { continue } - let changed = await Task.detached(priority: .userInitiated) { - Self.refreshDirectory(node: node) - }.value - if changed { - Self.bubbleUpSizes(from: node) - needsLayout = true + var workingTree = startingTree + var skippedUnspliceable = false + for changedPath in paths { + guard let spliced = Self.splicedTree(afterChangeAt: changedPath, in: workingTree) else { + if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { + FileHandle.standardError.write("REFRESH skipped-unspliceable path=\(changedPath)\n".data(using: .utf8)!) + } + skippedUnspliceable = true + continue } + workingTree = spliced } - if needsLayout { - await recomputeLayout() + if skippedUnspliceable { hasStaleResults = true } + + guard workingTree !== startingTree else { return } // nothing actually spliceable in this batch + + if ProcessInfo.processInfo.environment["MDS_DEBUG_TREE"] != nil { + FileHandle.standardError.write("REFRESH spliced changedPaths=\(paths.count) root=\(scanURL.path)\n".data(using: .utf8)!) + } + await applySplicedTree(workingTree, from: startingTree) + } + + // Rescans exactly the on-disk directory at `changedPath` (a full, + // synchronous walk via `scanSubtree`, the same per-directory rescan step + // the pre-Phase-2 refresh path used) and splices the result into `tree`, + // replacing its stale subtree in place — the cheap alternative to + // rescanning the whole root on every FSEvents notification. + // + // Returns nil when the splice can't be trusted, and the caller must fall + // back to a full rescan of the root instead: + // - `changedPath` (after normalizing away a trailing slash) resolves + // to the tree's own root — a changed root might mean the scanned + // directory itself was replaced or renamed, which no subtree splice + // can recover from. + // - `changedPath` doesn't resolve to any node in `tree` at all: it (or + // an ancestor) was deleted/renamed since the last refresh, or it + // lives inside an auto-summarized directory, whose children were + // never materialized in the first place — the path-component walk + // simply runs out of children to match partway down. + // - the resolved node is itself auto-summarized: it has no children + // array to splice into (see AtomicDirectorySummary.swift) — + // re-summarizing it in place is future work; falling back to a full + // rescan is correct and simple for now. + // - the freshly-rescanned replacement contains a directory the real + // scanner's auto-summarization would have collapsed (named + // "node_modules", mirroring `knownGeneratedDirectoryNames` in + // AtomicDirectorySummary.swift) — `scanSubtree` below doesn't + // implement that heuristic at all, so materializing it here would + // both be slow and disagree with the rest of the tree's + // summarization policy. + nonisolated static func splicedTree(afterChangeAt changedPath: String, in tree: FileTree) -> FileTree? { + let normalized = (changedPath.hasSuffix("/") && changedPath != "/") ? String(changedPath.dropLast()) : changedPath + + guard var index = findIndex(forPath: normalized, in: tree) else { return nil } + // FSEvents (without the FileEvents flag, which this app doesn't + // request) reports directories, but be defensive: if this ever + // resolves to a file, the directory that actually needs rescanning + // is its parent. + if !tree.records[index].isDirectory { + let parent = tree.parentIndex[index] + guard parent >= 0 else { return nil } + index = parent + } + guard index != tree.rootIndex else { return nil } + guard !tree.records[index].isAutoSummarized else { return nil } + + let node = FileNode(tree: tree, index: index) + let showHiddenFiles = UserDefaults.standard.bool(forKey: "showHiddenFiles") + let excludedNames = parseExcludedNames() + + // Cross-tree hardlink dedup (BUG 2 fix): `scanSubtree`'s own + // `treeRoot` parameter — meant to catch a hardlink whose twin lives + // outside the directory being rescanned via `firstNode(withRef:in:)` + // — only makes sense against a live FSNode tree, which no longer + // exists on this flat-store splice path, so it stays `nil` below and + // that check can never fire. Left alone, that means only hardlinks + // *within* this one rescanned directory would get deduped (via the + // fresh, otherwise-empty `seenRefs` below), and any twin living + // outside the spliced subtree would double-count. Pre-seeding + // `seenRefs` before the call replaces that dead check: it contains + // every `hardLinkRef` that appears anywhere in `tree` OUTSIDE the + // subtree being replaced, but ONLY when that outside occurrence is + // itself the size carrier (size > 0). That asymmetry matters — if + // the carrier instead lives INSIDE the subtree being replaced (i.e. + // the outside twin is the 0-size loser), seeding on the twin's mere + // presence would make the rescan zero its own copy too, and the + // inode's bytes would vanish from the tree entirely (both copies at + // 0) instead of correctly moving to whichever copy is now first-seen. + var seenRefs = Self.hardLinkRefsOutsideSubtree(rootedAt: index, in: tree) + + // FIX 1 setup: capture which `hardLinkRef`s the OLD (stale) subtree + // itself carried (size > 0), against the ORIGINAL tree, before it's + // replaced below. This is the only place that information survives — + // once `replacingSubtree` swaps the stale nodes out, there's no way + // to tell "this ref isn't carried by the new subtree" apart from + // "this ref was never carried by anything in this subtree at all". + // See the comment at the promotion call site below for why this + // matters: an external deletion (Finder/`rm`/a build tool — NOT this + // app's own Trash action, which `FileTree.removingSubtrees` already + // covers) of the carrier link inside this subtree leaves the inode + // still fully allocated via a twin elsewhere, and nothing else in + // this function would ever notice. + let oldCarriedRefs = Self.carriedHardLinkRefs(insideSubtreeRootedAt: index, in: tree) + + let freshNode = scanSubtree( + url: node.url, + parent: nil, + showHiddenFiles: showHiddenFiles, + excludedNames: excludedNames, + treeRoot: nil, + seenRefs: &seenRefs + ) + + guard freshNode.name != "node_modules", !containsUnsummarizedGeneratedDirectory(freshNode) else { return nil } + + let subtree = FileTreeBuilder.build(from: freshNode, rootPath: node.url.path) + var result = tree.replacingSubtree(at: index, with: subtree) + + // FIX 1: promote a surviving twin for every ref the OLD subtree + // carried but the freshly-rescanned NEW subtree does not — the + // on-disk link that carried the inode's bytes vanished for a reason + // this splice can't otherwise account for (an external delete, not + // this app's own Trash action). Without this, the bytes would simply + // drop out of the tree until the next full rescan, even though the + // inode is still fully allocated via a twin outside this subtree (or + // even inside it — see `promotingSurvivingTwin`'s doc comment for why + // a plain whole-tree search for the twin is correct either way). + // Multiple independent orphaned refs (two unrelated hardlink pairs + // both losing their carrier in the same splice) each promote their + // own twin, one call per ref. + if !oldCarriedRefs.isEmpty { + var newCarriedRefs = Set() + for record in subtree.records where record.hardLinkRef != nil && record.size > 0 { + newCarriedRefs.insert(record.hardLinkRef!) + } + for (ref, size) in oldCarriedRefs where !newCarriedRefs.contains(ref) { + result = result.promotingSurvivingTwin(ref: ref, size: size) + } + } + + return result + } + + // FIX 1 support for `splicedTree`: collects `hardLinkRef -> size` for + // every node INSIDE the subtree rooted at `subtreeRootIndex` that is + // itself the size carrier (`size > 0`) for its ref — the set of refs an + // external deletion inside this subtree could orphan. Mirrors + // `hardLinkRefsOutsideSubtree`'s traversal but walks IN rather than + // computing the complement, and needs the size (not just the ref) so the + // caller can promote a survivor to the exact right amount. + private nonisolated static func carriedHardLinkRefs(insideSubtreeRootedAt subtreeRootIndex: Int, in tree: FileTree) -> [HardLinkRef: Int64] { + var result: [HardLinkRef: Int64] = [:] + var stack = [subtreeRootIndex] + while let i = stack.popLast() { + if let ref = tree.records[i].hardLinkRef, tree.records[i].size > 0 { + result[ref] = tree.records[i].size + } + let start = tree.childStart[i] + let cnt = tree.childCount[i] + for offset in 0.. 0`). See the comment at the `splicedTree` call site + // for why the `size > 0` condition (rather than mere presence) matters. + private nonisolated static func hardLinkRefsOutsideSubtree(rootedAt subtreeRootIndex: Int, in tree: FileTree) -> Set { + let count = tree.records.count + var inside = [Bool](repeating: false, count: count) + var stack = [subtreeRootIndex] + while let i = stack.popLast() { + if inside[i] { continue } + inside[i] = true + let start = tree.childStart[i] + let cnt = tree.childCount[i] + for offset in 0..() + for i in 0.. 0 { + refs.insert(ref) + } } + return refs + } + + // See the last bullet of `splicedTree`'s doc comment above. + private nonisolated static func containsUnsummarizedGeneratedDirectory(_ node: FSNode) -> Bool { + for child in node.children where child.isDirectory { + if child.name == "node_modules" || containsUnsummarizedGeneratedDirectory(child) { + return true + } + } + return false + } + + // Repairs everything that referenced the old topology after one or more + // splices, mirroring `pruneTree(afterTrashing:from:)`: selection/drill + // stack are captured as paths beforehand and resolved back to indices + // afterward, since indices shift on every splice. The synthetic + // "Hidden & Unreadable Space" root child is never touched by any splice + // (it's a root-level child and a splice target is never the root — see + // `splicedTree`'s root guard — so it always survives untouched, no + // special-case re-appending needed here the way a full rescan needs + // `appendHiddenSpaceNodeIfNeeded`). + // + // Unlike a prune, a splice can introduce brand-new nodes (the freshly + // rescanned subtree), which start out `.caution`/no-duplicate-group from + // `FileTreeBuilder` — same as any fresh scan — so safety tagging and + // duplicate detection both re-run over the whole tree, exactly as they + // do after `scan(url:)`. Safety tagging is awaited synchronously before + // anything else touches `newTree.records`: it and `DuplicateDetector` + // both mutate that array in place, so — same reasoning as `scan(url:)` — + // they can't be allowed to run concurrently with each other. + private func applySplicedTree(_ newTree: FileTree, from oldTree: FileTree) async { + let selectedPath = selectedNode.map { oldTree.path(of: $0.index) } + let drillPaths = drillStack.map { oldTree.path(of: $0.index) } + + await Task.detached(priority: .userInitiated) { + Self.tagSafetyLevels(tree: newTree) + }.value + + self.tree = newTree + + if let selectedPath, let idx = Self.findIndex(forPath: selectedPath, in: newTree) { + selectedNode = FileNode(tree: newTree, index: idx) + } else { + selectedNode = nil + } + + var newDrillStack: [FileNode] = [] + for path in drillPaths { + guard let idx = Self.findIndex(forPath: path, in: newTree) else { break } + newDrillStack.append(FileNode(tree: newTree, index: idx)) + } + drillStack = newDrillStack + + let rootNode = FileNode(tree: newTree, index: newTree.rootIndex) + let map = ExtensionColorMap(root: rootNode) + colorMap = map + + extensionTask?.cancel() + extensionTask = Task.detached(priority: .userInitiated) { [weak self] in + let summaries = Self.buildExtensionSummaries(tree: newTree, map: map) + guard !Task.isCancelled else { return } + await MainActor.run { self?.extensionSummaries = summaries } + } + + duplicateTask?.cancel() + duplicateTask = Task.detached(priority: .utility) { [weak self] in + let detector = DuplicateDetector() + await detector.detect(in: newTree) + guard !Task.isCancelled else { return } + let groups = Self.buildDuplicateGroups(tree: newTree) + guard !Task.isCancelled else { return } + await MainActor.run { + self?.duplicatesReady = true + self?.duplicateGroups = groups + } + } + + isComputingLayout = true + await recomputeLayout() } // Walk the tree by path components to find the FSNode for a given path. @@ -249,45 +647,130 @@ public final class ScanViewModel: ObservableObject { return current } + // Returns the allocated disk size (st_blocks * 512) for a path via lstat, or nil if + // the path can't be stat'd or is a symlink. + private nonisolated static func lstatInfo(path: String) -> (isDir: Bool, isSymlink: Bool, allocatedSize: Int64, linkCount: Int, ref: HardLinkRef)? { + var st = stat() + guard lstat(path, &st) == 0 else { return nil } + let mode = st.st_mode & S_IFMT + let isSymlink = mode == S_IFLNK + let isDir = mode == S_IFDIR + let allocatedSize = Int64(st.st_blocks) * 512 + let ref = HardLinkRef(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) + return (isDir, isSymlink, allocatedSize, Int(st.st_nlink), ref) + } + + // Climbs the parent chain to the tree's root node. + private nonisolated static func rootNode(of node: FSNode) -> FSNode { + var current = node + while let parent = current.parent { current = parent } + return current + } + + // Iterative whole-tree search for a node representing the given inode. + // Only invoked when hardlinked entries (st_nlink > 1) appear or disappear, + // which is rare per refresh, so the O(tree) walk is acceptable. + private nonisolated static func firstNode(withRef ref: HardLinkRef, in root: FSNode, requireZeroSize: Bool = false) -> FSNode? { + var stack = [root] + while let n = stack.popLast() { + if n.hardLinkRef == ref, !requireZeroSize || n.size == 0 { return n } + stack.append(contentsOf: n.children) + } + return nil + } + + // Parses the excludedFolderNames default the same way FileScanner does. + private nonisolated static func parseExcludedNames() -> Set { + let raw = UserDefaults.standard.string(forKey: "excludedFolderNames") + ?? ".git,node_modules,DerivedData,.Trash" + return Set(raw.components(separatedBy: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty }) + } + // Re-stat the directory on disk and update children to match. // Returns true if anything changed (additions/removals/size changes). @discardableResult - private nonisolated static func refreshDirectory(node: FSNode) -> Bool { + nonisolated static func refreshDirectory(node: FSNode) -> Bool { guard node.isDirectory else { return false } let fm = FileManager.default + let showHiddenFiles = UserDefaults.standard.bool(forKey: "showHiddenFiles") + let excludedNames = parseExcludedNames() guard let entries = try? fm.contentsOfDirectory( at: node.url, - includingPropertiesForKeys: [.fileSizeKey, .isDirectoryKey, .isSymbolicLinkKey], - options: [.skipsHiddenFiles] + includingPropertiesForKeys: nil, + options: showHiddenFiles ? [] : [.skipsHiddenFiles] ) else { return false } - let onDisk = Dictionary(uniqueKeysWithValues: entries.map { ($0.lastPathComponent, $0) }) + // Filter out excluded folder names and symlinks up front, so both the + // removal pass and the add/update pass agree on what's "on disk". + var onDisk: [String: (url: URL, info: (isDir: Bool, isSymlink: Bool, allocatedSize: Int64, linkCount: Int, ref: HardLinkRef))] = [:] + for url in entries { + let name = url.lastPathComponent + if excludedNames.contains(name) { continue } + guard let info = lstatInfo(path: url.path) else { continue } + if info.isSymlink { continue } + onDisk[name] = (url, info) + } + var changed = false - // Remove children that no longer exist + // Remove children that no longer exist on disk (or are now excluded/symlinks), + // but never remove synthetic nodes (e.g. the "Hidden & Unreadable Space" + // reconciliation entry), which never correspond to a real path. + let removedSizeCarriers = node.children.filter { + !$0.isSynthetic && !onDisk.keys.contains($0.name) && $0.hardLinkRef != nil && $0.size > 0 + } let before = node.children.count - node.children.removeAll { !onDisk.keys.contains($0.name) } + node.children.removeAll { !$0.isSynthetic && !onDisk.keys.contains($0.name) } if node.children.count != before { changed = true } + // If a removed entry carried the representative size for a hardlinked inode + // that still exists via other links, promote a surviving 0-size link + // (anywhere in the tree) to carry the size, or the bytes vanish forever. + for removed in removedSizeCarriers { + guard let ref = removed.hardLinkRef else { continue } + let root = rootNode(of: node) + guard let survivor = firstNode(withRef: ref, in: root, requireZeroSize: true), + let survivorInfo = lstatInfo(path: survivor.url.path), !survivorInfo.isDir + else { continue } + survivor.size = survivorInfo.allocatedSize + bubbleUpSizes(from: survivor.parent ?? survivor) + changed = true + } + // Add or update children - for (name, url) in onDisk { + for (name, entry) in onDisk { + let (url, info) = entry if let existing = node.children.first(where: { $0.name == name }) { // Update size for files (directories update via recursive bubble) - if !existing.isDirectory, - let attrs = try? url.resourceValues(forKeys: [.fileSizeKey]), - let newSize = attrs.fileSize.map(Int64.init) { + if !existing.isDirectory { + if info.linkCount > 1 && existing.hardLinkRef == nil { existing.hardLinkRef = info.ref } + // A 0-byte node for a multi-link inode is a hardlink the initial scan + // already counted elsewhere — re-statting it would double-count. + if info.linkCount > 1 && existing.size == 0 { continue } + let newSize = info.allocatedSize if existing.size != newSize { existing.size = newSize changed = true } } + } else if info.isDir { + // New directory — scan its whole subtree so it isn't left as a 0-byte leaf. + var seenRefs = Set() + let child = scanSubtree(url: url, parent: node, showHiddenFiles: showHiddenFiles, excludedNames: excludedNames, treeRoot: rootNode(of: node), seenRefs: &seenRefs) + node.children.append(child) + changed = true } else { - // New entry — create a minimal FSNode - let attrs = try? url.resourceValues(forKeys: [.fileSizeKey, .isDirectoryKey]) - let isDir = attrs?.isDirectory ?? false - let size = (attrs?.fileSize).map(Int64.init) ?? 0 - let ext = isDir ? "" : url.pathExtension.lowercased() - let child = FSNode(url: url, name: name, isDirectory: isDir, size: size, fileExtension: ext, parent: node) + // New file. A new name for an inode the tree already accounts for + // (a hardlink created after the scan) must contribute 0 bytes. + let ext = url.pathExtension.lowercased() + var size = info.allocatedSize + var linkRef: HardLinkRef? + if info.linkCount > 1 { + linkRef = info.ref + if firstNode(withRef: info.ref, in: rootNode(of: node)) != nil { size = 0 } + } + let child = FSNode(url: url, name: name, isDirectory: false, size: size, fileExtension: ext, parent: node) + child.hardLinkRef = linkRef child.safetyLevel = SafetyAnalyzer.level(for: child) node.children.append(child) changed = true @@ -300,6 +783,110 @@ public final class ScanViewModel: ObservableObject { return changed } + // Synchronously walks a newly-discovered directory subtree, applying the same rules + // as the initial scan: skip symlinks, skip hidden files unless showHiddenFiles, skip + // excludedNames, allocated sizes via lstat, directory size = sum of children. + private nonisolated static func scanSubtree( + url: URL, + parent: FSNode?, + showHiddenFiles: Bool, + excludedNames: Set, + treeRoot: FSNode?, + seenRefs: inout Set + ) -> FSNode { + let name = url.lastPathComponent + guard let info = lstatInfo(path: url.path), info.isDir else { + // Not actually a directory (or vanished) — return an empty leaf; caller only + // invokes this when it already believes the entry is a directory. + return FSNode(url: url, name: name, isDirectory: false, size: 0, fileExtension: url.pathExtension.lowercased(), parent: parent) + } + + let node = FSNode(url: url, name: name, isDirectory: true, size: 0, fileExtension: "", parent: parent) + node.safetyLevel = SafetyAnalyzer.level(for: node) + + let fm = FileManager.default + guard let entries = try? fm.contentsOfDirectory( + at: url, + includingPropertiesForKeys: nil, + options: showHiddenFiles ? [] : [.skipsHiddenFiles] + ) else { return node } + + var children: [FSNode] = [] + var totalSize: Int64 = 0 + for childURL in entries { + let childName = childURL.lastPathComponent + if excludedNames.contains(childName) { continue } + guard let childInfo = lstatInfo(path: childURL.path) else { continue } + if childInfo.isSymlink { continue } + + let child: FSNode + if childInfo.isDir { + child = scanSubtree(url: childURL, parent: node, showHiddenFiles: showHiddenFiles, excludedNames: excludedNames, treeRoot: treeRoot, seenRefs: &seenRefs) + } else { + let ext = childURL.pathExtension.lowercased() + var size = childInfo.allocatedSize + var linkRef: HardLinkRef? + if childInfo.linkCount > 1 { + linkRef = childInfo.ref + if seenRefs.contains(childInfo.ref) { + size = 0 + } else { + seenRefs.insert(childInfo.ref) + if let treeRoot, firstNode(withRef: childInfo.ref, in: treeRoot) != nil { size = 0 } + } + } + child = FSNode(url: childURL, name: childName, isDirectory: false, size: size, fileExtension: ext, parent: node) + child.hardLinkRef = linkRef + child.safetyLevel = SafetyAnalyzer.level(for: child) + } + children.append(child) + totalSize += child.size + } + + children.sort { $0.size > $1.size } + node.children = children + node.size = totalSize + return node + } + + // Computes the gap between what the volume reports as used (total - available) + // and what the scanner actually accounted for. On APFS volumes this gap is + // never zero: snapshots, purgeable space, and unreadable/excluded areas all + // count toward "used" without ever appearing as a scannable file. Returns nil + // when inputs are invalid or the gap is small enough to be measurement noise. + nonisolated static func hiddenSpaceBytes(volumeTotal: Int64, volumeAvailable: Int64, scannedTotal: Int64) -> Int64? { + guard volumeTotal > 0 else { return nil } + let hidden = max(0, volumeTotal - volumeAvailable - scannedTotal) + let oneGB: Int64 = 1_000_000_000 + return hidden >= oneGB ? hidden : nil + } + + // When the scanned URL is itself a volume's mount point, returns a NEW + // tree with a synthetic "Hidden & Unreadable Space" child representing + // the portion of the volume's used space the scanner could never + // account for (nil for non-volume scans, e.g. scanning a subfolder, or + // when the gap is negligible). Topology is immutable on `FileTree`, so + // this can't append in place the way the old FSNode version did. + private nonisolated static func appendHiddenSpaceNodeIfNeeded(tree: FileTree, scannedURL: URL) -> FileTree? { + guard let values = try? scannedURL.resourceValues(forKeys: [.volumeURLKey]), + let volumeURL = values.volume, + volumeURL.standardizedFileURL.path == scannedURL.standardizedFileURL.path + else { return nil } + + guard let volumeValues = try? scannedURL.resourceValues(forKeys: [.volumeTotalCapacityKey, .volumeAvailableCapacityKey]), + let totalCapacity = volumeValues.volumeTotalCapacity, + let availableCapacity = volumeValues.volumeAvailableCapacity + else { return nil } + + guard let hidden = hiddenSpaceBytes( + volumeTotal: Int64(totalCapacity), + volumeAvailable: Int64(availableCapacity), + scannedTotal: tree.records[tree.rootIndex].size + ) else { return nil } + + return tree.appendingSyntheticRootChild(name: "Hidden & Unreadable Space", size: hidden) + } + // Walk up the parent chain recalculating folder sizes from their children. private nonisolated static func bubbleUpSizes(from node: FSNode) { var current: FSNode? = node @@ -324,7 +911,7 @@ public final class ScanViewModel: ObservableObject { } } - public func drillDown(into node: FSNode) { + public func drillDown(into node: FileNode) { guard node.isDirectory else { return } drillStack.append(node) Task { await recomputeLayout() } @@ -336,7 +923,7 @@ public final class ScanViewModel: ObservableObject { Task { await recomputeLayout() } } - public func select(_ node: FSNode?) { + public func select(_ node: FileNode?) { selectedNode = node } @@ -350,9 +937,133 @@ public final class ScanViewModel: ObservableObject { Task { await recomputeLayout() } } - private nonisolated static func buildDuplicateGroups(root: FSNode) -> [[FSNode]] { - var all: [FSNode] = [] - collectAll(node: root, into: &all) + // MARK: - Move to Trash (prune-in-place, no rescan) + + // Moves a single node to the Trash and, if that succeeds, prunes it out + // of the in-memory tree instead of triggering a full rescan. Returns + // whether anything was actually trashed (mirrors the old call sites' + // `try?`-and-check pattern). + @discardableResult + public func trashNode(_ node: FileNode) -> Bool { + trashNodes([node]) + } + + // Moves several nodes to the Trash (the "keep 1, delete N" / "Delete All + // Duplicates" case) and prunes every successfully-trashed one out of the + // tree in a single splice, rather than rescanning once per node. + // + // An opened `.mdscan` archive is a read-only snapshot of a scan that may + // no longer match what's on disk — trashing from it would either delete + // the wrong (current, possibly since-changed) file or silently no-op, + // neither of which is acceptable, so this is a hard no-op while a + // snapshot is loaded. + @discardableResult + public func trashNodes(_ nodes: [FileNode]) -> Bool { + guard !isReadOnlySnapshot, let currentTree = tree, !nodes.isEmpty else { return false } + + var trashedIndices: [Int] = [] + trashedIndices.reserveCapacity(nodes.count) + for node in nodes where ObjectIdentifier(node.tree) == ObjectIdentifier(currentTree) { + do { + try FileManager.default.trashItem(at: node.url, resultingItemURL: nil) + trashedIndices.append(node.index) + } catch { + // Swallow, same as the old per-call-site `try?` behavior — + // one failed delete (e.g. permissions) shouldn't block the + // rest of the batch or surface a blocking alert. + } + } + guard !trashedIndices.isEmpty else { return false } + + pruneTree(afterTrashing: trashedIndices, from: currentTree) + return true + } + + // Splices the trashed nodes out of `tree` and repairs everything that + // referenced the old topology: selection, drill stack, color map, + // extension summaries, duplicate groups, and layout. Indices shift on + // every prune, so selection/drill state is captured as paths beforehand + // and resolved back to indices in the new tree afterward. + private func pruneTree(afterTrashing indices: [Int], from oldTree: FileTree) { + let selectedPath = selectedNode.map { oldTree.path(of: $0.index) } + let drillPaths = drillStack.map { oldTree.path(of: $0.index) } + + let newTree = oldTree.removingSubtrees(at: indices) + self.tree = newTree + + if let selectedPath, let idx = Self.findIndex(forPath: selectedPath, in: newTree) { + selectedNode = FileNode(tree: newTree, index: idx) + } else { + selectedNode = nil + } + + // Keep every prefix of the drill stack that still resolves; the + // first missing entry (the trashed directory itself, if it was + // drilled into) truncates the stack back to its nearest surviving + // ancestor instead of leaving stale/dangling entries. + var newDrillStack: [FileNode] = [] + for path in drillPaths { + guard let idx = Self.findIndex(forPath: path, in: newTree) else { break } + newDrillStack.append(FileNode(tree: newTree, index: idx)) + } + drillStack = newDrillStack + + let rootNode = FileNode(tree: newTree, index: newTree.rootIndex) + let map = ExtensionColorMap(root: rootNode) + colorMap = map + + // Safety tags and duplicateGroupIDs are per-node fields that were + // already computed on the surviving records and carry over as-is + // through the splice (removingSubtrees copies whole `FileNodeRecord` + // values), so only the two aggregate derived passes need to re-run — + // and neither needs a full re-detect, just a re-group/re-bucket over + // the pruned record set. + extensionTask?.cancel() + extensionTask = Task.detached(priority: .userInitiated) { [weak self] in + let summaries = Self.buildExtensionSummaries(tree: newTree, map: map) + guard !Task.isCancelled else { return } + await MainActor.run { self?.extensionSummaries = summaries } + } + + duplicateTask?.cancel() + duplicateTask = Task.detached(priority: .utility) { [weak self] in + let groups = Self.buildDuplicateGroups(tree: newTree) + guard !Task.isCancelled else { return } + await MainActor.run { self?.duplicateGroups = groups } + } + + Task { await recomputeLayout() } + } + + // Resolves an absolute path back to an index in `tree` by walking + // root->leaf name components (no full-tree path map to build — this is + // only ever called for a handful of paths per prune: selection + drill + // stack). Returns nil if the path no longer exists (it was the node + // that got trashed, or a descendant of it). + private nonisolated static func findIndex(forPath path: String, in tree: FileTree) -> Int? { + if path == tree.rootPath { return tree.rootIndex } + let rootPrefix = tree.rootPath.hasSuffix("/") ? tree.rootPath : tree.rootPath + "/" + guard path.hasPrefix(rootPrefix) else { return nil } + + let relative = String(path.dropFirst(rootPrefix.count)) + let components = relative.split(separator: "/").map(String.init) + var current = tree.rootIndex + for component in components { + let start = tree.childStart[current] + let count = tree.childCount[current] + guard let offset = (0.. [[FileNode]] { + let all = (0.. 1 } @@ -372,10 +1083,13 @@ public final class ScanViewModel: ObservableObject { public let percentage: Double } - private nonisolated static func buildExtensionSummaries(root: FSNode, map: ExtensionColorMap) -> [ExtensionSummary] { + private nonisolated static func buildExtensionSummaries(tree: FileTree, map: ExtensionColorMap) -> [ExtensionSummary] { var groups: [String: (count: Int, size: Int64)] = [:] - collectExtensions(node: root, into: &groups) - let total = Double(root.size) + for record in tree.records where !record.isDirectory { + groups[record.fileExtension, default: (0, 0)].count += 1 + groups[record.fileExtension, default: (0, 0)].size += record.size + } + let total = Double(tree.records[tree.rootIndex].size) return groups.map { ext, stats in ExtensionSummary( id: ext, @@ -405,56 +1119,173 @@ public final class ScanViewModel: ObservableObject { self.isComputingLayout = false } - private nonisolated static func tagSafetyLevels(node: FSNode) { - var stack: [FSNode] = [node] - while !stack.isEmpty { - let n = stack.removeLast() - n.safetyLevel = SafetyAnalyzer.level(for: n) - stack.append(contentsOf: n.children) + // Flat loop over every index — order doesn't matter (each node's safety + // level only depends on its own reconstructed path/name, both already + // fully populated by the builder before this runs). + private nonisolated static func tagSafetyLevels(tree: FileTree) { + for index in 0.. $1.size } - stack.append(contentsOf: n.children) + // MARK: - Save / reopen scans (.mdscan archive) + + // Encodes the current tree + a small metadata block and writes it to + // `url` off the main actor (encoding a multi-million-record tree to + // JSON is real work). Errors surface through `errorMessage`, same as + // `exportCSV`. + public func saveScan(to url: URL) { + guard let tree else { return } + let metadata = ScanArchive.Metadata( + scannedPath: tree.rootPath, + scanDate: Date(), + deniedCount: deniedCount, + appVersion: Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "unknown" + ) + let archive = ScanArchive(tree: tree, metadata: metadata) + + Task.detached(priority: .utility) { [weak self] in + do { + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + let data = try encoder.encode(archive) + try data.write(to: url, options: .atomic) + } catch { + await MainActor.run { self?.errorMessage = "Couldn't save scan: \(error.localizedDescription)" } + } } } - private nonisolated static func collectAll(node: FSNode, into list: inout [FSNode]) { - var stack = [node] - while !stack.isEmpty { - let n = stack.removeLast() - list.append(n) - stack.append(contentsOf: n.children) + // Decodes and validates a `.mdscan` archive off the main actor, then + // installs it as a read-only snapshot. A malformed or doctored archive + // surfaces as `errorMessage` rather than crashing or hanging — see + // `ScanArchive.validate()`. + public func openArchive(from url: URL) { + scanTask?.cancel() + extensionTask?.cancel() + duplicateTask?.cancel() + watchTask?.cancel() + fileWatcher.stop() + isWatching = false + layoutGeneration += 1 + securityScopedURL?.stopAccessingSecurityScopedResource() + securityScopedURL = nil + errorMessage = nil + isScanning = true + isComputingLayout = false + + Task.detached(priority: .userInitiated) { [weak self] in + do { + let data = try Data(contentsOf: url) + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + let archive = try decoder.decode(ScanArchive.self, from: data) + try archive.validate() + let tree = archive.makeTree() + await MainActor.run { self?.applyOpenedArchive(tree: tree, metadata: archive.metadata) } + } catch { + await MainActor.run { + self?.isScanning = false + self?.errorMessage = "Couldn't open scan: \(error.localizedDescription)" + } + } } } - private nonisolated static func collectExtensions(node: FSNode, into groups: inout [String: (count: Int, size: Int64)]) { - var stack = [node] - while !stack.isEmpty { - let n = stack.removeLast() - if !n.isDirectory { - groups[n.fileExtension, default: (0, 0)].count += 1 - groups[n.fileExtension, default: (0, 0)].size += n.size + // Installs a validated, freshly-decoded tree as the active read-only + // snapshot: no live watching is started (this never calls `scan(url:)` + // or `startWatching`), and `isReadOnlySnapshot` gates trash actions off + // for the rest of this tree's lifetime. + private func applyOpenedArchive(tree: FileTree, metadata: ScanArchive.Metadata) { + isScanning = false + isReadOnlySnapshot = true + snapshotDate = metadata.scanDate + scanURL = URL(fileURLWithPath: metadata.scannedPath) + deniedCount = metadata.deniedCount + selectedNode = nil + drillStack = [] + highlightedExtension = nil + errorMessage = nil + UserDefaults.standard.set(metadata.scannedPath, forKey: "lastScannedPath") + + self.tree = tree + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + let map = ExtensionColorMap(root: rootNode) + colorMap = map + isComputingLayout = true + Task { await recomputeLayout() } + + // Safety tags and duplicateGroupIDs were already computed before the + // original scan was saved and travel with the archived records, so + // only the two aggregate derived views need to be rebuilt. + extensionTask?.cancel() + extensionTask = Task.detached(priority: .userInitiated) { [tree, map, weak self] in + let summaries = Self.buildExtensionSummaries(tree: tree, map: map) + guard !Task.isCancelled else { return } + await MainActor.run { self?.extensionSummaries = summaries } + } + + duplicateGroups = Self.buildDuplicateGroups(tree: tree) + duplicatesReady = true + } + + // MARK: - Compare two scans (4c) + + // What `ComparisonView` renders: the diff itself plus enough context + // about the "before" side (it was loaded from an archive, not the live + // tree) to label the report meaningfully. + public struct ComparisonResult: Sendable { + public let changes: [ScanChange] + public let beforeMetadata: ScanArchive.Metadata + public let afterRootPath: String + } + + // Loads `archiveURL` as the "before" snapshot and diffs it against the + // currently active tree ("after"), off the main actor — decoding a large + // archive and walking two multi-million-node trees is real work. Never + // mutates `tree`/`isReadOnlySnapshot`: this is a read-only side report, + // wholly independent of whatever is currently loaded/live-watched. + public func compareWithSavedScan(archiveURL: URL) { + guard let afterTree = tree else { return } + errorMessage = nil + isComputingComparison = true + + Task.detached(priority: .userInitiated) { [weak self] in + do { + let data = try Data(contentsOf: archiveURL) + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + let archive = try decoder.decode(ScanArchive.self, from: data) + try archive.validate() + let beforeTree = archive.makeTree() + let changes = ScanComparison.compare(before: beforeTree, after: afterTree) + let result = ComparisonResult(changes: changes, beforeMetadata: archive.metadata, afterRootPath: afterTree.rootPath) + await MainActor.run { + guard let self else { return } + self.comparisonResult = result + self.isComputingComparison = false + self.showComparisonSheet = true + } + } catch { + await MainActor.run { + self?.isComputingComparison = false + self?.errorMessage = "Couldn't compare scan: \(error.localizedDescription)" + } } - stack.append(contentsOf: n.children) } } public func exportCSV() { - guard let root else { return } + guard let tree else { return } let panel = NSSavePanel() panel.allowedContentTypes = [.commaSeparatedText] - panel.nameFieldStringValue = "\(root.name)-disk-usage.csv" + panel.nameFieldStringValue = "\(tree.records[tree.rootIndex].name)-disk-usage.csv" guard panel.runModal() == .OK, let url = panel.url else { return } - Task.detached(priority: .utility) { [root] in + Task.detached(priority: .utility) { [tree] in var lines = ["Path,Size (bytes),Human Size,Type,Duplicate Group"] - Self.appendCSV(node: root, to: &lines) + Self.appendCSV(tree: tree, to: &lines) let csv = lines.joined(separator: "\n") + "\n" do { try csv.write(to: url, atomically: true, encoding: .utf8) @@ -468,19 +1299,18 @@ public final class ScanViewModel: ObservableObject { "\"\(value.replacingOccurrences(of: "\"", with: "\"\""))\"" } - private nonisolated static func appendCSV(node: FSNode, to lines: inout [String]) { - var stack: [FSNode] = [node] - while let current = stack.popLast() { - let type = current.isDirectory ? "directory" : current.fileExtension - let group = current.duplicateGroupID?.uuidString ?? "" + private nonisolated static func appendCSV(tree: FileTree, to lines: inout [String]) { + for index in 0.. String { + let formatter = DateFormatter() + formatter.dateStyle = .medium + formatter.timeStyle = .short + return formatter.string(from: date) + } + + // MARK: - Summary bar + + private func summaryBar(changes: [ScanChange]) -> some View { + let added = changes.filter { $0.kind == .added }.count + let removed = changes.filter { $0.kind == .removed }.count + let grew = changes.filter { $0.kind == .grew }.count + let shrank = changes.filter { $0.kind == .shrank }.count + let replaced = changes.filter { $0.kind == .replaced }.count + let netDelta = changes.reduce(Int64(0)) { $0 + $1.delta } + + return HStack(spacing: 14) { + summaryPill(count: added, label: "added", color: .green, icon: "plus.circle") + summaryPill(count: removed, label: "removed", color: .red, icon: "minus.circle") + summaryPill(count: grew, label: "grew", color: .orange, icon: "arrow.up.circle") + summaryPill(count: shrank, label: "shrank", color: .teal, icon: "arrow.down.circle") + if replaced > 0 { + summaryPill(count: replaced, label: "replaced", color: .purple, icon: "arrow.triangle.swap") + } + + Spacer() + + Label(signedByteString(netDelta), systemImage: netDelta >= 0 ? "arrow.up.right" : "arrow.down.right") + .font(.system(size: 11, weight: .semibold, design: .monospaced)) + .foregroundStyle(netDelta >= 0 ? .orange : .teal) + } + .padding(.horizontal, 12) + .padding(.vertical, 8) + .background(.ultraThinMaterial) + } + + @ViewBuilder + private func summaryPill(count: Int, label: String, color: Color, icon: String) -> some View { + if count > 0 { + Label("\(count) \(label)", systemImage: icon) + .font(.caption) + .foregroundStyle(color) + } + } + + private func signedByteString(_ delta: Int64) -> String { + let sign = delta > 0 ? "+" : (delta < 0 ? "-" : "") + return sign + ByteFormatter.string(from: abs(delta)) + } + + // MARK: - Change list + + private func changeList(changes: [ScanChange]) -> some View { + ScrollView { + LazyVStack(spacing: 0) { + ForEach(changes) { change in + ChangeRow(change: change) + Divider().padding(.leading, 34) + } + } + } + } +} + +// MARK: - Change row + +private struct ChangeRow: View { + let change: ScanChange + + private var icon: String { + switch change.kind { + case .added: return "plus.circle.fill" + case .removed: return "minus.circle.fill" + case .grew: return "arrow.up.circle.fill" + case .shrank: return "arrow.down.circle.fill" + case .replaced: return "arrow.triangle.swap" + } + } + + private var color: Color { + switch change.kind { + case .added: return .green + case .removed: return .red + case .grew: return .orange + case .shrank: return .teal + case .replaced: return .purple + } + } + + private var deltaString: String { + let sign = change.delta > 0 ? "+" : (change.delta < 0 ? "-" : "") + return sign + ByteFormatter.string(from: abs(change.delta)) + } + + var body: some View { + HStack(spacing: 10) { + Image(systemName: icon) + .font(.system(size: 13)) + .foregroundStyle(color) + .frame(width: 16) + + Image(systemName: change.isDirectory ? "folder" : "doc") + .font(.system(size: 11)) + .foregroundStyle(.secondary) + .frame(width: 14) + + VStack(alignment: .leading, spacing: 1) { + Text(change.name) + .font(.system(size: 12.5)) + .lineLimit(1) + Text(change.relativePath) + .font(.system(size: 10.5)) + .foregroundStyle(.tertiary) + .lineLimit(1) + .truncationMode(.middle) + } + + Spacer(minLength: 8) + + switch change.kind { + case .added: + Text(ByteFormatter.string(from: change.afterSize)) + .font(.system(size: 11, design: .monospaced)) + .foregroundStyle(.secondary) + case .removed: + Text(ByteFormatter.string(from: change.beforeSize)) + .font(.system(size: 11, design: .monospaced)) + .foregroundStyle(.secondary) + case .grew, .shrank, .replaced: + Text("\(ByteFormatter.string(from: change.beforeSize)) → \(ByteFormatter.string(from: change.afterSize))") + .font(.system(size: 11, design: .monospaced)) + .foregroundStyle(.secondary) + } + + Text(deltaString) + .font(.system(size: 11, weight: .semibold, design: .monospaced)) + .foregroundStyle(color) + .frame(width: 70, alignment: .trailing) + } + .padding(.horizontal, 12) + .padding(.vertical, 6) + .contentShape(Rectangle()) + } +} diff --git a/Sources/Views/DirectoryTree/DirectoryTreeView.swift b/Sources/Views/DirectoryTree/DirectoryTreeView.swift index 3513428..187c675 100644 --- a/Sources/Views/DirectoryTree/DirectoryTreeView.swift +++ b/Sources/Views/DirectoryTree/DirectoryTreeView.swift @@ -40,14 +40,13 @@ struct DirectoryTreeView: View { Label("Open in Chart", systemImage: "arrow.down.right.circle") } } - Divider() - Button(role: .destructive) { - if let _ = try? FileManager.default.trashItem(at: node.url, resultingItemURL: nil), - let root = vm.root { - vm.scan(url: root.url) + if !vm.isReadOnlySnapshot { + Divider() + Button(role: .destructive) { + vm.trashNode(node) + } label: { + Label("Move to Trash", systemImage: "trash") } - } label: { - Label("Move to Trash", systemImage: "trash") } } } @@ -84,7 +83,7 @@ private struct ScanningPlaceholder: View { } private struct NodeRow: View { - let node: FSNode + let node: FileNode let isSelected: Bool var body: some View { @@ -131,6 +130,22 @@ private struct NodeRow: View { .help(SafetyAnalyzer.reason(for: node) ?? "Do not delete") } + if node.isAccessDenied { + Image(systemName: "lock.fill") + .font(.system(size: 9)) + .foregroundStyle(.secondary) + .help("Contents couldn't be read — Full Disk Access may be required") + } + + if node.isAutoSummarized { + Text("(\(node.descendantFileCount) files, summarized)") + .font(.system(size: 9.5)) + .foregroundStyle(isSelected ? Color.white.opacity(0.55) : Color.secondary.opacity(0.65)) + .lineLimit(1) + .fixedSize() + .help("Collapsed to keep the scan fast: contents aren't browsable individually") + } + // Size Text(ByteFormatter.string(from: node.size)) .font(.system(size: 11, design: .monospaced)) diff --git a/Sources/Views/Duplicates/DuplicatesView.swift b/Sources/Views/Duplicates/DuplicatesView.swift index 50979f7..1a64fc5 100644 --- a/Sources/Views/Duplicates/DuplicatesView.swift +++ b/Sources/Views/Duplicates/DuplicatesView.swift @@ -24,6 +24,7 @@ struct DuplicatesView: View { index: index, group: group, isExpanded: expanded.contains(index), + isReadOnly: vm.isReadOnlySnapshot, onToggle: { withAnimation(.easeInOut(duration: 0.15)) { if expanded.contains(index) { expanded.remove(index) } @@ -45,7 +46,7 @@ struct DuplicatesView: View { // MARK: - Summary bar @ViewBuilder - private func summaryBar(groups: [[FSNode]]) -> some View { + private func summaryBar(groups: [[FileNode]]) -> some View { let totalWasted = groups.reduce(Int64(0)) { $0 + $1[0].size * Int64($1.count - 1) } let totalFiles = groups.reduce(0) { $0 + $1.count - 1 } @@ -58,13 +59,15 @@ struct DuplicatesView: View { systemImage: "externaldrive.badge.minus") .font(.caption).foregroundStyle(.orange) Spacer() - Button(role: .destructive) { deleteAll(groups: groups) } label: { - Label("Delete All Duplicates", systemImage: "trash") - .font(.caption.weight(.medium)) + if !vm.isReadOnlySnapshot { + Button(role: .destructive) { deleteAll(groups: groups) } label: { + Label("Delete All Duplicates", systemImage: "trash") + .font(.caption.weight(.medium)) + } + .glassButton() + .tint(.red) + .controlSize(.small) } - .glassButton() - .tint(.red) - .controlSize(.small) } .padding(.horizontal, 12) .padding(.vertical, 8) @@ -73,35 +76,16 @@ struct DuplicatesView: View { // MARK: - Delete helpers - private func deleteAll(groups: [[FSNode]]) { - guard let rootURL = vm.root?.url else { return } - var deleted = false - for group in groups { - for node in group.dropFirst() { - if (try? FileManager.default.trashItem(at: node.url, resultingItemURL: nil)) != nil { - deleted = true - } - } - } - if deleted { vm.scan(url: rootURL) } + private func deleteAll(groups: [[FileNode]]) { + vm.trashNodes(groups.flatMap { $0.dropFirst() }) } - private func deleteGroup(_ group: [FSNode]) { - guard let rootURL = vm.root?.url else { return } - var deleted = false - for node in group.dropFirst() { - if (try? FileManager.default.trashItem(at: node.url, resultingItemURL: nil)) != nil { - deleted = true - } - } - if deleted { vm.scan(url: rootURL) } + private func deleteGroup(_ group: [FileNode]) { + vm.trashNodes(Array(group.dropFirst())) } - private func deleteNode(_ node: FSNode) { - guard let rootURL = vm.root?.url else { return } - if (try? FileManager.default.trashItem(at: node.url, resultingItemURL: nil)) != nil { - vm.scan(url: rootURL) - } + private func deleteNode(_ node: FileNode) { + vm.trashNode(node) } } @@ -109,11 +93,12 @@ struct DuplicatesView: View { private struct GroupSection: View { let index: Int - let group: [FSNode] + let group: [FileNode] let isExpanded: Bool + let isReadOnly: Bool let onToggle: () -> Void let onDeleteGroup: () -> Void - let onDeleteNode: (FSNode) -> Void + let onDeleteNode: (FileNode) -> Void private var wasted: Int64 { group[0].size * Int64(group.count - 1) } @@ -153,15 +138,17 @@ private struct GroupSection: View { .frame(maxWidth: 180, alignment: .trailing) } - Button(role: .destructive, action: onDeleteGroup) { - Label("Keep 1, Delete \(group.count - 1)", systemImage: "trash") - .font(.system(size: 11, weight: .medium)) + if !isReadOnly { + Button(role: .destructive, action: onDeleteGroup) { + Label("Keep 1, Delete \(group.count - 1)", systemImage: "trash") + .font(.system(size: 11, weight: .medium)) + } + .glassButton() + .tint(.red) + .controlSize(.mini) + // Don't let the delete button trigger the expand toggle + .onTapGesture {} } - .glassButton() - .tint(.red) - .controlSize(.mini) - // Don't let the delete button trigger the expand toggle - .onTapGesture {} } .padding(.horizontal, 12) .padding(.vertical, 7) @@ -173,7 +160,7 @@ private struct GroupSection: View { if isExpanded { VStack(spacing: 0) { ForEach(Array(group.enumerated()), id: \.element.id) { i, node in - FileRow(node: node, isKeep: i == 0) { onDeleteNode(node) } + FileRow(node: node, isKeep: i == 0, isReadOnly: isReadOnly) { onDeleteNode(node) } if i < group.count - 1 { Divider().padding(.leading, 52) } @@ -189,8 +176,9 @@ private struct GroupSection: View { // MARK: - File row private struct FileRow: View { - let node: FSNode + let node: FileNode let isKeep: Bool + let isReadOnly: Bool let onDelete: () -> Void @State private var isHovered = false @@ -228,7 +216,7 @@ private struct FileRow: View { .foregroundStyle(.secondary) .fixedSize() - if isKeep { + if isKeep || isReadOnly { Color.clear.frame(width: 24, height: 24) } else { Button(action: onDelete) { @@ -253,7 +241,7 @@ private struct FileRow: View { NSPasteboard.general.clearContents() NSPasteboard.general.setString(node.url.path, forType: .string) } - if !isKeep { + if !isKeep && !isReadOnly { Divider() Button(role: .destructive, action: onDelete) { Label("Move to Trash", systemImage: "trash") diff --git a/Sources/Views/ExtensionList/ExtensionListView.swift b/Sources/Views/ExtensionList/ExtensionListView.swift index 54e299e..afd42a6 100644 --- a/Sources/Views/ExtensionList/ExtensionListView.swift +++ b/Sources/Views/ExtensionList/ExtensionListView.swift @@ -8,7 +8,7 @@ struct ExtensionListView: View { private let topN = 5 // Show selected item, or fall back to the current chart root - private var displayNode: FSNode? { vm.selectedNode ?? vm.treemapRoot } + private var displayNode: FileNode? { vm.selectedNode ?? vm.treemapRoot } var body: some View { VStack(spacing: 0) { diff --git a/Sources/Views/Shared/FileTypeIcon.swift b/Sources/Views/Shared/FileTypeIcon.swift index 08737a8..ded2b46 100644 --- a/Sources/Views/Shared/FileTypeIcon.swift +++ b/Sources/Views/Shared/FileTypeIcon.swift @@ -1,7 +1,7 @@ import SwiftUI enum FileTypeIcon { - static func systemName(for node: FSNode) -> String { + static func systemName(for node: FileNode) -> String { node.isDirectory ? "folder.fill" : systemName(forExt: node.fileExtension) } @@ -48,7 +48,7 @@ enum FileTypeIcon { } } - static func color(for node: FSNode) -> Color { + static func color(for node: FileNode) -> Color { node.isDirectory ? .yellow : color(forExt: node.fileExtension) } diff --git a/Sources/Views/Treemap/HapticEngine.swift b/Sources/Views/Treemap/HapticEngine.swift index f36bf22..b72aebf 100644 --- a/Sources/Views/Treemap/HapticEngine.swift +++ b/Sources/Views/Treemap/HapticEngine.swift @@ -7,7 +7,7 @@ final class HapticEngine { static let shared = HapticEngine() private let p = NSHapticFeedbackManager.defaultPerformer - private var lastID: UUID? + private var lastID: Int? private var isEnabled: Bool { UserDefaults.standard.object(forKey: "hapticFeedbackEnabled") as? Bool ?? true @@ -15,7 +15,7 @@ final class HapticEngine { // ── Hover ──────────────────────────────────────────────────────────────── - func hoverEntered(_ node: FSNode) { + func hoverEntered(_ node: FileNode) { guard isEnabled, node.id != lastID else { return } lastID = node.id @@ -51,7 +51,7 @@ final class HapticEngine { /// Choose pattern + tap-count based on node size. /// Directories always feel like a firm "snap" (alignment). - private func feedback(for node: FSNode) -> (NSHapticFeedbackManager.FeedbackPattern, Int) { + private func feedback(for node: FileNode) -> (NSHapticFeedbackManager.FeedbackPattern, Int) { if node.isDirectory { return (.alignment, 1) } diff --git a/Sources/Views/Treemap/TreemapRenderer.swift b/Sources/Views/Treemap/TreemapRenderer.swift index c34eb7e..ec8864d 100644 --- a/Sources/Views/Treemap/TreemapRenderer.swift +++ b/Sources/Views/Treemap/TreemapRenderer.swift @@ -6,8 +6,8 @@ struct TreemapRenderer { static func draw( cells: [TreemapCell], - hoveredNode: FSNode?, - selectedNode: FSNode?, + hoveredNode: FileNode?, + selectedNode: FileNode?, highlightedExtension: String?, duplicatesReady: Bool, pulsePhase: Double, diff --git a/Sources/Views/Treemap/TreemapView.swift b/Sources/Views/Treemap/TreemapView.swift index cf7ba53..361a990 100644 --- a/Sources/Views/Treemap/TreemapView.swift +++ b/Sources/Views/Treemap/TreemapView.swift @@ -102,7 +102,15 @@ struct TreemapView: View { } // ── Live indicator ─────────────────────────────────────────────── - if vm.isWatching { + // When a change arrived that couldn't be folded in incrementally, + // say so and offer a rescan instead of restarting the scan on the + // user's behalf (which used to loop forever). + if vm.hasStaleResults { + StaleBadge { if let url = vm.scanURL { vm.scan(url: url) } } + .padding(10) + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topTrailing) + .transition(.opacity.animation(.easeInOut(duration: 0.3))) + } else if vm.isWatching { LiveBadge() .padding(10) .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topTrailing) @@ -158,7 +166,7 @@ struct TreemapView: View { @ViewBuilder private func contextMenuContent() -> some View { let c = chartCenter - let node: FSNode? = hoveredCell?.node + let node: FileNode? = hoveredCell?.node ?? (c.x > 0 ? TreemapRenderer.cell(at: cursorPos, center: c, in: vm.cells)?.node : nil) ?? vm.selectedNode @@ -202,22 +210,21 @@ struct TreemapView: View { } } - Divider() + if !vm.isReadOnlySnapshot { + Divider() - Button(role: .destructive) { - do { - try FileManager.default.trashItem(at: node.url, resultingItemURL: nil) - if let root = vm.root { vm.scan(url: root.url) } - } catch {} - } label: { - Label("Move to Trash", systemImage: "trash") + Button(role: .destructive) { + vm.trashNode(node) + } label: { + Label("Move to Trash", systemImage: "trash") + } } } } // MARK: - Center label - private func centerLabel(for root: FSNode) -> some View { + private func centerLabel(for root: FileNode) -> some View { VStack(spacing: 4) { if !vm.drillStack.isEmpty { Image(systemName: "chevron.backward.circle.fill") @@ -304,7 +311,7 @@ struct TreemapView: View { // MARK: - Hover tooltip private struct HoverTooltip: View { - let node: FSNode + let node: FileNode var body: some View { HStack(alignment: .top, spacing: 10) { @@ -368,6 +375,31 @@ private struct HoverTooltip: View { // MARK: - Live badge +// Shown instead of the Live badge once a filesystem change arrived that the +// incremental splice refused (a change to the scanned root itself, or inside +// an auto-summarized folder). The results on screen are still correct for +// everything else, so this offers a rescan rather than forcing one. +private struct StaleBadge: View { + let rescan: () -> Void + + var body: some View { + Button(action: rescan) { + HStack(spacing: 5) { + Image(systemName: "arrow.clockwise") + .font(.system(size: 9, weight: .semibold)) + Text("Rescan") + .font(.system(size: 10, weight: .semibold)) + } + .foregroundStyle(.secondary) + .padding(.horizontal, 7) + .padding(.vertical, 4) + .glassCapsule() + } + .buttonStyle(.plain) + .help("Some changes on disk couldn't be updated in place. Click to rescan.") + } +} + private struct LiveBadge: View { @State private var pulse = false diff --git a/Tests/AutoSummaryTests.swift b/Tests/AutoSummaryTests.swift new file mode 100644 index 0000000..53dae17 --- /dev/null +++ b/Tests/AutoSummaryTests.swift @@ -0,0 +1,517 @@ +import XCTest +@testable import MacDirStat + +final class AutoSummaryTests: XCTestCase { + + // MARK: - Helpers + + private func withAutoSummarize(_ enabled: Bool, _ body: () async throws -> T) async rethrows -> T { + let prior = UserDefaults.standard.object(forKey: "autoSummarizeEnabled") + UserDefaults.standard.set(enabled, forKey: "autoSummarizeEnabled") + defer { + if let prior { UserDefaults.standard.set(prior, forKey: "autoSummarizeEnabled") } + else { UserDefaults.standard.removeObject(forKey: "autoSummarizeEnabled") } + } + return try await body() + } + + private func withMinFileCountOverride(_ value: Int, _ body: () async throws -> T) async rethrows -> T { + setenv("MDS_SUMMARY_MIN_FILES", "\(value)", 1) + defer { unsetenv("MDS_SUMMARY_MIN_FILES") } + return try await body() + } + + // The app's default excludedFolderNames already contains "node_modules" + // (it is fully skipped, not just collapsed - see ScanConfig's default in + // FileScanner.swift), so tests exercising the node_modules named-layout + // shortcut must use a config where it is NOT excluded, exactly like a + // user who removed it from their exclusion list would see. + private func withExcludedFolderNames(_ names: String, _ body: () async throws -> T) async rethrows -> T { + let prior = UserDefaults.standard.string(forKey: "excludedFolderNames") + UserDefaults.standard.set(names, forKey: "excludedFolderNames") + defer { + if let prior { UserDefaults.standard.set(prior, forKey: "excludedFolderNames") } + else { UserDefaults.standard.removeObject(forKey: "excludedFolderNames") } + } + return try await body() + } + + private func scanTree(at url: URL) async -> FileNode? { + let scanner = FileScanner() + var root: FileNode? + for await progress in await scanner.scan(url: url) { + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } + } + return root + } + + // Recursively counts every non-directory FileNode (normal leaf files) plus + // every isAutoSummarized node's descendantFileCount, so summarized and + // non-summarized scans of the same tree can be compared on a "how many + // files did we account for" basis. + private func totalAccountedFiles(_ node: FileNode) -> Int { + if node.isAutoSummarized { return node.descendantFileCount } + if !node.isDirectory { return 1 } + return node.children.reduce(0) { $0 + totalAccountedFiles($1) } + } + + // Regression test for the summary-pool forward-progress hazard. + // + // `summarizeSubtree` used to keep a synchronous signature and bridge into + // its nested worker pool by blocking the calling thread on a + // DispatchSemaphore. Because the caller runs on a Swift-concurrency + // cooperative thread and the nested pool needs threads from that same + // pool, enough SIMULTANEOUS summaries could block every cooperative + // thread waiting on work that had no thread left to run on. A wedged + // summary also pins the main scan's in-flight count above zero, so + // `WorkQueue.isFinished` never trips and every other scan worker spins + // forever - one stuck summary hung the entire scan, and cancelling did + // not recover it. + // + // This builds many sibling directories that ALL trip summarization at + // once (far more than the worker-pool cap, so the scheduler is forced to + // overlap them) and asserts the scan still completes with correct totals. + // Against the old blocking bridge this is the shape that could hang; + // with the pool awaited directly it simply finishes. + func test_many_concurrent_summaries_complete_without_stalling() async throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + + // 24 siblings >> the 8-worker cap on either pool, so many summaries + // are guaranteed to be in flight at the same time. Each candidate is + // named `node_modules` so it summarizes via the deterministic + // named-layout shortcut rather than depending on the filesystem's + // allocated-size rounding to satisfy the average-size heuristic. + let siblingCount = 24 + let filesPerSibling = 12 + var expectedFiles = 0 + + for s in 0.. Int { + (node.isAutoSummarized ? 1 : 0) + node.children.reduce(0) { $0 + countSummarized($1) } + } + + // The other half of the forward-progress bug: with the old blocking + // bridge, the poll loop only forwarded cancellation to the detached pool + // and kept spinning, so a wedged summary could not be recovered by + // "Stop Scan" at all. + // + // Drives `summarizeSubtree` DIRECTLY with a cancel closure that throws + // partway through, rather than cancelling a whole scan. An end-to-end + // scan-then-cancel test looks like it covers this but does not: the + // cancellation almost always lands during the initial traversal, before + // any summary walk has begun, so the test passes even with every + // cancellation check inside the summary path deleted (verified by + // mutation). Calling the walk directly is deterministic and actually + // fails if the walk stops honouring `cancel`. + func test_summary_walk_propagates_cancellation() async throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + + // Enough nested directories that the walk makes many cancel() calls. + for d in 0..<40 { + let deep = root.appendingPathComponent("m\(d)", isDirectory: true) + try FileManager.default.createDirectory(at: deep, withIntermediateDirectories: true) + for f in 0..<10 { + try Data(repeating: 1, count: 32).write(to: deep.appendingPathComponent("f\(f).dat")) + } + } + + var st = stat() + XCTAssertEqual(lstat(root.path, &st), 0) + let rootDev = UInt64(bitPattern: Int64(st.st_dev)) + + let fd = open(root.path, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW) + XCTAssertGreaterThanOrEqual(fd, 0) + defer { close(fd) } + let rootEntries = try listDirectoryEntries(path: root.path, fd: fd, forceFallback: false) + + struct StopWalking: Error {} + // Let a few directories through, then refuse to continue. + let callCount = Counter() + let config = ScanConfig( + excludedNames: [], + showHiddenFiles: false, + forceFallbackEnum: false, + autoSummarizeEnabled: true + ) + + do { + _ = try await summarizeSubtree( + rootEntries: rootEntries, + rootPath: root.path, + rootDev: rootDev, + config: config, + visited: VisitedSet() + ) { + if callCount.increment() > 5 { throw StopWalking() } + } + XCTFail("summarizeSubtree must propagate the cancel closure's error, not swallow it") + } catch is StopWalking { + // Expected: the walk asked, was told to stop, and unwound. + } + } + + // Thread-safe call counter: the summary pool invokes `cancel` from + // several workers at once. + private final class Counter: @unchecked Sendable { + private var lock = os_unfair_lock() + private var value = 0 + func increment() -> Int { + os_unfair_lock_lock(&lock) + value += 1 + let result = value + os_unfair_lock_unlock(&lock) + return result + } + } + + // Complements the direct test above: a full scan with summarization + // enabled must still cancel cleanly and never emit `.completed`. + func test_cancelled_scan_with_summarization_never_completes() async throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + + // Enough summarizable siblings, each with real depth and file count, + // that a full uncancelled scan takes appreciable time. + for s in 0..<12 { + let holder = root.appendingPathComponent("pkg\(s)", isDirectory: true) + let target = holder.appendingPathComponent("node_modules", isDirectory: true) + for d in 0..<12 { + let deep = target.appendingPathComponent("m\(d)", isDirectory: true) + try FileManager.default.createDirectory(at: deep, withIntermediateDirectories: true) + for f in 0..<25 { + try Data(repeating: 9, count: 64).write(to: deep.appendingPathComponent("f\(f).dat")) + } + } + } + + let elapsed: Double = await withExcludedFolderNames(".git,DerivedData,.Trash") { + await withAutoSummarize(true) { + let scanner = FileScanner() + var sawCompleted = false + let stream = await scanner.scan(url: root) + let start = DispatchTime.now() + let consumeTask = Task { + for await progress in stream { + if case .completed = progress { sawCompleted = true } + } + } + await scanner.cancel() + _ = await consumeTask.value + let seconds = Double(DispatchTime.now().uptimeNanoseconds - start.uptimeNanoseconds) / 1_000_000_000 + + XCTAssertFalse(sawCompleted, "a scan cancelled during summarization must not emit .completed") + return seconds + } + } + + // Generous bound: the point is "it unwinds", not a precise deadline. + // Against a wedged pool this would never return at all. + XCTAssertLessThan(elapsed, 10.0, "cancellation during a summary walk should tear down promptly") + } + + private func findNode(_ root: FileNode, path: [String]) -> FileNode? { + var current = root + for name in path { + guard let next = current.children.first(where: { $0.name == name }) else { return nil } + current = next + } + return current + } + + // Creates `count` small files spread across a couple of nested + // subdirectories under `dir`, each `bytes` bytes long. Returns the total + // allocated size (via lstat) of every file created, so tests can assert + // an exact expected total without hardcoding filesystem block behavior. + @discardableResult + private func makeFiles(count: Int, bytes: Int, under dir: URL, nestedEvery: Int = 7) throws -> Int64 { + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + var total: Int64 = 0 + var currentDir = dir + for i in 0.. 0, i > 0, i % nestedEvery == 0 { + currentDir = currentDir.appendingPathComponent("nested\(i)") + try FileManager.default.createDirectory(at: currentDir, withIntermediateDirectories: true) + } + let file = currentDir.appendingPathComponent("f\(i).bin") + try Data(repeating: UInt8(i % 251), count: bytes).write(to: file) + var st = stat() + XCTAssertEqual(lstat(file.path, &st), 0) + total += Int64(st.st_blocks) * 512 + } + return total + } + + // MARK: - 1. node_modules named-layout shortcut + + func test_node_modules_is_summarized() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: tmp) } + + let nodeModules = tmp.appendingPathComponent("proj").appendingPathComponent("node_modules") + let expectedTotal = try makeFiles(count: 50, bytes: 16, under: nodeModules) + + guard let onRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(true, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan (autoSummarize on) produced no root") + } + guard let nmNode = findNode(onRoot, path: ["proj", "node_modules"]) else { + return XCTFail("node_modules node not found") + } + + XCTAssertTrue(nmNode.isAutoSummarized) + XCTAssertTrue(nmNode.children.isEmpty) + XCTAssertEqual(nmNode.descendantFileCount, 50) + XCTAssertEqual(nmNode.size, expectedTotal) + + guard let offRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(false, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan (autoSummarize off) produced no root") + } + XCTAssertEqual(onRoot.size, offRoot.size, "collapsing node_modules must not change the root's total size") + + guard let offNmNode = findNode(offRoot, path: ["proj", "node_modules"]) else { + return XCTFail("node_modules node not found in the non-summarized scan") + } + XCTAssertFalse(offNmNode.isAutoSummarized) + XCTAssertEqual(offNmNode.children.isEmpty, false) + } + + // MARK: - 2. General immediate-file-count heuristic (env-overridden threshold) + + func test_threshold_directory_summarized() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: tmp) } + + // depth: tmp=0, level1=1, tinyDir/bigDir=2 (>= minDepth). + let level1 = tmp.appendingPathComponent("level1") + let tinyDir = level1.appendingPathComponent("tinyDir") + let bigDir = level1.appendingPathComponent("bigDir") + + // Empty files: allocatedSize 0, average 0 < maxAverageFileSize. + try makeFiles(count: 10, bytes: 0, under: tinyDir, nestedEvery: 0) + // 8 KB files: average clearly above maxAverageFileSize (4096). + try makeFiles(count: 10, bytes: 8192, under: bigDir, nestedEvery: 0) + + guard let root = await withMinFileCountOverride(10, { + await withAutoSummarize(true, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan produced no root") + } + + guard let tinyNode = findNode(root, path: ["level1", "tinyDir"]) else { + return XCTFail("tinyDir node not found") + } + XCTAssertTrue(tinyNode.isAutoSummarized, "many-tiny-file directory below the average-size threshold should be summarized") + XCTAssertEqual(tinyNode.descendantFileCount, 10) + + guard let bigNode = findNode(root, path: ["level1", "bigDir"]) else { + return XCTFail("bigDir node not found") + } + XCTAssertFalse(bigNode.isAutoSummarized, "directory whose average file size exceeds the threshold must not be summarized") + XCTAssertEqual(bigNode.children.count, 10) + } + + // MARK: - 3. autoSummarize off matches a full scan's sizes and file accounting + + func test_summarize_off_matches_full_scan_sizes() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: tmp) } + + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + try Data(repeating: 1, count: 4096).write(to: tmp.appendingPathComponent("plain.bin")) + + let nodeModules = tmp.appendingPathComponent("node_modules") + try makeFiles(count: 20, bytes: 16, under: nodeModules) + + guard let onRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(true, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan (autoSummarize on) produced no root") + } + guard let offRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(false, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan (autoSummarize off) produced no root") + } + + XCTAssertEqual(onRoot.size, offRoot.size, "root size must be identical whether or not subtrees got collapsed") + XCTAssertEqual(totalAccountedFiles(onRoot), totalAccountedFiles(offRoot), "every file must be accounted for exactly once in both modes") + // 1 plain file + 20 files under node_modules. + XCTAssertEqual(totalAccountedFiles(offRoot), 21) + } + + // MARK: - 4. Hardlinks inside a summarized subtree are not double-counted + + func test_hardlinks_not_double_counted_in_summary() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: tmp) } + + let nodeModules = tmp.appendingPathComponent("node_modules") + try FileManager.default.createDirectory(at: nodeModules, withIntermediateDirectories: true) + + let hard1 = nodeModules.appendingPathComponent("hard1.bin") + let hard2 = nodeModules.appendingPathComponent("hard2.bin") + try Data(repeating: 3, count: 262_144).write(to: hard1) + try FileManager.default.linkItem(at: hard1, to: hard2) + try Data(repeating: 4, count: 16).write(to: nodeModules.appendingPathComponent("other.bin")) + + var st = stat() + XCTAssertEqual(lstat(hard1.path, &st), 0) + let hardlinkSize = Int64(st.st_blocks) * 512 + XCTAssertEqual(lstat(nodeModules.appendingPathComponent("other.bin").path, &st), 0) + let otherSize = Int64(st.st_blocks) * 512 + + guard let root = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(true, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan produced no root") + } + guard let nmNode = findNode(root, path: ["node_modules"]) else { + return XCTFail("node_modules node not found") + } + + XCTAssertTrue(nmNode.isAutoSummarized) + XCTAssertEqual(nmNode.descendantFileCount, 3, "both hardlink names and the unrelated file are each one descendant file") + XCTAssertEqual(nmNode.size, hardlinkSize + otherSize, "the hardlinked pair must contribute its allocated bytes exactly once") + } + + // MARK: - 5. Parallel walk matches a serial (autoSummarize off) scan + + // Builds a moderately deep/wide tree that trips summarization (via a low + // MDS_SUMMARY_MIN_FILES override), scans it with autoSummarize on (which + // now drives the parallel worker pool in `summarizeSubtree`) and off + // (full, per-file walk), and asserts the summarized node's aggregate + // size and descendantFileCount exactly match the full scan's totals. + // This is a determinism/parity check for the parallel accumulator: since + // multiple workers race to discover directories/hardlinks, this is the + // test that would catch any double-counting or dropped entries the + // parallelization could introduce. + func test_parallel_summary_matches_serial_for_nested_tree() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: tmp) } + + // Use the node_modules named-layout shortcut (rather than the + // immediate-file-count heuristic) so the tree can be wide - many + // independent "package" subdirectories, each with its own shallow + // nested chain - without needing files directly inside node_modules + // itself. Wide-and-shallow, rather than one long chained path, is + // what actually makes the parallel pool fan out across many workers + // while staying well under macOS's path-length limit. + let nodeModules = tmp.appendingPathComponent("proj").appendingPathComponent("node_modules") + var expectedTotal: Int64 = 0 + for pkg in 0..<20 { + expectedTotal += try makeFiles(count: 20, bytes: 32, under: nodeModules.appendingPathComponent("pkg\(pkg)"), nestedEvery: 3) + } + + guard let onRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(true, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan (autoSummarize on) produced no root") + } + guard let nmNode = findNode(onRoot, path: ["proj", "node_modules"]) else { + return XCTFail("node_modules node not found") + } + XCTAssertTrue(nmNode.isAutoSummarized, "node_modules-named directory should trip summarization") + + guard let offRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(false, { await scanTree(at: tmp) }) + }) else { + return XCTFail("scan (autoSummarize off) produced no root") + } + guard let offNmNode = findNode(offRoot, path: ["proj", "node_modules"]) else { + return XCTFail("node_modules node not found in the non-summarized scan") + } + + XCTAssertEqual(nmNode.size, expectedTotal, "parallel summary walk must total exactly the bytes on disk") + XCTAssertEqual(nmNode.descendantFileCount, 400, "parallel summary walk must count every file exactly once") + XCTAssertEqual(nmNode.size, offNmNode.size, "parallel summary result must match a full serial scan's size") + XCTAssertEqual(nmNode.descendantFileCount, totalAccountedFiles(offNmNode), "parallel summary result must match a full serial scan's file count") + } + + // MARK: - 6. Env-gated benchmark + + func test_benchmark_autosummary() async throws { + guard ProcessInfo.processInfo.environment["MDS_SUMMARY_BENCH"] == "1" else { + throw XCTSkip("set MDS_SUMMARY_BENCH=1 to run the benchmark") + } + guard let path = ProcessInfo.processInfo.environment["MDS_SUMMARY_BENCH_PATH"] else { + return XCTFail("set MDS_SUMMARY_BENCH_PATH to a directory containing a node_modules-heavy tree") + } + let url = URL(fileURLWithPath: path) + + func countNodes(_ node: FileNode) -> Int { + 1 + node.children.reduce(0) { $0 + countNodes($1) } + } + + // The app's default excludedFolderNames already fully excludes + // "node_modules" (see ScanConfig); drop it here so the benchmark + // actually exercises the named-layout summarization path instead of + // both runs skipping the tree identically. + let onStart = DispatchTime.now() + guard let onRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(true, { await scanTree(at: url) }) + }) else { + return XCTFail("summarize-on benchmark scan produced no root") + } + let onElapsed = Double(DispatchTime.now().uptimeNanoseconds - onStart.uptimeNanoseconds) / 1_000_000_000 + + let offStart = DispatchTime.now() + guard let offRoot = await withExcludedFolderNames(".git,DerivedData,.Trash", { + await withAutoSummarize(false, { await scanTree(at: url) }) + }) else { + return XCTFail("summarize-off benchmark scan produced no root") + } + let offElapsed = Double(DispatchTime.now().uptimeNanoseconds - offStart.uptimeNanoseconds) / 1_000_000_000 + + let nodesOn = countNodes(onRoot) + let nodesOff = countNodes(offRoot) + let rootBytesEqual = onRoot.size == offRoot.size + + print("MDS_SUMMARY_BENCH on=\(onElapsed) off=\(offElapsed) nodes_on=\(nodesOn) nodes_off=\(nodesOff) rootBytesEqual=\(rootBytesEqual)") + + XCTAssertTrue(rootBytesEqual, "auto-summarization must not change the total scanned size") + } +} diff --git a/Tests/BulkScannerTests.swift b/Tests/BulkScannerTests.swift new file mode 100644 index 0000000..b8b1daa --- /dev/null +++ b/Tests/BulkScannerTests.swift @@ -0,0 +1,411 @@ +import XCTest +@testable import MacDirStat + +final class BulkScannerTests: XCTestCase { + + // MARK: - Helpers + + // Sorted (relativePath, isDirectory, size) fingerprint of an entire tree, + // used to compare the bulk and fallback enumeration paths against each other. + // + // Hardlinked files need special handling: which specific link "wins" the + // non-zero size (VisitedSet.visit is a first-come-first-served race across + // concurrent workers) is real, expected nondeterminism of concurrent + // traversal, not a correctness bug - verified directly, it can differ + // between two runs of the *same* enumeration method on a hardlink-heavy + // real tree (e.g. Xcode.app, which hardlinks tens of thousands of files + // across sibling bundles), not just bulk vs fallback. Two consequences: + // - a hardlinked file's own size is normalized to 0 here and instead + // checked as a group: the same set of paths must share one inode, and + // that group's total allocated size must match. + // - a *directory's* aggregated size becomes nondeterministic too, if a + // hardlink's two links sit under different parents (the bytes get + // attributed to whichever parent's link happened to win). Only the + // grand total is guaranteed invariant. So directory entries carry no + // size at all here; only file entries and the root total do. + private struct FileFingerprintEntry: Hashable { + let relativePath: String + let size: Int64 + } + + private struct HardlinkGroupFingerprint: Hashable { + let paths: [String] + let total: Int64 + } + + private struct Fingerprint { + let fileEntries: [FileFingerprintEntry] + let directoryPaths: [String] + let hardlinkGroups: [HardlinkGroupFingerprint] + let total: Int64 + } + + private func fingerprint(root: FileNode, base: URL) -> Fingerprint { + var fileEntries: [FileFingerprintEntry] = [] + var directoryPaths: [String] = [] + var groups: [HardLinkRef: (paths: Set, total: Int64)] = [:] + + func visit(_ node: FileNode) { + let relativePath = String(node.url.path.dropFirst(base.path.count)) + if node.isDirectory { + directoryPaths.append(relativePath) + } else if let ref = node.hardLinkRef { + var group = groups[ref] ?? (paths: [], total: 0) + group.paths.insert(relativePath) + group.total += node.size + groups[ref] = group + fileEntries.append(FileFingerprintEntry(relativePath: relativePath, size: 0)) + } else { + fileEntries.append(FileFingerprintEntry(relativePath: relativePath, size: node.size)) + } + for child in node.children { visit(child) } + } + visit(root) + fileEntries.sort { $0.relativePath < $1.relativePath } + directoryPaths.sort() + + let hardlinkGroups = groups.values + .map { HardlinkGroupFingerprint(paths: $0.paths.sorted(), total: $0.total) } + .sorted { $0.paths.first ?? "" < $1.paths.first ?? "" } + + return Fingerprint(fileEntries: fileEntries, directoryPaths: directoryPaths, hardlinkGroups: hardlinkGroups, total: root.size) + } + + private func scanTree(at url: URL) async -> FileNode? { + let scanner = FileScanner() + var root: FileNode? + for await progress in await scanner.scan(url: url) { + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } + } + return root + } + + // Builds a tree exercising every scan semantic: nested dirs, hidden files, + // an excluded dir name, a symlink, a hardlink pair, and a file > 4 KB. + private func buildFixtureTree(at tmp: URL) throws { + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + + let sub = tmp.appendingPathComponent("sub") + let nested = sub.appendingPathComponent("nested") + try FileManager.default.createDirectory(at: nested, withIntermediateDirectories: true) + + try Data(repeating: 0xAB, count: 8192).write(to: tmp.appendingPathComponent("big.bin")) + try Data(repeating: 0, count: 16).write(to: sub.appendingPathComponent("small.txt")) + try Data(repeating: 0, count: 16).write(to: nested.appendingPathComponent("deep.txt")) + try Data(repeating: 0, count: 8).write(to: tmp.appendingPathComponent(".hidden")) + + let excludedDir = tmp.appendingPathComponent("node_modules") + try FileManager.default.createDirectory(at: excludedDir, withIntermediateDirectories: true) + try Data(repeating: 0, count: 4096).write(to: excludedDir.appendingPathComponent("junk.bin")) + + let real = tmp.appendingPathComponent("real.bin") + try Data(repeating: 1, count: 4096).write(to: real) + let link = tmp.appendingPathComponent("link.bin") + try FileManager.default.createSymbolicLink(at: link, withDestinationURL: real) + + let hard1 = sub.appendingPathComponent("hard1.bin") + let hard2 = sub.appendingPathComponent("hard2.bin") + try Data(repeating: 3, count: 262_144).write(to: hard1) + try FileManager.default.linkItem(at: hard1, to: hard2) + } + + // MARK: - 1. Parity fingerprint test + + func test_bulk_and_fallback_enumeration_produce_identical_fingerprint() async throws { + let priorValue = UserDefaults.standard.string(forKey: "excludedFolderNames") + UserDefaults.standard.set(".git,node_modules,DerivedData,.Trash", forKey: "excludedFolderNames") + defer { + if let priorValue { UserDefaults.standard.set(priorValue, forKey: "excludedFolderNames") } + else { UserDefaults.standard.removeObject(forKey: "excludedFolderNames") } + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try buildFixtureTree(at: tmp) + defer { try? FileManager.default.removeItem(at: tmp) } + + guard let bulkRoot = await scanTree(at: tmp) else { + return XCTFail("bulk scan produced no root") + } + let bulkFingerprint = fingerprint(root: bulkRoot, base: tmp) + + setenv("MDS_FORCE_FALLBACK_ENUM", "1", 1) + let fallbackRoot = await scanTree(at: tmp) + unsetenv("MDS_FORCE_FALLBACK_ENUM") + + guard let fallbackRoot else { + return XCTFail("fallback scan produced no root") + } + let fallbackFingerprint = fingerprint(root: fallbackRoot, base: tmp) + + XCTAssertEqual(bulkFingerprint.fileEntries, fallbackFingerprint.fileEntries, "bulk and fallback enumeration must see the same files with the same sizes") + XCTAssertEqual(bulkFingerprint.directoryPaths, fallbackFingerprint.directoryPaths, "bulk and fallback enumeration must see the same directories") + XCTAssertEqual(bulkFingerprint.hardlinkGroups, fallbackFingerprint.hardlinkGroups, "hardlink groupings and their total sizes must match") + XCTAssertEqual(bulkFingerprint.total, fallbackFingerprint.total, "bulk and fallback totals must match") + XCTAssertGreaterThan(bulkFingerprint.total, 0) + } + + // MARK: - 2. Bulk enumerator unit test + + func test_bulk_enumerator_matches_lstat_ground_truth() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let fileA = tmp.appendingPathComponent("a.bin") + let fileB = tmp.appendingPathComponent("b.bin") + let dirC = tmp.appendingPathComponent("c_dir") + try Data(repeating: 1, count: 12_345).write(to: fileA) + try Data(repeating: 2, count: 42).write(to: fileB) + try FileManager.default.createDirectory(at: dirC, withIntermediateDirectories: true) + + let fd = open(tmp.path, O_RDONLY | O_DIRECTORY | O_CLOEXEC) + XCTAssertGreaterThanOrEqual(fd, 0) + defer { close(fd) } + + let entries = try enumerateDirectoryBulk(fd: fd) + let byName = Dictionary(uniqueKeysWithValues: entries.map { ($0.name, $0) }) + + XCTAssertEqual(entries.count, 3) + + for name in ["a.bin", "b.bin", "c_dir"] { + guard let entry = byName[name] else { + XCTFail("missing entry for \(name)") + continue + } + var st = stat() + XCTAssertEqual(lstat(tmp.appendingPathComponent(name).path, &st), 0) + XCTAssertEqual(entry.dev, UInt64(bitPattern: Int64(st.st_dev)), "dev mismatch for \(name)") + XCTAssertEqual(entry.ino, UInt64(st.st_ino), "ino mismatch for \(name)") + if name == "c_dir" { + XCTAssertEqual(entry.kind, .directory) + } else { + XCTAssertEqual(entry.kind, .file) + XCTAssertEqual(entry.linkCount, UInt32(st.st_nlink), "linkCount mismatch for \(name)") + XCTAssertEqual(entry.allocatedSize, Int64(st.st_blocks) * 512, "allocatedSize mismatch for \(name)") + } + } + } + + // MARK: - 3. Hardlink dedup test + + func test_hardlinked_file_deduped_across_two_directories() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let dirA = tmp.appendingPathComponent("dirA") + let dirB = tmp.appendingPathComponent("dirB") + try FileManager.default.createDirectory(at: dirA, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: dirB, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let original = dirA.appendingPathComponent("orig.bin") + try Data(repeating: 9, count: 262_144).write(to: original) + let linked = dirB.appendingPathComponent("linked.bin") + try FileManager.default.linkItem(at: original, to: linked) + + var st = stat() + XCTAssertEqual(lstat(original.path, &st), 0) + let fullSize = Int64(st.st_blocks) * 512 + + guard let root = await scanTree(at: tmp) else { + return XCTFail("scan produced no root") + } + + let aNode = root.children.first { $0.name == "dirA" } + let bNode = root.children.first { $0.name == "dirB" } + let origNode = aNode?.children.first { $0.name == "orig.bin" } + let linkedNode = bNode?.children.first { $0.name == "linked.bin" } + + XCTAssertNotNil(origNode?.hardLinkRef) + XCTAssertNotNil(linkedNode?.hardLinkRef) + XCTAssertEqual(origNode?.hardLinkRef, linkedNode?.hardLinkRef) + + let sizes = [origNode?.size ?? -1, linkedNode?.size ?? -1].sorted() + XCTAssertEqual(sizes, [0, fullSize], "one link carries the full size, the other is zero") + XCTAssertEqual(root.size, fullSize, "the shared inode must be counted exactly once across both directories") + } + + // MARK: - 4. Access-denied test + + func test_unreadable_subdirectory_marked_access_denied() async throws { + guard getuid() != 0 else { + throw XCTSkip("running as root: chmod 000 does not block access") + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + let locked = tmp.appendingPathComponent("locked") + try FileManager.default.createDirectory(at: locked, withIntermediateDirectories: true) + try Data(repeating: 0, count: 4096).write(to: locked.appendingPathComponent("secret.bin")) + + chmod(locked.path, 0) + defer { + chmod(locked.path, 0o755) + try? FileManager.default.removeItem(at: tmp) + } + + var deniedCount = 0 + let scanner = FileScanner() + var root: FileNode? + for await progress in await scanner.scan(url: tmp) { + if case .completed(let tree, let denied) = progress { + root = FileNode(tree: tree, index: tree.rootIndex) + deniedCount = denied + } + } + + let lockedNode = root?.children.first { $0.name == "locked" } + XCTAssertNotNil(lockedNode) + XCTAssertTrue(lockedNode?.isAccessDenied ?? false) + XCTAssertEqual(lockedNode?.children.count, 0) + XCTAssertGreaterThanOrEqual(deniedCount, 1) + } + + // MARK: - 5. Cancellation test + + func test_cancellation_ends_stream_without_completed() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + // A moderately large synthetic tree so the scan is very unlikely to + // finish before cancellation takes effect. + for i in 0..<50 { + let dir = tmp.appendingPathComponent("d\(i)") + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + for j in 0..<50 { + try Data(repeating: 0, count: 256).write(to: dir.appendingPathComponent("f\(j).bin")) + } + } + + let scanner = FileScanner() + var sawCompleted = false + let stream = await scanner.scan(url: tmp) + let consumeTask = Task { + for await progress in stream { + if case .completed = progress { sawCompleted = true } + } + } + await scanner.cancel() + _ = await consumeTask.value + + XCTAssertFalse(sawCompleted, "a cancelled scan must not emit .completed") + } + + // MARK: - 7. Env-gated benchmark + + func test_benchmark_bulk_vs_fallback() async throws { + guard ProcessInfo.processInfo.environment["MDS_BENCH"] == "1" else { + throw XCTSkip("set MDS_BENCH=1 to run the benchmark") + } + let path = ProcessInfo.processInfo.environment["MDS_BENCH_PATH"] ?? "/Applications" + let url = URL(fileURLWithPath: path) + + let bulkStart = DispatchTime.now() + guard let bulkRoot = await scanTree(at: url) else { + return XCTFail("bulk benchmark scan produced no root") + } + let bulkElapsed = Double(DispatchTime.now().uptimeNanoseconds - bulkStart.uptimeNanoseconds) / 1_000_000_000 + + setenv("MDS_FORCE_FALLBACK_ENUM", "1", 1) + let fallbackStart = DispatchTime.now() + let fallbackRoot = await scanTree(at: url) + let fallbackElapsed = Double(DispatchTime.now().uptimeNanoseconds - fallbackStart.uptimeNanoseconds) / 1_000_000_000 + unsetenv("MDS_FORCE_FALLBACK_ENUM") + + guard let fallbackRoot else { + return XCTFail("fallback benchmark scan produced no root") + } + + let bulkFingerprint = fingerprint(root: bulkRoot, base: url) + let fallbackFingerprint = fingerprint(root: fallbackRoot, base: url) + + print("MDS_BENCH elapsed_bulk=\(bulkElapsed) elapsed_fallback=\(fallbackElapsed) path=\(path)") + + if bulkFingerprint.fileEntries != fallbackFingerprint.fileEntries { + let bulkSet = Set(bulkFingerprint.fileEntries.map { "\($0.relativePath)|\($0.size)" }) + let fallbackSet = Set(fallbackFingerprint.fileEntries.map { "\($0.relativePath)|\($0.size)" }) + let onlyBulk = bulkSet.subtracting(fallbackSet) + let onlyFallback = fallbackSet.subtracting(bulkSet) + print("MDS_BENCH_DIFF onlyBulk=\(onlyBulk.count) onlyFallback=\(onlyFallback.count)") + for line in onlyBulk.sorted().prefix(20) { print("MDS_BENCH_ONLY_BULK \(line)") } + for line in onlyFallback.sorted().prefix(20) { print("MDS_BENCH_ONLY_FALLBACK \(line)") } + } + if bulkFingerprint.directoryPaths != fallbackFingerprint.directoryPaths { + let bulkSet = Set(bulkFingerprint.directoryPaths) + let fallbackSet = Set(fallbackFingerprint.directoryPaths) + print("MDS_BENCH_DIFF onlyBulkDirs=\(bulkSet.subtracting(fallbackSet).count) onlyFallbackDirs=\(fallbackSet.subtracting(bulkSet).count)") + } + if bulkFingerprint.hardlinkGroups != fallbackFingerprint.hardlinkGroups { + print("MDS_BENCH_DIFF hardlinkGroups bulk=\(bulkFingerprint.hardlinkGroups.count) fallback=\(fallbackFingerprint.hardlinkGroups.count)") + } + + XCTAssertEqual(bulkFingerprint.fileEntries, fallbackFingerprint.fileEntries, "bulk and fallback enumeration must see the same files with the same sizes") + XCTAssertEqual(bulkFingerprint.directoryPaths, fallbackFingerprint.directoryPaths, "bulk and fallback enumeration must see the same directories") + XCTAssertEqual(bulkFingerprint.hardlinkGroups, fallbackFingerprint.hardlinkGroups, "hardlink groupings and their total sizes must match") + XCTAssertEqual(bulkFingerprint.total, fallbackFingerprint.total) + } + + // MARK: - Firmlinked directories must be scanned, not silently dropped + // + // Regression test for the worst bug this scanner has had. macOS firmlinks + // (/Users, /Applications, /Library, /opt, /private, /Volumes, /cores) are + // recorded on the sealed system volume with their own inode, and resolve + // on open to a different inode on the Data volume. The scanner used to + // compare the enumerated (dev, ino) against the opened one as a TOCTOU + // guard and drop the directory when they disagreed — which threw away + // every firmlink, i.e. essentially all user data. Scanning "/" reported + // 11.8 GB instead of 479.3 GB on the development machine. + // + // The existing bulk-vs-fallback parity test could never have caught this: + // its fixture is a plain temp directory, which has no firmlinks and no + // mount points. This one scans the real startup volume shallowly, so it + // exercises the actual platform behaviour. + // + // Kept cheap and robust: it only asserts that the well-known firmlinked + // directories are present with a non-zero size, which requires no + // knowledge of the machine's contents and no full-disk walk. + func test_startup_volume_firmlinks_are_scanned() async throws { + // /Users is a firmlink on every modern macOS install; if it isn't + // readable at all (sandboxed CI, no permissions) there is nothing + // meaningful to assert. + guard FileManager.default.isReadableFile(atPath: "/Users") else { + throw XCTSkip("/Users is not readable in this environment") + } + + var rootStat = stat() + var usersStat = stat() + guard lstat("/", &rootStat) == 0, lstat("/Users", &usersStat) == 0 else { + throw XCTSkip("could not stat / and /Users") + } + // The bug only exists where the enumerated and resolved identities + // differ, which is what makes a path a firmlink. On a volume layout + // without firmlinks there is nothing to regress. + guard rootStat.st_dev == usersStat.st_dev else { + throw XCTSkip("unexpected volume layout: /Users is on another device") + } + + // Scan "/" itself but keep it cheap: exclude the large subtrees, so + // this walks the top level and stops. The point is whether the + // firmlinked entries survive traversal at all, not their exact sizes. + let prior = UserDefaults.standard.string(forKey: "excludedFolderNames") + UserDefaults.standard.set("System,usr,bin,sbin,dev,.Trash", forKey: "excludedFolderNames") + defer { + if let prior { UserDefaults.standard.set(prior, forKey: "excludedFolderNames") } + else { UserDefaults.standard.removeObject(forKey: "excludedFolderNames") } + } + + let scanner = FileScanner() + var root: FileNode? + for await progress in await scanner.scan(url: URL(fileURLWithPath: "/")) { + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } + } + guard let root else { return XCTFail("scanning / produced no tree") } + + guard let users = root.children.first(where: { $0.name == "Users" }) else { + return XCTFail("/Users is missing from the scan entirely — firmlinks are being dropped") + } + XCTAssertGreaterThan( + users.size, 0, + "/Users scanned as 0 bytes — the firmlinked directory was traversed but produced nothing" + ) + } +} diff --git a/Tests/DiagScanTests.swift b/Tests/DiagScanTests.swift new file mode 100644 index 0000000..c0061a3 --- /dev/null +++ b/Tests/DiagScanTests.swift @@ -0,0 +1,41 @@ +import XCTest +@testable import MacDirStat + +// Diagnostic harness, not a CI test: scans a real directory with the production +// FileScanner and prints per-child totals for comparison against `du`. +// Enable with MDS_DIAG_PATH=/some/path. +final class DiagScanTests: XCTestCase { + + func test_diag_scan_prints_top_level_sizes() async throws { + guard let target = ProcessInfo.processInfo.environment["MDS_DIAG_PATH"] else { + throw XCTSkip("set MDS_DIAG_PATH to run the diagnostic scan") + } + + let scanner = FileScanner() + var completedRoot: FileNode? + var lastItems = 0 + var lastBytes: Int64 = 0 + for await progress in await scanner.scan(url: URL(fileURLWithPath: target)) { + switch progress { + case .update(let items, let bytes): + lastItems = items + lastBytes = bytes + case .completed(let tree, _): + completedRoot = FileNode(tree: tree, index: tree.rootIndex) + case .failed(let msg): + XCTFail("scan failed: \(msg)") + } + } + + guard let root = completedRoot else { + XCTFail("no completed root (last progress: \(lastItems) items, \(lastBytes) bytes)") + return + } + + func gb(_ v: Int64) -> String { String(format: "%.1f GB", Double(v) / 1_000_000_000) } + print("DIAG root=\(target) total=\(gb(root.size)) rawBytes=\(root.size) progressBytes=\(gb(lastBytes)) items=\(lastItems)") + for child in root.children.sorted(by: { $0.size > $1.size }).prefix(25) { + print("DIAG child \(gb(child.size)) \(child.name)\(child.isDirectory ? "/" : "")") + } + } +} diff --git a/Tests/DuplicateDetectorTests.swift b/Tests/DuplicateDetectorTests.swift index 2d66b0f..7912c2e 100644 --- a/Tests/DuplicateDetectorTests.swift +++ b/Tests/DuplicateDetectorTests.swift @@ -3,6 +3,13 @@ import XCTest final class DuplicateDetectorTests: XCTestCase { + // Finds the record index for a given file name — the FSNode -> FileTree + // conversion doesn't preserve any positional ordering guarantees the + // tests can rely on, so tests look nodes up by name. + private func index(in tree: FileTree, named name: String) -> Int? { + tree.records.firstIndex { $0.name == name } + } + func test_detects_identical_files() async throws { let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) @@ -24,18 +31,19 @@ final class DuplicateDetectorTests: XCTestCase { root.children.append(child) root.size += child.size } + let tree = FileTreeBuilder.build(from: root, rootPath: tmp.path) let detector = DuplicateDetector() - await detector.detect(in: root) + await detector.detect(in: tree) - let copy1 = root.children.first { $0.name == "copy1.bin" }! - let copy2 = root.children.first { $0.name == "copy2.bin" }! - let unique = root.children.first { $0.name == "unique.bin" }! + let copy1 = index(in: tree, named: "copy1.bin")! + let copy2 = index(in: tree, named: "copy2.bin")! + let unique = index(in: tree, named: "unique.bin")! - XCTAssertNotNil(copy1.duplicateGroupID) - XCTAssertNotNil(copy2.duplicateGroupID) - XCTAssertEqual(copy1.duplicateGroupID, copy2.duplicateGroupID) - XCTAssertNil(unique.duplicateGroupID, "unique file must not be grouped") + XCTAssertNotNil(tree.records[copy1].duplicateGroupID) + XCTAssertNotNil(tree.records[copy2].duplicateGroupID) + XCTAssertEqual(tree.records[copy1].duplicateGroupID, tree.records[copy2].duplicateGroupID) + XCTAssertNil(tree.records[unique].duplicateGroupID, "unique file must not be grouped") } func test_small_files_below_threshold_are_skipped() async throws { @@ -54,11 +62,133 @@ final class DuplicateDetectorTests: XCTestCase { let child = FSNode(url: url, name: url.lastPathComponent, isDirectory: false, size: Int64(tinyData.count), fileExtension: "txt", parent: root) root.children.append(child) } + let tree = FileTreeBuilder.build(from: root, rootPath: tmp.path) + + let detector = DuplicateDetector() + await detector.detect(in: tree) + + let tiny1 = index(in: tree, named: "tiny1.txt")! + let tiny2 = index(in: tree, named: "tiny2.txt")! + XCTAssertNil(tree.records[tiny1].duplicateGroupID, "files below threshold should not be grouped") + XCTAssertNil(tree.records[tiny2].duplicateGroupID) + } + + func test_same_prefix_different_tail_not_duplicates() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + // 128 KB files: identical first 64 KB (quick-hash prefix), different last bytes. + let sharedPrefix = Data(repeating: 7, count: 65_536) + let tail1 = Data(repeating: 1, count: 65_536) + let tail2 = Data(repeating: 2, count: 65_536) + let data1 = sharedPrefix + tail1 + let data2 = sharedPrefix + tail2 + + let f1 = tmp.appendingPathComponent("a.bin") + let f2 = tmp.appendingPathComponent("b.bin") + try data1.write(to: f1) + try data2.write(to: f2) + + let root = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + for url in [f1, f2] { + let child = FSNode(url: url, name: url.lastPathComponent, isDirectory: false, size: Int64(data1.count), fileExtension: "bin", parent: root) + root.children.append(child) + } + let tree = FileTreeBuilder.build(from: root, rootPath: tmp.path) let detector = DuplicateDetector() - await detector.detect(in: root) + await detector.detect(in: tree) + + let a = index(in: tree, named: "a.bin")! + let b = index(in: tree, named: "b.bin")! + XCTAssertNil(tree.records[a].duplicateGroupID, "files sharing only a quick-hash prefix must not be grouped") + XCTAssertNil(tree.records[b].duplicateGroupID) + } + + func test_small_file_duplicates_detected() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + // 8 KB files: above minSize (4 KB), below quickHashBytes (64 KB) -> quick hash IS full hash. + let data = Data(repeating: 55, count: 8192) + let other = Data(repeating: 66, count: 8192) + let f1 = tmp.appendingPathComponent("s1.bin") + let f2 = tmp.appendingPathComponent("s2.bin") + let f3 = tmp.appendingPathComponent("s3.bin") + try data.write(to: f1) + try data.write(to: f2) + try other.write(to: f3) - XCTAssertNil(root.children[0].duplicateGroupID, "files below threshold should not be grouped") - XCTAssertNil(root.children[1].duplicateGroupID) + let root = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + for url in [f1, f2, f3] { + let child = FSNode(url: url, name: url.lastPathComponent, isDirectory: false, size: Int64(data.count), fileExtension: "bin", parent: root) + root.children.append(child) + } + let tree = FileTreeBuilder.build(from: root, rootPath: tmp.path) + + let detector = DuplicateDetector() + await detector.detect(in: tree) + + let s1 = index(in: tree, named: "s1.bin")! + let s2 = index(in: tree, named: "s2.bin")! + let s3 = index(in: tree, named: "s3.bin")! + + XCTAssertNotNil(tree.records[s1].duplicateGroupID) + XCTAssertNotNil(tree.records[s2].duplicateGroupID) + XCTAssertEqual(tree.records[s1].duplicateGroupID, tree.records[s2].duplicateGroupID) + XCTAssertNil(tree.records[s3].duplicateGroupID) + } + + func test_many_duplicate_pairs_all_detected() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let root = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + + let pairCount = 20 + var pairNames: [(String, String)] = [] + + for i in 0..() + for (nameA, nameB) in pairNames { + let a = index(in: tree, named: nameA)! + let b = index(in: tree, named: nameB)! + XCTAssertNotNil(tree.records[a].duplicateGroupID, "\(nameA) should be grouped") + XCTAssertNotNil(tree.records[b].duplicateGroupID, "\(nameB) should be grouped") + XCTAssertEqual(tree.records[a].duplicateGroupID, tree.records[b].duplicateGroupID, "\(nameA) and \(nameB) should share a group") + if let gid = tree.records[a].duplicateGroupID { + XCTAssertFalse(seenGroupIDs.contains(gid), "group ID \(gid) reused across pairs") + seenGroupIDs.insert(gid) + } + } + XCTAssertEqual(seenGroupIDs.count, pairCount) } } diff --git a/Tests/FDAPromptTests.swift b/Tests/FDAPromptTests.swift new file mode 100644 index 0000000..f7906a2 --- /dev/null +++ b/Tests/FDAPromptTests.swift @@ -0,0 +1,97 @@ +import XCTest +@testable import MacDirStat + +@MainActor +final class FDAPromptTests: XCTestCase { + + // MARK: - shouldOfferFullDiskAccess + + func test_offers_when_fda_missing_and_folders_denied() { + XCTAssertTrue( + ScanViewModel.shouldOfferFullDiskAccess( + deniedCount: 241, + hasFullDiskAccess: false, + suppressed: false + ) + ) + } + + func test_does_not_offer_when_fda_already_granted() { + XCTAssertFalse( + ScanViewModel.shouldOfferFullDiskAccess( + deniedCount: 241, + hasFullDiskAccess: true, + suppressed: false + ) + ) + } + + func test_does_not_offer_when_no_folders_denied() { + XCTAssertFalse( + ScanViewModel.shouldOfferFullDiskAccess( + deniedCount: 0, + hasFullDiskAccess: false, + suppressed: false + ) + ) + } + + func test_does_not_offer_when_suppressed() { + XCTAssertFalse( + ScanViewModel.shouldOfferFullDiskAccess( + deniedCount: 241, + hasFullDiskAccess: false, + suppressed: true + ) + ) + } + + // MARK: - "Access granted!" must reflect an observed grant, not a probe + + // The dead end this guards against: the readability probe claims access + // while the scan is still being denied hundreds of folders, so opening the + // sheet from the warning banner landed on a congratulations screen whose + // only action was a relaunch that changed nothing — and the System + // Settings button was unreachable. + func test_granted_state_requires_access_to_have_flipped_while_sheet_open() { + // Probe already claimed access when the sheet opened: never celebrate. + XCTAssertFalse( + FullDiskAccessSheet.showsGrantedState(accessWasMissingOnOpen: false, hasAccessNow: true), + "a probe that was already true on open is not evidence of a grant" + ) + // Access was missing and has now appeared: this is the real grant. + XCTAssertTrue( + FullDiskAccessSheet.showsGrantedState(accessWasMissingOnOpen: true, hasAccessNow: true) + ) + // Still missing. + XCTAssertFalse( + FullDiskAccessSheet.showsGrantedState(accessWasMissingOnOpen: true, hasAccessNow: false) + ) + // Before onAppear has recorded the starting state. + XCTAssertFalse( + FullDiskAccessSheet.showsGrantedState(accessWasMissingOnOpen: nil, hasAccessNow: true) + ) + } + + // MARK: - The draggable icon must point at the RUNNING copy + + // macOS grants Full Disk Access to one exact copy of an app. The whole + // point of dragging the icon instead of using the "+" picker is that the + // payload cannot be some other build sitting in /Applications, so the + // dragged URL has to be this bundle and nothing else. + func test_drag_payload_is_the_running_bundle() { + XCTAssertEqual( + FullDiskAccessSheet.runningAppURL(), + Bundle.main.bundleURL, + "the drag must carry the running bundle, not a copy found elsewhere" + ) + XCTAssertTrue( + FileManager.default.fileExists(atPath: FullDiskAccessSheet.runningAppURL().path), + "the dragged URL must exist on disk or the drop silently does nothing" + ) + XCTAssertFalse( + FullDiskAccessSheet.runningAppName().isEmpty, + "the tile needs a name to label the icon with" + ) + } +} diff --git a/Tests/FileScannerTests.swift b/Tests/FileScannerTests.swift index c4ebd6a..8b84cab 100644 --- a/Tests/FileScannerTests.swift +++ b/Tests/FileScannerTests.swift @@ -31,6 +31,15 @@ final class FileScannerTests: XCTestCase { XCTAssertNil(child.parent) } + func test_optional_children_returns_all_children() { + let dir = FSNode(url: URL(fileURLWithPath: "/tmp"), name: "tmp", isDirectory: true, size: 0, fileExtension: "", parent: nil) + for i in 0..<2001 { + let child = FSNode(url: URL(fileURLWithPath: "/tmp/\(i)"), name: "\(i)", isDirectory: false, size: 1, fileExtension: "txt", parent: dir) + dir.children.append(child) + } + XCTAssertEqual(dir.optionalChildren?.count, 2001) + } + func test_scanner_builds_tree_from_temp_directory() async throws { let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) @@ -42,9 +51,9 @@ final class FileScannerTests: XCTestCase { try Data(repeating: 0, count: 2048).write(to: file2) let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } XCTAssertNotNil(root) @@ -65,9 +74,9 @@ final class FileScannerTests: XCTestCase { try Data(repeating: 0, count: 4096).write(to: sub.appendingPathComponent("big.bin")) let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } XCTAssertEqual(root?.size ?? 0, root?.children.first?.size ?? -1) @@ -84,12 +93,215 @@ final class FileScannerTests: XCTestCase { try FileManager.default.createSymbolicLink(at: link, withDestinationURL: real) let scanner = FileScanner() - var root: FSNode? + var root: FileNode? for await progress in await scanner.scan(url: tmp) { - if case .completed(let node) = progress { root = node } + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } } XCTAssertEqual(root?.children.count, 1, "symlink should be skipped") XCTAssertEqual(root?.children.first?.name, "real.txt") } + + func test_scanner_sets_hardlink_refs() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let hard1 = tmp.appendingPathComponent("hard1.bin") + let hard2 = tmp.appendingPathComponent("hard2.bin") + try Data(repeating: 7, count: 262_144).write(to: hard1) + try FileManager.default.linkItem(at: hard1, to: hard2) + var st = stat() + XCTAssertEqual(lstat(hard1.path, &st), 0) + let fullSize = Int64(st.st_blocks) * 512 + + let scanner = FileScanner() + var root: FileNode? + for await progress in await scanner.scan(url: tmp) { + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } + } + + let n1 = root?.children.first { $0.name == "hard1.bin" } + let n2 = root?.children.first { $0.name == "hard2.bin" } + XCTAssertNotNil(n1?.hardLinkRef, "hardlinked file must carry its inode identity") + XCTAssertEqual(n1?.hardLinkRef, n2?.hardLinkRef, "both links to one inode share the same ref") + XCTAssertEqual(root?.size, fullSize, "hardlinked inode counted exactly once") + } + + func test_scanner_marks_unreadable_directory_as_denied() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + let locked = tmp.appendingPathComponent("locked") + try FileManager.default.createDirectory(at: locked, withIntermediateDirectories: true) + try Data(repeating: 0, count: 4096).write(to: locked.appendingPathComponent("secret.bin")) + + chmod(locked.path, 0) + defer { + chmod(locked.path, 0o755) + try? FileManager.default.removeItem(at: tmp) + } + + let scanner = FileScanner() + var root: FileNode? + var deniedCount = 0 + for await progress in await scanner.scan(url: tmp) { + if case .completed(let tree, let denied) = progress { + root = FileNode(tree: tree, index: tree.rootIndex) + deniedCount = denied + } + } + + let lockedNode = root?.children.first { $0.name == "locked" } + XCTAssertNotNil(lockedNode) + XCTAssertTrue(lockedNode?.isAccessDenied ?? false, "locked directory should be marked as access denied") + XCTAssertGreaterThanOrEqual(deniedCount, 1) + } + + // MARK: - Pinning tests for perf refactor (ScanConfig hoisting + TaskBudget fan-out cap) + + func test_scanner_respects_excluded_folder_names() async throws { + let priorValue = UserDefaults.standard.string(forKey: "excludedFolderNames") + UserDefaults.standard.set("skipme", forKey: "excludedFolderNames") + defer { + if let priorValue { + UserDefaults.standard.set(priorValue, forKey: "excludedFolderNames") + } else { + UserDefaults.standard.removeObject(forKey: "excludedFolderNames") + } + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let skipDir = tmp.appendingPathComponent("skipme") + let keepDir = tmp.appendingPathComponent("keep") + try FileManager.default.createDirectory(at: skipDir, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: keepDir, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + try Data(repeating: 0, count: 100 * 1024).write(to: skipDir.appendingPathComponent("big.bin")) + try Data(repeating: 0, count: 4096).write(to: keepDir.appendingPathComponent("file.bin")) + + let scanner = FileScanner() + var root: FileNode? + for await progress in await scanner.scan(url: tmp) { + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } + } + + XCTAssertNotNil(root) + let names = Set(root?.children.map { $0.name } ?? []) + XCTAssertFalse(names.contains("skipme"), "excluded folder should not appear as a child") + XCTAssertTrue(names.contains("keep")) + + guard let keepNode = root?.children.first(where: { $0.name == "keep" }) else { + return XCTFail("keep dir missing from scan results") + } + XCTAssertEqual(root?.size, keepNode.size, "root size should exclude the skipped folder's contents") + } + + func test_scanner_honors_show_hidden_files_setting() async throws { + let priorValue = UserDefaults.standard.object(forKey: "showHiddenFiles") as? Bool + defer { + if let priorValue { + UserDefaults.standard.set(priorValue, forKey: "showHiddenFiles") + } else { + UserDefaults.standard.removeObject(forKey: "showHiddenFiles") + } + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + try Data(repeating: 0, count: 10).write(to: tmp.appendingPathComponent(".hidden")) + try Data(repeating: 0, count: 10).write(to: tmp.appendingPathComponent("visible.txt")) + + UserDefaults.standard.set(false, forKey: "showHiddenFiles") + do { + let scanner = FileScanner() + var root: FileNode? + for await progress in await scanner.scan(url: tmp) { + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } + } + let names = Set(root?.children.map { $0.name } ?? []) + XCTAssertFalse(names.contains(".hidden"), "hidden file should be excluded when showHiddenFiles is false") + XCTAssertTrue(names.contains("visible.txt")) + } + + UserDefaults.standard.set(true, forKey: "showHiddenFiles") + do { + let scanner = FileScanner() + var root: FileNode? + for await progress in await scanner.scan(url: tmp) { + if case .completed(let tree, _) = progress { root = FileNode(tree: tree, index: tree.rootIndex) } + } + let names = Set(root?.children.map { $0.name } ?? []) + XCTAssertTrue(names.contains(".hidden"), "hidden file should be included when showHiddenFiles is true") + XCTAssertTrue(names.contains("visible.txt")) + } + } + + func test_scanner_handles_deep_wide_tree_correctly() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + var allDirs: [URL] = [] + var allFiles: [URL] = [] + + // Builds a tree nested 5 levels deep with ~40 directories total, each + // holding exactly one 4 KB file. This stresses the bounded task-group + // fan-out (Issue 2) to make sure totals stay correct regardless of + // whether a subtree is scanned via the task group or inline recursion. + let branchingPerLevel = [2, 2, 2, 2, 1] + func buildLevel(base: URL, levelIndex: Int) throws { + guard levelIndex < branchingPerLevel.count else { return } + let branches = branchingPerLevel[levelIndex] + for i in 0.. (dirs: Int, files: Int) { + if node.isDirectory { + var dirs = 1 + var files = 0 + for child in node.children { + let c = countNodes(child) + dirs += c.dirs + files += c.files + } + return (dirs, files) + } else { + return (0, 1) + } + } + + let counts = countNodes(root!) + XCTAssertEqual(counts.dirs, allDirs.count + 1, "every directory (plus root) must be reachable in the tree") + XCTAssertEqual(counts.files, allFiles.count, "every file must be reachable in the tree") + } } diff --git a/Tests/FileTreePruneTests.swift b/Tests/FileTreePruneTests.swift new file mode 100644 index 0000000..7d56a78 --- /dev/null +++ b/Tests/FileTreePruneTests.swift @@ -0,0 +1,490 @@ +import XCTest +@testable import MacDirStat + +// Coverage for `FileTree.removingSubtree(at:)` / `removingSubtrees(at:)` — the +// prune-on-trash replacement for the old "trash then full rescan" flow (see +// `ScanViewModel.trashNode`/`trashNodes`). These are pure, synchronous tree +// operations, so they're tested directly against hand-built fixtures without +// touching the filesystem or the Trash. +final class FileTreePruneTests: XCTestCase { + + // Builds a small FSNode fixture: + // root (/scan) size 900 + // ├── big.bin (500) + // └── sub (dir) size 400 + // ├── a.txt (300) + // └── b.txt (100) + private func makeFixture() -> (root: FSNode, big: FSNode, sub: FSNode, a: FSNode, b: FSNode) { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big = FSNode(url: URL(fileURLWithPath: "/scan/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: root) + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 400, fileExtension: "", parent: root) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 300, fileExtension: "txt", parent: sub) + let b = FSNode(url: URL(fileURLWithPath: "/scan/sub/b.txt"), name: "b.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: sub) + sub.children = [a, b] + root.children = [big, sub] + root.size = big.size + sub.size + return (root, big, sub, a, b) + } + + private func node(named name: String, in tree: FileTree) -> FileNode { + for i in 0..= 0 && start + cnt <= tree.childIndices.count, "child span out of range at \(i)", file: file, line: line) + var previousSize: Int64? = nil + for offset in 0..= 0 && child < count, "child index \(child) out of range", file: file, line: line) + XCTAssertEqual(tree.parentIndex[child], i, "child \(child)'s parentIndex must point back to \(i)", file: file, line: line) + let size = tree.records[child].size + if let previousSize { + XCTAssertGreaterThanOrEqual(previousSize, size, "children must stay sorted size-desc", file: file, line: line) + } + previousSize = size + } + } + XCTAssertEqual(tree.parentIndex[tree.rootIndex], -1, "root must have no parent", file: file, line: line) + } + + // MARK: - removingSubtree: single removal + + func test_removingSubtree_leaf_reduces_record_count_by_one() { + let (root, _, sub, a, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let aNode = node(named: "a.txt", in: tree) + _ = sub; _ = a + + let pruned = tree.removingSubtree(at: aNode.index) + + XCTAssertEqual(pruned.records.count, tree.records.count - 1) + assertValidTopology(pruned) + } + + func test_removingSubtree_reduces_every_ancestor_by_exactly_the_removed_size() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let aNode = node(named: "a.txt", in: tree) + let removedSize = aNode.size // 300 + + let pruned = tree.removingSubtree(at: aNode.index) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(newSub.size, 400 - removedSize) + XCTAssertEqual(newRoot.size, 900 - removedSize) + } + + func test_removingSubtree_directory_removes_whole_subtree_and_reduces_only_direct_ancestors() { + let (root, _, sub, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let subNode = node(named: "sub", in: tree) + _ = sub + + let pruned = tree.removingSubtree(at: subNode.index) + + // sub + a.txt + b.txt all gone -> 3 fewer records than the original 5. + XCTAssertEqual(pruned.records.count, tree.records.count - 3) + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.children.map(\.name), ["big.bin"]) + XCTAssertEqual(newRoot.size, 500, "root size must drop by the whole removed subtree's size (400), not just sub's direct children") + assertValidTopology(pruned) + } + + func test_removingSubtree_preserves_sort_order_and_sibling_subtrees() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let aNode = node(named: "a.txt", in: tree) + + let pruned = tree.removingSubtree(at: aNode.index) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + // big.bin(500) must still sort ahead of the shrunk sub(100). + XCTAssertEqual(newRoot.children.map(\.name), ["big.bin", "sub"]) + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(newSub.children.map(\.name), ["b.txt"]) + } + + func test_removingSubtree_path_reconstruction_still_correct_for_survivors() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let aNode = node(named: "a.txt", in: tree) + + let pruned = tree.removingSubtree(at: aNode.index) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.url.path, "/scan") + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(newSub.url.path, "/scan/sub") + let newB = newSub.children.first { $0.name == "b.txt" }! + XCTAssertEqual(newB.url.path, "/scan/sub/b.txt") + let newBig = newRoot.children.first { $0.name == "big.bin" }! + XCTAssertEqual(newBig.url.path, "/scan/big.bin") + } + + func test_removingSubtree_leaves_original_tree_untouched() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let originalCount = tree.records.count + let originalRootSize = tree.records[tree.rootIndex].size + let aNode = node(named: "a.txt", in: tree) + + _ = tree.removingSubtree(at: aNode.index) + + XCTAssertEqual(tree.records.count, originalCount, "original tree must not mutate") + XCTAssertEqual(tree.records[tree.rootIndex].size, originalRootSize, "original tree's root size must not mutate") + } + + func test_removingSubtree_at_root_returns_self_unchanged() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + + let result = tree.removingSubtree(at: tree.rootIndex) + + XCTAssertTrue(result === tree, "removing the root must be a no-op, returning the same instance") + } + + func test_removingSubtree_carries_over_duplicate_group_and_safety_fields_on_survivors() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let bigNode = node(named: "big.bin", in: tree) + let groupID = UUID() + tree.setDuplicateGroupID(groupID, at: bigNode.index) + tree.setSafety(.safe, at: bigNode.index) + let aNode = node(named: "a.txt", in: tree) + + let pruned = tree.removingSubtree(at: aNode.index) + + let prunedBig = node(named: "big.bin", in: pruned) + XCTAssertEqual(prunedBig.duplicateGroupID, groupID) + XCTAssertEqual(prunedBig.safetyLevel, .safe) + } + + // MARK: - removingSubtrees: multiple removals + + func test_removingSubtrees_disjoint_nodes_reduces_each_ancestor_chain_independently() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let aNode = node(named: "a.txt", in: tree) // under sub + let bigNode = node(named: "big.bin", in: tree) // under root directly + + let pruned = tree.removingSubtrees(at: [aNode.index, bigNode.index]) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.children.map(\.name), ["sub"]) + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(newSub.size, 400 - 300, "sub must lose exactly a.txt's size") + XCTAssertEqual(newRoot.size, 900 - 300 - 500, "root must lose both removed subtrees' sizes, once each") + assertValidTopology(pruned) + } + + func test_removingSubtrees_nested_indices_does_not_double_count_ancestor_size() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let subNode = node(named: "sub", in: tree) + let aNode = node(named: "a.txt", in: tree) // already inside sub's subtree + + // Passing both sub and one of its own descendants must behave exactly + // like removing sub alone — a.txt's size must not be subtracted twice. + let pruned = tree.removingSubtrees(at: [subNode.index, aNode.index]) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.children.map(\.name), ["big.bin"]) + XCTAssertEqual(newRoot.size, 900 - 400) + assertValidTopology(pruned) + } + + func test_removingSubtrees_empty_indices_returns_self() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + + let result = tree.removingSubtrees(at: []) + + XCTAssertTrue(result === tree) + } + + func test_removingSubtrees_out_of_range_index_is_ignored() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + + let result = tree.removingSubtrees(at: [999]) + + XCTAssertTrue(result === tree, "an out-of-range seed must be a no-op, not a crash") + } + + func test_removingSubtrees_duplicate_index_in_list_removed_once() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let aNode = node(named: "a.txt", in: tree) + + let pruned = tree.removingSubtrees(at: [aNode.index, aNode.index]) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.size, 900 - 300, "duplicate seeds in the list must not subtract twice") + assertValidTopology(pruned) + } + + // MARK: - removingSubtrees: hardlink survivor promotion (BUG 1) + + // A single hardlinked pair split across two sibling directories: + // root (/scan) size 500 + // ├── A (dir) size 500 + // │ └── big.bin (500, ref R, carrier) + // └── B (dir) size 0 + // └── twin.bin (0, ref R) + private func makeHardlinkPairFixture() -> (root: FSNode, dirA: FSNode, big: FSNode, dirB: FSNode, twin: FSNode) { + let ref = HardLinkRef(dev: 1, ino: 42) + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let dirA = FSNode(url: URL(fileURLWithPath: "/scan/A"), name: "A", isDirectory: true, size: 500, fileExtension: "", parent: root) + let big = FSNode(url: URL(fileURLWithPath: "/scan/A/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: dirA) + big.hardLinkRef = ref + dirA.children = [big] + let dirB = FSNode(url: URL(fileURLWithPath: "/scan/B"), name: "B", isDirectory: true, size: 0, fileExtension: "", parent: root) + let twin = FSNode(url: URL(fileURLWithPath: "/scan/B/twin.bin"), name: "twin.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirB) + twin.hardLinkRef = ref + dirB.children = [twin] + root.children = [dirA, dirB] + root.size = dirA.size + dirB.size + return (root, dirA, big, dirB, twin) + } + + func test_removingSubtree_trashing_size_carrier_promotes_surviving_twin_root_total_unchanged() { + let (root, _, big, _, _) = makeHardlinkPairFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let bigNode = node(named: "big.bin", in: tree) + let originalRootSize = tree.records[tree.rootIndex].size // 500 + + let pruned = tree.removingSubtree(at: bigNode.index) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.size, originalRootSize, "root total must be unchanged — the disk usage didn't change, only which path carries it") + let newDirA = newRoot.children.first { $0.name == "A" }! + XCTAssertEqual(newDirA.size, 0, "carrier's own ancestor (A) must still lose the size") + let newDirB = newRoot.children.first { $0.name == "B" }! + XCTAssertEqual(newDirB.size, 500, "twin's ancestor (B) must gain exactly the promoted size") + let newTwin = newDirB.children.first { $0.name == "twin.bin" }! + XCTAssertEqual(newTwin.size, 500, "surviving twin must be promoted to the carrier's full size") + assertValidTopology(pruned) + } + + func test_removingSubtree_trashing_zero_size_twin_does_not_promote_and_leaves_root_unchanged() { + let (root, _, big, _, twin) = makeHardlinkPairFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let twinNode = node(named: "twin.bin", in: tree) + let originalRootSize = tree.records[tree.rootIndex].size // 500 + _ = big; _ = twin + + let pruned = tree.removingSubtree(at: twinNode.index) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.size, originalRootSize, "subtracting a 0-size twin must leave the root total unchanged") + let newDirA = newRoot.children.first { $0.name == "A" }! + XCTAssertEqual(newDirA.size, 500, "the surviving carrier must be untouched — no promotion should have happened") + let newBig = newDirA.children.first { $0.name == "big.bin" }! + XCTAssertEqual(newBig.size, 500) + assertValidTopology(pruned) + } + + func test_removingSubtrees_both_hardlink_links_removed_together_drops_size_exactly_once() { + let (root, _, big, _, twin) = makeHardlinkPairFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let bigNode = node(named: "big.bin", in: tree) + let twinNode = node(named: "twin.bin", in: tree) + _ = big; _ = twin + + let pruned = tree.removingSubtrees(at: [bigNode.index, twinNode.index]) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.size, 0, "with no surviving link, the inode's size must drop out exactly once (500 - 500 - 0)") + assertValidTopology(pruned) + } + + func test_removingSubtrees_two_independent_hardlink_pairs_each_promote_their_own_twin() { + let refR1 = HardLinkRef(dev: 1, ino: 100) + let refR2 = HardLinkRef(dev: 1, ino: 200) + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + + let dirA1 = FSNode(url: URL(fileURLWithPath: "/scan/A1"), name: "A1", isDirectory: true, size: 500, fileExtension: "", parent: root) + let bigA = FSNode(url: URL(fileURLWithPath: "/scan/A1/bigA.bin"), name: "bigA.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: dirA1) + bigA.hardLinkRef = refR1 + dirA1.children = [bigA] + + let dirA2 = FSNode(url: URL(fileURLWithPath: "/scan/A2"), name: "A2", isDirectory: true, size: 0, fileExtension: "", parent: root) + let twinA = FSNode(url: URL(fileURLWithPath: "/scan/A2/twinA.bin"), name: "twinA.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirA2) + twinA.hardLinkRef = refR1 + dirA2.children = [twinA] + + let dirB1 = FSNode(url: URL(fileURLWithPath: "/scan/B1"), name: "B1", isDirectory: true, size: 300, fileExtension: "", parent: root) + let bigB = FSNode(url: URL(fileURLWithPath: "/scan/B1/bigB.bin"), name: "bigB.bin", isDirectory: false, size: 300, fileExtension: "bin", parent: dirB1) + bigB.hardLinkRef = refR2 + dirB1.children = [bigB] + + let dirB2 = FSNode(url: URL(fileURLWithPath: "/scan/B2"), name: "B2", isDirectory: true, size: 0, fileExtension: "", parent: root) + let twinB = FSNode(url: URL(fileURLWithPath: "/scan/B2/twinB.bin"), name: "twinB.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirB2) + twinB.hardLinkRef = refR2 + dirB2.children = [twinB] + + root.children = [dirA1, dirA2, dirB1, dirB2] + root.size = 500 + 0 + 300 + 0 // 800 + + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let bigANode = node(named: "bigA.bin", in: tree) + let bigBNode = node(named: "bigB.bin", in: tree) + + let pruned = tree.removingSubtrees(at: [bigANode.index, bigBNode.index]) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + XCTAssertEqual(newRoot.size, 800, "root total must be unchanged: each pair's bytes just moved to its own surviving twin") + let newTwinA = newRoot.children.first { $0.name == "A2" }!.children.first { $0.name == "twinA.bin" }! + XCTAssertEqual(newTwinA.size, 500, "twinA must be promoted with its own pair's size") + let newTwinB = newRoot.children.first { $0.name == "B2" }!.children.first { $0.name == "twinB.bin" }! + XCTAssertEqual(newTwinB.size, 300, "twinB must be promoted with its own pair's size, independently of twinA") + assertValidTopology(pruned) + } + + // MARK: - removingSubtrees: removed node's own ancestor chain must stay sorted (BUG 2 / FIX 2) + + // The plan's exact repro: + // root + // ├─ P size 900 + // │ ├─ A (dir, size 500) + // │ │ └─ big.bin (500, ref R, carrier) + // │ └─ C (dir, size 400, untouched) + // │ └─ other.bin (400, unrelated) + // └─ Q (dir, size 0) + // └─ twin.bin (0, ref R) + // `P.children` starts `[A(500), C(400)]`. Removing `big.bin` promotes + // twin.bin to 500 (A's own promotion-resort logic already covers Q's and + // root's spans), but A itself drops straight to 0 — nothing in the old + // code re-sorted P's span to reflect that, leaving `P.children == + // [A(0), C(400)]`, which violates the size-desc invariant even though + // this has nothing to do with hardlinks (a plain size *shrink* would + // trigger the exact same gap; the hardlink promotion path just makes a + // 500->0 drop easy to construct in one step). + private func makeAncestorResortFixture() -> (root: FSNode, dirP: FSNode, dirA: FSNode, big: FSNode, dirC: FSNode, other: FSNode, dirQ: FSNode, twin: FSNode) { + let ref = HardLinkRef(dev: 1, ino: 77) + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + + let dirP = FSNode(url: URL(fileURLWithPath: "/scan/P"), name: "P", isDirectory: true, size: 900, fileExtension: "", parent: root) + let dirA = FSNode(url: URL(fileURLWithPath: "/scan/P/A"), name: "A", isDirectory: true, size: 500, fileExtension: "", parent: dirP) + let big = FSNode(url: URL(fileURLWithPath: "/scan/P/A/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: dirA) + big.hardLinkRef = ref + dirA.children = [big] + let dirC = FSNode(url: URL(fileURLWithPath: "/scan/P/C"), name: "C", isDirectory: true, size: 400, fileExtension: "", parent: dirP) + let other = FSNode(url: URL(fileURLWithPath: "/scan/P/C/other.bin"), name: "other.bin", isDirectory: false, size: 400, fileExtension: "bin", parent: dirC) + dirC.children = [other] + dirP.children = [dirA, dirC] // sorted size-desc: A(500), C(400) + + let dirQ = FSNode(url: URL(fileURLWithPath: "/scan/Q"), name: "Q", isDirectory: true, size: 0, fileExtension: "", parent: root) + let twin = FSNode(url: URL(fileURLWithPath: "/scan/Q/twin.bin"), name: "twin.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirQ) + twin.hardLinkRef = ref + dirQ.children = [twin] + + root.children = [dirP, dirQ] + root.size = dirP.size + dirQ.size + return (root, dirP, dirA, big, dirC, other, dirQ, twin) + } + + func test_removingSubtree_resorts_removed_nodes_own_ancestor_chain_after_carrier_shrinks_to_zero() { + let (root, dirP, _, big, _, _, _, _) = makeAncestorResortFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let bigNode = node(named: "big.bin", in: tree) + _ = dirP + + let pruned = tree.removingSubtree(at: bigNode.index) + + let newRoot = FileNode(tree: pruned, index: pruned.rootIndex) + let newP = newRoot.children.first { $0.name == "P" }! + XCTAssertEqual( + newP.children.map(\.name), + ["C", "A"], + "P's children must stay sorted size-desc after A drops from 500 to 0 (C is now the bigger sibling)" + ) + let newA = newP.children.first { $0.name == "A" }! + XCTAssertEqual(newA.size, 0) + let newC = newP.children.first { $0.name == "C" }! + XCTAssertEqual(newC.size, 400, "C must be completely untouched") + + // The promoted twin's side must still be correct too (BUG 1's own + // invariant), and every node in the whole tree — not just P — must + // satisfy the size-desc + topology invariants. + let newQ = newRoot.children.first { $0.name == "Q" }! + XCTAssertEqual(newQ.size, 500) + XCTAssertEqual(newRoot.size, 900, "root total must be unchanged — the bytes just moved from A's side to Q's side") + assertValidTopology(pruned) + } +} + +// MARK: - ScanViewModel-level integration + +// Unlike the pure-tree tests above, this one exercises the real +// `ScanViewModel.trashNode` entry point end to end: a real scan of a temp +// directory, a real `FileManager.trashItem` call, and the resulting prune +// updating `tree`/`selectedNode`/`extensionSummaries`/`duplicateGroups` +// in place instead of the old "trash then full rescan" behavior. +@MainActor +final class ScanViewModelPruneTests: XCTestCase { + + private func waitUntil(timeout: TimeInterval = 5, _ condition: () -> Bool) async { + let deadline = Date().addingTimeInterval(timeout) + while !condition(), Date() < deadline { + try? await Task.sleep(nanoseconds: 10_000_000) + } + } + + func test_trashNode_prunes_tree_in_place_without_rescanning() async throws { + let prior = UserDefaults.standard.object(forKey: "realtimeMonitoring") as? Bool + UserDefaults.standard.set(false, forKey: "realtimeMonitoring") + defer { + if let prior { UserDefaults.standard.set(prior, forKey: "realtimeMonitoring") } + else { UserDefaults.standard.removeObject(forKey: "realtimeMonitoring") } + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + try Data(repeating: 1, count: 4096).write(to: tmp.appendingPathComponent("keep.bin")) + let toDeleteURL = tmp.appendingPathComponent("delete-me.bin") + try Data(repeating: 2, count: 2048).write(to: toDeleteURL) + + let vm = ScanViewModel() + vm.updateLayoutSize(CGSize(width: 400, height: 400)) + vm.scan(url: tmp) + await waitUntil { !vm.isScanning && !vm.isComputingLayout } + + guard let root = vm.root else { return XCTFail("scan should populate a root") } + let originalTotal = root.size + guard let target = root.children.first(where: { $0.name == "delete-me.bin" }) else { + return XCTFail("expected delete-me.bin among the scanned children") + } + let targetSize = target.size + vm.select(target) + + let trashed = vm.trashNode(target) + XCTAssertTrue(trashed, "trashNode should report success for a real, trashable file") + + // The prune itself (tree/selection/drillStack/colorMap) is synchronous; + // only the off-thread extension-summary/duplicate-group passes need + // waiting for. + XCTAssertFalse(FileManager.default.fileExists(atPath: toDeleteURL.path), "trashItem should have actually moved the file out of tmp") + guard let newRoot = vm.root else { return XCTFail("tree must still exist after a prune") } + XCTAssertEqual(newRoot.children.map(\.name), ["keep.bin"], "the trashed node must be gone from the live tree") + XCTAssertEqual(newRoot.size, originalTotal - targetSize, "root size must shrink by exactly the trashed node's size") + XCTAssertNil(vm.selectedNode, "the node that was selected and then trashed must be deselected, not left dangling") + + await waitUntil { vm.extensionSummaries.first(where: { $0.ext == ".bin" })?.fileCount == 1 } + let binSummary = vm.extensionSummaries.first { $0.ext == ".bin" } + XCTAssertEqual(binSummary?.fileCount, 1, "extension summaries must be recomputed against the pruned tree") + } +} diff --git a/Tests/FileTreeTests.swift b/Tests/FileTreeTests.swift new file mode 100644 index 0000000..3271eb4 --- /dev/null +++ b/Tests/FileTreeTests.swift @@ -0,0 +1,268 @@ +import XCTest +@testable import MacDirStat + +final class FileTreeTests: XCTestCase { + + // Builds a small FSNode fixture: + // root (/scan) + // ├── big.bin (500) + // └── sub (dir) + // ├── a.txt (300) + // └── b.txt (100) + private func makeFixture() -> (root: FSNode, big: FSNode, sub: FSNode, a: FSNode, b: FSNode) { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big = FSNode(url: URL(fileURLWithPath: "/scan/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: root) + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 400, fileExtension: "", parent: root) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 300, fileExtension: "txt", parent: sub) + let b = FSNode(url: URL(fileURLWithPath: "/scan/sub/b.txt"), name: "b.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: sub) + sub.children = [a, b] // deliberately unsorted (a=300 before b=100 is fine, already desc) + root.children = [sub, big] // deliberately NOT size-desc (big=500 should come first) + root.size = big.size + sub.size + return (root, big, sub, a, b) + } + + // MARK: - Builder correctness + + func test_builder_root_index_is_zero() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + XCTAssertEqual(tree.rootIndex, 0) + XCTAssertEqual(tree.records[tree.rootIndex].name, "scan") + XCTAssertEqual(tree.parentIndex[tree.rootIndex], -1) + } + + func test_builder_parent_links_are_correct() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + for child in rootNode.children { + XCTAssertEqual(child.parent?.index, tree.rootIndex) + } + let subNode = rootNode.children.first { $0.name == "sub" }! + for grandchild in subNode.children { + XCTAssertEqual(grandchild.parent?.index, subNode.index) + } + } + + func test_builder_children_are_sorted_size_desc_regardless_of_fsnode_order() { + let (root, _, _, _, _) = makeFixture() + // FSNode fixture deliberately has children in [sub(400), big(500)] order. + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + let sizes = rootNode.children.map(\.size) + XCTAssertEqual(sizes, sizes.sorted(by: >), "builder must sort each node's children size-desc") + XCTAssertEqual(rootNode.children.first?.name, "big.bin") + + let subNode = rootNode.children.first { $0.name == "sub" }! + XCTAssertEqual(subNode.children.map(\.name), ["a.txt", "b.txt"], "a.txt (300) must sort before b.txt (100)") + } + + func test_builder_child_spans_do_not_overlap() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + // Every index reachable from root's children arrays must be unique and + // within bounds — a cheap proxy for "spans don't alias each other". + var seen = Set() + for index in 0..= 0 && childIndex < tree.records.count) + XCTAssertTrue(seen.insert(childIndex).inserted, "child index \(childIndex) referenced by more than one parent") + } + } + } + + func test_builder_leaf_has_no_children() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + let bigNode = rootNode.children.first { $0.name == "big.bin" }! + XCTAssertEqual(bigNode.children.count, 0) + XCTAssertNil(bigNode.optionalChildren) + } + + // MARK: - Path reconstruction + + func test_path_of_root_is_root_path() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + XCTAssertEqual(tree.path(of: tree.rootIndex), "/scan") + } + + func test_path_of_nested_node_joins_names() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + let subNode = rootNode.children.first { $0.name == "sub" }! + let aNode = subNode.children.first { $0.name == "a.txt" }! + XCTAssertEqual(subNode.url.path, "/scan/sub") + XCTAssertEqual(aNode.url.path, "/scan/sub/a.txt") + } + + func test_path_handles_trailing_slash_root_path() { + let root = FSNode(url: URL(fileURLWithPath: "/"), name: "/", isDirectory: true, size: 10, fileExtension: "", parent: nil) + let child = FSNode(url: URL(fileURLWithPath: "/Users"), name: "Users", isDirectory: true, size: 10, fileExtension: "", parent: root) + root.children = [child] + let tree = FileTreeBuilder.build(from: root, rootPath: "/") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + XCTAssertEqual(rootNode.children.first?.url.path, "/Users", "must not produce a double slash when rootPath already ends in '/'") + } + + // MARK: - Record field carry-over + + func test_builder_carries_over_hardlink_ref_and_synthetic_and_autosummarized_flags() { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + + let hardlinked = FSNode(url: URL(fileURLWithPath: "/scan/h.bin"), name: "h.bin", isDirectory: false, size: 200, fileExtension: "bin", parent: root) + hardlinked.hardLinkRef = HardLinkRef(dev: 1, ino: 42) + + let summarized = FSNode(url: URL(fileURLWithPath: "/scan/node_modules"), name: "node_modules", isDirectory: true, size: 900, fileExtension: "", parent: root) + summarized.isAutoSummarized = true + summarized.descendantFileCount = 123 + + let synthetic = FSNode(url: URL(fileURLWithPath: "/scan/synthetic"), name: "synthetic", isDirectory: false, size: 50, fileExtension: "", parent: root) + synthetic.isSynthetic = true + + let denied = FSNode(url: URL(fileURLWithPath: "/scan/denied"), name: "denied", isDirectory: true, size: 0, fileExtension: "", parent: root) + denied.isAccessDenied = true + + root.children = [hardlinked, summarized, synthetic, denied] + root.size = 1200 + + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + + let hNode = rootNode.children.first { $0.name == "h.bin" }! + XCTAssertEqual(hNode.hardLinkRef, HardLinkRef(dev: 1, ino: 42)) + + let nmNode = rootNode.children.first { $0.name == "node_modules" }! + XCTAssertTrue(nmNode.isAutoSummarized) + XCTAssertEqual(nmNode.descendantFileCount, 123) + + let synNode = rootNode.children.first { $0.name == "synthetic" }! + XCTAssertTrue(synNode.isSynthetic) + + let deniedNode = rootNode.children.first { $0.name == "denied" }! + XCTAssertTrue(deniedNode.isAccessDenied) + + // Per the design, safetyLevel/duplicateGroupID always start at their + // defaults at assembly time — they're filled by a separate pass + // (tagSafetyLevels / DuplicateDetector) over the finished FileTree. + XCTAssertNil(hNode.duplicateGroupID) + XCTAssertEqual(hNode.safetyLevel, .caution) + } + + // MARK: - Mutation + + func test_set_duplicate_group_id_and_safety_and_size() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + let bigNode = rootNode.children.first { $0.name == "big.bin" }! + + let groupID = UUID() + tree.setDuplicateGroupID(groupID, at: bigNode.index) + XCTAssertEqual(FileNode(tree: tree, index: bigNode.index).duplicateGroupID, groupID) + + tree.setSafety(.danger, at: bigNode.index) + XCTAssertEqual(FileNode(tree: tree, index: bigNode.index).safetyLevel, .danger) + + tree.setSize(999, at: bigNode.index) + XCTAssertEqual(FileNode(tree: tree, index: bigNode.index).size, 999) + } + + // MARK: - Synthetic root child (topology-changing splice used for hidden-space) + + func test_appending_synthetic_root_child_inserts_in_sorted_position() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + // Root's children before: [big.bin(500), sub(400)]. A synthetic entry + // of 450 bytes should land between them. + let newTree = tree.appendingSyntheticRootChild(name: "Hidden & Unreadable Space", size: 450) + let newRoot = FileNode(tree: newTree, index: newTree.rootIndex) + + XCTAssertEqual(newRoot.children.map(\.name), ["big.bin", "Hidden & Unreadable Space", "sub"]) + XCTAssertTrue(newRoot.children[1].isSynthetic) + XCTAssertEqual(newRoot.children[1].safetyLevel, .danger) + XCTAssertEqual(newRoot.size, 500 + 400 + 450, "root's aggregate size must include the synthetic child") + } + + func test_appending_synthetic_root_child_leaves_original_tree_untouched() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let originalChildCount = FileNode(tree: tree, index: tree.rootIndex).children.count + let originalSize = tree.records[tree.rootIndex].size + + _ = tree.appendingSyntheticRootChild(name: "Hidden & Unreadable Space", size: 450) + + XCTAssertEqual(FileNode(tree: tree, index: tree.rootIndex).children.count, originalChildCount, "original tree's topology must not mutate") + XCTAssertEqual(tree.records[tree.rootIndex].size, originalSize, "original tree's root size must not mutate") + } + + func test_appending_synthetic_root_child_preserves_other_subtrees() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let newTree = tree.appendingSyntheticRootChild(name: "Hidden & Unreadable Space", size: 450) + let newRoot = FileNode(tree: newTree, index: newTree.rootIndex) + + let subNode = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(subNode.children.map(\.name), ["a.txt", "b.txt"], "unrelated subtree's children/order must survive the splice") + XCTAssertEqual(subNode.url.path, "/scan/sub") + } + + // The synthetic node used to be appended to `records` and `parentIndex` + // without a matching entry in `childStart`/`childCount`, leaving the tree + // one span short of its own record count. Nothing noticed until some later + // pass iterated every record and read its span — which crashed the app + // with "Index out of range" (see the splice test below). Every per-node + // array must stay the same length. + func test_appending_synthetic_root_child_keeps_all_per_node_arrays_in_step() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + let newTree = tree.appendingSyntheticRootChild(name: "Hidden & Unreadable Space", size: 450) + + let count = newTree.records.count + XCTAssertEqual(newTree.parentIndex.count, count, "parentIndex must have one entry per record") + XCTAssertEqual(newTree.childStart.count, count, "childStart must have one entry per record") + XCTAssertEqual(newTree.childCount.count, count, "childCount must have one entry per record") + + // Reading every node's span must be in range and internally sane. + for i in 0..= 0 && start + cnt <= newTree.childIndices.count, "span out of range for node \(i)") + } + } + + // The crash a user actually hit: scan a volume root (which appends the + // synthetic "Hidden & Unreadable Space" child), then let a live + // filesystem change splice a subtree. `replacingSubtree` walks every + // record and reads its span, so the missing entry blew up on the main + // thread and killed the app. + func test_splicing_a_tree_that_has_a_synthetic_child_does_not_crash() { + let (root, _, _, _, _) = makeFixture() + let tree = FileTreeBuilder + .build(from: root, rootPath: "/scan") + .appendingSyntheticRootChild(name: "Hidden & Unreadable Space", size: 450) + + let subIndex = (0.. 10 GB hidden + let hundredGB: Int64 = 100_000_000_000 + let fortyGB: Int64 = 40_000_000_000 + let fiftyGB: Int64 = 50_000_000_000 + XCTAssertEqual( + ScanViewModel.hiddenSpaceBytes(volumeTotal: hundredGB, volumeAvailable: fortyGB, scannedTotal: fiftyGB), + 10_000_000_000 + ) + } + + func test_hidden_space_below_1gb_is_nil() { + // Gap is under 1 GB — treated as noise, not worth surfacing. + let total: Int64 = 100_000_000_000 + let available: Int64 = 50_000_000_000 + let scanned: Int64 = 49_500_000_000 // gap = 500,000,000 (0.5 GB) + XCTAssertNil(ScanViewModel.hiddenSpaceBytes(volumeTotal: total, volumeAvailable: available, scannedTotal: scanned)) + } + + func test_hidden_space_zero_total_is_nil() { + XCTAssertNil(ScanViewModel.hiddenSpaceBytes(volumeTotal: 0, volumeAvailable: 0, scannedTotal: 0)) + } + + func test_hidden_space_negative_total_is_nil() { + XCTAssertNil(ScanViewModel.hiddenSpaceBytes(volumeTotal: -1, volumeAvailable: 0, scannedTotal: 0)) + } + + func test_hidden_space_negative_gap_clamps_to_nil() { + // scanned + available exceeds total (e.g. race/measurement skew) -> clamps to 0, below threshold -> nil + let total: Int64 = 100_000_000_000 + let available: Int64 = 60_000_000_000 + let scanned: Int64 = 60_000_000_000 + XCTAssertNil(ScanViewModel.hiddenSpaceBytes(volumeTotal: total, volumeAvailable: available, scannedTotal: scanned)) + } + + // MARK: - refresh preserves synthetic children + + func test_refresh_preserves_synthetic_children() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let realFileURL = tmp.appendingPathComponent("real.bin") + try Data(repeating: 0, count: 4096).write(to: realFileURL) + + let root = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + let realChild = FSNode(url: realFileURL, name: "real.bin", isDirectory: false, size: 4096, fileExtension: "bin", parent: root) + + let syntheticURL = tmp.appendingPathComponent("#hidden-space") + let syntheticChild = FSNode(url: syntheticURL, name: "Hidden & Unreadable Space", isDirectory: false, size: 5_000_000_000, fileExtension: "", parent: root) + syntheticChild.isSynthetic = true + + root.children = [realChild, syntheticChild] + root.size = realChild.size + syntheticChild.size + + let changed = ScanViewModel.refreshDirectory(node: root) + + XCTAssertTrue(root.children.contains { $0.isSynthetic }, "synthetic child must survive a refresh pass") + _ = changed + } +} diff --git a/Tests/IncrementalRefreshTests.swift b/Tests/IncrementalRefreshTests.swift new file mode 100644 index 0000000..647dc32 --- /dev/null +++ b/Tests/IncrementalRefreshTests.swift @@ -0,0 +1,677 @@ +import XCTest +@testable import MacDirStat + +// Coverage for the Phase 4d incremental splice refresh: `FileTree.replacingSubtree(at:with:)` +// (the pure tree-topology algorithm) and `ScanViewModel.splicedTree(afterChangeAt:in:)` (the +// per-directory-rescan-and-splice orchestration that `handleFileSystemChanges` now uses instead +// of a full rescan on every FSEvents notification). +final class IncrementalRefreshTests: XCTestCase { + + // MARK: - FileTree.replacingSubtree: pure algorithm + + // Builds the same small fixture FileTreePruneTests uses: + // root (/scan) size 900 + // ├── big.bin (500) + // └── sub (dir) size 400 + // ├── a.txt (300) + // └── b.txt (100) + private func makeFixtureTree() -> FileTree { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big = FSNode(url: URL(fileURLWithPath: "/scan/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: root) + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 400, fileExtension: "", parent: root) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 300, fileExtension: "txt", parent: sub) + let b = FSNode(url: URL(fileURLWithPath: "/scan/sub/b.txt"), name: "b.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: sub) + sub.children = [a, b] + root.children = [big, sub] + root.size = big.size + sub.size + return FileTreeBuilder.build(from: root, rootPath: "/scan") + } + + private func node(named name: String, in tree: FileTree) -> FileNode { + for i in 0..= 0 && start + cnt <= tree.childIndices.count, "child span out of range at \(i)", file: file, line: line) + var previousSize: Int64? + for offset in 0..= 0 && child < count, "child index \(child) out of range", file: file, line: line) + XCTAssertEqual(tree.parentIndex[child], i, "child \(child)'s parentIndex must point back to \(i)", file: file, line: line) + let size = tree.records[child].size + if let previousSize { + XCTAssertGreaterThanOrEqual(previousSize, size, "children must stay sorted size-desc", file: file, line: line) + } + previousSize = size + } + } + XCTAssertEqual(tree.parentIndex[tree.rootIndex], -1, "root must have no parent", file: file, line: line) + } + + // A "freshly rescanned sub" replacement: a.txt shrinks 300->200, b.txt is + // gone, c.txt (50) is new -> new sub totals 250 (was 400). + private func makeShrunkSubtree() -> FileTree { + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 200, fileExtension: "txt", parent: sub) + let c = FSNode(url: URL(fileURLWithPath: "/scan/sub/c.txt"), name: "c.txt", isDirectory: false, size: 50, fileExtension: "txt", parent: sub) + sub.children = [a, c] + sub.size = 250 + return FileTreeBuilder.build(from: sub, rootPath: "/scan/sub") + } + + // A "freshly rescanned sub" replacement that grew past big.bin's 500, + // to exercise the ancestor re-sort path. + private func makeGrownSubtree() -> FileTree { + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big2 = FSNode(url: URL(fileURLWithPath: "/scan/sub/big2.bin"), name: "big2.bin", isDirectory: false, size: 600, fileExtension: "bin", parent: sub) + sub.children = [big2] + sub.size = 600 + return FileTreeBuilder.build(from: sub, rootPath: "/scan/sub") + } + + func test_replacingSubtree_shrinks_ancestor_sizes_by_exact_delta() { + let tree = makeFixtureTree() + let subNode = node(named: "sub", in: tree) + let replacement = makeShrunkSubtree() + + let spliced = tree.replacingSubtree(at: subNode.index, with: replacement) + + let newRoot = FileNode(tree: spliced, index: spliced.rootIndex) + XCTAssertEqual(newRoot.size, 900 - 400 + 250, "root size must reflect exactly the size delta of the replaced subtree") + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(newSub.size, 250) + assertValidTopology(spliced) + } + + func test_replacingSubtree_replaces_node_set_under_target() { + let tree = makeFixtureTree() + let subNode = node(named: "sub", in: tree) + let replacement = makeShrunkSubtree() + + let spliced = tree.replacingSubtree(at: subNode.index, with: replacement) + + let newRoot = FileNode(tree: spliced, index: spliced.rootIndex) + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(Set(newSub.children.map(\.name)), Set(["a.txt", "c.txt"]), "b.txt must be gone, c.txt must be present") + XCTAssertNil(newSub.children.first { $0.name == "b.txt" }) + let newA = newSub.children.first { $0.name == "a.txt" }! + XCTAssertEqual(newA.size, 200, "a.txt's size must reflect the fresh rescan, not the stale one") + } + + func test_replacingSubtree_preserves_untouched_sibling() { + let tree = makeFixtureTree() + let subNode = node(named: "sub", in: tree) + let replacement = makeShrunkSubtree() + + let spliced = tree.replacingSubtree(at: subNode.index, with: replacement) + + let newRoot = FileNode(tree: spliced, index: spliced.rootIndex) + let bigStill = newRoot.children.first { $0.name == "big.bin" } + XCTAssertNotNil(bigStill) + XCTAssertEqual(bigStill?.size, 500, "an untouched sibling subtree must be completely unaffected by the splice") + } + + func test_replacingSubtree_path_reconstruction_correct_for_spliced_and_survivors() { + let tree = makeFixtureTree() + let subNode = node(named: "sub", in: tree) + let replacement = makeShrunkSubtree() + + let spliced = tree.replacingSubtree(at: subNode.index, with: replacement) + + let newRoot = FileNode(tree: spliced, index: spliced.rootIndex) + XCTAssertEqual(newRoot.url.path, "/scan") + let newSub = newRoot.children.first { $0.name == "sub" }! + XCTAssertEqual(newSub.url.path, "/scan/sub") + let newC = newSub.children.first { $0.name == "c.txt" }! + XCTAssertEqual(newC.url.path, "/scan/sub/c.txt", "a brand-new spliced-in node must still reconstruct its path correctly") + let bigStill = newRoot.children.first { $0.name == "big.bin" }! + XCTAssertEqual(bigStill.url.path, "/scan/big.bin", "a surviving node's path must still be correct after the splice") + } + + func test_replacingSubtree_resorts_ancestor_chain_when_target_grows_past_a_sibling() { + let tree = makeFixtureTree() + let subNode = node(named: "sub", in: tree) + let replacement = makeGrownSubtree() // 600, now bigger than big.bin's 500 + + let spliced = tree.replacingSubtree(at: subNode.index, with: replacement) + + let newRoot = FileNode(tree: spliced, index: spliced.rootIndex) + XCTAssertEqual(newRoot.children.map(\.name), ["sub", "big.bin"], "sub must now sort ahead of big.bin") + assertValidTopology(spliced) + } + + func test_replacingSubtree_leaves_original_tree_untouched() { + let tree = makeFixtureTree() + let subNode = node(named: "sub", in: tree) + let replacement = makeShrunkSubtree() + let originalRootSize = tree.records[tree.rootIndex].size + let originalCount = tree.records.count + + _ = tree.replacingSubtree(at: subNode.index, with: replacement) + + XCTAssertEqual(tree.records[tree.rootIndex].size, originalRootSize, "the original tree must not mutate") + XCTAssertEqual(tree.records.count, originalCount) + } + + func test_replacingSubtree_at_root_returns_self_unchanged() { + let tree = makeFixtureTree() + let replacement = makeShrunkSubtree() + + let result = tree.replacingSubtree(at: tree.rootIndex, with: replacement) + + XCTAssertTrue(result === tree, "replacing the root must be a no-op, returning the same instance") + } + + func test_replacingSubtree_carries_over_duplicate_group_and_safety_on_untouched_survivors() { + let tree = makeFixtureTree() + let bigNode = node(named: "big.bin", in: tree) + let groupID = UUID() + tree.setDuplicateGroupID(groupID, at: bigNode.index) + tree.setSafety(.safe, at: bigNode.index) + let subNode = node(named: "sub", in: tree) + let replacement = makeShrunkSubtree() + + let spliced = tree.replacingSubtree(at: subNode.index, with: replacement) + + let splicedBig = node(named: "big.bin", in: spliced) + XCTAssertEqual(splicedBig.duplicateGroupID, groupID) + XCTAssertEqual(splicedBig.safetyLevel, .safe) + } + + // MARK: - ScanViewModel.splicedTree: fallback cases + + func test_splicedTree_root_change_falls_back_to_nil() { + let tree = makeFixtureTree() + + XCTAssertNil(ScanViewModel.splicedTree(afterChangeAt: "/scan", in: tree)) + } + + func test_splicedTree_root_change_with_trailing_slash_falls_back_to_nil() { + let tree = makeFixtureTree() + + XCTAssertNil(ScanViewModel.splicedTree(afterChangeAt: "/scan/", in: tree)) + } + + func test_splicedTree_unresolvable_path_falls_back_to_nil() { + let tree = makeFixtureTree() + + XCTAssertNil(ScanViewModel.splicedTree(afterChangeAt: "/scan/does/not/exist", in: tree)) + } + + func test_splicedTree_path_outside_tree_falls_back_to_nil() { + let tree = makeFixtureTree() + + XCTAssertNil(ScanViewModel.splicedTree(afterChangeAt: "/somewhere/else", in: tree)) + } + + func test_splicedTree_autosummarized_target_falls_back_to_nil() { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let bigDir = FSNode(url: URL(fileURLWithPath: "/scan/node_modules"), name: "node_modules", isDirectory: true, size: 12_345, fileExtension: "", parent: root) + bigDir.isAutoSummarized = true + bigDir.descendantFileCount = 9_999 + root.children = [bigDir] + root.size = bigDir.size + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + + XCTAssertNil(ScanViewModel.splicedTree(afterChangeAt: "/scan/node_modules", in: tree), "a summarized node has no children to splice into") + } + + func test_splicedTree_path_inside_autosummarized_node_falls_back_to_nil() { + // Same fixture as above, but the "changed" path is one level deeper + // than the summarized node itself — never materialized in the tree + // at all, so the path-component walk simply can't resolve it. + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let bigDir = FSNode(url: URL(fileURLWithPath: "/scan/node_modules"), name: "node_modules", isDirectory: true, size: 12_345, fileExtension: "", parent: root) + bigDir.isAutoSummarized = true + root.children = [bigDir] + root.size = bigDir.size + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + + XCTAssertNil(ScanViewModel.splicedTree(afterChangeAt: "/scan/node_modules/some-package", in: tree)) + } + + // MARK: - Key correctness check: splice result == full fresh rescan + + @MainActor + private func waitUntil(timeout: TimeInterval = 5, _ condition: () -> Bool) async { + let deadline = Date().addingTimeInterval(timeout) + while !condition(), Date() < deadline { + try? await Task.sleep(nanoseconds: 10_000_000) + } + } + + private struct NodeSnapshot: Hashable { + let relativePath: String + let isDirectory: Bool + let size: Int64 + } + + // Flattens every descendant of `node` (node included, as "") into a + // set keyed by its path relative to `node`'s own tree root — order- and + // index-independent, so it's safe to compare across two entirely + // separate `FileTree` instances built from independent scans. + private func snapshot(_ root: FileNode) -> Set { + var result: Set = [] + let rootPath = root.tree.rootPath + func walk(_ n: FileNode) { + let full = n.url.path + let relative = full == rootPath ? "" : String(full.dropFirst(rootPath.count)) + result.insert(NodeSnapshot(relativePath: relative, isDirectory: n.isDirectory, size: n.size)) + for child in n.children { walk(child) } + } + walk(root) + return result + } + + // MARK: - A live refresh must never restart the whole scan + // + // Regression test for a user-visible glitch: the scan would finish, the + // treemap would start drawing, then everything vanished and scanning + // began again from zero, forever. Cause: any FSEvents change the splice + // refused escalated to a full `scan(url:)`, which clears `tree`/`cells`; + // completing that scan restarted watching, which delivered another + // refused change, and so on. + // + // The refused cases are reachable in ordinary use — this watcher reports + // directory granularity, so a change to the scanned root itself is + // routine, as is any change inside an auto-summarized folder. + @MainActor + func test_unspliceable_change_does_not_restart_the_scan() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + try Data(repeating: 1, count: 4096).write(to: tmp.appendingPathComponent("a.bin")) + + let vm = await scanAndWait(tmp) + guard let treeBefore = vm.tree else { return XCTFail("initial scan should populate a tree") } + XCTAssertFalse(vm.isScanning) + + // The scanned root itself — the single most common FSEvents report, + // and one `splicedTree` always refuses. + await vm.handleFileSystemChanges([tmp.path]) + + XCTAssertFalse(vm.isScanning, "an unspliceable change must not kick off a full rescan") + XCTAssertNotNil(vm.tree, "the finished tree must survive an unspliceable change") + XCTAssertTrue(vm.tree === treeBefore, "the tree should be left untouched, not rebuilt") + XCTAssertTrue(vm.hasStaleResults, "the UI should be told results may be stale") + } + + // The other refused case: a path inside an auto-summarized directory. + @MainActor + func test_change_inside_autosummarized_directory_does_not_restart_the_scan() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let nodeModules = tmp.appendingPathComponent("node_modules") + try FileManager.default.createDirectory(at: nodeModules, withIntermediateDirectories: true) + for i in 0..<5 { + try Data(repeating: 2, count: 64).write(to: nodeModules.appendingPathComponent("f\(i).js")) + } + + let priorExcluded = UserDefaults.standard.string(forKey: "excludedFolderNames") + UserDefaults.standard.set(".git,DerivedData,.Trash", forKey: "excludedFolderNames") + defer { + if let priorExcluded { UserDefaults.standard.set(priorExcluded, forKey: "excludedFolderNames") } + else { UserDefaults.standard.removeObject(forKey: "excludedFolderNames") } + } + + let vm = await scanAndWait(tmp) + guard let treeBefore = vm.tree else { return XCTFail("initial scan should populate a tree") } + + await vm.handleFileSystemChanges([nodeModules.appendingPathComponent("f0.js").path]) + + XCTAssertFalse(vm.isScanning, "a change inside a summarized folder must not kick off a full rescan") + XCTAssertTrue(vm.tree === treeBefore, "the tree should be left untouched, not rebuilt") + } + + @MainActor + private func scanAndWait(_ url: URL) async -> ScanViewModel { + let prior = UserDefaults.standard.object(forKey: "realtimeMonitoring") as? Bool + UserDefaults.standard.set(false, forKey: "realtimeMonitoring") + defer { + if let prior { UserDefaults.standard.set(prior, forKey: "realtimeMonitoring") } + else { UserDefaults.standard.removeObject(forKey: "realtimeMonitoring") } + } + let vm = ScanViewModel() + vm.updateLayoutSize(CGSize(width: 400, height: 400)) + vm.scan(url: url) + await waitUntil { !vm.isScanning && !vm.isComputingLayout } + return vm + } + + // The key correctness test the plan calls for: build a real tree from a + // real temp directory (via the actual FileScanner, exactly like a real + // scan), mutate a subdirectory on disk (add a file, remove a file, + // resize a file, add a nested directory), splice just that directory via + // `ScanViewModel.splicedTree`, and compare the result against an + // entirely independent, from-scratch full rescan of the same now-mutated + // directory tree (a second real `FileScanner` run via a second + // `ScanViewModel`). The two must describe exactly the same node set and + // sizes — a splice is only a cheaper way to arrive at the same state a + // full rescan would. + func test_splice_result_matches_full_fresh_rescan() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + try Data(repeating: 1, count: 8192).write(to: tmp.appendingPathComponent("keep.bin")) + let sub = tmp.appendingPathComponent("sub") + try FileManager.default.createDirectory(at: sub, withIntermediateDirectories: true) + try Data(repeating: 2, count: 4096).write(to: sub.appendingPathComponent("a.txt")) + let toRemove = sub.appendingPathComponent("b.txt") + try Data(repeating: 3, count: 2048).write(to: toRemove) + let toResize = sub.appendingPathComponent("c.txt") + try Data(repeating: 4, count: 1024).write(to: toResize) + + let before = await scanAndWait(tmp) + guard let beforeTree = await before.tree else { return XCTFail("initial scan should populate a tree") } + guard let subIndex = (0.. `dirY/link.bin`), the exact + // shape of the plan's BUG 2 scenario. Returns the inode's `HardLinkRef` + // so the tests below can find out, after the initial scan, which side + // the scanner's first-seen-wins picked as the carrier (deterministic per + // run, but not something the test should hard-code). + private func makeHardlinkAcrossDirsFixture() throws -> (tmp: URL, ref: HardLinkRef) { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + let dirX = tmp.appendingPathComponent("dirX") + let dirY = tmp.appendingPathComponent("dirY") + try FileManager.default.createDirectory(at: dirX, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: dirY, withIntermediateDirectories: true) + + let bigPath = dirX.appendingPathComponent("big.bin") + try Data(repeating: 9, count: 262_144).write(to: bigPath) + try FileManager.default.linkItem(at: bigPath, to: dirY.appendingPathComponent("link.bin")) + // An unrelated file in each directory so a splice of either side has + // something else in it besides the hardlink half. + try Data(repeating: 1, count: 128).write(to: dirX.appendingPathComponent("other.txt")) + try Data(repeating: 2, count: 128).write(to: dirY.appendingPathComponent("other.txt")) + + var st = stat() + XCTAssertEqual(lstat(bigPath.path, &st), 0) + let ref = HardLinkRef(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) + return (tmp, ref) + } + + // Finds the node sharing `ref` whose size does (or doesn't, per + // `wantCarrier`) match the first-seen-wins carrier convention, then + // climbs to its direct child-of-root directory name — "the directory + // holding the carrier/twin" from the plan's scenario. + private func topLevelDirName(carryingRef wantCarrier: Bool, ref: HardLinkRef, tree: FileTree) -> String? { + for i in 0.. 0) == wantCarrier else { continue } + var current = i + while tree.parentIndex[current] != tree.rootIndex { + let p = tree.parentIndex[current] + guard p >= 0 else { return nil } + current = p + } + return tree.records[current].name + } + return nil + } + + // Splicing the directory holding the NON-carrier (0-size) twin must not + // re-materialize the full size a second time: the carrier lives outside + // the spliced subtree, so it must be pre-seeded into `seenRefs` and the + // rescanned copy must come back at 0, exactly as it was. + func test_splice_of_directory_holding_hardlink_loser_matches_full_rescan_total() async throws { + let (tmp, ref) = try makeHardlinkAcrossDirsFixture() + defer { try? FileManager.default.removeItem(at: tmp) } + + let before = await scanAndWait(tmp) + guard let beforeTree = await before.tree else { return XCTFail("initial scan should populate a tree") } + guard let loserDirName = topLevelDirName(carryingRef: false, ref: ref, tree: beforeTree) else { + return XCTFail("expected to find the hardlink's 0-size twin in the initial scan") + } + + guard let spliced = ScanViewModel.splicedTree(afterChangeAt: tmp.appendingPathComponent(loserDirName).path, in: beforeTree) else { + return XCTFail("splice should succeed for a plain, non-summarized subdirectory") + } + + let after = await scanAndWait(tmp) + guard let afterTree = await after.tree else { return XCTFail("second full rescan should populate a tree") } + + let splicedRoot = FileNode(tree: spliced, index: spliced.rootIndex) + let afterRoot = FileNode(tree: afterTree, index: afterTree.rootIndex) + XCTAssertEqual(splicedRoot.size, afterRoot.size, "splicing the directory holding the hardlink LOSER must not double-count the carrier living outside it") + } + + // The mirrored case: splicing the directory holding the CARRIER must let + // the rescan re-take the full size for that (still 0-seeded-locally) + // ref, while the untouched outside 0-size twin correctly stays at 0. + func test_splice_of_directory_holding_hardlink_carrier_matches_full_rescan_total() async throws { + let (tmp, ref) = try makeHardlinkAcrossDirsFixture() + defer { try? FileManager.default.removeItem(at: tmp) } + + let before = await scanAndWait(tmp) + guard let beforeTree = await before.tree else { return XCTFail("initial scan should populate a tree") } + guard let carrierDirName = topLevelDirName(carryingRef: true, ref: ref, tree: beforeTree) else { + return XCTFail("expected to find the hardlink carrier in the initial scan") + } + + guard let spliced = ScanViewModel.splicedTree(afterChangeAt: tmp.appendingPathComponent(carrierDirName).path, in: beforeTree) else { + return XCTFail("splice should succeed for a plain, non-summarized subdirectory") + } + + let after = await scanAndWait(tmp) + guard let afterTree = await after.tree else { return XCTFail("second full rescan should populate a tree") } + + let splicedRoot = FileNode(tree: spliced, index: spliced.rootIndex) + let afterRoot = FileNode(tree: afterTree, index: afterTree.rootIndex) + XCTAssertEqual(splicedRoot.size, afterRoot.size, "splicing the directory holding the hardlink CARRIER must still match a full fresh rescan's total") + } + + // The mirror of the trash-path promotion bug, on the live-refresh path: + // an EXTERNAL process (Finder, rm, a build tool) deletes the link that + // happened to be the size carrier, while the inode stays alive through a + // twin in an untouched sibling directory. FSEvents reports only the + // carrier's directory, so only that subtree is rescanned — and nothing + // promotes the surviving twin, so the still-allocated bytes silently + // vanish from the total. + // + // The bar is the same one the other splice tests use: the spliced tree + // must agree with a full fresh rescan of the same on-disk state. + func test_splice_after_external_deletion_of_carrier_promotes_surviving_twin() async throws { + let (tmp, ref) = try makeHardlinkAcrossDirsFixture() + defer { try? FileManager.default.removeItem(at: tmp) } + + let before = await scanAndWait(tmp) + guard let beforeTree = await before.tree else { return XCTFail("initial scan should populate a tree") } + guard let carrierDirName = topLevelDirName(carryingRef: true, ref: ref, tree: beforeTree) else { + return XCTFail("expected to find the hardlink's size carrier in the initial scan") + } + + // Delete only the carrier's link. The inode's blocks are still fully + // allocated, reachable via the twin in the other directory. + let carrierDir = tmp.appendingPathComponent(carrierDirName) + let carrierLink = try FileManager.default + .contentsOfDirectory(at: carrierDir, includingPropertiesForKeys: nil) + .first { url in + var st = stat() + guard lstat(url.path, &st) == 0 else { return false } + return HardLinkRef(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) == ref + } + guard let carrierLink else { return XCTFail("could not locate the carrier link on disk") } + try FileManager.default.removeItem(at: carrierLink) + + guard let spliced = ScanViewModel.splicedTree(afterChangeAt: carrierDir.path, in: beforeTree) else { + return XCTFail("splice should succeed for a plain, non-summarized subdirectory") + } + + let after = await scanAndWait(tmp) + guard let afterTree = await after.tree else { return XCTFail("rescan should populate a tree") } + + let splicedRoot = FileNode(tree: spliced, index: spliced.rootIndex) + let afterRoot = FileNode(tree: afterTree, index: afterTree.rootIndex) + XCTAssertEqual( + splicedRoot.size, + afterRoot.size, + "deleting a hardlink's size carrier must promote the surviving twin, not drop the inode's bytes" + ) + } + + // Two entirely independent hardlink pairs, each split across its own two + // sibling directories, with BOTH carriers deleted externally out of the + // SAME directory in one splice — each orphaned ref must promote its own + // twin; nothing should get mixed up or dropped between the two. + // + // Both pairs' first halves live in "dirX", both second halves in "dirY", + // so whichever of the two directories the scanner happens to visit first + // ends up carrying BOTH pairs' sizes (first-seen-wins is decided at the + // whole-directory level here, since each pair's two links are never in + // the same directory as each other). The test discovers that carrier + // directory rather than assuming which one it is, so it isn't coupled to + // `FileManager.contentsOfDirectory`'s (unspecified) enumeration order. + func test_splice_after_external_deletion_of_two_carriers_in_same_directory_promotes_both_twins() async throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let dirX = tmp.appendingPathComponent("dirX") + let dirY = tmp.appendingPathComponent("dirY") + try FileManager.default.createDirectory(at: dirX, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: dirY, withIntermediateDirectories: true) + + // Pair 1: dirX/firstBig.bin <-> dirY/firstLink.bin + let firstBigPath = dirX.appendingPathComponent("firstBig.bin") + try Data(repeating: 9, count: 262_144).write(to: firstBigPath) + try FileManager.default.linkItem(at: firstBigPath, to: dirY.appendingPathComponent("firstLink.bin")) + var firstSt = stat() + XCTAssertEqual(lstat(firstBigPath.path, &firstSt), 0) + let firstRef = HardLinkRef(dev: UInt64(bitPattern: Int64(firstSt.st_dev)), ino: UInt64(firstSt.st_ino)) + + // Pair 2: dirX/secondBig.bin <-> dirY/secondLink.bin — a completely + // independent inode, unrelated to pair 1. + let secondBigPath = dirX.appendingPathComponent("secondBig.bin") + try Data(repeating: 3, count: 131_072).write(to: secondBigPath) + try FileManager.default.linkItem(at: secondBigPath, to: dirY.appendingPathComponent("secondLink.bin")) + var secondSt = stat() + XCTAssertEqual(lstat(secondBigPath.path, &secondSt), 0) + let secondRef = HardLinkRef(dev: UInt64(bitPattern: Int64(secondSt.st_dev)), ino: UInt64(secondSt.st_ino)) + + let before = await scanAndWait(tmp) + guard let beforeTree = await before.tree else { return XCTFail("initial scan should populate a tree") } + guard let firstCarrierDirName = topLevelDirName(carryingRef: true, ref: firstRef, tree: beforeTree), + let secondCarrierDirName = topLevelDirName(carryingRef: true, ref: secondRef, tree: beforeTree), + firstCarrierDirName == secondCarrierDirName + else { return XCTFail("expected both pairs to share the same first-seen-wins carrier directory") } + let carrierDirName = firstCarrierDirName + let twinDirName = carrierDirName == "dirX" ? "dirY" : "dirX" + let carrierDir = tmp.appendingPathComponent(carrierDirName) + + // Delete BOTH carrier links out of the shared carrier directory. + // Both inodes are still fully allocated via their respective twins + // in the other directory. + for url in try FileManager.default.contentsOfDirectory(at: carrierDir, includingPropertiesForKeys: nil) { + var st = stat() + guard lstat(url.path, &st) == 0 else { continue } + let ref = HardLinkRef(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) + if ref == firstRef || ref == secondRef { + try FileManager.default.removeItem(at: url) + } + } + + guard let spliced = ScanViewModel.splicedTree(afterChangeAt: carrierDir.path, in: beforeTree) else { + return XCTFail("splice should succeed for a plain, non-summarized subdirectory") + } + + let after = await scanAndWait(tmp) + guard let afterTree = await after.tree else { return XCTFail("rescan should populate a tree") } + + let splicedRoot = FileNode(tree: spliced, index: spliced.rootIndex) + let afterRoot = FileNode(tree: afterTree, index: afterTree.rootIndex) + XCTAssertEqual( + splicedRoot.size, + afterRoot.size, + "deleting two independent hardlinks' size carriers from the same directory must promote both surviving twins" + ) + + // Confirm both promotions actually happened (not just a total that + // happens to match by coincidence): the untouched sibling directory's + // two remaining links must now each carry their own pair's full size. + guard let splicedTwinDir = splicedRoot.children.first(where: { $0.name == twinDirName }) else { + return XCTFail("the twin directory must survive the splice untouched (it wasn't the spliced one)") + } + let firstTwin = splicedTwinDir.children.first { $0.hardLinkRef == firstRef } + let secondTwin = splicedTwinDir.children.first { $0.hardLinkRef == secondRef } + XCTAssertEqual(firstTwin?.size, 262_144, "the first pair's surviving twin must be promoted to its own pair's full size") + XCTAssertEqual(secondTwin?.size, 131_072, "the second pair's surviving twin must be promoted to its own pair's full size, independently of the first pair") + } +} diff --git a/Tests/ScanArchiveTests.swift b/Tests/ScanArchiveTests.swift new file mode 100644 index 0000000..c49dc9a --- /dev/null +++ b/Tests/ScanArchiveTests.swift @@ -0,0 +1,232 @@ +import XCTest +@testable import MacDirStat + +final class ScanArchiveTests: XCTestCase { + + // Same shape as the FileTreeTests fixture: + // root (/scan) + // ├── big.bin (500) + // └── sub (dir) + // ├── a.txt (300) + // └── b.txt (100) + private func makeFixtureTree() -> FileTree { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big = FSNode(url: URL(fileURLWithPath: "/scan/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: root) + big.hardLinkRef = HardLinkRef(dev: 1, ino: 42) + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 400, fileExtension: "", parent: root) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 300, fileExtension: "txt", parent: sub) + let b = FSNode(url: URL(fileURLWithPath: "/scan/sub/b.txt"), name: "b.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: sub) + sub.children = [a, b] + root.children = [sub, big] + root.size = big.size + sub.size + + let tree = FileTreeBuilder.build(from: root, rootPath: "/scan") + // Give a couple of fields their post-scan-pass values so the round + // trip test actually exercises non-default data. + let rootNode = FileNode(tree: tree, index: tree.rootIndex) + let bigNode = rootNode.children.first { $0.name == "big.bin" }! + let aNode = rootNode.children.first { $0.name == "sub" }!.children.first { $0.name == "a.txt" }! + tree.setSafety(.danger, at: bigNode.index) + let groupID = UUID() + tree.setDuplicateGroupID(groupID, at: aNode.index) + return tree + } + + private func makeMetadata() -> ScanArchive.Metadata { + ScanArchive.Metadata(scannedPath: "/scan", scanDate: Date(timeIntervalSince1970: 1_700_000_000), deniedCount: 3, appVersion: "1.1") + } + + // MARK: - Round trip + + func test_round_trip_preserves_records_topology_and_sizes() throws { + let tree = makeFixtureTree() + let archive = ScanArchive(tree: tree, metadata: makeMetadata()) + + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + let data = try encoder.encode(archive) + + let decoder = JSONDecoder() + decoder.dateDecodingStrategy = .iso8601 + let decoded = try decoder.decode(ScanArchive.self, from: data) + try decoded.validate() + + XCTAssertEqual(decoded.records.count, tree.records.count) + XCTAssertEqual(decoded.parentIndex, tree.parentIndex) + XCTAssertEqual(decoded.childStart, tree.childStart) + XCTAssertEqual(decoded.childCount, tree.childCount) + XCTAssertEqual(decoded.childIndices, tree.childIndices) + XCTAssertEqual(decoded.rootIndex, tree.rootIndex) + XCTAssertEqual(decoded.rootPath, tree.rootPath) + + for i in 0.. 0 }) else { + return XCTFail("fixture must have at least one node with children") + } + childCount[parentWithChildren] -= 1 + let doctored = ScanArchive( + records: archive.records, parentIndex: archive.parentIndex, + childStart: archive.childStart, childCount: childCount, + childIndices: archive.childIndices, rootIndex: archive.rootIndex, + rootPath: archive.rootPath, metadata: archive.metadata + ) + XCTAssertThrowsError(try doctored.validate()) + } + + func test_validate_rejects_empty_archive() { + let archive = ScanArchive( + records: [], parentIndex: [], childStart: [], childCount: [], + childIndices: [], rootIndex: 0, rootPath: "/scan", metadata: makeMetadata() + ) + XCTAssertThrowsError(try archive.validate()) + } +} + +// Test-only initializer mirroring every stored property, so validation +// tests can construct a deliberately-doctored archive without going through +// `ScanArchive(tree:metadata:)`. +private extension ScanArchive { + init( + records: [FileNodeRecord], parentIndex: [Int], childStart: [Int], + childCount: [Int], childIndices: [Int], rootIndex: Int, + rootPath: String, metadata: ScanArchive.Metadata + ) { + self = try! JSONDecoder().decode(ScanArchive.self, from: JSONEncoder().encode( + ScanArchiveTestPayload( + records: records, parentIndex: parentIndex, childStart: childStart, + childCount: childCount, childIndices: childIndices, rootIndex: rootIndex, + rootPath: rootPath, metadata: metadata + ) + )) + } +} + +// `ScanArchive`'s memberwise fields aren't independently settable (its only +// public initializer takes a `FileTree`), so this mirrors its `Codable` +// shape exactly and round-trips through JSON to construct arbitrary/doctored +// instances for the validator tests above. +private struct ScanArchiveTestPayload: Codable { + let records: [FileNodeRecord] + let parentIndex: [Int] + let childStart: [Int] + let childCount: [Int] + let childIndices: [Int] + let rootIndex: Int + let rootPath: String + let metadata: ScanArchive.Metadata +} diff --git a/Tests/ScanComparisonTests.swift b/Tests/ScanComparisonTests.swift new file mode 100644 index 0000000..5c5260c --- /dev/null +++ b/Tests/ScanComparisonTests.swift @@ -0,0 +1,191 @@ +import XCTest +@testable import MacDirStat + +final class ScanComparisonTests: XCTestCase { + + // "Before" fixture: + // root (/scan) + // ├── big.bin (500) -- shrinks to 100 in "after" + // ├── sub (dir, 400) + // │ ├── a.txt (300) -- grows to 350 in "after" + // │ └── b.txt (100) -- unchanged + // └── oldDir (dir, 200) -- entirely removed in "after" + // └── oldFile.txt (200) + private func makeBeforeTree() -> FileTree { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big = FSNode(url: URL(fileURLWithPath: "/scan/big.bin"), name: "big.bin", isDirectory: false, size: 500, fileExtension: "bin", parent: root) + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 400, fileExtension: "", parent: root) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 300, fileExtension: "txt", parent: sub) + let b = FSNode(url: URL(fileURLWithPath: "/scan/sub/b.txt"), name: "b.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: sub) + sub.children = [a, b] + + let oldDir = FSNode(url: URL(fileURLWithPath: "/scan/oldDir"), name: "oldDir", isDirectory: true, size: 200, fileExtension: "", parent: root) + let oldFile = FSNode(url: URL(fileURLWithPath: "/scan/oldDir/oldFile.txt"), name: "oldFile.txt", isDirectory: false, size: 200, fileExtension: "txt", parent: oldDir) + oldDir.children = [oldFile] + + root.children = [big, sub, oldDir] + root.size = big.size + sub.size + oldDir.size + return FileTreeBuilder.build(from: root, rootPath: "/scan") + } + + // "After" fixture: big.bin shrank, sub/a.txt grew, oldDir is gone + // entirely, and a brand-new top-level file appeared. + private func makeAfterTree() -> FileTree { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let big = FSNode(url: URL(fileURLWithPath: "/scan/big.bin"), name: "big.bin", isDirectory: false, size: 100, fileExtension: "bin", parent: root) + let sub = FSNode(url: URL(fileURLWithPath: "/scan/sub"), name: "sub", isDirectory: true, size: 450, fileExtension: "", parent: root) + let a = FSNode(url: URL(fileURLWithPath: "/scan/sub/a.txt"), name: "a.txt", isDirectory: false, size: 350, fileExtension: "txt", parent: sub) + let b = FSNode(url: URL(fileURLWithPath: "/scan/sub/b.txt"), name: "b.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: sub) + sub.children = [a, b] + + let newFile = FSNode(url: URL(fileURLWithPath: "/scan/newFile.txt"), name: "newFile.txt", isDirectory: false, size: 250, fileExtension: "txt", parent: root) + + root.children = [big, sub, newFile] + root.size = big.size + sub.size + newFile.size + return FileTreeBuilder.build(from: root, rootPath: "/scan") + } + + func test_compare_reports_exactly_the_four_expected_changes() { + let before = makeBeforeTree() + let after = makeAfterTree() + + let changes = ScanComparison.compare(before: before, after: after) + + XCTAssertEqual(changes.count, 4, "must not list oldDir's descendant separately, or unrelated unchanged nodes") + + let byPath = Dictionary(uniqueKeysWithValues: changes.map { ($0.relativePath, $0) }) + + let added = byPath["newFile.txt"] + XCTAssertEqual(added?.kind, .added) + XCTAssertEqual(added?.isDirectory, false) + XCTAssertEqual(added?.beforeSize, 0) + XCTAssertEqual(added?.afterSize, 250) + XCTAssertEqual(added?.delta, 250) + + let removed = byPath["oldDir"] + XCTAssertEqual(removed?.kind, .removed) + XCTAssertEqual(removed?.isDirectory, true) + XCTAssertEqual(removed?.beforeSize, 200) + XCTAssertEqual(removed?.afterSize, 0) + XCTAssertEqual(removed?.delta, -200) + XCTAssertNil(byPath["oldDir/oldFile.txt"], "descendant of a removed directory must be collapsed into the directory's own row") + + let grew = byPath["sub/a.txt"] + XCTAssertEqual(grew?.kind, .grew) + XCTAssertEqual(grew?.beforeSize, 300) + XCTAssertEqual(grew?.afterSize, 350) + XCTAssertEqual(grew?.delta, 50) + + let shrank = byPath["big.bin"] + XCTAssertEqual(shrank?.kind, .shrank) + XCTAssertEqual(shrank?.beforeSize, 500) + XCTAssertEqual(shrank?.afterSize, 100) + XCTAssertEqual(shrank?.delta, -400) + + // Unchanged node must not appear at all. + XCTAssertNil(byPath["sub/b.txt"]) + // The directory that merely contains a changed file must not be + // reported itself (its size delta is implied by sub/a.txt's row). + XCTAssertNil(byPath["sub"]) + } + + func test_compare_sorts_by_absolute_delta_descending() { + let changes = ScanComparison.compare(before: makeBeforeTree(), after: makeAfterTree()) + let deltas = changes.map { abs($0.delta) } + XCTAssertEqual(deltas, deltas.sorted(by: >)) + // big.bin (400) > newFile.txt (250) > oldDir (200) > sub/a.txt (50) + XCTAssertEqual(changes.map(\.relativePath), ["big.bin", "newFile.txt", "oldDir", "sub/a.txt"]) + } + + func test_compare_identical_trees_reports_no_changes() { + let tree = makeBeforeTree() + XCTAssertTrue(ScanComparison.compare(before: tree, after: tree).isEmpty) + } + + func test_compare_collapses_a_newly_added_directory_to_a_single_row() { + // "before": just a root with one file. + let beforeRoot = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 100, fileExtension: "", parent: nil) + let keep = FSNode(url: URL(fileURLWithPath: "/scan/keep.txt"), name: "keep.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: beforeRoot) + beforeRoot.children = [keep] + let before = FileTreeBuilder.build(from: beforeRoot, rootPath: "/scan") + + // "after": same file, plus a whole new directory with two files inside. + let afterRoot = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 400, fileExtension: "", parent: nil) + let keep2 = FSNode(url: URL(fileURLWithPath: "/scan/keep.txt"), name: "keep.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: afterRoot) + let newDir = FSNode(url: URL(fileURLWithPath: "/scan/newDir"), name: "newDir", isDirectory: true, size: 300, fileExtension: "", parent: afterRoot) + let n1 = FSNode(url: URL(fileURLWithPath: "/scan/newDir/n1.txt"), name: "n1.txt", isDirectory: false, size: 200, fileExtension: "txt", parent: newDir) + let n2 = FSNode(url: URL(fileURLWithPath: "/scan/newDir/n2.txt"), name: "n2.txt", isDirectory: false, size: 100, fileExtension: "txt", parent: newDir) + newDir.children = [n1, n2] + afterRoot.children = [keep2, newDir] + let after = FileTreeBuilder.build(from: afterRoot, rootPath: "/scan") + + let changes = ScanComparison.compare(before: before, after: after) + XCTAssertEqual(changes.count, 1, "adding a whole directory tree must produce exactly one collapsed row") + XCTAssertEqual(changes[0].relativePath, "newDir") + XCTAssertEqual(changes[0].kind, .added) + XCTAssertEqual(changes[0].afterSize, 300) + } + + // MARK: - Type changes at the same path (file <-> directory) + // + // A path present in BOTH trees is skipped by the added/removed loops, and + // the grew/shrank loop only compares file-to-file. A node that changed + // kind therefore used to fall through every branch and vanish from the + // report entirely, while its new children were reported as plain "added" + // rows (their parent path technically existed before, as a file). + + private func singleNodeTree(isDirectory: Bool, size: Int64, withChild child: (name: String, size: Int64)?) -> FileTree { + let root = FSNode(url: URL(fileURLWithPath: "/scan"), name: "scan", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let a = FSNode(url: URL(fileURLWithPath: "/scan/A"), name: "A", isDirectory: isDirectory, size: size, fileExtension: isDirectory ? "" : "bin", parent: root) + if let child { + let inner = FSNode(url: URL(fileURLWithPath: "/scan/A/\(child.name)"), name: child.name, isDirectory: false, size: child.size, fileExtension: "txt", parent: a) + a.children = [inner] + } + root.children = [a] + root.size = a.size + return FileTreeBuilder.build(from: root, rootPath: "/scan") + } + + func test_file_becoming_directory_is_reported_once_and_children_suppressed() { + let before = singleNodeTree(isDirectory: false, size: 10, withChild: nil) + let after = singleNodeTree(isDirectory: true, size: 2_000, withChild: ("inner.txt", 2_000)) + + let changes = ScanComparison.compare(before: before, after: after) + + XCTAssertEqual(changes.count, 1, "expected exactly one row for the replaced node, got: \(changes.map { "\($0.relativePath):\($0.kind)" })") + XCTAssertEqual(changes[0].relativePath, "A") + XCTAssertEqual(changes[0].kind, .replaced) + XCTAssertTrue(changes[0].isDirectory) + XCTAssertEqual(changes[0].beforeSize, 10) + XCTAssertEqual(changes[0].afterSize, 2_000) + XCTAssertEqual(changes[0].delta, 1_990) + } + + func test_directory_becoming_file_is_reported_once_and_old_children_suppressed() { + let before = singleNodeTree(isDirectory: true, size: 100, withChild: ("inner.txt", 100)) + let after = singleNodeTree(isDirectory: false, size: 500, withChild: nil) + + let changes = ScanComparison.compare(before: before, after: after) + + XCTAssertEqual(changes.count, 1, "expected exactly one row for the replaced node, got: \(changes.map { "\($0.relativePath):\($0.kind)" })") + XCTAssertEqual(changes[0].relativePath, "A") + XCTAssertEqual(changes[0].kind, .replaced) + XCTAssertFalse(changes[0].isDirectory) + XCTAssertEqual(changes[0].beforeSize, 100) + XCTAssertEqual(changes[0].afterSize, 500) + } + + // A type change with an identical size still matters: neither size-based + // branch would fire, so this specifically guards the "reported regardless + // of size" rule. + func test_type_change_with_identical_size_is_still_reported() { + let before = singleNodeTree(isDirectory: false, size: 100, withChild: nil) + let after = singleNodeTree(isDirectory: true, size: 100, withChild: ("inner.txt", 100)) + + let changes = ScanComparison.compare(before: before, after: after) + + XCTAssertEqual(changes.count, 1) + XCTAssertEqual(changes[0].kind, .replaced) + XCTAssertEqual(changes[0].delta, 0) + } +} diff --git a/Tests/ScanRefreshTests.swift b/Tests/ScanRefreshTests.swift new file mode 100644 index 0000000..203aba0 --- /dev/null +++ b/Tests/ScanRefreshTests.swift @@ -0,0 +1,250 @@ +import XCTest +@testable import MacDirStat + +final class ScanRefreshTests: XCTestCase { + + // Returns allocated disk size (st_blocks * 512) for a path, matching the scanner's metric. + private func allocatedSize(at path: String) -> Int64 { + var st = stat() + guard lstat(path, &st) == 0 else { return -1 } + return Int64(st.st_blocks) * 512 + } + + func test_refresh_uses_allocated_size() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let fileURL = tmp.appendingPathComponent("a.bin") + try Data([0x42]).write(to: fileURL) + + let expectedSize = allocatedSize(at: fileURL.path) + XCTAssertGreaterThan(expectedSize, 0) + + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 999999, fileExtension: "", parent: nil) + let staleChild = FSNode(url: fileURL, name: "a.bin", isDirectory: false, size: 999999, fileExtension: "bin", parent: dirNode) + dirNode.children = [staleChild] + + ScanViewModel.refreshDirectory(node: dirNode) + + let updated = dirNode.children.first { $0.name == "a.bin" } + XCTAssertNotNil(updated) + XCTAssertEqual(updated?.size, expectedSize, "refresh must use allocated size (st_blocks * 512), not logical size") + } + + // Returns the (dev, ino) identity for a path the way the scanner records it. + private func ref(at path: String) -> HardLinkRef { + var st = stat() + precondition(lstat(path, &st) == 0) + return HardLinkRef(dev: UInt64(bitPattern: Int64(st.st_dev)), ino: UInt64(st.st_ino)) + } + + func test_refresh_new_hardlink_not_double_counted() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let orig = tmp.appendingPathComponent("orig.bin") + try Data(repeating: 1, count: 262_144).write(to: orig) + let fullSize = allocatedSize(at: orig.path) + + // Tree state as the initial scan left it: only orig.bin existed then. + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: fullSize, fileExtension: "", parent: nil) + let origNode = FSNode(url: orig, name: "orig.bin", isDirectory: false, size: fullSize, fileExtension: "bin", parent: dirNode) + origNode.hardLinkRef = ref(at: orig.path) + dirNode.children = [origNode] + + // A new hardlink appears after the scan. + let newLink = tmp.appendingPathComponent("newlink.bin") + try FileManager.default.linkItem(at: orig, to: newLink) + + ScanViewModel.refreshDirectory(node: dirNode) + + let linkNode = dirNode.children.first { $0.name == "newlink.bin" } + XCTAssertNotNil(linkNode) + XCTAssertEqual(linkNode?.size, 0, "a new name for an already-counted inode must not add size") + XCTAssertNotNil(linkNode?.hardLinkRef) + let total = dirNode.children.reduce(Int64(0)) { $0 + $1.size } + XCTAssertEqual(total, fullSize, "directory total must not double-count the inode") + } + + func test_refresh_promotes_survivor_when_winner_deleted() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let hard1 = tmp.appendingPathComponent("hard1.bin") + let hard2 = tmp.appendingPathComponent("hard2.bin") + let hard3 = tmp.appendingPathComponent("hard3.bin") + try Data(repeating: 2, count: 262_144).write(to: hard1) + try FileManager.default.linkItem(at: hard1, to: hard2) + try FileManager.default.linkItem(at: hard1, to: hard3) + let fullSize = allocatedSize(at: hard1.path) + let inodeRef = ref(at: hard1.path) + + // Tree as the initial scan left it: hard1 carries the size, the others are 0. + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: fullSize, fileExtension: "", parent: nil) + let winner = FSNode(url: hard1, name: "hard1.bin", isDirectory: false, size: fullSize, fileExtension: "bin", parent: dirNode) + let loser2 = FSNode(url: hard2, name: "hard2.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirNode) + let loser3 = FSNode(url: hard3, name: "hard3.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirNode) + for n in [winner, loser2, loser3] { n.hardLinkRef = inodeRef } + dirNode.children = [winner, loser2, loser3] + + // The size-carrying link is deleted; the inode still exists via the survivors. + try FileManager.default.removeItem(at: hard1) + + ScanViewModel.refreshDirectory(node: dirNode) + + XCTAssertEqual(dirNode.children.count, 2) + let sizes = dirNode.children.map(\.size).sorted(by: >) + XCTAssertEqual(sizes, [fullSize, 0], "one survivor must be promoted to carry the inode's size") + } + + func test_scan_subtree_dedupes_internal_hardlinks() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + // Tree as scanned: empty directory. + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + + // A new directory appears containing two links to one inode. + let newDir = tmp.appendingPathComponent("newdir") + try FileManager.default.createDirectory(at: newDir, withIntermediateDirectories: true) + let a = newDir.appendingPathComponent("a.bin") + try Data(repeating: 3, count: 262_144).write(to: a) + try FileManager.default.linkItem(at: a, to: newDir.appendingPathComponent("b.bin")) + let fullSize = allocatedSize(at: a.path) + + ScanViewModel.refreshDirectory(node: dirNode) + + let newDirNode = dirNode.children.first { $0.name == "newdir" } + XCTAssertNotNil(newDirNode) + XCTAssertEqual(newDirNode?.size, fullSize, "hardlinked pair inside a new directory must count once") + } + + func test_refresh_preserves_hardlink_dedup() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let hard1 = tmp.appendingPathComponent("hard1.bin") + let hard2 = tmp.appendingPathComponent("hard2.bin") + try Data(repeating: 1, count: 262_144).write(to: hard1) + try FileManager.default.linkItem(at: hard1, to: hard2) + let fullSize = allocatedSize(at: hard1.path) + XCTAssertGreaterThan(fullSize, 0) + + // Initial scan counts a hardlinked inode once: hard1 keeps the size, hard2 is 0. + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: fullSize, fileExtension: "", parent: nil) + let winner = FSNode(url: hard1, name: "hard1.bin", isDirectory: false, size: fullSize, fileExtension: "bin", parent: dirNode) + let loser = FSNode(url: hard2, name: "hard2.bin", isDirectory: false, size: 0, fileExtension: "bin", parent: dirNode) + dirNode.children = [winner, loser] + + // Unrelated change in the same directory triggers a refresh. + try Data(repeating: 9, count: 4096).write(to: tmp.appendingPathComponent("other.bin")) + + ScanViewModel.refreshDirectory(node: dirNode) + + let refreshedLoser = dirNode.children.first { $0.name == "hard2.bin" } + let refreshedWinner = dirNode.children.first { $0.name == "hard1.bin" } + XCTAssertEqual(refreshedLoser?.size, 0, "refresh must not resurrect a deduplicated hardlink to full size") + XCTAssertEqual(refreshedWinner?.size, fullSize) + } + + func test_refresh_scans_new_directory_recursively() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let newDir = tmp.appendingPathComponent("newdir") + let subDir = newDir.appendingPathComponent("sub") + try FileManager.default.createDirectory(at: subDir, withIntermediateDirectories: true) + let fileURL = subDir.appendingPathComponent("file.bin") + try Data(repeating: 7, count: 8192).write(to: fileURL) + + let expectedFileSize = allocatedSize(at: fileURL.path) + XCTAssertGreaterThan(expectedFileSize, 0) + + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + + ScanViewModel.refreshDirectory(node: dirNode) + + let newDirNode = dirNode.children.first { $0.name == "newdir" } + XCTAssertNotNil(newDirNode, "new directory should appear as a child") + XCTAssertEqual(newDirNode?.isDirectory, true) + XCTAssertEqual(newDirNode?.size, expectedFileSize, "new directory size should reflect the recursively-scanned subtree") + + let subDirNode = newDirNode?.children.first { $0.name == "sub" } + XCTAssertNotNil(subDirNode, "nested subdirectory should be scanned") + let fileNode = subDirNode?.children.first { $0.name == "file.bin" } + XCTAssertNotNil(fileNode, "nested file should be present") + XCTAssertEqual(fileNode?.size, expectedFileSize) + } + + func test_refresh_skips_symlinks() throws { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let realURL = tmp.appendingPathComponent("real.bin") + try Data(repeating: 1, count: 4096).write(to: realURL) + let linkURL = tmp.appendingPathComponent("link") + try FileManager.default.createSymbolicLink(at: linkURL, withDestinationURL: realURL) + + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + + ScanViewModel.refreshDirectory(node: dirNode) + + XCTAssertNil(dirNode.children.first { $0.name == "link" }, "symlinks must not be added by refresh") + XCTAssertNotNil(dirNode.children.first { $0.name == "real.bin" }) + } + + func test_refresh_honors_hidden_files_setting() throws { + let defaults = UserDefaults.standard + let priorValue = defaults.object(forKey: "showHiddenFiles") + defaults.set(true, forKey: "showHiddenFiles") + defer { + if let priorValue { defaults.set(priorValue, forKey: "showHiddenFiles") } else { defaults.removeObject(forKey: "showHiddenFiles") } + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let hiddenURL = tmp.appendingPathComponent(".dotfile") + try Data(repeating: 3, count: 4096).write(to: hiddenURL) + let expectedSize = allocatedSize(at: hiddenURL.path) + + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + + ScanViewModel.refreshDirectory(node: dirNode) + + let hiddenChild = dirNode.children.first { $0.name == ".dotfile" } + XCTAssertNotNil(hiddenChild, "when showHiddenFiles is true, refresh should include dotfiles") + XCTAssertEqual(hiddenChild?.size, expectedSize) + } + + func test_refresh_skips_excluded_folders() throws { + let defaults = UserDefaults.standard + let priorValue = defaults.object(forKey: "excludedFolderNames") + defaults.set(".git,node_modules,DerivedData,.Trash", forKey: "excludedFolderNames") + defer { + if let priorValue { defaults.set(priorValue, forKey: "excludedFolderNames") } else { defaults.removeObject(forKey: "excludedFolderNames") } + } + + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + let excludedDir = tmp.appendingPathComponent("node_modules") + try FileManager.default.createDirectory(at: excludedDir, withIntermediateDirectories: true) + try Data(repeating: 5, count: 4096).write(to: excludedDir.appendingPathComponent("junk.bin")) + + let dirNode = FSNode(url: tmp, name: tmp.lastPathComponent, isDirectory: true, size: 0, fileExtension: "", parent: nil) + + ScanViewModel.refreshDirectory(node: dirNode) + + XCTAssertNil(dirNode.children.first { $0.name == "node_modules" }, "excluded folder names must not be added by refresh") + } +} diff --git a/Tests/ScanViewModelIntegrationTests.swift b/Tests/ScanViewModelIntegrationTests.swift new file mode 100644 index 0000000..478d179 --- /dev/null +++ b/Tests/ScanViewModelIntegrationTests.swift @@ -0,0 +1,92 @@ +import XCTest +@testable import MacDirStat + +// End-to-end coverage for ScanViewModel.scan(url:) itself — the biggest +// rewritten code path in the Phase 2 flat-store migration (FileScanner -> +// FileTree -> safety tagging -> extension summaries -> duplicate detection -> +// layout), which none of the other test files exercise directly (they either +// drive FileScanner/FileTree/DuplicateDetector in isolation, or call the +// FSNode-based static refresh helpers directly). +@MainActor +final class ScanViewModelIntegrationTests: XCTestCase { + + private func withRealtimeMonitoringDisabled(_ body: () async throws -> T) async rethrows -> T { + let prior = UserDefaults.standard.object(forKey: "realtimeMonitoring") as? Bool + UserDefaults.standard.set(false, forKey: "realtimeMonitoring") + defer { + if let prior { UserDefaults.standard.set(prior, forKey: "realtimeMonitoring") } + else { UserDefaults.standard.removeObject(forKey: "realtimeMonitoring") } + } + return try await body() + } + + // Polls a condition instead of a fixed sleep, since scan completion runs + // through several hops (scanner -> Task.detached tagging -> recomputeLayout). + private func waitUntil(timeout: TimeInterval = 5, _ condition: () -> Bool) async { + let deadline = Date().addingTimeInterval(timeout) + while !condition(), Date() < deadline { + try? await Task.sleep(nanoseconds: 10_000_000) + } + } + + func test_scan_populates_tree_layout_and_extension_summaries() async throws { + try await withRealtimeMonitoringDisabled { + let tmp = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: tmp, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: tmp) } + + try Data(repeating: 1, count: 8192).write(to: tmp.appendingPathComponent("a.bin")) + try Data(repeating: 2, count: 4096).write(to: tmp.appendingPathComponent("b.txt")) + let sub = tmp.appendingPathComponent("sub") + try FileManager.default.createDirectory(at: sub, withIntermediateDirectories: true) + try Data(repeating: 3, count: 2048).write(to: sub.appendingPathComponent("c.bin")) + + let vm = ScanViewModel() + vm.updateLayoutSize(CGSize(width: 400, height: 400)) + vm.scan(url: tmp) + + await waitUntil { !vm.isScanning && !vm.isComputingLayout } + + XCTAssertNotNil(vm.tree) + guard let root = vm.root else { return XCTFail("scan did not populate a root") } + XCTAssertTrue(root.isDirectory) + XCTAssertEqual(root.children.map(\.name).sorted(), ["a.bin", "b.txt", "sub"]) + // Children must be size-desc (a.bin=8192 > sub=2048 > b.txt=4096... sub aggregates + // c.bin's 2048, so expected order is a.bin(8192), b.txt(4096), sub(2048)). + XCTAssertEqual(root.children.map(\.name), ["a.bin", "b.txt", "sub"]) + + // Safety tagging ran over the whole tree: a plain temp-dir file/subdir + // matches none of the danger/safe path rules, so it settles at .caution + // (the default an untagged record would also read as — the real signal + // this checks is that tagging ran without crashing across a synthetic-free + // tree with directories and files mixed together). + for child in root.children { + XCTAssertEqual(child.safetyLevel, .caution) + } + + // Layout: a non-trivial layoutSize was set before scanning, so recomputeLayout + // should have produced cells once the scan settled. + await waitUntil { !vm.cells.isEmpty } + XCTAssertFalse(vm.cells.isEmpty, "treemap layout should be computed after a scan completes") + + // Extension summaries run off-thread; wait for them to land. + await waitUntil(timeout: 5) { !vm.extensionSummaries.isEmpty } + XCTAssertFalse(vm.extensionSummaries.isEmpty) + let binSummary = vm.extensionSummaries.first { $0.ext == ".bin" } + XCTAssertNotNil(binSummary) + XCTAssertEqual(binSummary?.fileCount, 2, "both a.bin and sub/c.bin should be counted") + } + } + + func test_scan_of_deleted_root_reports_failure_not_crash() async throws { + let missing = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + // Never created — the scanner should treat it as a nonexistent leaf, + // not throw or crash the view model. + let vm = ScanViewModel() + vm.scan(url: missing) + await waitUntil { !vm.isScanning } + // Whatever the scanner decides (empty leaf vs failure), the view model + // must reach a settled, non-scanning state without crashing. + XCTAssertFalse(vm.isScanning) + } +} diff --git a/Tests/TreemapLayoutTests.swift b/Tests/TreemapLayoutTests.swift index 51c77b2..ff0cc79 100644 --- a/Tests/TreemapLayoutTests.swift +++ b/Tests/TreemapLayoutTests.swift @@ -26,6 +26,16 @@ final class TreemapLayoutTests: XCTestCase { XCTAssertNotEqual(map.color(for: "pdf"), map.color(for: "mp4")) } + func test_directory_hue_is_deterministic_and_bounded() { + let name = "Documents" + let expected = name.utf8.reduce(UInt32(5381)) { ($0 &<< 5) &+ $0 &+ UInt32($1) } + let hue = TreemapLayout.directoryHue(for: name) + XCTAssertEqual(hue, Double(expected % 360) / 360.0) + XCTAssertGreaterThanOrEqual(hue, 0) + XCTAssertLessThan(hue, 1) + XCTAssertNotEqual(TreemapLayout.directoryHue(for: "aaa"), TreemapLayout.directoryHue(for: "zzz")) + } + func test_byte_formatter_kb() { XCTAssertEqual(ByteFormatter.string(from: 1_000), "1.0 KB") } @@ -44,7 +54,10 @@ final class TreemapLayoutTests: XCTestCase { // MARK: - Helpers - func makeTree(_ files: [(String, Int64)]) -> FSNode { + // Builds an FSNode fixture (as before) and converts it to a FileTree, + // returning a FileNode handle onto its root — TreemapLayout/ExtensionColorMap + // are now FileNode-based, but the fixture construction itself is unchanged. + func makeTree(_ files: [(String, Int64)]) -> FileNode { let root = FSNode(url: URL(fileURLWithPath: "/"), name: "/", isDirectory: true, size: 0, fileExtension: "", parent: nil) for (name, size) in files { let ext = (name as NSString).pathExtension.lowercased() @@ -52,7 +65,8 @@ final class TreemapLayoutTests: XCTestCase { root.children.append(child) root.size += size } - return root + let tree = FileTreeBuilder.build(from: root, rootPath: "/") + return FileNode(tree: tree, index: tree.rootIndex) } // The layout is a radial sunburst: a node's children fill the angular range @@ -110,7 +124,7 @@ final class TreemapLayoutTests: XCTestCase { } func test_layout_empty_children_returns_no_cells() { - let root = FSNode(url: URL(fileURLWithPath: "/"), name: "/", isDirectory: true, size: 0, fileExtension: "", parent: nil) + let root = makeTree([]) let map = ExtensionColorMap(root: root) let cells = TreemapLayout.compute(root: root, in: CGRect(x: 0, y: 0, width: 400, height: 400), colorMap: map) XCTAssertTrue(cells.isEmpty) diff --git a/docs/adr/0002-getattrlistbulk-batch-enumeration.md b/docs/adr/0002-getattrlistbulk-batch-enumeration.md new file mode 100644 index 0000000..cd74c3a --- /dev/null +++ b/docs/adr/0002-getattrlistbulk-batch-enumeration.md @@ -0,0 +1,72 @@ +# 0002. Enumerate directories with getattrlistbulk, not per-entry stat + +## Status + +Accepted, 2026-07-24. + +## Context + +The scanner's previous directory walk did what most POSIX code does: open a +directory, call `readdir` to get one entry's name, then call `fstatat` on +that name to get its metadata (size, type, inode). That is two syscalls per +entry, and on a directory with thousands of files (a `node_modules`, a +Photos library, a build output folder) that means thousands of syscalls +just to enumerate one directory, before any of the actual scanning work +happens. + +Traversal itself compounded the problem: each directory spawned a new +recursive task for every subdirectory it found, with no cap. On a wide, +shallow tree (again, `node_modules` is the canonical case) this could fan +out into far more concurrent tasks than the machine has cores to run them +on, all contending for the same thread pool. + +macOS offers `getattrlistbulk(2)`, a single syscall that returns a batch of +directory entries with their names and requested attributes (size, type, +inode, flags) together, amortizing the syscall cost across the whole batch +instead of paying it per entry. Not every filesystem or mount supports it, +though (some network and legacy filesystems don't), so it cannot be the +only path. + +## Decision + +Replace the `readdir`/`fstatat` pair with `getattrlistbulk(2)` as the +primary enumeration path in `BulkDirectoryEnumerator`, requesting name and +all previously-`fstatat`'d metadata in one call per batch. Keep a +`readdir`-based fallback path for volumes that reject `getattrlistbulk` +(detected by the syscall itself failing, not by pre-checking filesystem +type), so no volume becomes unscannable. + +Change traversal from unbounded recursive task fan-out to an iterative work +queue drained by a bounded worker pool sized +`min(max(2, cores/2), 8)`: enough parallelism to keep disk and CPU busy, +capped so a wide directory tree cannot spawn more concurrent work than the +machine can usefully run. + +All existing scan semantics (allocated vs. logical size, symlink/hidden/ +exclusion skips, mount-point and hardlink dedup, access-denied surfacing) +are preserved identically in both the bulk and fallback paths, and a +fingerprint test scans the same fixture tree once with bulk enumeration and +once with the fallback forced, asserting the resulting trees are identical +byte-for-byte in every field that matters. The public `FileScanner`/ +`FSNode`/`ScanProgress` API is unchanged, so this is purely an internal +rewrite. + +Measured on `/Applications`: 1.63s with `getattrlistbulk` vs. 2.02s with +the `readdir` fallback, on the same machine and directory. + +## Consequences + +- Two enumeration code paths now exist and must be kept in sync; the + parity test is the thing that makes that safe; anyone changing what a + scan captures (a new attribute, a new skip rule) must update both paths + or the parity test will (correctly) fail. +- The bounded worker pool means a scan of a very wide, shallow tree no + longer transiently spikes to thousands of concurrent tasks, at a small + cost in wall-clock time on such trees compared to fully unbounded + fan-out; this trade was accepted because unbounded fan-out was also + the root cause of thread-pool exhaustion bugs fixed later on this branch + (see the async summary-walk fix). +- `getattrlistbulk` returning inode/device information here is what later + interacted badly with macOS firmlinks (see ADR 0004): the enumerator's + own report of an entry's identity cannot be trusted for firmlinked + directories, only what you get back from actually opening them. diff --git a/docs/adr/0003-flat-file-tree-store.md b/docs/adr/0003-flat-file-tree-store.md new file mode 100644 index 0000000..6411012 --- /dev/null +++ b/docs/adr/0003-flat-file-tree-store.md @@ -0,0 +1,92 @@ +# 0003. Replace the FSNode class tree with a flat struct-of-arrays store + +## Status + +Accepted, 2026-07-24. + +## Context + +The app's in-memory representation of a scan was a tree of `FSNode` class +instances: one heap-allocated object per file and folder, each holding a +`URL`, a `UUID`, a size, a type, an array of child references, and a weak +parent reference. That is a natural way to model a filesystem tree, and it +made the early implementation straightforward, but it does not scale well: + +- Every file and folder in a scan is a separate heap allocation and a + separate reference-counted object, which is slow to allocate in bulk and + slow for ARC to tear down when a scan is discarded or replaced. +- Sorting children by size, walking the tree for layout, or searching by + path all chase pointers through scattered heap objects instead of + scanning contiguous memory, which is unfriendly to the CPU cache on + scans with hundreds of thousands of nodes. +- A `URL` and a `UUID` per node is more retained state per node than a + treemap actually needs to render and interact with a chart: identity for + UI purposes doesn't require a real `URL`, and the path can always be + rebuilt from parent links when it's actually needed (opening in Finder, + computing a delete target). + +On the auto-summarization work (ADR-adjacent, not its own ADR) this became +a hard blocker: summarizing away a folder full of tiny files back into +individual `FSNode`s only for them to be discarded again the moment the +summary collapses them was pure waste, both in time and peak memory. + +## Decision + +Replace `FSNode` as the model layer's tree representation with `FileTree`, +a flat struct-of-arrays store: a `FileNodeRecord` per node holds only what +the UI and business logic actually need (name, size, type, safety level, +hardlink ref), stored in a single contiguous `records` array. Parent/child +relationships are represented as an index into that array (a +`UInt32`-style parent index per record) plus a `childStart`/`childCount` +span describing a contiguous run of children, rather than child arrays or +object references. `FileNode` is a lightweight handle (effectively an +index plus a reference to the owning `FileTree`) that call sites use in +place of the old class reference; paths are reconstructed on demand by +walking parent indices up to the root, not stored per node. + +`FSNode` is kept, deliberately, as the scanner's internal build type: the +scanner still constructs a tree incrementally while walking the +filesystem, where a class tree with real parent/child object references is +the natural shape for that job (nodes get created, reparented, and +summarized away before the scan finishes). Once a scan completes, it is +converted once into a `FileTree`, which is what the rest of the app (view +model, layout, duplicates, compare, archive) operates on from then on. +Everything downstream of the scanner only ever sees `FileTree`. + +The split that makes later live-refresh and hardlink work tractable is +that a `FileTree`'s topology (the parent indices and child spans) is +treated as effectively immutable once built, and reshaped only through a +small number of whole-operation functions (`replacingSubtree`, +`removingSubtrees`, `promotingSurvivingTwin`) that rebuild the affected +spans and ancestor sizes in one pass, rather than through ad hoc mutation +of individual records. Per-record fields that do change in place, size +above all, are mutated directly in the `records` array without touching +topology at all. + +## Consequences + +- Real numbers from the migration: `swift build`/`swift test` stayed fully + green throughout (the migration touched essentially every test file), + and the store change alone, independent of auto-summarization, is what + made scans of hundreds of thousands of nodes practical to hold in memory + and sort/lay out responsively. +- Every consumer of the tree (layout, duplicates, compare, archive, + live-refresh) works in terms of `FileNode` handles and `FileTree` + queries, not object references; code that assumes it can hold a stable + reference to "a node" across a topology-mutating operation is wrong by + construction; handles must be re-resolved after any `replacingSubtree`/ + `removingSubtrees` call. +- The immutable-topology/mutable-fields split is what several later fixes + on this branch depend on being respected: the synthetic "Hidden & + Unreadable Space" node bug (records/parentIndex extended but childStart/ + childCount left one entry short) and the ancestor re-sort gap after + hardlink removal were both violations of this split slipping through + because a new code path mutated part of the arrays without going through + the shared span-rebuilding helpers. `FileTree`'s initializer now asserts + all per-node arrays are the same length specifically to catch the first + kind of gap at construction time instead of at some distant read. +- `.mdscan` archives serialize `FileNodeRecord`/`HardLinkRef`/ + `SafetyLevel` directly (they're `Codable`), which is only straightforward + because they are plain value types with no object graph to break out of; + this would have been considerably more awkward to do safely with the old + `FSNode` class tree. diff --git a/docs/adr/0004-directory-identity-from-the-opened-fd.md b/docs/adr/0004-directory-identity-from-the-opened-fd.md new file mode 100644 index 0000000..803f16f --- /dev/null +++ b/docs/adr/0004-directory-identity-from-the-opened-fd.md @@ -0,0 +1,92 @@ +# 0004. Decide directory identity, mount boundaries, and dedup from the opened directory, never the parent listing + +## Status + +Accepted, 2026-07-27. + +## Context + +The bulk enumerator (ADR 0002) added a TOCTOU (time-of-check-to-time-of-use) +guard: when a parent directory's listing reports an entry's `(dev, ino)`, +and the scanner later opens that entry to recurse into it, the guard +compared the opened directory's `(dev, ino)` against what the listing had +claimed and treated a mismatch as evidence that the entry changed out from +under the scan (swapped for a symlink, replaced, etc.), and dropped it. +That is a reasonable-sounding safety check, and it shipped as part of the +same commit that introduced batch enumeration. + +It broke nearly the entire scan. Running MacDirStat on `/` reported +**11.8 GB used** instead of the real **479.3 GB** on the same machine. +`/Users`, `/Applications`, `/Library`, `/opt`, `/private`, `/Volumes`, and +`/cores` all came back as 0 bytes, and the missing ~818 GB was silently +folded into the synthetic "Hidden & Unreadable Space" node, so nothing +about the failure looked like an error, it just looked like a very +restricted disk. Since Users and Applications carry essentially all of a +typical Mac's non-system data, this was not an edge case: it was +functionally "the scanner doesn't work." + +The cause: every one of those paths is a macOS firmlink. Since Catalina, +the boot volume is split into a read-only system volume and a writable +data volume, joined by firmlinks, kernel-level directory aliases that +present as ordinary directories in one namespace but resolve to a +different location, and a different inode, on the other volume. A parent +listing's `getattrlistbulk`/`readdir` entry for `/Users` reports the +firmlink's own identity; opening `/Users` and asking the open file +descriptor for its identity reports the Data volume's identity underneath. +Those two `(dev, ino)` pairs are supposed to disagree, that disagreement is +what a firmlink *is*, not evidence of a race. The TOCTOU guard, applied at +the listing level, could not tell a real swapped-directory attack apart +from completely ordinary firmlink resolution, and rejected both. + +The existing bulk-vs-fallback parity test (ADR 0002) could not have caught +this: its fixture is a temporary directory with no firmlinks or mount +points, so both enumeration paths agreed with each other and both were +equally wrong about the real world. + +## Decision + +Directory identity, the mount-boundary check (is this entry on a different +volume than its parent, and should the scan cross it), and alias +deduplication (has this physical location already been visited under a +different path, e.g. `/Users` vs. `/System/Volumes/Data/Users`) are now all +decided from the *opened* directory, never from the parent listing's report +of it. Concretely: open the directory first, ask the open descriptor for +its `(dev, ino)`, and make every identity-sensitive decision from that +value. The listing is used only to know what names exist and to get a +cheap first-pass size/type hint; it is never treated as authoritative about +identity. + +This is not just a bug fix but a strictly more correct model for two +things that were already true before firmlinks entered the picture: + +- `/dev` enumerates with the root filesystem's device number, but is + actually `devfs`; only the opened identity reveals that. +- Firmlink aliases only collide (resolve to the same underlying location) + *after* opening; comparing pre-open identities can never detect the + aliasing that alias-dedup exists to catch in the first place. + +A regression test scans the real startup volume shallowly and asserts +`/Users` (or the equivalent well-known firmlink on the test machine) is +reported with nonzero size; it fails against the pre-fix commit with +`/Users` at 0 bytes, and passes after. + +## Consequences + +- Any future scanner change that adds a new identity-sensitive decision + (a new dedup rule, a new mount-crossing rule, a new "have I seen this + before" check) must key it off the identity obtained after `open()`, + not off anything read from a parent directory's listing. This is the + single most important invariant in the scanner: violating it silently + drops firmlinked data with no error, no crash, and a plausible-looking + (merely wrong) total. +- There is deliberately no TOCTOU guard left at the listing level anymore. + A real swap-after-list race is not defended against by comparing + listing-time identity to open-time identity, because that comparison + cannot distinguish an attack from a firmlink. If TOCTOU hardening is + wanted again, it needs a different signal than pre-open vs. post-open + identity. +- Whoever touches this code next should re-run the shallow `/` scan + regression test (not just the synthetic-fixture parity test) before + trusting any change to enumeration, identity, or dedup logic; a synthetic + temp-directory fixture will never exercise firmlinks and cannot catch + this class of bug.