From 203e3cb4d8255e28809e57a23093666b606e7185 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:56:33 +0100 Subject: [PATCH 1/3] perf(cache-utils): use native fs traversal Switches the following: - `locate-path` -> use a regular `for...of` around `fs.access` - `move-file` -> use `fs.rename` and `fs.cp` for cross-device - `readdirp` -> use `fs.readdir` --- package-lock.json | 9 ++++---- packages/cache-utils/package.json | 5 +---- packages/cache-utils/src/fs.ts | 24 +++++++++++++++++++-- packages/cache-utils/src/hash.ts | 16 +++++++++++--- packages/cache-utils/src/list.ts | 36 +++++++++++++++++++++++++------ 5 files changed, 70 insertions(+), 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9facd1e11c..d44e098fe6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12753,6 +12753,9 @@ }, "node_modules/fsevents": { "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, "license": "MIT", "optional": true, "os": [ @@ -16465,6 +16468,7 @@ }, "node_modules/move-file": { "version": "3.1.0", + "dev": true, "license": "MIT", "dependencies": { "path-exists": "^5.0.0" @@ -23077,10 +23081,7 @@ "cpy": "^11.0.0", "get-stream": "^9.0.0", "globby": "^14.0.0", - "junk": "^4.0.0", - "locate-path": "^7.0.0", - "move-file": "^3.0.0", - "readdirp": "^4.0.0" + "junk": "^4.0.0" }, "devDependencies": { "@types/node": "^22.12.0", diff --git a/packages/cache-utils/package.json b/packages/cache-utils/package.json index 068003ff2b..0acf707291 100644 --- a/packages/cache-utils/package.json +++ b/packages/cache-utils/package.json @@ -53,10 +53,7 @@ "cpy": "^11.0.0", "get-stream": "^9.0.0", "globby": "^14.0.0", - "junk": "^4.0.0", - "locate-path": "^7.0.0", - "move-file": "^3.0.0", - "readdirp": "^4.0.0" + "junk": "^4.0.0" }, "devDependencies": { "@types/node": "^22.12.0", diff --git a/packages/cache-utils/src/fs.ts b/packages/cache-utils/src/fs.ts index 62101a9ee4..62a698f482 100644 --- a/packages/cache-utils/src/fs.ts +++ b/packages/cache-utils/src/fs.ts @@ -4,7 +4,6 @@ import { basename, dirname, join } from 'path' import cpy from 'cpy' import { type Options, globby } from 'globby' import { isNotJunk } from 'junk' -import { moveFile } from 'move-file' /** * Move or copy a cached file/directory from/to a local one @@ -15,7 +14,28 @@ import { moveFile } from 'move-file' export const moveCacheFile = async function (src: string, dest: string, move = false) { // Moving is faster but removes the source files locally if (move) { - return moveFile(src, dest, { overwrite: false }) + try { + await fs.access(dest) + throw new Error(`The destination file exists: ${dest}`) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error + } + } + + await fs.mkdir(dirname(dest), { recursive: true }) + + try { + await fs.rename(src, dest) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EXDEV') { + throw error + } + // Only happens when moving cross-device + await fs.cp(src, dest, { recursive: true, force: false, errorOnExist: true }) + await fs.rm(src, { recursive: true, force: true }) + } + return } const { srcGlob, dest: matchedDest, ...options } = await getSrcAndDest(src, dirname(dest)) diff --git a/packages/cache-utils/src/hash.ts b/packages/cache-utils/src/hash.ts index ad504c3632..1500711c77 100644 --- a/packages/cache-utils/src/hash.ts +++ b/packages/cache-utils/src/hash.ts @@ -1,8 +1,7 @@ import { createHash } from 'crypto' -import { createReadStream } from 'fs' +import { createReadStream, promises as fs } from 'fs' import getStream from 'get-stream' -import { locatePath } from 'locate-path' // We need a hashing algorithm that's as fast as possible. // Userland CRC32 implementations are actually slower than Node.js SHA1. @@ -17,7 +16,18 @@ export const getHash = async function (digests, move) { return } - const digestPath = await locatePath(digests) + let digestPath = undefined + + for (const digest of digests) { + try { + await fs.access(digest) + digestPath = digest + break + } catch { + continue + } + } + if (digestPath === undefined) { return } diff --git a/packages/cache-utils/src/list.ts b/packages/cache-utils/src/list.ts index c9e7c04adb..639ba9c868 100644 --- a/packages/cache-utils/src/list.ts +++ b/packages/cache-utils/src/list.ts @@ -1,6 +1,6 @@ -import { join } from 'path' - -import { readdirpPromise } from 'readdirp' +import { join, relative } from 'path' +import { readdir } from 'node:fs/promises' +import type { Dirent } from 'node:fs' import { getCacheDir } from './dir.js' import { isManifest } from './manifest.js' @@ -36,11 +36,33 @@ const listBase = async function ({ cacheDir: string depth?: number }) { - const files = await readdirpPromise(`${cacheDir}/${name}`, { fileFilter, depth, type: 'files_directories' }) - const filesA = files.map(({ path }) => join(base, path)) - return filesA + let queue = [{ path: `${cacheDir}/${name}`, depth: 0 }] + const results: string[] = [] + let queueEntry: { path: string; depth: number } | undefined + while ((queueEntry = queue.pop())) { + const { path: currentPath, depth: currentDepth } = queueEntry + let children: Dirent[] + try { + children = await readdir(currentPath, { withFileTypes: true }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + continue + } + throw error + } + for (const child of children) { + const childPath = join(child.parentPath, child.name) + if ((child.isDirectory() || child.isFile()) && fileFilter(child.name)) { + results.push(join(base, relative(`${cacheDir}/${name}`, childPath))) + } + if ((depth === undefined || currentDepth < depth) && child.isDirectory()) { + queue.push({ path: childPath, depth: currentDepth + 1 }) + } + } + } + return results } -const fileFilter = function ({ basename }) { +const fileFilter = function (basename) { return !isManifest(basename) } From f6010b66cde6da1778466cad19af7a705001472c Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:54:28 +0100 Subject: [PATCH 2/3] chore: add strong types to getHash --- packages/cache-utils/src/hash.ts | 4 ++-- packages/cache-utils/src/main.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cache-utils/src/hash.ts b/packages/cache-utils/src/hash.ts index 1500711c77..feb28610df 100644 --- a/packages/cache-utils/src/hash.ts +++ b/packages/cache-utils/src/hash.ts @@ -10,13 +10,13 @@ const HASH_ALGO = 'sha1' // Caching a big directory like `node_modules` is slow. However, those can // sometime be represented by a digest file such as `package-lock.json`. If this // has not changed, we don't need to save cache again. -export const getHash = async function (digests, move) { +export const getHash = async function (digests: string[], move: boolean) { // Moving files is faster than computing hashes if (move || digests.length === 0) { return } - let digestPath = undefined + let digestPath: string | undefined = undefined for (const digest of digests) { try { diff --git a/packages/cache-utils/src/main.ts b/packages/cache-utils/src/main.ts index 61a0164c62..e13f5a8265 100644 --- a/packages/cache-utils/src/main.ts +++ b/packages/cache-utils/src/main.ts @@ -23,7 +23,7 @@ const saveOne = async function ( cwd?: string move?: boolean ttl?: number - digests?: any[] + digests?: string[] } = {}, ) { const { srcPath, cachePath } = await parsePath({ path, cacheDir, cwdOpt }) From 42d4d52a10ecbb587acc472e0a91b060a28123d8 Mon Sep 17 00:00:00 2001 From: James Garbutt <43081j@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:21:51 +0100 Subject: [PATCH 3/3] test: add digest-as-directory test --- packages/cache-utils/src/hash.ts | 8 +++++--- packages/cache-utils/tests/digests.test.ts | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/cache-utils/src/hash.ts b/packages/cache-utils/src/hash.ts index feb28610df..58184f2723 100644 --- a/packages/cache-utils/src/hash.ts +++ b/packages/cache-utils/src/hash.ts @@ -20,9 +20,11 @@ export const getHash = async function (digests: string[], move: boolean) { for (const digest of digests) { try { - await fs.access(digest) - digestPath = digest - break + const stat = await fs.stat(digest) + if (stat.isFile()) { + digestPath = digest + break + } } catch { continue } diff --git a/packages/cache-utils/tests/digests.test.ts b/packages/cache-utils/tests/digests.test.ts index d2b5c065d3..db29b6d741 100644 --- a/packages/cache-utils/tests/digests.test.ts +++ b/packages/cache-utils/tests/digests.test.ts @@ -41,6 +41,23 @@ test('Should allow caching according to several potential digests files', async } }) +test('Should ignore directory digests', async () => { + const [cacheDir, srcDir] = await Promise.all([createTmpDir(), createTmpDir()]) + try { + const srcFile = `${srcDir}/test` + const digestDir = `${srcDir}/digestDir` + await Promise.all([fs.writeFile(srcFile, 'test'), fs.mkdir(digestDir)]) + expect(await save(srcDir, { cacheDir, digests: [digestDir] })).toBe(true) + await fs.writeFile(srcFile, 'newTest') + expect(await save(srcDir, { cacheDir, digests: [digestDir] })).toBe(true) + expect(await restore(srcDir, { cacheDir, digests: [digestDir] })).toBe(true) + const content = await fs.readFile(srcFile, 'utf8') + expect(content).toBe('newTest') + } finally { + await removeFiles([cacheDir, srcDir]) + } +}) + test('Should ignore non-existing digests files', async () => { const [cacheDir, srcDir] = await Promise.all([createTmpDir(), createTmpDir()]) try {