diff --git a/package-lock.json b/package-lock.json index f0fb618770..473caf8026 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16548,6 +16548,7 @@ }, "node_modules/move-file": { "version": "3.1.0", + "dev": true, "license": "MIT", "dependencies": { "path-exists": "^5.0.0" @@ -23157,10 +23158,7 @@ "cpy": "^11.0.0", "get-stream": "^9.0.0", "globby": "^14.0.0", - "junk": "^4.0.0", - "locate-path": "^7.0.0", - "move-file": "^3.0.0", - "readdirp": "^4.0.0" + "junk": "^4.0.0" }, "devDependencies": { "@types/node": "^22.12.0", diff --git a/packages/cache-utils/package.json b/packages/cache-utils/package.json index b7f60f98b8..3be9e000e2 100644 --- a/packages/cache-utils/package.json +++ b/packages/cache-utils/package.json @@ -53,10 +53,7 @@ "cpy": "^11.0.0", "get-stream": "^9.0.0", "globby": "^14.0.0", - "junk": "^4.0.0", - "locate-path": "^7.0.0", - "move-file": "^3.0.0", - "readdirp": "^4.0.0" + "junk": "^4.0.0" }, "devDependencies": { "@types/node": "^22.12.0", diff --git a/packages/cache-utils/src/fs.ts b/packages/cache-utils/src/fs.ts index 62101a9ee4..62a698f482 100644 --- a/packages/cache-utils/src/fs.ts +++ b/packages/cache-utils/src/fs.ts @@ -4,7 +4,6 @@ import { basename, dirname, join } from 'path' import cpy from 'cpy' import { type Options, globby } from 'globby' import { isNotJunk } from 'junk' -import { moveFile } from 'move-file' /** * Move or copy a cached file/directory from/to a local one @@ -15,7 +14,28 @@ import { moveFile } from 'move-file' export const moveCacheFile = async function (src: string, dest: string, move = false) { // Moving is faster but removes the source files locally if (move) { - return moveFile(src, dest, { overwrite: false }) + try { + await fs.access(dest) + throw new Error(`The destination file exists: ${dest}`) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error + } + } + + await fs.mkdir(dirname(dest), { recursive: true }) + + try { + await fs.rename(src, dest) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EXDEV') { + throw error + } + // Only happens when moving cross-device + await fs.cp(src, dest, { recursive: true, force: false, errorOnExist: true }) + await fs.rm(src, { recursive: true, force: true }) + } + return } const { srcGlob, dest: matchedDest, ...options } = await getSrcAndDest(src, dirname(dest)) diff --git a/packages/cache-utils/src/hash.ts b/packages/cache-utils/src/hash.ts index ad504c3632..58184f2723 100644 --- a/packages/cache-utils/src/hash.ts +++ b/packages/cache-utils/src/hash.ts @@ -1,8 +1,7 @@ import { createHash } from 'crypto' -import { createReadStream } from 'fs' +import { createReadStream, promises as fs } from 'fs' import getStream from 'get-stream' -import { locatePath } from 'locate-path' // We need a hashing algorithm that's as fast as possible. // Userland CRC32 implementations are actually slower than Node.js SHA1. @@ -11,13 +10,26 @@ const HASH_ALGO = 'sha1' // Caching a big directory like `node_modules` is slow. However, those can // sometime be represented by a digest file such as `package-lock.json`. If this // has not changed, we don't need to save cache again. -export const getHash = async function (digests, move) { +export const getHash = async function (digests: string[], move: boolean) { // Moving files is faster than computing hashes if (move || digests.length === 0) { return } - const digestPath = await locatePath(digests) + let digestPath: string | undefined = undefined + + for (const digest of digests) { + try { + const stat = await fs.stat(digest) + if (stat.isFile()) { + digestPath = digest + break + } + } catch { + continue + } + } + if (digestPath === undefined) { return } diff --git a/packages/cache-utils/src/list.ts b/packages/cache-utils/src/list.ts index c9e7c04adb..639ba9c868 100644 --- a/packages/cache-utils/src/list.ts +++ b/packages/cache-utils/src/list.ts @@ -1,6 +1,6 @@ -import { join } from 'path' - -import { readdirpPromise } from 'readdirp' +import { join, relative } from 'path' +import { readdir } from 'node:fs/promises' +import type { Dirent } from 'node:fs' import { getCacheDir } from './dir.js' import { isManifest } from './manifest.js' @@ -36,11 +36,33 @@ const listBase = async function ({ cacheDir: string depth?: number }) { - const files = await readdirpPromise(`${cacheDir}/${name}`, { fileFilter, depth, type: 'files_directories' }) - const filesA = files.map(({ path }) => join(base, path)) - return filesA + let queue = [{ path: `${cacheDir}/${name}`, depth: 0 }] + const results: string[] = [] + let queueEntry: { path: string; depth: number } | undefined + while ((queueEntry = queue.pop())) { + const { path: currentPath, depth: currentDepth } = queueEntry + let children: Dirent[] + try { + children = await readdir(currentPath, { withFileTypes: true }) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + continue + } + throw error + } + for (const child of children) { + const childPath = join(child.parentPath, child.name) + if ((child.isDirectory() || child.isFile()) && fileFilter(child.name)) { + results.push(join(base, relative(`${cacheDir}/${name}`, childPath))) + } + if ((depth === undefined || currentDepth < depth) && child.isDirectory()) { + queue.push({ path: childPath, depth: currentDepth + 1 }) + } + } + } + return results } -const fileFilter = function ({ basename }) { +const fileFilter = function (basename) { return !isManifest(basename) } diff --git a/packages/cache-utils/src/main.ts b/packages/cache-utils/src/main.ts index 61a0164c62..e13f5a8265 100644 --- a/packages/cache-utils/src/main.ts +++ b/packages/cache-utils/src/main.ts @@ -23,7 +23,7 @@ const saveOne = async function ( cwd?: string move?: boolean ttl?: number - digests?: any[] + digests?: string[] } = {}, ) { const { srcPath, cachePath } = await parsePath({ path, cacheDir, cwdOpt }) diff --git a/packages/cache-utils/tests/digests.test.ts b/packages/cache-utils/tests/digests.test.ts index d2b5c065d3..db29b6d741 100644 --- a/packages/cache-utils/tests/digests.test.ts +++ b/packages/cache-utils/tests/digests.test.ts @@ -41,6 +41,23 @@ test('Should allow caching according to several potential digests files', async } }) +test('Should ignore directory digests', async () => { + const [cacheDir, srcDir] = await Promise.all([createTmpDir(), createTmpDir()]) + try { + const srcFile = `${srcDir}/test` + const digestDir = `${srcDir}/digestDir` + await Promise.all([fs.writeFile(srcFile, 'test'), fs.mkdir(digestDir)]) + expect(await save(srcDir, { cacheDir, digests: [digestDir] })).toBe(true) + await fs.writeFile(srcFile, 'newTest') + expect(await save(srcDir, { cacheDir, digests: [digestDir] })).toBe(true) + expect(await restore(srcDir, { cacheDir, digests: [digestDir] })).toBe(true) + const content = await fs.readFile(srcFile, 'utf8') + expect(content).toBe('newTest') + } finally { + await removeFiles([cacheDir, srcDir]) + } +}) + test('Should ignore non-existing digests files', async () => { const [cacheDir, srcDir] = await Promise.all([createTmpDir(), createTmpDir()]) try {