diff --git a/__tests__/batch-lint.spec.ts b/__tests__/batch-lint.spec.ts index 968394f..6d3ab1a 100644 --- a/__tests__/batch-lint.spec.ts +++ b/__tests__/batch-lint.spec.ts @@ -3,7 +3,7 @@ import { availableParallelism, tmpdir } from 'os'; import * as path from 'path'; import { Piscina } from 'piscina'; import type { LintMdRulesConfig } from '@lint-md/core'; -import { batchLint, getMaxFileSize, resolveAdaptiveConcurrency, runTasksWithLimit } from '../src/utils/batch-lint'; +import { STAT_CONCURRENCY_LIMIT, batchLint, getMaxFileSize, resolveAdaptiveConcurrency, runTasksWithLimit } from '../src/utils/batch-lint'; const RULES_NO_EMPTY_LIST: LintMdRulesConfig = { 'no-empty-list': 2, @@ -214,6 +214,41 @@ describe('getMaxFileSize', () => { const missing = path.join(tmpDir, 'missing.md'); await expect(getMaxFileSize([missing])).rejects.toThrow(); }); + + test('bounds concurrent stat calls to STAT_CONCURRENCY_LIMIT', async () => { + const fileCount = STAT_CONCURRENCY_LIMIT * 3; + const filePaths = Array.from({ length: fileCount }, (_, index) => + path.join(tmpDir, `bounded-${index}.md`) + ); + const sizeMap = new Map(filePaths.map((filePath, index) => [filePath, index + 1])); + + let inFlight = 0; + let maxInFlight = 0; + + const fsPromises = require('fs/promises'); + const statSpy = jest.spyOn(fsPromises, 'stat').mockImplementation((filePath: string) => { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + return new Promise((resolve) => { + setTimeout(() => { + inFlight--; + resolve({ size: sizeMap.get(filePath) ?? 0 }); + }, 20); + }); + }); + + try { + const result = await getMaxFileSize(filePaths); + + expect(result).toBe(fileCount); + expect(statSpy).toHaveBeenCalledTimes(fileCount); + expect(maxInFlight).toBeGreaterThan(1); + expect(maxInFlight).toBeLessThanOrEqual(STAT_CONCURRENCY_LIMIT); + } + finally { + statSpy.mockRestore(); + } + }); }); describe('resolveAdaptiveConcurrency', () => { diff --git a/src/utils/batch-lint.ts b/src/utils/batch-lint.ts index e39137f..88fbd43 100644 --- a/src/utils/batch-lint.ts +++ b/src/utils/batch-lint.ts @@ -11,6 +11,8 @@ const FIVE_MIB = 5 * ONE_MIB; const ADAPTIVE_MEDIUM_CAP = 2; const ADAPTIVE_LARGE_FILE_THRESHOLD = ONE_MIB; const ADAPTIVE_HUGE_FILE_THRESHOLD = FIVE_MIB; +// Upper bound on concurrent stat() fds in getMaxFileSize (see #80). +export const STAT_CONCURRENCY_LIMIT = 128; export async function runTasksWithLimit( tasks: (() => Promise)[], @@ -39,15 +41,23 @@ const resolveWorkerFilename = (): string => { return path.resolve(__dirname, '../../lib/src/utils/lint-worker.js'); }; -// TODO(perf, #78): getMaxFileSize() currently stats every file with one -// Promise.all. For very large repositories, consider chunking stat calls -// or short-circuiting once a file >= ADAPTIVE_HUGE_FILE_THRESHOLD is found. +// getMaxFileSize() stats every file but bounds the in-flight stat calls to +// STAT_CONCURRENCY_LIMIT via runTasksWithLimit, avoiding an N-fd filesystem +// burst on very large repositories (see #80). +// We bound concurrency rather than short-circuit on a >= 5 MiB file: this +// function must return the TRUE maximum size, because src/lint-md.ts logs +// `max file ${maxMiB} MiB` under --threads auto --dev. An early return would +// make that log (and any future consumer) inaccurate. STAT_CONCURRENCY_LIMIT +// is an empirical cap (the issue suggests 64/128), not benchmark-derived. export const getMaxFileSize = async (filePaths: string[]): Promise => { if (filePaths.length === 0) { return 0; } - const stats = await Promise.all(filePaths.map(filePath => stat(filePath))); - return stats.reduce((max, current) => (current.size > max ? current.size : max), 0); + const sizes = await runTasksWithLimit( + filePaths.map(filePath => () => stat(filePath).then(stats => stats.size)), + STAT_CONCURRENCY_LIMIT + ); + return sizes.reduce((max, current) => (current > max ? current : max), 0); }; export const resolveAdaptiveConcurrency = async (