diff --git a/lib/index.js b/lib/index.js index c99114b..a2c2b42 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,19 +1,141 @@ -const crypto = require('node:crypto'); -const fs = require('node:fs'); +const os = require('node:os'); const path = require('node:path'); -const zlib = require('node:zlib'); +const { Worker } = require('node:worker_threads'); -const babel = require('@babel/core'); const chokidar = require('chokidar'); -const mime = require('mime'); const Negotiator = require('negotiator'); -const sass = require('sass'); -const sassGraph = require('sass-graph'); -const Snockets = require('snockets'); -const UglifyJS = require('uglify-js'); -const UglifyCss = require('uglifycss'); -const gzipContentTypes = require('./gzipContentTypes.js'); +const { createProcessor, hashifyUrl, parseUrlPath } = require('./processor.js'); + +/** + * Restores Buffers lost to structured cloning. Binary content posted from the + * worker arrives as a Uint8Array; wrap it back into a Buffer so it behaves like + * a synchronously-read file. + * @param {object} file + */ +function reviveFile(file) { + if (typeof file.content !== 'string') { + file.content = Buffer.from(file.content.buffer, file.content.byteOffset, file.content.byteLength); + } + + if (file.gzip && typeof file.gzip.content !== 'string') { + file.gzip.content = Buffer.from(file.gzip.content.buffer, file.gzip.content.byteOffset, file.gzip.content.byteLength); + } + + return file; +} + +// A shared pool of worker threads warms every electricity.static() instance in +// the process. The pool is created lazily on the first warm job and reused from +// then on, so the heavy transform libraries (Babel/Sass/UglifyJS) load once per +// worker rather than once per instance. It's sized to leave a core free for the +// event loop that serves live traffic. +const WARM_POOL_SIZE = Math.max(1, os.availableParallelism() - 1); + +let warmPool = null; +let warmPoolFailed = false; +let nextWarmJobId = 0; + +// Maps a job id to the processor receiving that job's entries and the number of +// shards still working on it. +const warmJobs = new Map(); + +function handleWarmMessage({ done, file, jobId, urlPath }) { + const job = warmJobs.get(jobId); + + if (!job) { + return; + } + + if (done) { + job.pending--; + + if (job.pending === 0) { + warmJobs.delete(jobId); + } + + return; + } + + // Don't clobber an entry a request already populated via the synchronous + // fallback, or one another shard already posted; all produce identical + // content. + if (!job.processor.files[urlPath]) { + job.processor.files[urlPath] = reviveFile(file); + } +} + +/** + * Returns the shared warm worker pool, creating it on first use. Returns null + * if the pool can't be created, in which case warming is skipped and the + * synchronous read remains the only path. + */ +function getWarmPool() { + if (warmPool || warmPoolFailed) { + return warmPool; + } + + const pool = []; + + try { + for (let i = 0; i < WARM_POOL_SIZE; i++) { + const worker = new Worker(path.join(__dirname, 'warmer.js')); + + worker.on('message', handleWarmMessage); + worker.on('error', (err) => { + //eslint-disable-next-line no-console + console.warn(`Electricity cache warming error:\n ${err}`); + }); + + // Never let warming hold the process open. + worker.unref(); + + pool.push(worker); + } + } catch (err) { + warmPoolFailed = true; + pool.forEach(worker => worker.terminate()); + //eslint-disable-next-line no-console + console.warn(`Electricity skipping cache warming:\n ${err}`); + return null; + } + + warmPool = pool; + + return warmPool; +} + +/** + * Queues a warm job across the worker pool. Each worker reads and processes a + * disjoint shard of the directory's files, posting finished cache entries back + * so the expensive first read (Sass/Babel/UglifyJS/gzip) stays off the event + * loop. + * @param {string} directory + * @param {object} options + * @param {object} processor + */ +function startWarmer(directory, options, processor) { + const pool = getWarmPool(); + + if (!pool) { + return; + } + + const jobId = nextWarmJobId++; + + warmJobs.set(jobId, { pending: pool.length, processor }); + + try { + pool.forEach((worker, shardIndex) => { + worker.postMessage({ directory, jobId, options, shardCount: pool.length, shardIndex }); + }); + } catch (err) { + // options aren't structured-cloneable (e.g. a Sass importer or a Babel + // plugin passed as a function); leave warming off for this instance and + // rely on the synchronous read. + warmJobs.delete(jobId); + } +} exports.static = (directory, options) => { // Default to 'public' if the directory is not specified @@ -73,329 +195,34 @@ exports.static = (directory, options) => { }; } - // Create a local cache to hold the files - const files = {}; - - const snockets = new Snockets(); - let watcher; if (options.watch.enabled) { // Setup the watcher watcher = chokidar.watch(directory, { ignoreInitial: true }); - - watcher.on('all', (eventName, filePath) => { - removeFile(filePath); - }); } - /** - * Tries to read a file from local cache. - * Reads the file from disk if it's not present in the local cache. - * @param {string} urlPath - */ - function fetchFile(urlPath) { - // Try to get the file from local cache - let file = files[urlPath]; - - // Return the file from cache if found - if (file) { - return file; - } - - // Read the file from disk - file = readFile(urlPath); - - // Put the file in local cache - files[urlPath] = file; - - return file; - } - - /** - * Converts a URL (/robots.txt) to a URL that includes the file's hash (/robots-3f54004ef6fc21b24a9e6069fc114fd9070b77a1.txt) - * @param {string} url - * @param {string} hash - */ - function hashifyUrl(url, hash) { - if (!url.includes('.')) { - return url.replace(/([?#].*)?$/, `-${hash}$1`); - } - - return url.replace(/\.([^.]*)([?#].*)?$/, `-${hash}.$1$2`); - } - - /** - * Parses a URL path potentially containing a hash (/robots-3f54004ef6fc21b24a9e6069fc114fd9070b77a1.txt) - * into an object with a hash and path properties ({ hash: '3f54004ef6fc21b24a9e6069fc114fd9070b77a1', path: '/robots.txt' }) - * @param {object} req - */ - function parseUrlPath(urlPath) { - // https://regex101.com/r/j5hvRj/2 - const regex = /\/.+(-([0-9a-f]{32,40}))/; - const matches = urlPath.match(regex); - - if (!matches) { - return { - path: urlPath - }; - } + const processor = createProcessor(directory, options, watcher); - return { - hash: matches[2], - path: urlPath.replace(matches[1], '') - }; - } - - function readCascadingStyleSheetsFile(filePath) { - let data; - - // CSS - try { - data = fs.readFileSync(filePath).toString(); - } catch (err) { - // Handle ENOENT (No such file or directory): https://nodejs.org/api/errors.html#common-system-errors - if (err.code !== 'ENOENT') { - throw err; - } - - // SASS - const basename = path.basename(filePath, path.extname(filePath)); - const sassFile = path.join(path.dirname(filePath), `${basename}.scss`); - const result = sass.compile(sassFile, options.sass); - - data = result.css; - - // SASS (watcher) - if (watcher) { - result.loadedUrls.forEach(file => { - watcher.add(file.pathname); - }); - } - } - - // Update URLs in CSS: https://regex101.com/r/FxrppP/4 - data = data.replace(/url\(['"]?(.*?)['"]?\)/g, (match, p1) => { - return `url(${urlBuilder(p1)})`; + if (watcher) { + watcher.on('all', (eventName, filePath) => { + processor.removeFile(filePath); }); - - // UglifyCSS - if (options.uglifycss.enabled) { - data = UglifyCss.processString(data, options.uglifycss); - } - - return data; } - function readFile(urlPath) { - let filePath = toFilePath(urlPath); - let extension = path.extname(filePath); - let data; - - if (extension === '.css') { - data = readCascadingStyleSheetsFile(filePath); - } else if (extension === '.js') { - data = readJavaScriptFile(filePath); - } else { - data = fs.readFileSync(filePath); - } - - const file = { - content: data, - contentLength: data.length, - contentType: mime.getType(urlPath), - hash: crypto.createHash('sha1').update(data).digest('hex') - }; - - // Don't gzip any content less that 1500 bytes (the size of a TCP packet). Only gzip specific content types. - if (options.gzip.enabled && file.contentLength > 1500 && gzipContentTypes.includes(file.contentType)) { - const gzipContent = zlib.gzipSync(file.content); - - file.gzip = { - content: gzipContent, - contentLength: gzipContent.length - }; - } - - return file; - } - - function readJavaScriptFile(filePath) { - let data; - - // Snockets - try { - data = snockets.getConcatenation(filePath, options.snockets); - } catch(err) { - // Snockets can't parse, so just pass the js file along - //eslint-disable-next-line no-console - console.warn(`Snockets skipping ${filePath}:\n ${err}`); - } - - // Snockets (watcher) - if (watcher) { - try { - // Get all files in the snockets chain - const compiledChain = snockets.getCompiledChain(filePath, options.snockets); - - // Add each file of the snockets chain to the watcher - compiledChain.forEach(c => { - watcher.add(c.filename); - }); - } catch(err) { - // Snockets can't parse, so skip watch - //eslint-disable-next-line no-console - console.warn(`Snockets skipping watch for ${filePath}:\n ${err}`); - } - } - - // If Snockets didn't parse the file, read it from disk - if (!data) { - data = fs.readFileSync(filePath).toString(); - } - - // Babel - try { - let result = babel.transformSync(data, { - ...options.babel, - presets: [require('@babel/preset-react')] - }); - - data = result.code; - } catch(err) { - // Babel can't transform, so just pass the file along - //eslint-disable-next-line no-console - console.warn(`Babel skipping ${filePath}:\n ${err}`); - } - - // UglifyJS - if (options.uglifyjs.enabled) { - const uglifyjsOptions = JSON.parse(JSON.stringify(options.uglifyjs)); - delete uglifyjsOptions.enabled; - - const result = UglifyJS.minify(data, uglifyjsOptions); - - if (result.error) { - //eslint-disable-next-line no-console - console.warn(`UglifyJS skipping ${filePath}:\n ${JSON.stringify(result.error)}`); - } else { - data = result.code; - } - } - - return data; - } - - /** - * Removes a file from the local cache. - * @param {*} filePath - */ - function removeFile(filePath) { - let extension = path.extname(filePath); - - if (extension === '.js') { - return removeJavaScriptFile(filePath); - } else if (extension === '.scss') { - return removeSassFile(filePath); - } - - // Remove the changed file from the local cache - delete files[toUrlPath(filePath)]; - } - - /** - * Removes a JavaScript file from the local cache. - * @param {string} filePath - */ - function removeJavaScriptFile(filePath) { - // Remove the changed file from the local cache - delete files[toUrlPath(filePath)]; - - // Resolve the absolute file path for the changed file - const absoluteFilePath = path.resolve(filePath); - - // Find any parents that have a dependency on this file and remove them too - snockets.depGraph.parentsOf(absoluteFilePath).forEach(removeJavaScriptFile); - } - - /** - * Removes a SASS file from the local cache. - * @param {string} filePath - */ - function removeSassFile(filePath) { - const basename = path.basename(filePath, path.extname(filePath)); - const cssFilePath = path.join(path.dirname(filePath), `${basename}.css`); - const urlPath = toUrlPath(cssFilePath); - - // Remove the changed file from the local cache - delete files[urlPath]; - - // Resolve the absolute file path for the changed file - let absoluteFilePath = path.resolve(filePath); - - // Try to resolve symlinks - try { - absoluteFilePath = fs.realpathSync(filePath); - } catch (e) { - // ignore error - } - - const graph = sassGraph.parseDir(directory); - const sassFile = graph.index[absoluteFilePath]; - - if (sassFile) { - sassFile.importedBy.forEach(removeSassFile); - } - } - - function urlBuilder(urlPath) { - let file; - const request = parseUrlPath(urlPath); - let url = urlPath; - - try { - file = fetchFile(request.path); - } catch(err) { - // If we don't have a file that matches the specified URL path simply return the original URL path - return urlPath; - } - - if (options.hashify) { - url = hashifyUrl(request.path, file.hash); - } - - if (options.hostname) { - url = `https://${options.hostname}${url}`; - } - - return url; - } - - /** - * Converts a URL path (/robots.txt) to a file path (/Users/username/site/public/robots.txt). - * @param {string} urlPath - */ - function toFilePath(urlPath) { - const myURL = new URL(urlPath, 'https://example.org/'); - const pathname = myURL.pathname.replace(/^\//, ''); - return path.resolve(directory, pathname); - } - - /** - * Converts a file path (/Users/username/site/public/robots.txt) to a URL path (/robots.txt). - * @param {string} urlPath - */ - function toUrlPath(filePath) { - const urlPath = path.posix.relative(directory, path.resolve(filePath)); - - return `/${urlPath}`; + // Warm the cache on a background thread so the first request for a file + // doesn't read and process it synchronously on the event loop. Watching + // keeps processing on the main thread (it registers discovered dependencies + // with the watcher), so warming is skipped while watching. + if (!options.watch.enabled) { + startWarmer(directory, options, processor); } return function staticMiddleware(req, res, next) { // Register function in app.locals to help views build URLs: https://expressjs.com/en/api.html#app.locals if (req.app && !req.app.locals.electricity) { req.app.locals.electricity = { - url: urlBuilder + url: processor.urlBuilder }; } @@ -408,7 +235,7 @@ exports.static = (directory, options) => { const request = parseUrlPath(req.path); try { - file = fetchFile(request.path); + file = processor.fetchFile(request.path); } catch (err) { // Handle EISDIR (Is a directory): https://nodejs.org/api/errors.html#common-system-errors if (err.code === 'EISDIR') { @@ -491,4 +318,4 @@ exports.static = (directory, options) => { res.send(content); }; -}; \ No newline at end of file +}; diff --git a/lib/processor.js b/lib/processor.js new file mode 100644 index 0000000..380b4aa --- /dev/null +++ b/lib/processor.js @@ -0,0 +1,342 @@ +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const zlib = require('node:zlib'); + +const babel = require('@babel/core'); +const mime = require('mime'); +const sass = require('sass'); +const sassGraph = require('sass-graph'); +const Snockets = require('snockets'); +const UglifyJS = require('uglify-js'); +const UglifyCss = require('uglifycss'); + +const gzipContentTypes = require('./gzipContentTypes.js'); + +/** + * Converts a URL (/robots.txt) to a URL that includes the file's hash (/robots-3f54004ef6fc21b24a9e6069fc114fd9070b77a1.txt) + * @param {string} url + * @param {string} hash + */ +exports.hashifyUrl = function hashifyUrl(url, hash) { + if (!url.includes('.')) { + return url.replace(/([?#].*)?$/, `-${hash}$1`); + } + + return url.replace(/\.([^.]*)([?#].*)?$/, `-${hash}.$1$2`); +}; + +/** + * Parses a URL path potentially containing a hash (/robots-3f54004ef6fc21b24a9e6069fc114fd9070b77a1.txt) + * into an object with a hash and path properties ({ hash: '3f54004ef6fc21b24a9e6069fc114fd9070b77a1', path: '/robots.txt' }) + * @param {string} urlPath + */ +exports.parseUrlPath = function parseUrlPath(urlPath) { + // https://regex101.com/r/j5hvRj/2 + const regex = /\/.+(-([0-9a-f]{32,40}))/; + const matches = urlPath.match(regex); + + if (!matches) { + return { + path: urlPath + }; + } + + return { + hash: matches[2], + path: urlPath.replace(matches[1], '') + }; +}; + +/** + * Creates a file processor bound to a directory and a set of options. + * The processor owns its own in-memory cache, so the same logic can run on + * the main thread (serving requests) or inside a worker thread (warming the + * cache). When no watcher is supplied (the production / worker case) the + * watch-only side effects are skipped, so both contexts produce identical + * cache entries for the same file. + * @param {string} directory + * @param {object} options + * @param {object} [watcher] + */ +exports.createProcessor = function createProcessor(directory, options, watcher) { + // Create a local cache to hold the files + const files = {}; + + const snockets = new Snockets(); + + /** + * Tries to read a file from local cache. + * Reads the file from disk if it's not present in the local cache. + * @param {string} urlPath + */ + function fetchFile(urlPath) { + // Try to get the file from local cache + let file = files[urlPath]; + + // Return the file from cache if found + if (file) { + return file; + } + + // Read the file from disk + file = readFile(urlPath); + + // Put the file in local cache + files[urlPath] = file; + + return file; + } + + function readCascadingStyleSheetsFile(filePath) { + let data; + + // CSS + try { + data = fs.readFileSync(filePath).toString(); + } catch (err) { + // Handle ENOENT (No such file or directory): https://nodejs.org/api/errors.html#common-system-errors + if (err.code !== 'ENOENT') { + throw err; + } + + // SASS + const basename = path.basename(filePath, path.extname(filePath)); + const sassFile = path.join(path.dirname(filePath), `${basename}.scss`); + const result = sass.compile(sassFile, options.sass); + + data = result.css; + + // SASS (watcher) + if (watcher) { + result.loadedUrls.forEach(file => { + watcher.add(file.pathname); + }); + } + } + + // Update URLs in CSS: https://regex101.com/r/FxrppP/4 + data = data.replace(/url\(['"]?(.*?)['"]?\)/g, (match, p1) => { + return `url(${urlBuilder(p1)})`; + }); + + // UglifyCSS + if (options.uglifycss.enabled) { + data = UglifyCss.processString(data, options.uglifycss); + } + + return data; + } + + function readFile(urlPath) { + let filePath = toFilePath(urlPath); + let extension = path.extname(filePath); + let data; + + if (extension === '.css') { + data = readCascadingStyleSheetsFile(filePath); + } else if (extension === '.js') { + data = readJavaScriptFile(filePath); + } else { + data = fs.readFileSync(filePath); + } + + const file = { + content: data, + contentLength: data.length, + contentType: mime.getType(urlPath), + hash: crypto.createHash('sha1').update(data).digest('hex') + }; + + // Don't gzip any content less that 1500 bytes (the size of a TCP packet). Only gzip specific content types. + if (options.gzip.enabled && file.contentLength > 1500 && gzipContentTypes.includes(file.contentType)) { + const gzipContent = zlib.gzipSync(file.content); + + file.gzip = { + content: gzipContent, + contentLength: gzipContent.length + }; + } + + return file; + } + + function readJavaScriptFile(filePath) { + let data; + + // Snockets + try { + data = snockets.getConcatenation(filePath, options.snockets); + } catch(err) { + // Snockets can't parse, so just pass the js file along + //eslint-disable-next-line no-console + console.warn(`Snockets skipping ${filePath}:\n ${err}`); + } + + // Snockets (watcher) + if (watcher) { + try { + // Get all files in the snockets chain + const compiledChain = snockets.getCompiledChain(filePath, options.snockets); + + // Add each file of the snockets chain to the watcher + compiledChain.forEach(c => { + watcher.add(c.filename); + }); + } catch(err) { + // Snockets can't parse, so skip watch + //eslint-disable-next-line no-console + console.warn(`Snockets skipping watch for ${filePath}:\n ${err}`); + } + } + + // If Snockets didn't parse the file, read it from disk + if (!data) { + data = fs.readFileSync(filePath).toString(); + } + + // Babel + try { + let result = babel.transformSync(data, { + ...options.babel, + presets: [require('@babel/preset-react')] + }); + + data = result.code; + } catch(err) { + // Babel can't transform, so just pass the file along + //eslint-disable-next-line no-console + console.warn(`Babel skipping ${filePath}:\n ${err}`); + } + + // UglifyJS + if (options.uglifyjs.enabled) { + const uglifyjsOptions = JSON.parse(JSON.stringify(options.uglifyjs)); + delete uglifyjsOptions.enabled; + + const result = UglifyJS.minify(data, uglifyjsOptions); + + if (result.error) { + //eslint-disable-next-line no-console + console.warn(`UglifyJS skipping ${filePath}:\n ${JSON.stringify(result.error)}`); + } else { + data = result.code; + } + } + + return data; + } + + /** + * Removes a file from the local cache. + * @param {*} filePath + */ + function removeFile(filePath) { + let extension = path.extname(filePath); + + if (extension === '.js') { + return removeJavaScriptFile(filePath); + } else if (extension === '.scss') { + return removeSassFile(filePath); + } + + // Remove the changed file from the local cache + delete files[toUrlPath(filePath)]; + } + + /** + * Removes a JavaScript file from the local cache. + * @param {string} filePath + */ + function removeJavaScriptFile(filePath) { + // Remove the changed file from the local cache + delete files[toUrlPath(filePath)]; + + // Resolve the absolute file path for the changed file + const absoluteFilePath = path.resolve(filePath); + + // Find any parents that have a dependency on this file and remove them too + snockets.depGraph.parentsOf(absoluteFilePath).forEach(removeJavaScriptFile); + } + + /** + * Removes a SASS file from the local cache. + * @param {string} filePath + */ + function removeSassFile(filePath) { + const basename = path.basename(filePath, path.extname(filePath)); + const cssFilePath = path.join(path.dirname(filePath), `${basename}.css`); + const urlPath = toUrlPath(cssFilePath); + + // Remove the changed file from the local cache + delete files[urlPath]; + + // Resolve the absolute file path for the changed file + let absoluteFilePath = path.resolve(filePath); + + // Try to resolve symlinks + try { + absoluteFilePath = fs.realpathSync(filePath); + } catch (e) { + // ignore error + } + + const graph = sassGraph.parseDir(directory); + const sassFile = graph.index[absoluteFilePath]; + + if (sassFile) { + sassFile.importedBy.forEach(removeSassFile); + } + } + + function urlBuilder(urlPath) { + let file; + const request = exports.parseUrlPath(urlPath); + let url = urlPath; + + try { + file = fetchFile(request.path); + } catch(err) { + // If we don't have a file that matches the specified URL path simply return the original URL path + return urlPath; + } + + if (options.hashify) { + url = exports.hashifyUrl(request.path, file.hash); + } + + if (options.hostname) { + url = `https://${options.hostname}${url}`; + } + + return url; + } + + /** + * Converts a URL path (/robots.txt) to a file path (/Users/username/site/public/robots.txt). + * @param {string} urlPath + */ + function toFilePath(urlPath) { + const myURL = new URL(urlPath, 'https://example.org/'); + const pathname = myURL.pathname.replace(/^\//, ''); + return path.resolve(directory, pathname); + } + + /** + * Converts a file path (/Users/username/site/public/robots.txt) to a URL path (/robots.txt). + * @param {string} filePath + */ + function toUrlPath(filePath) { + const urlPath = path.posix.relative(directory, path.resolve(filePath)); + + return `/${urlPath}`; + } + + return { + files, + fetchFile, + removeFile, + toUrlPath, + urlBuilder + }; +}; diff --git a/lib/warmer.js b/lib/warmer.js new file mode 100644 index 0000000..73f95d3 --- /dev/null +++ b/lib/warmer.js @@ -0,0 +1,98 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const { parentPort } = require('node:worker_threads'); + +const { createProcessor } = require('./processor.js'); + +/** + * Recursively yields every regular file beneath a directory. Reads are + * synchronous on purpose: this runs on a worker thread, so blocking here never + * touches the main event loop. + * @param {string} dir + */ +function* walk(dir) { + let entries; + + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch (err) { + // Missing or unreadable directory: nothing to warm. + return; + } + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + + if (entry.isDirectory()) { + yield* walk(fullPath); + } else if (entry.isFile()) { + yield fullPath; + } + } +} + +/** + * Maps a file on disk to the URL path(s) that should be warmed for it. + * Sass sources are served as their compiled `.css` URL, and partials + * (`_foo.scss`) are never served on their own, so they're skipped. + * @param {object} processor + * @param {string} filePath + */ +function urlPathsToWarm(processor, filePath) { + const extension = path.extname(filePath); + + if (extension === '.scss') { + if (path.basename(filePath).startsWith('_')) { + return []; + } + + const cssFilePath = `${filePath.slice(0, -extension.length)}.css`; + + return [processor.toUrlPath(cssFilePath)]; + } + + return [processor.toUrlPath(filePath)]; +} + +// A shared pool of workers warms every electricity.static() instance in the +// process. Each warm job is fanned out to the whole pool, with each worker +// handling a disjoint shard of the directory's files (every shardCount-th file +// starting at shardIndex). Files are sorted so the shards are deterministic and +// cover the tree exactly once. Each worker creates its own processor, so a CSS +// file's url() dependencies resolve within that worker even when the referenced +// asset belongs to another shard; the entries stay byte-identical to a +// synchronous read because hashing is deterministic. +parentPort.on('message', ({ directory, jobId, options, shardCount, shardIndex }) => { + const processor = createProcessor(directory, options); + const filePaths = [...walk(directory)].sort(); + const warmed = new Set(); + + for (let i = 0; i < filePaths.length; i++) { + if (i % shardCount !== shardIndex) { + continue; + } + + for (const urlPath of urlPathsToWarm(processor, filePaths[i])) { + if (warmed.has(urlPath)) { + continue; + } + + warmed.add(urlPath); + + let file; + + try { + file = processor.fetchFile(urlPath); + } catch (err) { + // Couldn't process this file (e.g. it's a directory or + // unreadable). The main thread's synchronous fallback will + // surface any real error when the file is actually requested. + continue; + } + + parentPort.postMessage({ file, jobId, urlPath }); + } + } + + parentPort.postMessage({ done: true, jobId }); +}); diff --git a/test/index.js b/test/index.js index 6880000..812179b 100644 --- a/test/index.js +++ b/test/index.js @@ -84,6 +84,34 @@ test('electricity.static', { concurrency: true }, async (t) => { }); }); + t.test('should not throw when options can not be cloned to the warm worker', async () => { + // A Sass importer is a function, which can't be structured-cloned to + // the background warm worker. static() must not throw; warming is + // simply skipped for this instance and the synchronous read still + // serves the file. + await new Promise((resolve) => { + const middleware = electricity.static('test/public', { + sass: { importers: [() => null] } + }); + + const req = { + get: () => {}, + method: 'GET', + path: '/robots.txt' + }; + + const res = { + redirect: (path) => { + assert.strictEqual(path, '/robots-423251d722a53966eb9368c65bfd14b39649105d.txt'); + resolve(); + }, + set: () => {} + }; + + middleware(req, res); + }); + }); + t.test('babel', async (t) => { t.test('preset-react', { concurrency: true }, async (t) => { t.test('should transform JSX files', async () => {