From 216c3d742ad416d359fbf33f70c0d6304d00886d Mon Sep 17 00:00:00 2001 From: Mikko Tiihonen Date: Sun, 10 Apr 2016 02:43:02 +0300 Subject: [PATCH 1/2] Add support for precompressed (gzip) content --- .gitattributes | 2 + HISTORY.md | 5 ++ README.md | 36 ++++++++++++ index.js | 100 ++++++++++++++++++++++++++++++--- package.json | 4 +- test/fixtures/name.html.bz2 | Bin 0 -> 50 bytes test/fixtures/name.html.gz | Bin 0 -> 31 bytes test/send.js | 107 ++++++++++++++++++++++++++++++++++++ 8 files changed, 245 insertions(+), 9 deletions(-) create mode 100644 .gitattributes create mode 100644 test/fixtures/name.html.bz2 create mode 100644 test/fixtures/name.html.gz diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..b7979741 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.bz2 binary +*.gz binary diff --git a/HISTORY.md b/HISTORY.md index 1eacafa2..aae988fd 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,8 @@ +unreleased +========== + + * Send precompressed variant of content based on `Accept-Encoding` + 0.15.1 / 2017-03-04 =================== diff --git a/README.md b/README.md index 0c8d11df..f581d2ea 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,42 @@ Provide a max-age in milliseconds for http caching, defaults to 0. This can also be a string accepted by the [ms](https://www.npmjs.org/package/ms#readme) module. +##### precompressed + +Precompressed files are extra static files that are compressed before +they are requested, as opposed to compressing on the fly. Compressing +files once offline (for example during site build) allows using +stronger compression methods and both reduces latency and lowers cpu +usage when serving files. + +The `precompressed` option enables or disables serving of precompressed +content variants. The option defaults to `false`, if set to `true` checks +for existence of gzip compressed files with `.gz` extensions. + +Example scenario: + +The file `site.css` has both `site.css.gz` and `site.css.bz2` +precompressed versions available in the same directory. The server is configured +to serve both `.bz2` and `.gz` files in that prefence order. +When a request comes with an `Accept-Encoding` header with value `gzip, bz2` +requesting `site.css` the contents of `site.css.bz2` is sent instead and +a header `Content-Encoding` with value `br` is added to the response. +In addition a `Vary: Accept-Encoding` header is added to response allowing +caching proxies to work correctly. + +Custom configuration: + +It is also possible to customize the searched file extensions and header +values (used with Accept-Encoding and Content-Encoding headers) by specifying +them explicitly in an array in the preferred priority order. For example: +`[{encoding: 'bzip2', extension: '.bz2'}, {encoding: 'gzip', extension: '.gz'}]`. + +Compression tips: + * Precompress at least all static `js`, `css` and `svg` files. + * Precompress using both brotli (supported by Firefox and Chrome) and + gzip encoders. Brotli compresses generally 15-20% better than gzip. + * Use zopfli for gzip compression for and extra 5% benefit for all browsers. + ##### root Serve files relative to `path`. diff --git a/index.js b/index.js index 37568412..5e5cfa25 100644 --- a/index.js +++ b/index.js @@ -30,6 +30,8 @@ var path = require('path') var statuses = require('statuses') var Stream = require('stream') var util = require('util') +var vary = require('vary') +var Negotiator = require('negotiator') /** * Path function references. @@ -153,6 +155,21 @@ function SendStream (req, path, options) { ? normalizeList(opts.extensions, 'extensions option') : [] + if (Array.isArray(opts.precompressed)) { + if (opts.precompressed.length > 0) { + this._precompressionFormats = opts.precompressed + this._precompressionEncodings = this._precompressionFormats.map(function (format) { return format.encoding }) + this._precompressionEncodings.push('identity') + } + } else if (opts.precompressed) { + this._precompressionFormats = [{ encoding: 'gzip', extension: '.gz' }] + this._precompressionEncodings = ['gzip', 'identity'] + } + + this._precompressionFormats = opts.precompressionFormats !== undefined + ? opts.precompressionFormats + : this._precompressionFormats + this._index = opts.index !== undefined ? normalizeList(opts.index, 'index option') : ['index.html'] @@ -360,6 +377,33 @@ SendStream.prototype.isPreconditionFailure = function isPreconditionFailure () { return false } +/** + * Return the array of file precompressed file extensions to serve in preference order. + * + * @return {Array} + * @api private + */ + +SendStream.prototype.getAcceptEncodingExtensions = function () { + var self = this + var negotiatedEncodings = new Negotiator(this.req).encodings(self._precompressionEncodings) + var accepted = [] + for (var e = 0; e < negotiatedEncodings.length; e++) { + var encoding = negotiatedEncodings[e] + if (encoding === 'identity') { + break + } + for (var f = 0; f < self._precompressionFormats.length; f++) { + var format = self._precompressionFormats[f] + if (format.encoding === encoding) { + accepted.push(format.extension) + break + } + } + } + return accepted +} + /** * Strip content-* header fields. * @@ -611,8 +655,10 @@ SendStream.prototype.pipe = function pipe (res) { * @api public */ -SendStream.prototype.send = function send (path, stat) { - var len = stat.size +SendStream.prototype.send = function send (path, stat, contentPath, contentStat) { + contentStat = contentStat || stat + contentPath = contentPath || path + var len = contentStat.size var options = this.options var opts = {} var res = this.res @@ -626,7 +672,7 @@ SendStream.prototype.send = function send (path, stat) { return } - debug('pipe "%s"', path) + debug('pipe "%s"', contentPath) // set header fields this.setHeader(path, stat) @@ -712,7 +758,7 @@ SendStream.prototype.send = function send (path, stat) { return } - this.stream(path, opts) + this.stream(contentPath, opts) } /** @@ -733,8 +779,7 @@ SendStream.prototype.sendFile = function sendFile (path) { } if (err) return self.onStatError(err) if (stat.isDirectory()) return self.redirect(path) - self.emit('file', path, stat) - self.send(path, stat) + checkPrecompressionAndSendFile(path, stat) }) function next (err) { @@ -750,10 +795,49 @@ SendStream.prototype.sendFile = function sendFile (path) { fs.stat(p, function (err, stat) { if (err) return next(err) if (stat.isDirectory()) return next() - self.emit('file', p, stat) - self.send(p, stat) + checkPrecompressionAndSendFile(p, stat) + }) + } + + function checkPrecompressionAndSendFile (p, stat) { + self.emit('file', p, stat) + if (!self._precompressionFormats) return self.send(p, stat) + + var state = { + contents: [], + extensionsToCheck: self._precompressionFormats.length + } + + self._precompressionFormats.forEach(function (format) { + debug('stat "%s%s"', p, format.extension) + fs.stat(p + format.extension, function onstat (err, contentStat) { + if (!err) state.contents.push({ext: format.extension, encoding: format.encoding, contentStat: contentStat}) + if (--state.extensionsToCheck === 0) sendPreferredContent(p, stat, state.contents) + }) }) } + + function sendPreferredContent (p, stat, contents) { + if (contents.length) { + vary(self.res, 'Accept-Encoding') + } + + var preferredContent + var extensions = self.getAcceptEncodingExtensions() + for (var e = 0; e < extensions.length && !preferredContent; e++) { + for (var c = 0; c < contents.length; c++) { + if (extensions[e] === contents[c].ext) { + preferredContent = contents[c] + break + } + } + } + + if (!preferredContent) return self.send(p, stat) + + self.res.setHeader('Content-Encoding', preferredContent.encoding) + self.send(p, stat, p + preferredContent.ext, preferredContent.contentStat) + } } /** diff --git a/package.json b/package.json index 8bd72a71..6d82ef2f 100644 --- a/package.json +++ b/package.json @@ -26,9 +26,11 @@ "http-errors": "~1.6.1", "mime": "1.3.4", "ms": "0.7.2", + "negotiator": "jshttp/negotiator#d9907aec0585476d9a0c4271e464f7c6e4633049", "on-finished": "~2.3.0", "range-parser": "~1.2.0", - "statuses": "~1.3.1" + "statuses": "~1.3.1", + "vary": "~1.1.0" }, "devDependencies": { "after": "0.8.2", diff --git a/test/fixtures/name.html.bz2 b/test/fixtures/name.html.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..b22473eb12771e091da0e537bfa0f02d18fed68d GIT binary patch literal 50 zcmZ>Y%CIzaj8qGb{K#tS&cMJZ*}%Ziz$&0{gh7G9kcUHoUo*OeX?gcfr8ycN6;1ON GHvj-P84cP1 literal 0 HcmV?d00001 diff --git a/test/fixtures/name.html.gz b/test/fixtures/name.html.gz new file mode 100644 index 0000000000000000000000000000000000000000..c0aa1e02d9eca380ab632c4732e8e483c4b5547b GIT binary patch literal 31 ncmb2|=3qEn$r;YXyjgRD_DTO!TQ2HvU}mU2!@7%`fq?-4t33*5 literal 0 HcmV?d00001 diff --git a/test/send.js b/test/send.js index 8efe9c51..4a2e47f5 100644 --- a/test/send.js +++ b/test/send.js @@ -1232,6 +1232,113 @@ describe('send(file, options)', function () { }) }) + describe('precompressed', function () { + it('should not include vary header when no precompressed variants exist', function (done) { + request(createServer({precompressed: true, root: fixtures})) + .get('/name.txt') + .set('Accept-Encoding', 'gzip') + .expect(shouldNotHaveHeader('Vary')) + .expect(shouldNotHaveHeader('Content-Encoding')) + .expect(200, done) + }) + + it('should include vary header when precompressed variants exist even when accept-encoding not present', function (done) { + request(createServer({precompressed: true, root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', '') + .expect('Content-Length', '11') + .expect(shouldNotHaveHeader('Content-Encoding')) + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Vary', 'Accept-Encoding', done) + }) + + it('should prefer server encoding order (bzip2,gzip) when present with equal weight in accept-encoding', function (done) { + request(createServer({precompressed: [{encoding: 'bzip2', extension: '.bz2'}, {encoding: 'gzip', extension: '.gz'}], root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', 'gzip, deflate, bzip2') + .expect('Vary', 'Accept-Encoding') + .expect('Content-Encoding', 'bzip2') + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Content-Length', '50', done) + }) + + it('should prefer server encoding order (gzip,bzip2) when present with equal weight in accept-encoding', function (done) { + request(createServer({precompressed: [{encoding: 'gzip', extension: '.gz'}, {encoding: 'bzip2', extension: '.bz2'}], root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', 'bzip2, deflate, gzip') + .expect('Vary', 'Accept-Encoding') + .expect('Content-Encoding', 'gzip') + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Content-Length', '31', done) + }) + + it('should send gzip when preferred in accept-encoding', function (done) { + request(createServer({precompressed: true, root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', ' gzip , deflate') + .expect('Vary', 'Accept-Encoding') + .expect('Content-Encoding', 'gzip') + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Content-Length', '31', done) + }) + + it('should not send gzip when no-gzip encoding is used', function (done) { + request(createServer({precompressed: true, root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', 'no-gzip, deflate') + .expect('Content-Length', '11') + .expect('Vary', 'Accept-Encoding', done) + }) + + it('should consider empty array of precompressed configuration as disabled', function (done) { + request(createServer({precompressed: [], root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', 'gzip') + .expect(shouldNotHaveHeader('Content-Encoding')) + .expect('Content-Length', '11', done) + }) + + it('should append to existing Vary header', function (done) { + request(http.createServer(function (req, res) { + res.setHeader('Vary', 'custom') + send(req, req.url, {precompressed: true, root: fixtures}) + .pipe(res) + })) + .get('/name.html') + .expect('Vary', 'custom, Accept-Encoding', done) + }) + + it('should honour accept-encoding quality values', function (done) { + request(createServer({precompressed: true, root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', 'gzip;q=0.9, deflate;q=1, bzip2;q=0.1') + .expect('Vary', 'Accept-Encoding') + .expect('Content-Encoding', 'gzip') + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Content-Length', '31', done) + }) + + it('should return no encoding if identity encoding preferred in accept-encoding', function (done) { + request(createServer({precompressed: true, root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', 'gzip;q=0.8, identity') + .expect('Vary', 'Accept-Encoding') + .expect(shouldNotHaveHeader('Content-Encoding')) + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Content-Length', '11', done) + }) + + it('should return server preferred format for accept-encoding *', function (done) { + request(createServer({precompressed: [{encoding: 'bzip2', extension: '.bz2'}, {encoding: 'gzip', extension: '.gz'}], root: fixtures})) + .get('/name.html') + .set('Accept-Encoding', '*;q=0.9; gzip;q=0.8') + .expect('Vary', 'Accept-Encoding') + .expect('Content-Encoding', 'bzip2') + .expect('Content-Type', 'text/html; charset=UTF-8') + .expect('Content-Length', '50', done) + }) + }) + describe('index', function () { it('should reject numbers', function (done) { request(createServer({root: fixtures, index: 42})) From fddade577e0f46a53945017811e5d9bb79352e94 Mon Sep 17 00:00:00 2001 From: Mikko Tiihonen Date: Thu, 15 Sep 2016 17:09:18 +0300 Subject: [PATCH 2/2] Use new configurable negotiator api for encoding sorting --- README.md | 8 ++++++++ index.js | 6 +++++- package.json | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f581d2ea..ca367337 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,14 @@ Compression tips: gzip encoders. Brotli compresses generally 15-20% better than gzip. * Use zopfli for gzip compression for and extra 5% benefit for all browsers. +Performance of serving static files is lower due to extra stats – worst case +20% with 1 byte files to loopback client. Compared to on-the-fly compression +the precompression is still a large win. + +##### encodingNegotiatorOptions + +Allows configuring the [encoding negotation options](https://github.com/jshttp/negotiator#sort-options). + ##### root Serve files relative to `path`. diff --git a/index.js b/index.js index 5e5cfa25..15602571 100644 --- a/index.js +++ b/index.js @@ -170,6 +170,10 @@ function SendStream (req, path, options) { ? opts.precompressionFormats : this._precompressionFormats + this._encodingNegotiatorOptions = opts.encodingNegotiatorOptions !== undefined + ? opts.encodingNegotiatorOptions + : { sortPreference: 'clientThenServer' } + this._index = opts.index !== undefined ? normalizeList(opts.index, 'index option') : ['index.html'] @@ -386,7 +390,7 @@ SendStream.prototype.isPreconditionFailure = function isPreconditionFailure () { SendStream.prototype.getAcceptEncodingExtensions = function () { var self = this - var negotiatedEncodings = new Negotiator(this.req).encodings(self._precompressionEncodings) + var negotiatedEncodings = new Negotiator(this.req).encodings(self._precompressionEncodings, this._encodingNegotiatorOptions) var accepted = [] for (var e = 0; e < negotiatedEncodings.length; e++) { var encoding = negotiatedEncodings[e] diff --git a/package.json b/package.json index 6d82ef2f..22e2848e 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "http-errors": "~1.6.1", "mime": "1.3.4", "ms": "0.7.2", - "negotiator": "jshttp/negotiator#d9907aec0585476d9a0c4271e464f7c6e4633049", + "negotiator": "jshttp/negotiator#6038bf698c522c1883a1113c834e53256b35584f", "on-finished": "~2.3.0", "range-parser": "~1.2.0", "statuses": "~1.3.1",