Add support for precompressed (gzip) content

Author: Mikko Tiihonen

.gitattributes

@@ -0,0 +1,2 @@
+*.bz2 binary
+*.gz binary

HISTORY.md

@@ -1,3 +1,8 @@
+unreleased
+==========
+
+  * Send precompressed variant of content based on `Accept-Encoding`
+
 0.14.1 / 2016-06-09
 ===================
 

README.md

@@ -96,6 +96,42 @@ Provide a max-age in milliseconds for http caching, defaults to 0.
 This can also be a string accepted by the
 [ms](https://www.npmjs.org/package/ms#readme) module.
 
+##### precompressed
+
+Precompressed files are extra static files that are compressed before
+they are requested, as opposed to compressing on the fly. Compressing
+files once offline (for example during site build) allows using
+stronger compression methods and both reduces latency and lowers cpu
+usage when serving files.
+
+The `precompressed` option enables or disables serving of precompressed
+content variants. The option defaults to `false`, if set to `true` checks
+for existence of gzip compressed files with `.gz` extensions.
+
+Example scenario:
+
+The file `site.css` has both `site.css.gz` and `site.css.bz2`
+precompressed versions available in the same directory. The server is configured
+to serve both `.bz2` and `.gz` files in that prefence order.
+When a request comes with an `Accept-Encoding` header with value `gzip, bz2`
+requesting `site.css` the contents of `site.css.bz2` is sent instead and
+a header `Content-Encoding` with value `br` is added to the response.
+In addition a `Vary: Accept-Encoding` header is added to response allowing
+caching proxies to work correctly.
+
+Custom configuration:
+
+It is also possible to customize the searched file extensions and header
+values (used with Accept-Encoding and Content-Encoding headers) by specifying
+them explicitly in an array in the preferred priority order. For example:
+`[{encoding: 'bzip2', extension: '.bz2'}, {encoding: 'gzip', extension: '.gz'}]`.
+
+Compression tips:
+   * Precompress at least all static `js`, `css`  and `svg` files.
+   * Precompress using both brotli (supported by Firefox and Chrome) and
+     gzip encoders. Brotli compresses generally 15-20% better than gzip.
+   * Use zopfli for gzip compression for and extra 5% benefit for all browsers.
+
 ##### root
 
 Serve files relative to `path`.

index.js

@@ -30,6 +30,8 @@ var path = require('path')
 var statuses = require('statuses')
 var Stream = require('stream')
 var util = require('util')
+var vary = require('vary')
+var Negotiator = require('negotiator')
 
 /**
  * Path function references.
@@ -146,6 +148,21 @@ function SendStream (req, path, options) {
     ? normalizeList(opts.extensions, 'extensions option')
     : []
 
+  if (Array.isArray(opts.precompressed)) {
+    if (opts.precompressed.length > 0) {
+      this._precompressionFormats = opts.precompressed
+      this._precompressionEncodings = this._precompressionFormats.map(function (format) { return format.encoding; })
+      this._precompressionEncodings.push('identity')
+    }
+  } else if (opts.precompressed) {
+    this._precompressionFormats = [{encoding:'gzip', extension:'.gz'}]
+    this._precompressionEncodings = ['gzip', 'identity']
+  }
+
+  this._precompressionFormats = opts.precompressionFormats !== undefined
+      ? opts.precompressionFormats
+      : this._precompressionFormats
+
   this._index = opts.index !== undefined
     ? normalizeList(opts.index, 'index option')
     : ['index.html']
@@ -319,6 +336,33 @@ SendStream.prototype.isConditionalGET = function isConditionalGET () {
     this.req.headers['if-modified-since']
 }
 
+/**
+ * Return the array of file precompressed file extensions to serve in preference order.
+ *
+ * @return {Array}
+ * @api private
+ */
+
+SendStream.prototype.getAcceptEncodingExtensions = function() {
+  var self = this
+  var negotiatedEncodings = new Negotiator(this.req).encodings(self._precompressionEncodings)
+  var accepted = []
+  for (var e = 0; e < negotiatedEncodings.length; e++) {
+    var encoding = negotiatedEncodings[e]
+    if (encoding === 'identity') {
+      break
+    }
+    for (var f = 0; f < self._precompressionFormats.length; f++) {
+      var format = self._precompressionFormats[f]
+      if (format.encoding === encoding) {
+        accepted.push(format.extension)
+        break
+      }
+    }
+  }
+  return accepted;
+}
+
 /**
  * Strip content-* header fields.
  *
@@ -558,8 +602,10 @@ SendStream.prototype.pipe = function pipe (res) {
  * @api public
  */
 
-SendStream.prototype.send = function send (path, stat) {
-  var len = stat.size
+SendStream.prototype.send = function send (path, stat, contentPath, contentStat) {
+  contentStat = contentStat || stat
+  contentPath = contentPath || path
+  var len = contentStat.size;
   var options = this.options
   var opts = {}
   var res = this.res
@@ -573,7 +619,7 @@ SendStream.prototype.send = function send (path, stat) {
     return
   }
 
-  debug('pipe "%s"', path)
+  debug('pipe "%s"', contentPath)
 
   // set header fields
   this.setHeader(path, stat)
@@ -652,7 +698,7 @@ SendStream.prototype.send = function send (path, stat) {
     return
   }
 
-  this.stream(path, opts)
+  this.stream(contentPath, opts)
 }
 
 /**
@@ -673,8 +719,7 @@ SendStream.prototype.sendFile = function sendFile (path) {
     }
     if (err) return self.onStatError(err)
     if (stat.isDirectory()) return self.redirect(self.path)
-    self.emit('file', path, stat)
-    self.send(path, stat)
+    checkPrecompressionAndSendFile(path, stat)
   })
 
   function next (err) {
@@ -690,10 +735,49 @@ SendStream.prototype.sendFile = function sendFile (path) {
     fs.stat(p, function (err, stat) {
       if (err) return next(err)
       if (stat.isDirectory()) return next()
-      self.emit('file', p, stat)
-      self.send(p, stat)
+      checkPrecompressionAndSendFile(p, stat)
+    })
+  }
+
+  function checkPrecompressionAndSendFile(p, stat) {
+    self.emit('file', p, stat)
+    if (!self._precompressionFormats) return self.send(p, stat)
+
+    var state = {
+      contents: [],
+      extensionsToCheck: self._precompressionFormats.length
+    }
+
+    self._precompressionFormats.forEach(function (format) {
+      debug('stat "%s%s"', p, format.extension);
+      fs.stat(p + format.extension, function onstat(err, contentStat) {
+        if (!err) state.contents.push({ext: format.extension, encoding: format.encoding, contentStat: contentStat})
+        if (--state.extensionsToCheck == 0) sendPreferredContent(p, stat, state.contents)
+      })
     })
   }
+
+  function sendPreferredContent(p, stat, contents) {
+    if (contents.length) {
+      vary(self.res, 'Accept-Encoding')
+    }
+
+    var preferredContent
+    var extensions = self.getAcceptEncodingExtensions()
+    for (var e = 0; e < extensions.length && !preferredContent; e++) {
+      for (var c = 0; c < contents.length; c++) {
+        if (extensions[e] === contents[c].ext) {
+          preferredContent = contents[c]
+          break
+        }
+      }
+    }
+
+    if (!preferredContent) return self.send(p, stat)
+
+    self.res.setHeader('Content-Encoding', preferredContent.encoding)
+    self.send(p, stat, p + preferredContent.ext, preferredContent.contentStat)
+  }
 }
 
 /**

package.json

@@ -24,9 +24,11 @@
     "http-errors": "~1.5.0",
     "mime": "1.3.4",
     "ms": "0.7.1",
+    "negotiator": "jshttp/negotiator#d9907aec0585476d9a0c4271e464f7c6e4633049",
     "on-finished": "~2.3.0",
     "range-parser": "~1.2.0",
-    "statuses": "~1.3.0"
+    "statuses": "~1.3.0",
+    "vary": "~1.1.0"
   },
   "devDependencies": {
     "after": "0.8.1",

test/fixtures/name.html.bz2

@@ -1090,6 +1090,113 @@ describe('send(file, options)', function () {
     })
   })
 
+  describe('precompressed', function () {
+    it('should not include vary header when no precompressed variants exist', function (done) {
+      request(createServer({precompressed: true, root: fixtures}))
+      .get('/name.txt')
+      .set('Accept-Encoding', 'gzip')
+      .expect(shouldNotHaveHeader('Vary'))
+      .expect(shouldNotHaveHeader('Content-Encoding'))
+      .expect(200, done)
+    })
+
+    it('should include vary header when precompressed variants exist even when accept-encoding not present', function (done) {
+      request(createServer({precompressed: true, root: fixtures}))
+      .get('/name.html')
+      .set('Accept-Encoding', '')
+      .expect('Content-Length', '11')
+      .expect(shouldNotHaveHeader('Content-Encoding'))
+      .expect('Content-Type', 'text/html; charset=UTF-8')
+      .expect('Vary', 'Accept-Encoding', done)
+    })
+
+    it('should prefer server encoding order (bzip2,gzip) when present with equal weight in accept-encoding', function (done) {
+      request(createServer({precompressed: [{encoding: 'bzip2', extension: '.bz2'}, {encoding: 'gzip', extension: '.gz'}], root: fixtures}))
+      .get('/name.html')
+      .set('Accept-Encoding', 'gzip, deflate, bzip2')
+      .expect('Vary', 'Accept-Encoding')
+      .expect('Content-Encoding', 'bzip2')
+      .expect('Content-Type', 'text/html; charset=UTF-8')
+      .expect('Content-Length', '50', done)
+    })
+
+    it('should prefer server encoding order (gzip,bzip2) when present with equal weight in accept-encoding', function (done) {
+      request(createServer({precompressed: [{encoding: 'gzip', extension: '.gz'}, {encoding: 'bzip2', extension: '.bz2'}], root: fixtures}))
+      .get('/name.html')
+      .set('Accept-Encoding', 'bzip2, deflate, gzip')
+      .expect('Vary', 'Accept-Encoding')
+      .expect('Content-Encoding', 'gzip')
+      .expect('Content-Type', 'text/html; charset=UTF-8')
+      .expect('Content-Length', '31', done)
+    })
+
+    it('should send gzip when preferred in accept-encoding', function (done) {
+      request(createServer({precompressed: true, root: fixtures}))
+      .get('/name.html')
+      .set('Accept-Encoding', ' gzip , deflate')
+      .expect('Vary', 'Accept-Encoding')
+      .expect('Content-Encoding', 'gzip')
+      .expect('Content-Type', 'text/html; charset=UTF-8')
+      .expect('Content-Length', '31', done)
+    })
+
+    it('should not send gzip when no-gzip encoding is used', function (done) {
+      request(createServer({precompressed: true, root: fixtures}))
+      .get('/name.html')
+      .set('Accept-Encoding', 'no-gzip, deflate')
+      .expect('Content-Length', '11')
+      .expect('Vary', 'Accept-Encoding', done)
+    })
+
+    it('should consider empty array of precompressed configuration as disabled', function (done) {
+      request(createServer({precompressed: [], root: fixtures}))
+      .get('/name.html')
+      .set('Accept-Encoding', 'gzip')
+      .expect(shouldNotHaveHeader('Content-Encoding'))
+      .expect('Content-Length', '11', done)
+    })
+
+    it('should append to existing Vary header', function (done) {
+      request(http.createServer(function (req, res) {
+        res.setHeader('Vary', 'custom')
+        send(req, req.url, {precompressed: true, root: fixtures})
+        .pipe(res)
+      }))
+      .get('/name.html')
+      .expect('Vary', 'custom, Accept-Encoding', done)
+    })
+
+    it('should honour accept-encoding quality values', function (done) {
+      request(createServer({precompressed: true, root: fixtures}))
+      .get('/name.html')
+      .set('Accept-Encoding', 'gzip;q=0.9, deflate;q=1, bzip2;q=0.1')
+      .expect('Vary', 'Accept-Encoding')
+      .expect('Content-Encoding', 'gzip')
+      .expect('Content-Type', 'text/html; charset=UTF-8')
+      .expect('Content-Length', '31', done)
+    })
+
+    it('should return no encoding if identity encoding preferred in accept-encoding', function (done) {
+      request(createServer({precompressed: true, root: fixtures}))
+      .get('/name.html')
+      .set('Accept-Encoding', 'gzip;q=0.8, identity')
+      .expect('Vary', 'Accept-Encoding')
+      .expect(shouldNotHaveHeader('Content-Encoding'))
+      .expect('Content-Type', 'text/html; charset=UTF-8')
+      .expect('Content-Length', '11', done)
+    })
+
+    it('should return server preferred format for accept-encoding *', function (done) {
+      request(createServer({precompressed: [{encoding: 'bzip2', extension: '.bz2'}, {encoding: 'gzip', extension: '.gz'}], root: fixtures}))
+      .get('/name.html')
+      .set('Accept-Encoding', '*;q=0.9; gzip;q=0.8')
+      .expect('Vary', 'Accept-Encoding')
+      .expect('Content-Encoding', 'bzip2')
+      .expect('Content-Type', 'text/html; charset=UTF-8')
+      .expect('Content-Length', '50', done)
+    })
+  })
+
   describe('index', function () {
     it('should reject numbers', function (done) {
       request(createServer({root: fixtures, index: 42}))