Add brotli (br) compression support

Update packages to latest
Migrate to ES Modules
Update major version to 2 since these are breaking changes
Update node version. Replace TravisCI with Github Actions

Author: sonyseng

.github/workflows/node.js.yml

@@ -0,0 +1,25 @@
+# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on: push
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [14.x]
+        
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v2
+      with:
+        node-version: ${{ matrix.node-version }}
+        cache: 'npm'
+    - run: npm ci
+    - run: npm test

.travis.yml

@@ -1,9 +1,10 @@
-# json-caching-proxy [![Build Status](https://travis-ci.org/sonyseng/json-caching-proxy.svg?branch=master)](https://travis-ci.org/sonyseng/json-caching-proxy) [![NPM Version](http://img.shields.io/npm/v/json-caching-proxy.svg?style=flat)](https://www.npmjs.org/package/json-caching-proxy) [![NPM Downloads](https://img.shields.io/npm/dm/json-caching-proxy.svg?style=flat)](https://www.npmjs.org/package/json-caching-proxy)
+# json-caching-proxy ![Build Status](https://github.com/sonyseng/json-caching-proxy/actions/workflows/node.js.yml/badge.svg) [![NPM Version](http://img.shields.io/npm/v/json-caching-proxy.svg?style=flat)](https://www.npmjs.org/package/json-caching-proxy) [![NPM Downloads](https://img.shields.io/npm/dm/json-caching-proxy.svg?style=flat)](https://www.npmjs.org/package/json-caching-proxy)
 
 Node caching HTTP proxy built on top of [express-http-proxy](https://github.com/villadora/express-http-proxy). Persists requests and responses to an in-memory HAR-like data structure based on [HAR1.2](http://www.softwareishard.com/blog/har-12-spec/) . Caches JSON content-type responses by default with the ability to cache an entire site; including content-types describing images. Useful for testing front end code, mocking api, and saving the cache to a HAR file which can be used for further tests.
+
 ## Installation
 
-Requires Node >= 10
+Requires Node >= 14
 
 Command line tool:
 ```
@@ -226,7 +227,7 @@ cacheBustingParams: ['time', 'dc', 'cacheSlayer', '_']
 ```
 
 ## Controlling the Proxy
-Once the proxy has started, you may point your browser to the following urls to affect the state of the proxy:
+Once the proxy has started on a port (e.g. 3001), you may point your browser to the following urls to affect the state of the proxy:
 ```
 http://localhost:3001/proxy/playback?enabled=[true|false] - Start/Stop replaying cached requests.
 http://localhost:3001/proxy/record?enabled=[true|false] - Start/Stop recording request/responses to the cache.

README.md

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 
-const version = require('./package.json').version;
-const JsonCachingProxy = require('./');
+import { program } from "commander";
+import fs from "fs";
+import path from "path";
+import stripJsonComments from "strip-json-comments";
 
-const fs = require('fs');
-const url = require('url');
-const path = require('path');
-const program = require('commander');
-const stripJsonComments = require('strip-json-comments');
+import JsonCachingProxy from "./index.js";
 
+const npmPackage = JSON.parse(fs.readFileSync('./package.json'));
+const version = npmPackage.version;
 const cwd = process.cwd();
 
 function list (val) {

bin.js

@@ -1,14 +1,18 @@
-const npmPackage = require('./package.json');
-const crypto = require('crypto');
-const express = require('express');
-const proxy = require('express-http-proxy');
-const cors = require('cors');
-const bodyParser = require('body-parser');
-const urlUtil = require('url');
-const chalk = require('chalk');
-
-/** The caching proxy server. */
-class JsonCachingProxy {
+import bodyParser from 'body-parser';
+import chalk from 'chalk';
+import cors from 'cors';
+import crypto from 'crypto';
+import express from 'express';
+import fs from 'fs';
+import proxy from 'express-http-proxy';
+import urlUtil from 'url';
+import { promisify } from 'util';
+import zlib from 'zlib';
+
+const brotliDecompress = promisify(zlib.brotliDecompress);
+const npmPackage = JSON.parse(fs.readFileSync('package.json'));
+
+export default class JsonCachingProxy {
 	/**
 	 * @param {Object} options - Options passed into the ctor will override defaults if defined
 	 */
@@ -130,10 +134,10 @@ class JsonCachingProxy {
 	 * @param {string} startedDateTime - An ISO Datetime String
 	 * @param {Object} req - An express IncomingMessage request
 	 * @param {Object} res - An express ServerResponse response
-	 * @param {Object} data - An express response body (the content)
+	 * @param {Buffer} data - An express response body (the content buffer)
 	 * @returns {Object} A HAR entry object
 	 */
-	createHarEntry(startedDateTime, req, res, data) {
+	async createHarEntry(startedDateTime, req, res, data) {
 		let reqMimeType = req.get('Content-Type');
 		let resMimeType = res.get('Content-Type') || 'text/plain';
 		let encoding = (/^text\/|^application\/(javascript|json)/).test(resMimeType) ? 'utf8' : 'base64';
@@ -153,11 +157,16 @@ class JsonCachingProxy {
 				status: res.statusCode,
 				statusText: res.statusMessage,
 				cookies: this.convertToNameValueList(res.cookies),
-				headers: this.convertToNameValueList(res._headers).filter(header => header.name.toLowerCase() !== 'content-encoding'), // Not  compressed
+				headers: this.convertToNameValueList(res.getHeaders()).filter(header => header.name.toLowerCase() !== 'content-encoding'), // Not  compressed
 				content: {
 					size: -1,
 					mimeType: resMimeType,
-					text: data.toString(encoding),
+
+					// Workaround for http-express-proxy not handling brotli encoding. TODO: remove this code when HEP fixes this
+					text: (res.getHeaders()['content-encoding'] || '').toLowerCase() == 'br' ?
+						(await brotliDecompress(data)).toString(encoding) :
+						data.toString(encoding),
+
 					encoding: encoding
 				},
 				headersSize: -1,
@@ -301,7 +310,7 @@ class JsonCachingProxy {
 	}
 
 	/**
-	 * Add Request body parsing into RAW if there is actual body content
+	 * Add Request body parsing into bodyParser if there is actual body content
 	 * @returns {JsonCachingProxy}
 	 */
 	addBodyParser() {
@@ -376,9 +385,11 @@ class JsonCachingProxy {
 	 * Modifies locations on redirects.
 	 * @returns {JsonCachingProxy}
 	 */
-	addProxyRoute() {
+	async addProxyRoute() {
 		this.app.use('/', proxy(this.options.remoteServerUrl, {
-			userResDecorator: (rsp, rspData, req, res) => {
+			userResDecorator: async (rsp, rspData, req, res) => {
+				let headers = res.getHeaders();
+
 				// Handle Redirects by modifying the location property of the response header
 				let location = res.get('location');
 				if (location) {
@@ -389,20 +400,21 @@ class JsonCachingProxy {
 					res.header('access-control-allow-origin', this.options.overrideCors);
 				}
 
-				if (this.options.deleteCookieDomain && res._headers['set-cookie']) {
-					res.header('set-cookie', this.removeCookiesDomain(res._headers['set-cookie'] || []));
+				if (this.options.deleteCookieDomain && headers['set-cookie']) {
+					res.header('set-cookie', this.removeCookiesDomain(headers['set-cookie'] || []));
 				}
 
 				if (this.isRouteExcluded(req.method, req.url)) {
 					this.log(chalk.red('Exclude Proxied Resource', chalk.bold(req.method, req.url)));
 				} else if (this.isStatusExcluded(res.statusCode)) {
 					this.log(chalk.red('Exclude Proxied Resource', chalk.bold(req.method, req.url, `\tStatus: ${res.statusCode}`)));
 				} else {
-					let mimeType = res._headers['content-type'];
+					let mimeType = headers['content-type'];
 
 					if (this.options.dataRecord && (this.options.cacheEverything || !this.options.cacheEverything && mimeType && mimeType.indexOf('application/json') >= 0)) {
 						let { key, hash } = this.genKeyFromExpressReq(req);
-						let entry = this.createHarEntry(new Date().toISOString(), req, res, rspData);
+						let entry = await this.createHarEntry(new Date().toISOString(), req, res, rspData);
+
 						this.routeCache[key] = entry;
 						this.log(chalk.yellow('Saved to Cache', hash, chalk.bold(entry.request.method, entry.request.url)));
 					} else {
@@ -532,5 +544,3 @@ class JsonCachingProxy {
 	 */
 	isRecording() { return this.options.dataRecord; }
 }
-
-module.exports = JsonCachingProxy;