[fix] Handle oversized messages with designated error (#2285)

Ensure that the correct error is surfaced when the max message size is
exceeded.

Author: Amirali-Amirifar

lib/permessage-deflate.js

@@ -494,6 +494,14 @@ function inflateOnData(chunk) {
   this[kError].code = 'WS_ERR_UNSUPPORTED_MESSAGE_LENGTH';
   this[kError][kStatusCode] = 1009;
   this.removeListener('data', inflateOnData);
+
+  //
+  // The choice to employ `zlib.reset()` over `zlib.close()` is dictated by the
+  // fact that in Node.js versions prior to 13.10.0, the callback for
+  // `zlib.flush()` is not called if `zlib.close()` is used. Utilizing
+  // `zlib.reset()` ensures that either the callback is invoked or an error is
+  // emitted.
+  //
   this.reset();
 }
 
@@ -509,6 +517,12 @@ function inflateOnError(err) {
   // closed when an error is emitted.
   //
   this[kPerMessageDeflate]._inflate = null;
+
+  if (this[kError]) {
+    this[kCallback](this[kError]);
+    return;
+  }
+
   err[kStatusCode] = 1007;
   this[kCallback](err);
 }

test/permessage-deflate.test.js

@@ -590,9 +590,27 @@ describe('PerMessageDeflate', () => {
       });
     });
 
-    it("doesn't call the callback twice when `maxPayload` is exceeded", (done) => {
+    it('calls the callback when `maxPayload` is exceeded (1/2)', (done) => {
       const perMessageDeflate = new PerMessageDeflate({}, false, 25);
-      const buf = Buffer.from('A'.repeat(50));
+      const buf = Buffer.alloc(50, 'A');
+
+      perMessageDeflate.accept([{}]);
+      perMessageDeflate.compress(buf, true, (err, data) => {
+        if (err) return done(err);
+
+        perMessageDeflate.decompress(data, true, (err) => {
+          assert.ok(err instanceof RangeError);
+          assert.strictEqual(err.message, 'Max payload size exceeded');
+          done();
+        });
+      });
+    });
+
+    it('calls the callback when `maxPayload` is exceeded (2/2)', (done) => {
+      // A copy of the previous test but with a larger input. See
+      // https://github.com/websockets/ws/pull/2285.
+      const perMessageDeflate = new PerMessageDeflate({}, false, 25);
+      const buf = Buffer.alloc(1024 * 1024, 'A');
 
       perMessageDeflate.accept([{}]);
       perMessageDeflate.compress(buf, true, (err, data) => {