Collect samples for attack wave detection

library/agent/Agent.test.ts

@@ -1,7 +1,6 @@
 import * as FakeTimers from "@sinonjs/fake-timers";
 import { hostname, platform, release } from "os";
 import * as t from "tap";
-import * as fetch from "../helpers/fetch";
 import { getSemverNodeVersion } from "../helpers/getNodeVersion";
 import { ip } from "../helpers/ipAddress";
 import { wrap } from "../helpers/wrap";
@@ -1265,17 +1264,14 @@ t.test("attack wave detected event", async (t) => {
       route: "/posts/:id",
       routeParams: {},
     },
-    metadata: {
-      x: "test",
-    },
   });
 
   t.match(api.getEvents(), [
     {
       type: "detected_attack_wave",
       attack: {
         metadata: {
-          x: "test",
+          samples: "[]",
         },
       },
       request: {

library/agent/Agent.ts

@@ -647,18 +647,27 @@ export class Agent {
   /**
    * This function gets called when an attack wave is detected, it reports this attack wave to the API
    */
-  onDetectedAttackWave({
-    request,
-    metadata,
-  }: {
-    request: Context;
-    metadata: Record<string, string>;
-  }) {
+  onDetectedAttackWave({ request }: { request: Context }) {
+    if (!request.remoteAddress) {
+      // Cannot report attack wave without IP address
+      // Should not happen since AttackWaveDetector checks for remoteAddress
+      return;
+    }
+
+    const samples = this.attackWaveDetector.getSamplesForIP(
+      request.remoteAddress
+    );
+
     const attack: DetectedAttackWave = {
       type: "detected_attack_wave",
       time: Date.now(),
       attack: {
-        metadata: limitLengthMetadata(metadata, 4096),
+        metadata: limitLengthMetadata(
+          {
+            samples: JSON.stringify(samples),
+          },
+          4096
+        ),
         user: request.user,
       },
       request: {

library/agent/api/Event.ts

@@ -158,7 +158,7 @@ type Heartbeat = {
 export type DetectedAttackWave = {
   type: "detected_attack_wave";
   request: {
-    ipAddress: string | undefined;
+    ipAddress: string;
     userAgent: string | undefined;
     source: string;
   };

library/agent/api/ReportingAPIRateLimitedClientSide.test.ts

@@ -62,7 +62,7 @@ function generateAttackWaveEvent(): Event {
     type: "detected_attack_wave",
     time: Date.now(),
     request: {
-      ipAddress: undefined,
+      ipAddress: "::1",
       userAgent: undefined,
       source: "express",
     },

library/sources/FunctionsFramework.test.ts

@@ -299,7 +299,6 @@ t.test("it waits for attack events to be sent before returning", async (t) => {
 
     agent.onDetectedAttackWave({
       request: getContext()!,
-      metadata: {},
     });
 
     res.sendStatus(200);

library/sources/HTTPServer.test.ts

@@ -1103,7 +1103,34 @@ t.test("it reports attack waves", async (t) => {
         {
           type: "detected_attack_wave",
           attack: {
-            metadata: {},
+            metadata: {
+              samples: JSON.stringify([
+                {
+                  method: "GET",
+                  url: "/../package.json",
+                },
+                {
+                  method: "GET",
+                  url: "/.env",
+                },
+                {
+                  method: "GET",
+                  url: "/wp-config.php",
+                },
+                {
+                  method: "GET",
+                  url: "/etc/passwd",
+                },
+                {
+                  method: "GET",
+                  url: "/.git/config",
+                },
+                {
+                  method: "GET",
+                  url: "/%systemroot%/system32/cmd.exe",
+                },
+              ]),
+            },
             user: undefined,
           },
           request: {

library/sources/Lambda.test.ts

@@ -558,7 +558,6 @@ t.test("it waits for attack events to be sent before returning", async (t) => {
 
     agent.onDetectedAttackWave({
       request: getContext()!,
-      metadata: {},
     });
 
     return { statusCode: 200 };

library/sources/http-server/createRequestListener.ts

@@ -113,8 +113,12 @@ function onFinishRequestHandler(
       agent.onRouteRateLimited(context.rateLimitedEndpoint);
     }
 
-    if (agent.getAttackWaveDetector().check(context)) {
-      agent.onDetectedAttackWave({ request: context, metadata: {} });
+    const attackWaveDetector = agent.getAttackWaveDetector();
+
+    if (attackWaveDetector.check(context) && context.remoteAddress) {
+      agent.onDetectedAttackWave({
+        request: context,
+      });
       agent.getInspectionStatistics().onAttackWaveDetected();
     }
   }

library/sources/http-server/http2/createStreamListener.ts

@@ -76,8 +76,12 @@ function discoverRouteFromStream(
         agent.onRouteRateLimited(context.rateLimitedEndpoint);
       }
 
-      if (agent.getAttackWaveDetector().check(context)) {
-        agent.onDetectedAttackWave({ request: context, metadata: {} });
+      const attackWaveDetector = agent.getAttackWaveDetector();
+
+      if (attackWaveDetector.check(context) && context.remoteAddress) {
+        agent.onDetectedAttackWave({
+          request: context,
+        });
         agent.getInspectionStatistics().onAttackWaveDetected();
       }
     }

library/vulnerabilities/attack-wave-detection/AttackWaveDetector.test.ts

@@ -30,6 +30,7 @@ function newAttackWaveDetector() {
     attackWaveTimeFrame: 60 * 1000,
     minTimeBetweenEvents: 60 * 60 * 1000,
     maxLRUEntries: 10_000,
+    maxSamplesPerIP: 5,
   });
 }
 
@@ -150,3 +151,55 @@ t.test("a slow web scanner that triggers in the third interval", async (t) => {
 
   clock.uninstall();
 });
+
+t.test("it collects samples correctly", async (t) => {
+  const detector = newAttackWaveDetector();
+  const ip = "::1";
+  detector.check(getTestContext(ip, "/wp-config.php", "GET"));
+  detector.check(getTestContext(ip, "/wp-config.php.bak", "GET"));
+  detector.check(getTestContext(ip, "/.git/config", "GET"));
+  detector.check(getTestContext(ip, "/.env", "GET"));
+  detector.check(getTestContext(ip, "/.htaccess", "GET"));
+
+  detector.check(getTestContext(ip, "/.htaccess", "GET")); // Duplicate
+  detector.check(getTestContext("::2", "/test/.env", "GET")); // Different IP
+
+  const samples = detector.getSamplesForIP(ip);
+  t.equal(samples.length, 5, "should have collected 5 samples");
+
+  t.same(
+    samples,
+    [
+      { method: "GET", url: "http://localhost:4000/wp-config.php" },
+      { method: "GET", url: "http://localhost:4000/wp-config.php.bak" },
+      { method: "GET", url: "http://localhost:4000/.git/config" },
+      { method: "GET", url: "http://localhost:4000/.env" },
+      { method: "GET", url: "http://localhost:4000/.htaccess" },
+    ],
+    "should have collected the correct samples"
+  );
+});
+
+t.test("it limits samples correctly", async (t) => {
+  const detector = newAttackWaveDetector();
+  const ip = "::1";
+
+  for (let i = 0; i < 10; i++) {
+    detector.check(getTestContext(ip, `/${i}/.env`, "GET"));
+  }
+
+  const samples = detector.getSamplesForIP(ip);
+  t.equal(samples.length, 5, "should have collected maximum 5 samples");
+
+  t.same(
+    samples,
+    [
+      { method: "GET", url: "http://localhost:4000/0/.env" },
+      { method: "GET", url: "http://localhost:4000/1/.env" },
+      { method: "GET", url: "http://localhost:4000/2/.env" },
+      { method: "GET", url: "http://localhost:4000/3/.env" },
+      { method: "GET", url: "http://localhost:4000/4/.env" },
+    ],
+    "should have collected the correct samples"
+  );
+});

library/vulnerabilities/attack-wave-detection/AttackWaveDetector.ts

@@ -2,8 +2,14 @@ import { LRUMap } from "../../ratelimiting/LRUMap";
 import type { Context } from "../../agent/Context";
 import { isWebScanner } from "./isWebScanner";
 
+export type SuspiciousRequest = {
+  method: string;
+  url: string;
+};
+
 export class AttackWaveDetector {
-  private suspiciousRequestsMap: LRUMap<string, number>;
+  private suspiciousRequestsCounts: LRUMap<string, number>;
+  private suspiciousRequestsSamples: LRUMap<string, SuspiciousRequest[]>;
   private sentEventsMap: LRUMap<string, number>;
 
   // How many suspicious requests are allowed before triggering an alert
@@ -14,21 +20,29 @@ export class AttackWaveDetector {
   private readonly minTimeBetweenEvents: number;
   // Maximum number of entries in the LRU cache
   private readonly maxLRUEntries: number;
+  // Maximum number of samples to keep per IP, can not be higher than attackWaveThreshold
+  private readonly maxSamplesPerIP: number;
 
   constructor(
     options: {
       attackWaveThreshold?: number;
       attackWaveTimeFrame?: number;
       minTimeBetweenEvents?: number;
       maxLRUEntries?: number;
+      maxSamplesPerIP?: number;
     } = {}
   ) {
     this.attackWaveThreshold = options.attackWaveThreshold ?? 15; // Default: 15 requests
     this.attackWaveTimeFrame = options.attackWaveTimeFrame ?? 60 * 1000; // Default: 1 minute
     this.minTimeBetweenEvents = options.minTimeBetweenEvents ?? 20 * 60 * 1000; // Default: 20 minutes
     this.maxLRUEntries = options.maxLRUEntries ?? 10_000; // Default: 10,000 entries
+    this.maxSamplesPerIP = options.maxSamplesPerIP ?? 15; // Default: 15 samples
 
-    this.suspiciousRequestsMap = new LRUMap(
+    this.suspiciousRequestsCounts = new LRUMap(
+      this.maxLRUEntries,
+      this.attackWaveTimeFrame
+    );
+    this.suspiciousRequestsSamples = new LRUMap(
       this.maxLRUEntries,
       this.attackWaveTimeFrame
     );
@@ -57,12 +71,22 @@ export class AttackWaveDetector {
       return false;
     }
 
+    // In isWebScanner we use `context.route`, `context.route` is always created from `context.url`
+    if (!context.method || !context.url) {
+      return false;
+    }
+
     if (!isWebScanner(context)) {
       return false;
     }
 
-    const suspiciousRequests = (this.suspiciousRequestsMap.get(ip) || 0) + 1;
-    this.suspiciousRequestsMap.set(ip, suspiciousRequests);
+    const suspiciousRequests = (this.suspiciousRequestsCounts.get(ip) || 0) + 1;
+    this.suspiciousRequestsCounts.set(ip, suspiciousRequests);
+
+    this.trackSample(ip, {
+      method: context.method,
+      url: context.url,
+    });
 
     if (suspiciousRequests < this.attackWaveThreshold) {
       return false;
@@ -72,4 +96,28 @@ export class AttackWaveDetector {
 
     return true;
   }
+
+  getSamplesForIP(ip: string): SuspiciousRequest[] {
+    return this.suspiciousRequestsSamples.get(ip) || [];
+  }
+
+  trackSample(ip: string, request: SuspiciousRequest) {
+    const samples = this.suspiciousRequestsSamples.get(ip) || [];
+    if (samples.length >= this.maxSamplesPerIP) {
+      return;
+    }
+
+    // Only store unique samples
+    // We can't use a Set because we have objects
+    if (
+      samples.some(
+        (sample) =>
+          sample.method === request.method && sample.url === request.url
+      )
+    ) {
+      return;
+    }
+    samples.push(request);
+    this.suspiciousRequestsSamples.set(ip, samples);
+  }
 }