feat: monitoring retention policy and ingest hardening (#540, #543)

apps/web/src/app/api/internal/monitoring/ingest/route.ts

@@ -4,6 +4,7 @@ import { db, systemLogs } from '@pagespace/db';
 import { writeApiMetrics, writeError } from '@pagespace/lib/logger-database';
 import { loggers } from '@pagespace/lib/server';
 import { secureCompare } from '@pagespace/lib';
+import { sanitizeIngestPayload } from '@/lib/monitoring/ingest-sanitizer';
 
 interface ApiRequestPayload {
   type: 'api-request';
@@ -45,9 +46,10 @@ export async function POST(request: Request) {
 
   const ingestKey = process.env.MONITORING_INGEST_KEY;
   if (!ingestKey) {
-    loggers.system.error(
-      'MONITORING_INGEST_KEY is not configured and MONITORING_INGEST_DISABLED is not set. ' +
-      'Monitoring ingest cannot process requests. Set the key or set MONITORING_INGEST_DISABLED=true to opt out.'
+    loggers.system.warn(
+      'MONITORING_INGEST_KEY is not configured. ' +
+      'Monitoring ingest cannot authenticate requests. ' +
+      'Set MONITORING_INGEST_KEY to enable or MONITORING_INGEST_DISABLED=true to opt out.'
     );
     return NextResponse.json({ error: 'Monitoring ingest not configured' }, { status: 503 });
   }
@@ -58,17 +60,19 @@ export async function POST(request: Request) {
     return unauthorized('Unauthorized');
   }
 
-  let payload: IngestPayload;
+  let rawPayload: IngestPayload;
   try {
-    payload = await request.json();
+    rawPayload = await request.json();
   } catch {
     return NextResponse.json({ error: 'Invalid JSON payload' }, { status: 400 });
   }
 
-  if (payload.type !== 'api-request') {
+  if (rawPayload.type !== 'api-request') {
     return NextResponse.json({ error: 'Unsupported payload type' }, { status: 400 });
   }
 
+  const payload = sanitizeIngestPayload(rawPayload);
+
   const timestamp = payload.timestamp ? new Date(payload.timestamp) : new Date();
   type HttpMethod = NonNullable<(typeof systemLogs.$inferInsert)['method']>;
   type SystemLogLevel = (typeof systemLogs.$inferInsert)['level'];
@@ -118,7 +122,6 @@ export async function POST(request: Request) {
         statusCode,
         requestSize: payload.requestSize,
         responseSize: payload.responseSize,
-        query: payload.query,
       },
     });
 

apps/web/src/lib/monitoring/__tests__/ingest-sanitizer.test.ts

@@ -0,0 +1,174 @@
+import { describe, it, expect } from 'vitest';
+import {
+  redactUrlQueryParams,
+  sanitizeEndpoint,
+  sanitizeIngestPayload,
+  truncateString,
+} from '../ingest-sanitizer';
+
+describe('redactUrlQueryParams', () => {
+  it('strips query parameters from URLs', () => {
+    expect(redactUrlQueryParams('/api/pages?token=secret&id=123')).toBe('/api/pages');
+  });
+
+  it('returns path unchanged when no query params', () => {
+    expect(redactUrlQueryParams('/api/pages')).toBe('/api/pages');
+  });
+
+  it('strips fragment identifiers', () => {
+    expect(redactUrlQueryParams('/api/pages#section')).toBe('/api/pages');
+  });
+
+  it('strips both query params and fragments', () => {
+    expect(redactUrlQueryParams('/api/pages?key=val#section')).toBe('/api/pages');
+  });
+
+  it('handles empty string', () => {
+    expect(redactUrlQueryParams('')).toBe('');
+  });
+
+  it('handles full URLs by extracting pathname', () => {
+    expect(redactUrlQueryParams('https://example.com/api/test?secret=abc')).toBe('/api/test');
+  });
+});
+
+describe('sanitizeEndpoint', () => {
+  it('normalizes double slashes', () => {
+    expect(sanitizeEndpoint('//api//pages//')).toBe('/api/pages/');
+  });
+
+  it('truncates excessively long endpoints', () => {
+    const longPath = '/api/' + 'a'.repeat(500);
+    const result = sanitizeEndpoint(longPath);
+    expect(result.length).toBeLessThanOrEqual(256);
+  });
+
+  it('removes query parameters', () => {
+    expect(sanitizeEndpoint('/api/pages?token=secret')).toBe('/api/pages');
+  });
+
+  it('handles undefined', () => {
+    expect(sanitizeEndpoint(undefined as unknown as string)).toBe('');
+  });
+});
+
+describe('truncateString', () => {
+  it('returns short strings unchanged', () => {
+    expect(truncateString('hello', 10)).toBe('hello');
+  });
+
+  it('truncates long strings', () => {
+    expect(truncateString('hello world', 5)).toBe('hello');
+  });
+
+  it('handles undefined', () => {
+    expect(truncateString(undefined, 10)).toBeUndefined();
+  });
+});
+
+describe('sanitizeIngestPayload', () => {
+  it('redacts query params from endpoint', () => {
+    const payload = {
+      type: 'api-request' as const,
+      method: 'GET',
+      endpoint: '/api/pages?token=secret',
+      statusCode: 200,
+      duration: 100,
+    };
+    const result = sanitizeIngestPayload(payload);
+    expect(result.endpoint).toBe('/api/pages');
+  });
+
+  it('removes query field from payload', () => {
+    const payload = {
+      type: 'api-request' as const,
+      method: 'GET',
+      endpoint: '/api/test',
+      statusCode: 200,
+      duration: 100,
+      query: { token: 'secret', password: '123' },
+    };
+    const result = sanitizeIngestPayload(payload);
+    expect(result.query).toBeUndefined();
+  });
+
+  it('truncates long error messages', () => {
+    const payload = {
+      type: 'api-request' as const,
+      method: 'GET',
+      endpoint: '/api/test',
+      statusCode: 500,
+      duration: 100,
+      error: 'x'.repeat(2000),
+    };
+    const result = sanitizeIngestPayload(payload);
+    expect(result.error!.length).toBeLessThanOrEqual(1024);
+  });
+
+  it('truncates long error stacks', () => {
+    const payload = {
+      type: 'api-request' as const,
+      method: 'GET',
+      endpoint: '/api/test',
+      statusCode: 500,
+      duration: 100,
+      errorStack: 'x'.repeat(10000),
+    };
+    const result = sanitizeIngestPayload(payload);
+    expect(result.errorStack!.length).toBeLessThanOrEqual(4096);
+  });
+
+  it('truncates long user agents', () => {
+    const payload = {
+      type: 'api-request' as const,
+      method: 'GET',
+      endpoint: '/api/test',
+      statusCode: 200,
+      duration: 100,
+      userAgent: 'x'.repeat(1000),
+    };
+    const result = sanitizeIngestPayload(payload);
+    expect(result.userAgent!.length).toBeLessThanOrEqual(512);
+  });
+
+  it('clamps negative duration to 0', () => {
+    const payload = {
+      type: 'api-request' as const,
+      method: 'GET',
+      endpoint: '/api/test',
+      statusCode: 200,
+      duration: -5,
+    };
+    const result = sanitizeIngestPayload(payload);
+    expect(result.duration).toBe(0);
+  });
+
+  it('clamps excessively large duration', () => {
+    const payload = {
+      type: 'api-request' as const,
+      method: 'GET',
+      endpoint: '/api/test',
+      statusCode: 200,
+      duration: 999999999,
+    };
+    const result = sanitizeIngestPayload(payload);
+    expect(result.duration).toBeLessThanOrEqual(300000);
+  });
+
+  it('preserves valid payload fields', () => {
+    const payload = {
+      type: 'api-request' as const,
+      method: 'POST',
+      endpoint: '/api/pages',
+      statusCode: 201,
+      duration: 45,
+      userId: 'user-123',
+      requestId: 'req-456',
+    };
+    const result = sanitizeIngestPayload(payload);
+    expect(result.method).toBe('POST');
+    expect(result.statusCode).toBe(201);
+    expect(result.userId).toBe('user-123');
+    expect(result.requestId).toBe('req-456');
+  });
+});

apps/web/src/lib/monitoring/ingest-sanitizer.ts

@@ -0,0 +1,87 @@
+const MAX_ENDPOINT_LENGTH = 256;
+const MAX_ERROR_LENGTH = 1024;
+const MAX_STACK_LENGTH = 4096;
+const MAX_USER_AGENT_LENGTH = 512;
+const MAX_DURATION_MS = 300000; // 5 minutes
+
+interface IngestPayload {
+  type: 'api-request';
+  requestId?: string;
+  timestamp?: string;
+  method: string;
+  endpoint: string;
+  statusCode: number;
+  duration: number;
+  requestSize?: number;
+  responseSize?: number;
+  userId?: string;
+  sessionId?: string;
+  ip?: string;
+  userAgent?: string;
+  error?: string;
+  errorName?: string;
+  errorStack?: string;
+  cacheHit?: boolean;
+  cacheKey?: string;
+  driveId?: string;
+  pageId?: string;
+  message?: string;
+  metadata?: Record<string, unknown>;
+  query?: Record<string, unknown>;
+}
+
+export function redactUrlQueryParams(url: string): string {
+  if (!url) return '';
+
+  // Handle full URLs
+  if (url.startsWith('http://') || url.startsWith('https://')) {
+    try {
+      const parsed = new URL(url);
+      return parsed.pathname;
+    } catch {
+      // Fall through to simple stripping
+    }
+  }
+
+  // Strip query string and fragment from path
+  const queryIdx = url.indexOf('?');
+  const hashIdx = url.indexOf('#');
+  const cutIdx = Math.min(
+    queryIdx >= 0 ? queryIdx : url.length,
+    hashIdx >= 0 ? hashIdx : url.length,
+  );
+  return url.slice(0, cutIdx);
+}
+
+export function sanitizeEndpoint(endpoint: string): string {
+  if (!endpoint) return '';
+  let clean = redactUrlQueryParams(endpoint);
+  // Normalize double slashes (but preserve leading slash)
+  clean = clean.replace(/\/\/+/g, '/');
+  if (clean.length > MAX_ENDPOINT_LENGTH) {
+    clean = clean.slice(0, MAX_ENDPOINT_LENGTH);
+  }
+  return clean;
+}
+
+export function truncateString(value: string | undefined, maxLength: number): string | undefined {
+  if (value === undefined) return undefined;
+  if (value.length <= maxLength) return value;
+  return value.slice(0, maxLength);
+}
+
+function clampNumber(value: number, min: number, max: number): number {
+  return Math.max(min, Math.min(max, value));
+}
+
+export function sanitizeIngestPayload(payload: IngestPayload): IngestPayload {
+  return {
+    ...payload,
+    endpoint: sanitizeEndpoint(payload.endpoint),
+    duration: clampNumber(payload.duration, 0, MAX_DURATION_MS),
+    error: truncateString(payload.error, MAX_ERROR_LENGTH),
+    errorStack: truncateString(payload.errorStack, MAX_STACK_LENGTH),
+    userAgent: truncateString(payload.userAgent, MAX_USER_AGENT_LENGTH),
+    query: undefined,
+  };
+}

apps/web/src/middleware/monitoring.ts

@@ -10,6 +10,7 @@ import {
   REQUEST_ID_HEADER,
 } from '@/lib/request-id/request-id';
 import { db, apiMetrics } from '@pagespace/db';
+import { sanitizeEndpoint } from '@/lib/monitoring/ingest-sanitizer';
 
 // In-memory buffer for metrics (flushed to database every 30s or when buffer is full)
 interface RequestMetrics {
@@ -180,7 +181,6 @@ interface MonitoringIngestPayload {
   driveId?: string;
   pageId?: string;
   metadata?: Record<string, unknown>;
-  query?: Record<string, unknown>;
 }
 
 const DEFAULT_INGEST_PATH = '/api/internal/monitoring/ingest';
@@ -366,9 +366,11 @@ export async function monitoringMiddleware(
     const responseSize = getResponseSize(response);
     const statusCode = response.status;
 
+    const cleanEndpoint = sanitizeEndpoint(pathname);
+
     // Track metrics
     await metricsCollector.track({
-      endpoint: pathname,
+      endpoint: cleanEndpoint,
       method: request.method,
       statusCode,
       duration,
@@ -389,7 +391,7 @@ export async function monitoringMiddleware(
         requestId,
         timestamp: startedAt.toISOString(),
         method: request.method.toUpperCase(),
-        endpoint: pathname,
+        endpoint: cleanEndpoint,
         statusCode,
         duration,
         requestSize,
@@ -398,7 +400,6 @@ export async function monitoringMiddleware(
         sessionId: context.sessionId,
         ip: context.ip,
         userAgent: context.userAgent,
-        query: context.query as Record<string, unknown> | undefined,
         error: isServerError ? `HTTP ${statusCode}` : undefined,
         errorName: isServerError ? 'HttpError' : undefined,
       });
@@ -431,9 +432,11 @@ export async function monitoringMiddleware(
     const errorName = error instanceof Error ? error.name : 'Error';
     const errorStack = error instanceof Error ? error.stack : undefined;
 
+    const cleanEndpointErr = sanitizeEndpoint(pathname);
+
     // Track error metrics
     await metricsCollector.track({
-      endpoint: pathname,
+      endpoint: cleanEndpointErr,
       method: request.method,
       statusCode: 500,
       duration,
@@ -452,7 +455,7 @@ export async function monitoringMiddleware(
         requestId,
         timestamp: startedAt.toISOString(),
         method: request.method.toUpperCase(),
-        endpoint: pathname,
+        endpoint: cleanEndpointErr,
         statusCode: 500,
         duration,
         requestSize: getRequestSize(request),
@@ -464,7 +467,6 @@ export async function monitoringMiddleware(
         error: errorMessage,
         errorName,
         errorStack,
-        query: context.query as Record<string, unknown> | undefined,
       });
     }