diff --git a/apps/web/src/app/api/internal/monitoring/ingest/route.ts b/apps/web/src/app/api/internal/monitoring/ingest/route.ts index f4cbeb594..bb19b8016 100644 --- a/apps/web/src/app/api/internal/monitoring/ingest/route.ts +++ b/apps/web/src/app/api/internal/monitoring/ingest/route.ts @@ -4,34 +4,7 @@ import { db, systemLogs } from '@pagespace/db'; import { writeApiMetrics, writeError } from '@pagespace/lib/logger-database'; import { loggers } from '@pagespace/lib/server'; import { secureCompare } from '@pagespace/lib'; - -interface ApiRequestPayload { - type: 'api-request'; - requestId?: string; - timestamp?: string; - method: string; - endpoint: string; - statusCode: number; - duration: number; - requestSize?: number; - responseSize?: number; - userId?: string; - sessionId?: string; - ip?: string; - userAgent?: string; - error?: string; - errorName?: string; - errorStack?: string; - cacheHit?: boolean; - cacheKey?: string; - driveId?: string; - pageId?: string; - message?: string; - metadata?: Record; - query?: Record; -} - -type IngestPayload = ApiRequestPayload; +import { sanitizeIngestPayload, type IngestPayload } from '@/lib/monitoring/ingest-sanitizer'; function unauthorized(message: string, status: number = 401) { return NextResponse.json({ error: message }, { status }); @@ -45,9 +18,10 @@ export async function POST(request: Request) { const ingestKey = process.env.MONITORING_INGEST_KEY; if (!ingestKey) { - loggers.system.error( - 'MONITORING_INGEST_KEY is not configured and MONITORING_INGEST_DISABLED is not set. ' + - 'Monitoring ingest cannot process requests. Set the key or set MONITORING_INGEST_DISABLED=true to opt out.' + loggers.system.warn( + 'MONITORING_INGEST_KEY is not configured. ' + + 'Monitoring ingest cannot authenticate requests. ' + + 'Set MONITORING_INGEST_KEY to enable or MONITORING_INGEST_DISABLED=true to opt out.' ); return NextResponse.json({ error: 'Monitoring ingest not configured' }, { status: 503 }); } @@ -58,17 +32,19 @@ export async function POST(request: Request) { return unauthorized('Unauthorized'); } - let payload: IngestPayload; + let rawPayload: IngestPayload; try { - payload = await request.json(); + rawPayload = await request.json(); } catch { return NextResponse.json({ error: 'Invalid JSON payload' }, { status: 400 }); } - if (payload.type !== 'api-request') { + if (rawPayload.type !== 'api-request') { return NextResponse.json({ error: 'Unsupported payload type' }, { status: 400 }); } + const payload = sanitizeIngestPayload(rawPayload); + const timestamp = payload.timestamp ? new Date(payload.timestamp) : new Date(); type HttpMethod = NonNullable<(typeof systemLogs.$inferInsert)['method']>; type SystemLogLevel = (typeof systemLogs.$inferInsert)['level']; @@ -78,52 +54,52 @@ export async function POST(request: Request) { const level = (statusCode >= 500 ? 'error' : statusCode >= 400 ? 'warn' : 'info') as SystemLogLevel; try { - await writeApiMetrics({ - endpoint: payload.endpoint, - method, - statusCode, - duration: payload.duration, - requestSize: payload.requestSize, - responseSize: payload.responseSize, - userId: payload.userId, - sessionId: payload.sessionId, - ip: payload.ip, - userAgent: payload.userAgent, - error: payload.error, - requestId: payload.requestId, - cacheHit: payload.cacheHit, - cacheKey: payload.cacheKey, - timestamp, - }); - - await db.insert(systemLogs).values({ - id: createId(), - timestamp, - level, - message: - payload.message || `${method} ${payload.endpoint} ${statusCode} ${payload.duration}ms`, - category: 'api', - userId: payload.userId, - sessionId: payload.sessionId, - requestId: payload.requestId, - driveId: payload.driveId, - pageId: payload.pageId, - endpoint: payload.endpoint, - method, - ip: payload.ip, - userAgent: payload.userAgent, - duration: payload.duration, - metadata: { - ...payload.metadata, + const operations: Promise[] = [ + writeApiMetrics({ + endpoint: payload.endpoint, + method, statusCode, + duration: payload.duration, requestSize: payload.requestSize, responseSize: payload.responseSize, - query: payload.query, - }, - }); + userId: payload.userId, + sessionId: payload.sessionId, + ip: payload.ip, + userAgent: payload.userAgent, + error: payload.error, + requestId: payload.requestId, + cacheHit: payload.cacheHit, + cacheKey: payload.cacheKey, + timestamp, + }), + db.insert(systemLogs).values({ + id: createId(), + timestamp, + level, + message: + payload.message || `${method} ${payload.endpoint} ${statusCode} ${payload.duration}ms`, + category: 'api', + userId: payload.userId, + sessionId: payload.sessionId, + requestId: payload.requestId, + driveId: payload.driveId, + pageId: payload.pageId, + endpoint: payload.endpoint, + method, + ip: payload.ip, + userAgent: payload.userAgent, + duration: payload.duration, + metadata: { + ...payload.metadata, + statusCode, + requestSize: payload.requestSize, + responseSize: payload.responseSize, + }, + }), + ]; if (payload.error) { - await writeError({ + operations.push(writeError({ name: payload.errorName || 'RequestError', message: payload.error, stack: payload.errorStack, @@ -139,9 +115,11 @@ export async function POST(request: Request) { statusCode, duration: payload.duration, }, - }); + })); } + await Promise.all(operations); + return NextResponse.json({ success: true }); } catch (error) { loggers.api.error('Failed to ingest monitoring payload', error as Error, { diff --git a/apps/web/src/lib/monitoring/__tests__/ingest-sanitizer.test.ts b/apps/web/src/lib/monitoring/__tests__/ingest-sanitizer.test.ts new file mode 100644 index 000000000..55f7c8c3a --- /dev/null +++ b/apps/web/src/lib/monitoring/__tests__/ingest-sanitizer.test.ts @@ -0,0 +1,174 @@ +import { describe, it, expect } from 'vitest'; +import { + redactUrlQueryParams, + sanitizeEndpoint, + sanitizeIngestPayload, + truncateString, +} from '../ingest-sanitizer'; + +describe('redactUrlQueryParams', () => { + it('strips query parameters from URLs', () => { + expect(redactUrlQueryParams('/api/pages?token=secret&id=123')).toBe('/api/pages'); + }); + + it('returns path unchanged when no query params', () => { + expect(redactUrlQueryParams('/api/pages')).toBe('/api/pages'); + }); + + it('strips fragment identifiers', () => { + expect(redactUrlQueryParams('/api/pages#section')).toBe('/api/pages'); + }); + + it('strips both query params and fragments', () => { + expect(redactUrlQueryParams('/api/pages?key=val#section')).toBe('/api/pages'); + }); + + it('handles empty string', () => { + expect(redactUrlQueryParams('')).toBe(''); + }); + + it('handles full URLs by extracting pathname', () => { + expect(redactUrlQueryParams('https://example.com/api/test?secret=abc')).toBe('/api/test'); + }); +}); + +describe('sanitizeEndpoint', () => { + it('normalizes double slashes', () => { + expect(sanitizeEndpoint('//api//pages//')).toBe('/api/pages/'); + }); + + it('truncates excessively long endpoints', () => { + const longPath = '/api/' + 'a'.repeat(500); + const result = sanitizeEndpoint(longPath); + expect(result.length).toBeLessThanOrEqual(256); + }); + + it('removes query parameters', () => { + expect(sanitizeEndpoint('/api/pages?token=secret')).toBe('/api/pages'); + }); + + it('handles undefined', () => { + expect(sanitizeEndpoint(undefined as unknown as string)).toBe(''); + }); +}); + +describe('truncateString', () => { + it('returns short strings unchanged', () => { + expect(truncateString('hello', 10)).toBe('hello'); + }); + + it('truncates long strings', () => { + expect(truncateString('hello world', 5)).toBe('hello'); + }); + + it('handles undefined', () => { + expect(truncateString(undefined, 10)).toBeUndefined(); + }); +}); + +describe('sanitizeIngestPayload', () => { + it('redacts query params from endpoint', () => { + const payload = { + type: 'api-request' as const, + method: 'GET', + endpoint: '/api/pages?token=secret', + statusCode: 200, + duration: 100, + }; + const result = sanitizeIngestPayload(payload); + expect(result.endpoint).toBe('/api/pages'); + }); + + it('removes query field from payload', () => { + const payload = { + type: 'api-request' as const, + method: 'GET', + endpoint: '/api/test', + statusCode: 200, + duration: 100, + query: { token: 'secret', password: '123' }, + }; + const result = sanitizeIngestPayload(payload); + expect(result.query).toBeUndefined(); + }); + + it('truncates long error messages', () => { + const payload = { + type: 'api-request' as const, + method: 'GET', + endpoint: '/api/test', + statusCode: 500, + duration: 100, + error: 'x'.repeat(2000), + }; + const result = sanitizeIngestPayload(payload); + expect(result.error!.length).toBeLessThanOrEqual(1024); + }); + + it('truncates long error stacks', () => { + const payload = { + type: 'api-request' as const, + method: 'GET', + endpoint: '/api/test', + statusCode: 500, + duration: 100, + errorStack: 'x'.repeat(10000), + }; + const result = sanitizeIngestPayload(payload); + expect(result.errorStack!.length).toBeLessThanOrEqual(4096); + }); + + it('truncates long user agents', () => { + const payload = { + type: 'api-request' as const, + method: 'GET', + endpoint: '/api/test', + statusCode: 200, + duration: 100, + userAgent: 'x'.repeat(1000), + }; + const result = sanitizeIngestPayload(payload); + expect(result.userAgent!.length).toBeLessThanOrEqual(512); + }); + + it('clamps negative duration to 0', () => { + const payload = { + type: 'api-request' as const, + method: 'GET', + endpoint: '/api/test', + statusCode: 200, + duration: -5, + }; + const result = sanitizeIngestPayload(payload); + expect(result.duration).toBe(0); + }); + + it('clamps excessively large duration', () => { + const payload = { + type: 'api-request' as const, + method: 'GET', + endpoint: '/api/test', + statusCode: 200, + duration: 999999999, + }; + const result = sanitizeIngestPayload(payload); + expect(result.duration).toBeLessThanOrEqual(300000); + }); + + it('preserves valid payload fields', () => { + const payload = { + type: 'api-request' as const, + method: 'POST', + endpoint: '/api/pages', + statusCode: 201, + duration: 45, + userId: 'user-123', + requestId: 'req-456', + }; + const result = sanitizeIngestPayload(payload); + expect(result.method).toBe('POST'); + expect(result.statusCode).toBe(201); + expect(result.userId).toBe('user-123'); + expect(result.requestId).toBe('req-456'); + }); +}); diff --git a/apps/web/src/lib/monitoring/ingest-sanitizer.ts b/apps/web/src/lib/monitoring/ingest-sanitizer.ts new file mode 100644 index 000000000..51292f132 --- /dev/null +++ b/apps/web/src/lib/monitoring/ingest-sanitizer.ts @@ -0,0 +1,87 @@ +const MAX_ENDPOINT_LENGTH = 256; +const MAX_ERROR_LENGTH = 1024; +const MAX_STACK_LENGTH = 4096; +const MAX_USER_AGENT_LENGTH = 512; +const MAX_DURATION_MS = 300000; // 5 minutes + +export interface IngestPayload { + type: 'api-request'; + requestId?: string; + timestamp?: string; + method: string; + endpoint: string; + statusCode: number; + duration: number; + requestSize?: number; + responseSize?: number; + userId?: string; + sessionId?: string; + ip?: string; + userAgent?: string; + error?: string; + errorName?: string; + errorStack?: string; + cacheHit?: boolean; + cacheKey?: string; + driveId?: string; + pageId?: string; + message?: string; + metadata?: Record; + query?: Record; +} + +export function redactUrlQueryParams(url: string): string { + if (!url) return ''; + + // Handle full URLs + if (url.startsWith('http://') || url.startsWith('https://')) { + try { + const parsed = new URL(url); + return parsed.pathname; + } catch { + // Fall through to simple stripping + } + } + + // Strip query string and fragment from path + const queryIdx = url.indexOf('?'); + const hashIdx = url.indexOf('#'); + const cutIdx = Math.min( + queryIdx >= 0 ? queryIdx : url.length, + hashIdx >= 0 ? hashIdx : url.length, + ); + return url.slice(0, cutIdx); +} + +export function sanitizeEndpoint(endpoint: string): string { + if (!endpoint) return ''; + let clean = redactUrlQueryParams(endpoint); + // Normalize double slashes (but preserve leading slash) + clean = clean.replace(/\/\/+/g, '/'); + if (clean.length > MAX_ENDPOINT_LENGTH) { + clean = clean.slice(0, MAX_ENDPOINT_LENGTH); + } + return clean; +} + +export function truncateString(value: string | undefined, maxLength: number): string | undefined { + if (value === undefined) return undefined; + if (value.length <= maxLength) return value; + return value.slice(0, maxLength); +} + +function clampNumber(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); +} + +export function sanitizeIngestPayload(payload: IngestPayload): IngestPayload { + return { + ...payload, + endpoint: sanitizeEndpoint(payload.endpoint), + duration: clampNumber(payload.duration, 0, MAX_DURATION_MS), + error: truncateString(payload.error, MAX_ERROR_LENGTH), + errorStack: truncateString(payload.errorStack, MAX_STACK_LENGTH), + userAgent: truncateString(payload.userAgent, MAX_USER_AGENT_LENGTH), + query: undefined, + }; +} diff --git a/apps/web/src/middleware/monitoring.ts b/apps/web/src/middleware/monitoring.ts index fe0afa44c..b66d887d7 100644 --- a/apps/web/src/middleware/monitoring.ts +++ b/apps/web/src/middleware/monitoring.ts @@ -10,6 +10,7 @@ import { REQUEST_ID_HEADER, } from '@/lib/request-id/request-id'; import { db, apiMetrics } from '@pagespace/db'; +import { sanitizeEndpoint } from '@/lib/monitoring/ingest-sanitizer'; // In-memory buffer for metrics (flushed to database every 30s or when buffer is full) interface RequestMetrics { @@ -180,7 +181,6 @@ interface MonitoringIngestPayload { driveId?: string; pageId?: string; metadata?: Record; - query?: Record; } const DEFAULT_INGEST_PATH = '/api/internal/monitoring/ingest'; @@ -345,9 +345,11 @@ export async function monitoringMiddleware( // Extract context const context = extractRequestContext(request); const userId = await extractUserId(request); - + const cleanEndpoint = sanitizeEndpoint(pathname); + const requestSize = getRequestSize(request); + // Create request-scoped logger - const requestLogger = logger.child({ + const requestLogger = logger.child({ requestId, userId, ...context @@ -359,16 +361,15 @@ export async function monitoringMiddleware( try { // Execute the request const response = await next(); - + // Calculate metrics const duration = Date.now() - startTime; - const requestSize = getRequestSize(request); const responseSize = getResponseSize(response); const statusCode = response.status; // Track metrics await metricsCollector.track({ - endpoint: pathname, + endpoint: cleanEndpoint, method: request.method, statusCode, duration, @@ -389,7 +390,7 @@ export async function monitoringMiddleware( requestId, timestamp: startedAt.toISOString(), method: request.method.toUpperCase(), - endpoint: pathname, + endpoint: cleanEndpoint, statusCode, duration, requestSize, @@ -398,7 +399,6 @@ export async function monitoringMiddleware( sessionId: context.sessionId, ip: context.ip, userAgent: context.userAgent, - query: context.query as Record | undefined, error: isServerError ? `HTTP ${statusCode}` : undefined, errorName: isServerError ? 'HttpError' : undefined, }); @@ -433,7 +433,7 @@ export async function monitoringMiddleware( // Track error metrics await metricsCollector.track({ - endpoint: pathname, + endpoint: cleanEndpoint, method: request.method, statusCode: 500, duration, @@ -452,10 +452,10 @@ export async function monitoringMiddleware( requestId, timestamp: startedAt.toISOString(), method: request.method.toUpperCase(), - endpoint: pathname, + endpoint: cleanEndpoint, statusCode: 500, duration, - requestSize: getRequestSize(request), + requestSize, responseSize: 0, userId, sessionId: context.sessionId, @@ -464,7 +464,6 @@ export async function monitoringMiddleware( error: errorMessage, errorName, errorStack, - query: context.query as Record | undefined, }); } diff --git a/packages/lib/package.json b/packages/lib/package.json index c6647609a..763350927 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -263,6 +263,11 @@ "import": "./dist/compliance/retention/retention-engine.js", "require": "./dist/compliance/retention/retention-engine.js" }, + "./compliance/retention/monitoring-retention": { + "types": "./dist/compliance/retention/monitoring-retention.d.ts", + "import": "./dist/compliance/retention/monitoring-retention.js", + "require": "./dist/compliance/retention/monitoring-retention.js" + }, "./compliance/file-cleanup/orphan-detector": { "types": "./dist/compliance/file-cleanup/orphan-detector.d.ts", "import": "./dist/compliance/file-cleanup/orphan-detector.js", @@ -423,6 +428,9 @@ "compliance/retention/retention-engine": [ "./dist/compliance/retention/retention-engine.d.ts" ], + "compliance/retention/monitoring-retention": [ + "./dist/compliance/retention/monitoring-retention.d.ts" + ], "compliance/file-cleanup/orphan-detector": [ "./dist/compliance/file-cleanup/orphan-detector.d.ts" ], diff --git a/packages/lib/src/compliance/retention/monitoring-retention.test.ts b/packages/lib/src/compliance/retention/monitoring-retention.test.ts new file mode 100644 index 000000000..724ddd21c --- /dev/null +++ b/packages/lib/src/compliance/retention/monitoring-retention.test.ts @@ -0,0 +1,163 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +// Mock drizzle-orm before any imports that use it +vi.mock('drizzle-orm', () => ({ + lt: vi.fn((col, val) => ({ operator: 'lt', column: col, value: val })), +})); + +// Mock @pagespace/db +const mockReturning = vi.fn(); + +vi.mock('@pagespace/db', () => ({ + db: { + delete: vi.fn(() => ({ + where: vi.fn(() => ({ + returning: mockReturning, + })), + })), + }, + apiMetrics: { id: 'id', timestamp: 'timestamp' }, + systemLogs: { id: 'id', timestamp: 'timestamp' }, + securityAuditLog: { id: 'id', timestamp: 'timestamp' }, +})); + +import { + getRetentionConfig, + getRetentionCutoff, + cleanupApiMetrics, + cleanupSystemLogs, + cleanupSecurityAuditLog, + runMonitoringRetentionCleanup, +} from './monitoring-retention'; +import { db } from '@pagespace/db'; + +// Mock environment +const originalEnv = process.env; + +beforeEach(() => { + process.env = { ...originalEnv }; + delete process.env.RETENTION_API_METRICS_DAYS; + delete process.env.RETENTION_SYSTEM_LOGS_DAYS; + delete process.env.RETENTION_SECURITY_AUDIT_DAYS; + vi.clearAllMocks(); + mockReturning.mockResolvedValue([]); +}); + +afterEach(() => { + process.env = originalEnv; +}); + +describe('getRetentionConfig', () => { + it('returns default retention periods when no env vars set', () => { + const config = getRetentionConfig(); + expect(config.apiMetricsDays).toBe(90); + expect(config.systemLogsDays).toBe(30); + expect(config.securityAuditDays).toBe(365); + }); + + it('uses env vars when set', () => { + process.env.RETENTION_API_METRICS_DAYS = '60'; + process.env.RETENTION_SYSTEM_LOGS_DAYS = '14'; + process.env.RETENTION_SECURITY_AUDIT_DAYS = '730'; + + const config = getRetentionConfig(); + expect(config.apiMetricsDays).toBe(60); + expect(config.systemLogsDays).toBe(14); + expect(config.securityAuditDays).toBe(730); + }); + + it('falls back to defaults for invalid env vars', () => { + process.env.RETENTION_API_METRICS_DAYS = 'not-a-number'; + process.env.RETENTION_SYSTEM_LOGS_DAYS = '-5'; + process.env.RETENTION_SECURITY_AUDIT_DAYS = '0'; + + const config = getRetentionConfig(); + expect(config.apiMetricsDays).toBe(90); + expect(config.systemLogsDays).toBe(30); + expect(config.securityAuditDays).toBe(365); + }); +}); + +describe('getRetentionCutoff', () => { + it('returns a date N days in the past', () => { + const now = Date.now(); + const cutoff = getRetentionCutoff(30); + const expectedMs = 30 * 24 * 60 * 60 * 1000; + const diff = now - cutoff.getTime(); + // Allow 1 second tolerance + expect(Math.abs(diff - expectedMs)).toBeLessThan(1000); + }); +}); + +describe('cleanupApiMetrics', () => { + it('calls delete on api_metrics table', async () => { + const result = await cleanupApiMetrics({ retentionDays: 90 }); + expect(result.table).toBe('api_metrics'); + expect(result.deleted).toBe(0); + expect(db.delete).toHaveBeenCalled(); + }); + + it('returns count of deleted rows', async () => { + mockReturning.mockResolvedValueOnce([{ id: '1' }, { id: '2' }]); + const result = await cleanupApiMetrics({ retentionDays: 90 }); + expect(result.deleted).toBe(2); + }); +}); + +describe('cleanupSystemLogs', () => { + it('calls delete on system_logs table', async () => { + const result = await cleanupSystemLogs({ retentionDays: 30 }); + expect(result.table).toBe('system_logs'); + expect(result.deleted).toBe(0); + expect(db.delete).toHaveBeenCalled(); + }); + + it('returns count of deleted rows', async () => { + mockReturning.mockResolvedValueOnce([{ id: '1' }, { id: '2' }, { id: '3' }]); + const result = await cleanupSystemLogs({ retentionDays: 30 }); + expect(result.deleted).toBe(3); + }); +}); + +describe('cleanupSecurityAuditLog', () => { + it('calls delete on security_audit_log table', async () => { + const result = await cleanupSecurityAuditLog({ retentionDays: 365 }); + expect(result.table).toBe('security_audit_log'); + expect(result.deleted).toBe(0); + expect(db.delete).toHaveBeenCalled(); + }); + + it('returns count of deleted rows', async () => { + mockReturning.mockResolvedValueOnce([{ id: '1' }]); + const result = await cleanupSecurityAuditLog({ retentionDays: 365 }); + expect(result.deleted).toBe(1); + }); +}); + +describe('runMonitoringRetentionCleanup', () => { + it('cleans up all three monitoring tables', async () => { + const results = await runMonitoringRetentionCleanup(); + expect(results).toHaveLength(3); + const tables = results.map(r => r.table).sort(); + expect(tables).toEqual(['api_metrics', 'security_audit_log', 'system_logs']); + }); + + it('uses default retention config', async () => { + const results = await runMonitoringRetentionCleanup(); + expect(results).toHaveLength(3); + for (const result of results) { + expect(result).toHaveProperty('table'); + expect(result).toHaveProperty('deleted'); + expect(typeof result.deleted).toBe('number'); + } + }); + + it('uses custom retention config from env vars', async () => { + process.env.RETENTION_API_METRICS_DAYS = '45'; + process.env.RETENTION_SYSTEM_LOGS_DAYS = '7'; + process.env.RETENTION_SECURITY_AUDIT_DAYS = '180'; + + const results = await runMonitoringRetentionCleanup(); + expect(results).toHaveLength(3); + }); +}); diff --git a/packages/lib/src/compliance/retention/monitoring-retention.ts b/packages/lib/src/compliance/retention/monitoring-retention.ts new file mode 100644 index 000000000..189bcbe71 --- /dev/null +++ b/packages/lib/src/compliance/retention/monitoring-retention.ts @@ -0,0 +1,69 @@ +import { lt } from 'drizzle-orm'; +import { db, apiMetrics, systemLogs, securityAuditLog } from '@pagespace/db'; +import type { CleanupResult } from './retention-engine'; + +const DEFAULT_API_METRICS_DAYS = 90; +const DEFAULT_SYSTEM_LOGS_DAYS = 30; +const DEFAULT_SECURITY_AUDIT_DAYS = 365; + +export interface RetentionConfig { + apiMetricsDays: number; + systemLogsDays: number; + securityAuditDays: number; +} + +function parsePositiveInt(value: string | undefined, fallback: number): number { + if (!value) return fallback; + const parsed = parseInt(value, 10); + if (isNaN(parsed) || parsed <= 0) return fallback; + return parsed; +} + +export function getRetentionConfig(): RetentionConfig { + return { + apiMetricsDays: parsePositiveInt(process.env.RETENTION_API_METRICS_DAYS, DEFAULT_API_METRICS_DAYS), + systemLogsDays: parsePositiveInt(process.env.RETENTION_SYSTEM_LOGS_DAYS, DEFAULT_SYSTEM_LOGS_DAYS), + securityAuditDays: parsePositiveInt(process.env.RETENTION_SECURITY_AUDIT_DAYS, DEFAULT_SECURITY_AUDIT_DAYS), + }; +} + +export function getRetentionCutoff(days: number): Date { + return new Date(Date.now() - days * 24 * 60 * 60 * 1000); +} + +export async function cleanupApiMetrics(opts: { retentionDays: number }): Promise { + const cutoff = getRetentionCutoff(opts.retentionDays); + const result = await db + .delete(apiMetrics) + .where(lt(apiMetrics.timestamp, cutoff)) + .returning({ id: apiMetrics.id }); + return { table: 'api_metrics', deleted: result.length }; +} + +export async function cleanupSystemLogs(opts: { retentionDays: number }): Promise { + const cutoff = getRetentionCutoff(opts.retentionDays); + const result = await db + .delete(systemLogs) + .where(lt(systemLogs.timestamp, cutoff)) + .returning({ id: systemLogs.id }); + return { table: 'system_logs', deleted: result.length }; +} + +export async function cleanupSecurityAuditLog(opts: { retentionDays: number }): Promise { + const cutoff = getRetentionCutoff(opts.retentionDays); + const result = await db + .delete(securityAuditLog) + .where(lt(securityAuditLog.timestamp, cutoff)) + .returning({ id: securityAuditLog.id }); + return { table: 'security_audit_log', deleted: result.length }; +} + +export async function runMonitoringRetentionCleanup(): Promise { + const config = getRetentionConfig(); + const results = await Promise.all([ + cleanupApiMetrics({ retentionDays: config.apiMetricsDays }), + cleanupSystemLogs({ retentionDays: config.systemLogsDays }), + cleanupSecurityAuditLog({ retentionDays: config.securityAuditDays }), + ]); + return results; +} diff --git a/packages/lib/src/compliance/retention/retention-engine.test.ts b/packages/lib/src/compliance/retention/retention-engine.test.ts index b883cc32d..ac4855b4e 100644 --- a/packages/lib/src/compliance/retention/retention-engine.test.ts +++ b/packages/lib/src/compliance/retention/retention-engine.test.ts @@ -346,7 +346,7 @@ describe('runRetentionCleanup', () => { const results = await runRetentionCleanup(db); expect(Array.isArray(results)).toBe(true); - expect(results.length).toBe(9); + expect(results.length).toBe(12); for (const result of results) { expect(result).toHaveProperty('table'); @@ -362,13 +362,16 @@ describe('runRetentionCleanup', () => { expect(tableNames).toEqual([ 'ai_usage_logs', + 'api_metrics', 'drive_backups', 'email_unsubscribe_tokens', 'page_permissions', 'page_versions', 'pulse_summaries', + 'security_audit_log', 'sessions', 'socket_tokens', + 'system_logs', 'verification_tokens', ]); }); diff --git a/packages/lib/src/compliance/retention/retention-engine.ts b/packages/lib/src/compliance/retention/retention-engine.ts index 4c5b4c5f6..6580f158d 100644 --- a/packages/lib/src/compliance/retention/retention-engine.ts +++ b/packages/lib/src/compliance/retention/retention-engine.ts @@ -11,6 +11,7 @@ import { pagePermissions, pulseSummaries, } from '@pagespace/db'; +import { runMonitoringRetentionCleanup } from './monitoring-retention'; export interface CleanupResult { table: string; @@ -121,16 +122,19 @@ export async function cleanupExpiredAiUsageLogs(database: DB): Promise { - const results = await Promise.all([ - cleanupExpiredSessions(database), - cleanupExpiredVerificationTokens(database), - cleanupExpiredSocketTokens(database), - cleanupExpiredEmailUnsubscribeTokens(database), - cleanupExpiredPulseSummaries(database), - cleanupExpiredPageVersions(database), - cleanupExpiredDriveBackups(database), - cleanupExpiredPagePermissions(database), - cleanupExpiredAiUsageLogs(database), + const [expiryResults, monitoringResults] = await Promise.all([ + Promise.all([ + cleanupExpiredSessions(database), + cleanupExpiredVerificationTokens(database), + cleanupExpiredSocketTokens(database), + cleanupExpiredEmailUnsubscribeTokens(database), + cleanupExpiredPulseSummaries(database), + cleanupExpiredPageVersions(database), + cleanupExpiredDriveBackups(database), + cleanupExpiredPagePermissions(database), + cleanupExpiredAiUsageLogs(database), + ]), + runMonitoringRetentionCleanup(), ]); - return results; + return [...expiryResults, ...monitoringResults]; }