From 7a3eabccd91d44c57262c6f4101cf1fd42b63273 Mon Sep 17 00:00:00 2001 From: Kaj Kowalski Date: Sat, 12 Jul 2025 01:03:52 +0200 Subject: [PATCH] feat: enhance security, performance, and stability This commit introduces a range of improvements across the application: - **Security:** - Adds authentication to the CSP metrics endpoint. - Hardens CSP bypass detection regex to prevent ReDoS attacks. - Improves CORS headers for the CSP metrics API. - Adds filtering for acknowledged alerts in security monitoring. - **Performance:** - Optimizes database connection pooling for NeonDB. - Improves session fetching with abort controller. - **Stability:** - Adds error handling to the tRPC demo component. - Fixes type inconsistencies in session data mapping. - **Docs & DX:** - Ignores files in git. - Fixes a token placeholder in the documentation. --- .gitignore | 1 + .../admin/security-monitoring/alerts/route.ts | 10 ++- app/api/csp-metrics/route.ts | 19 +++-- app/api/dashboard/session/[id]/route.ts | 8 +- app/platform/settings/page.tsx | 45 +++++++--- components/examples/TRPCDemo.tsx | 34 +++++++- docs/neon-database-optimization.md | 84 +++++++++---------- lib/csp.ts | 48 +++++------ lib/database-pool.ts | 21 +++-- 9 files changed, 173 insertions(+), 97 deletions(-) diff --git a/.gitignore b/.gitignore index 0579e83..a05bdf5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *-PROGRESS.md +pr-comments*.json # Created by https://www.toptal.com/developers/gitignore/api/node,nextjs,react # Edit at https://www.toptal.com/developers/gitignore?templates=node,nextjs,react diff --git a/app/api/admin/security-monitoring/alerts/route.ts b/app/api/admin/security-monitoring/alerts/route.ts index 382c40f..47bd2ca 100644 --- a/app/api/admin/security-monitoring/alerts/route.ts +++ b/app/api/admin/security-monitoring/alerts/route.ts @@ -45,10 +45,18 @@ export async function GET(request: NextRequest) { const context = await createAuditContext(request, session); // Get alerts based on filters - const alerts = securityMonitoring.getActiveAlerts( + let alerts = securityMonitoring.getActiveAlerts( query.severity as AlertSeverity ); + // Apply acknowledged filter if provided + if (query.acknowledged !== undefined) { + const showAcknowledged = query.acknowledged === "true"; + alerts = alerts.filter((alert) => + showAcknowledged ? alert.acknowledged : !alert.acknowledged + ); + } + // Apply pagination const limit = query.limit || 50; const offset = query.offset || 0; diff --git a/app/api/csp-metrics/route.ts b/app/api/csp-metrics/route.ts index 47a4aed..34fbba1 100644 --- a/app/api/csp-metrics/route.ts +++ b/app/api/csp-metrics/route.ts @@ -1,12 +1,19 @@ import { type NextRequest, NextResponse } from "next/server"; +import { getServerSession } from "next-auth"; +import { authOptions } from "@/lib/auth"; import { cspMonitoring } from "@/lib/csp-monitoring"; -import { rateLimiter } from "@/lib/rateLimiter"; +import { extractClientIP, rateLimiter } from "@/lib/rateLimiter"; export async function GET(request: NextRequest) { try { + // Authentication check for security metrics endpoint + const session = await getServerSession(authOptions); + + if (!session?.user || !session.user.isPlatformUser) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } // Rate limiting for metrics endpoint - const ip = - request.ip || request.headers.get("x-forwarded-for") || "unknown"; + const ip = extractClientIP(request); const rateLimitResult = await rateLimiter.check( `csp-metrics:${ip}`, 30, // 30 requests @@ -102,9 +109,11 @@ export async function OPTIONS() { return new NextResponse(null, { status: 200, headers: { - "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Origin": + process.env.ALLOWED_ORIGINS || "https://livedash.notso.ai", "Access-Control-Allow-Methods": "GET, OPTIONS", - "Access-Control-Allow-Headers": "Content-Type", + "Access-Control-Allow-Headers": "Content-Type, Authorization", + "Access-Control-Allow-Credentials": "true", }, }); } diff --git a/app/api/dashboard/session/[id]/route.ts b/app/api/dashboard/session/[id]/route.ts index bf3ae53..1987c8b 100644 --- a/app/api/dashboard/session/[id]/route.ts +++ b/app/api/dashboard/session/[id]/route.ts @@ -51,11 +51,11 @@ function mapPrismaSessionToChatSession(prismaSession: { country: prismaSession.country ?? null, ipAddress: prismaSession.ipAddress ?? null, sentiment: prismaSession.sentiment ?? null, - messagesSent: prismaSession.messagesSent ?? undefined, // Use undefined if ChatSession expects number | undefined + messagesSent: prismaSession.messagesSent ?? null, // Maintain consistency with other nullable fields avgResponseTime: prismaSession.avgResponseTime ?? null, - escalated: prismaSession.escalated ?? undefined, - forwardedHr: prismaSession.forwardedHr ?? undefined, - initialMsg: prismaSession.initialMsg ?? undefined, + escalated: prismaSession.escalated, + forwardedHr: prismaSession.forwardedHr, + initialMsg: prismaSession.initialMsg ?? null, fullTranscriptUrl: prismaSession.fullTranscriptUrl ?? null, summary: prismaSession.summary ?? null, // New field transcriptContent: null, // Not available in Session model diff --git a/app/platform/settings/page.tsx b/app/platform/settings/page.tsx index 343f719..f7e7f6a 100644 --- a/app/platform/settings/page.tsx +++ b/app/platform/settings/page.tsx @@ -25,6 +25,8 @@ function usePlatformSession() { name?: string; role: string; companyId?: string; + isPlatformUser?: boolean; + platformRole?: string; }; } | null>(null); const [status, setStatus] = useState< @@ -32,26 +34,47 @@ function usePlatformSession() { >("loading"); useEffect(() => { + const abortController = new AbortController(); + + const handleAuthSuccess = (sessionData: any) => { + if (sessionData?.user?.isPlatformUser) { + setSession(sessionData); + setStatus("authenticated"); + } else { + handleAuthFailure(); + } + }; + + const handleAuthFailure = (error?: unknown) => { + if (error instanceof Error && error.name === "AbortError") return; + if (error) console.error("Platform session fetch error:", error); + setSession(null); + setStatus("unauthenticated"); + }; + const fetchSession = async () => { try { - const response = await fetch("/api/platform/auth/session"); - const sessionData = await response.json(); + const response = await fetch("/api/platform/auth/session", { + signal: abortController.signal, + }); - if (sessionData?.user?.isPlatformUser) { - setSession(sessionData); - setStatus("authenticated"); - } else { - setSession(null); - setStatus("unauthenticated"); + if (!response.ok) { + if (response.status === 401) return handleAuthFailure(); + throw new Error(`Failed to fetch session: ${response.status}`); } + + const sessionData = await response.json(); + handleAuthSuccess(sessionData); } catch (error) { - console.error("Platform session fetch error:", error); - setSession(null); - setStatus("unauthenticated"); + handleAuthFailure(error); } }; fetchSession(); + + return () => { + abortController.abort(); + }; }, []); return { data: session, status }; diff --git a/components/examples/TRPCDemo.tsx b/components/examples/TRPCDemo.tsx index c727e73..c47a260 100644 --- a/components/examples/TRPCDemo.tsx +++ b/components/examples/TRPCDemo.tsx @@ -31,11 +31,17 @@ export function TRPCDemo() { refetch: refetchSessions, } = trpc.dashboard.getSessions.useQuery(sessionFilters); - const { data: overview, isLoading: overviewLoading } = - trpc.dashboard.getOverview.useQuery({}); + const { + data: overview, + isLoading: overviewLoading, + error: overviewError, + } = trpc.dashboard.getOverview.useQuery({}); - const { data: topQuestions, isLoading: questionsLoading } = - trpc.dashboard.getTopQuestions.useQuery({ limit: 3 }); + const { + data: topQuestions, + isLoading: questionsLoading, + error: questionsError, + } = trpc.dashboard.getTopQuestions.useQuery({ limit: 3 }); // Mutations const refreshSessionsMutation = trpc.dashboard.refreshSessions.useMutation({ @@ -84,6 +90,11 @@ export function TRPCDemo() { + {overviewError && ( +
+ Error: {overviewError.message} +
+ )} {overviewLoading ? (
@@ -102,6 +113,11 @@ export function TRPCDemo() { Avg Messages + {overviewError && ( +
+ Error: {overviewError.message} +
+ )} {overviewLoading ? (
@@ -122,6 +138,11 @@ export function TRPCDemo() { + {overviewError && ( +
+ Error: {overviewError.message} +
+ )} {overviewLoading ? (
@@ -150,6 +171,11 @@ export function TRPCDemo() { Top Questions + {questionsError && ( +
+ Error loading questions: {questionsError.message} +
+ )} {questionsLoading ? (
diff --git a/docs/neon-database-optimization.md b/docs/neon-database-optimization.md index 1ae702b..3c072ba 100644 --- a/docs/neon-database-optimization.md +++ b/docs/neon-database-optimization.md @@ -6,7 +6,7 @@ This document provides specific recommendations for optimizing database connecti From your logs, we can see: -``` +```bash Can't reach database server at `ep-tiny-math-a2zsshve-pooler.eu-central-1.aws.neon.tech:5432` [NODE-CRON] [WARN] missed execution at Sun Jun 29 2025 12:00:00 GMT+0200! Possible blocking IO or high CPU ``` @@ -15,21 +15,21 @@ Can't reach database server at `ep-tiny-math-a2zsshve-pooler.eu-central-1.aws.ne ### 1. Neon Connection Limits -- **Free Tier**: 20 concurrent connections -- **Pro Tier**: 100 concurrent connections -- **Multiple schedulers** can quickly exhaust connections +- **Free Tier**: 20 concurrent connections +- **Pro Tier**: 100 concurrent connections +- **Multiple schedulers** can quickly exhaust connections ### 2. Connection Pooling Issues -- Each scheduler was creating separate PrismaClient instances -- No connection reuse between operations -- No retry logic for temporary failures +- Each scheduler was creating separate PrismaClient instances +- No connection reuse between operations +- No retry logic for temporary failures ### 3. Neon-Specific Challenges -- **Auto-pause**: Databases pause after inactivity -- **Cold starts**: First connection after pause takes longer -- **Regional latency**: eu-central-1 may have variable latency +- **Auto-pause**: Databases pause after inactivity +- **Cold starts**: First connection after pause takes longer +- **Regional latency**: eu-central-1 may have variable latency ## Solutions Implemented @@ -100,15 +100,15 @@ SESSION_PROCESSING_INTERVAL="0 */2 * * *" # Every 2 hours instead of 1 ```bash # Check connection health -curl -H "Authorization: Bearer your-token" \ +curl -H "Authorization: Bearer YOUR_API_TOKEN" \ http://localhost:3000/api/admin/database-health ``` ### 2. Neon Dashboard Monitoring -- Monitor "Active connections" in Neon dashboard -- Check for connection spikes during scheduler runs -- Review query performance and slow queries +- Monitor "Active connections" in Neon dashboard +- Check for connection spikes during scheduler runs +- Review query performance and slow queries ### 3. Application Logs @@ -158,44 +158,44 @@ const prisma = new PrismaClient({ **Causes:** -- Neon database auto-paused -- Connection limit exceeded -- Network issues +- Neon database auto-paused +- Connection limit exceeded +- Network issues **Solutions:** -1. Enable enhanced pooling: `USE_ENHANCED_POOLING=true` -2. Reduce connection limit: `DATABASE_CONNECTION_LIMIT=15` -3. Implement retry logic (already done) -4. Check Neon dashboard for database status +1. Enable enhanced pooling: `USE_ENHANCED_POOLING=true` +2. Reduce connection limit: `DATABASE_CONNECTION_LIMIT=15` +3. Implement retry logic (already done) +4. Check Neon dashboard for database status ### "Connection terminated" **Causes:** -- Idle connection timeout -- Neon maintenance -- Long-running transactions +- Idle connection timeout +- Neon maintenance +- Long-running transactions **Solutions:** -1. Increase pool timeout: `DATABASE_POOL_TIMEOUT=30` -2. Add connection cycling -3. Break large operations into smaller batches +1. Increase pool timeout: `DATABASE_POOL_TIMEOUT=30` +2. Add connection cycling +3. Break large operations into smaller batches ### "Missed cron execution" **Causes:** -- Blocking database operations -- Scheduler overlap -- High CPU usage +- Blocking database operations +- Scheduler overlap +- High CPU usage **Solutions:** -1. Reduce scheduler frequency -2. Add concurrency limits -3. Monitor scheduler execution time +1. Reduce scheduler frequency +2. Add concurrency limits +3. Monitor scheduler execution time ## Recommended Production Settings @@ -223,17 +223,17 @@ SESSION_PROCESSING_INTERVAL="0 */2 * * *" ## Next Steps -1. **Immediate**: Apply the new environment variables -2. **Short-term**: Monitor connection usage via health endpoint -3. **Long-term**: Consider upgrading to Neon Pro for more connections -4. **Optional**: Implement read replicas for analytics queries +1. **Immediate**: Apply the new environment variables +2. **Short-term**: Monitor connection usage via health endpoint +3. **Long-term**: Consider upgrading to Neon Pro for more connections +4. **Optional**: Implement read replicas for analytics queries ## Monitoring Checklist -- [ ] Check Neon dashboard for connection spikes -- [ ] Monitor scheduler execution times -- [ ] Review error logs for connection patterns -- [ ] Test health endpoint regularly -- [ ] Set up alerts for connection failures +- [ ] Check Neon dashboard for connection spikes +- [ ] Monitor scheduler execution times +- [ ] Review error logs for connection patterns +- [ ] Test health endpoint regularly +- [ ] Set up alerts for connection failures With these optimizations, your Neon database connections should be much more stable and efficient! diff --git a/lib/csp.ts b/lib/csp.ts index 779ee63..bbf9bf9 100644 --- a/lib/csp.ts +++ b/lib/csp.ts @@ -401,30 +401,30 @@ export function parseCSPViolation(report: CSPViolationReport): { } /** - * CSP bypass detection patterns + * CSP bypass detection patterns - optimized to prevent ReDoS attacks */ export const CSP_BYPASS_PATTERNS = [ - // Common XSS bypass attempts - /javascript:/i, - /data:text\/html/i, - /vbscript:/i, - /livescript:/i, + // Common XSS bypass attempts (exact matches to prevent ReDoS) + /^javascript:/i, + /^data:text\/html/i, + /^vbscript:/i, + /^livescript:/i, - // Base64 encoded attempts - /data:.*base64.*script/i, - /data:text\/javascript/i, - /data:application\/javascript/i, + // Base64 encoded attempts (limited quantifiers to prevent ReDoS) + /^data:[^;]{0,50};base64[^,]{0,100},.*script/i, + /^data:text\/javascript/i, + /^data:application\/javascript/i, - // JSONP callback manipulation - /callback=.*script/i, + // JSONP callback manipulation (limited lookahead) + /callback=[^&]{0,200}script/i, - // Common CSP bypass techniques - /location\.href.*javascript/i, - /document\.write.*script/i, - /eval\(/i, + // Common CSP bypass techniques (limited quantifiers) + /location\.href[^;]{0,100}javascript/i, + /document\.write[^;]{0,100}script/i, + /\beval\s*\(/i, /\bnew\s+Function\s*\(/i, - /setTimeout\s*\(\s*['"`].*['"`]/i, - /setInterval\s*\(\s*['"`].*['"`]/i, + /setTimeout\s*\(\s*['"`][^'"`]{0,500}['"`]/i, + /setInterval\s*\(\s*['"`][^'"`]{0,500}['"`]/i, ]; /** @@ -550,14 +550,14 @@ export function detectCSPBypass(content: string): { } } - // Determine risk level based on pattern types + // Determine risk level based on pattern types (ReDoS-safe patterns) const highRiskPatterns = [ - /javascript:/i, - /eval\(/i, + /^javascript:/i, + /\beval\s*\(/i, /\bnew\s+Function\s*\(/i, - /data:text\/javascript/i, - /data:application\/javascript/i, - /data:.*base64.*script/i, + /^data:text\/javascript/i, + /^data:application\/javascript/i, + /^data:[^;]{0,50};base64[^,]{0,100},.*script/i, ]; const hasHighRiskPattern = detectedPatterns.some((pattern) => diff --git a/lib/database-pool.ts b/lib/database-pool.ts index 62efc83..767b073 100644 --- a/lib/database-pool.ts +++ b/lib/database-pool.ts @@ -21,15 +21,24 @@ export const createEnhancedPrismaClient = () => { ? { rejectUnauthorized: false } : undefined, - // Connection pool settings - max: env.DATABASE_CONNECTION_LIMIT || 20, // Maximum number of connections + // Connection pool settings optimized for Neon + max: env.DATABASE_CONNECTION_LIMIT || 15, // Maximum number of connections (reduced for Neon) + min: 2, // Minimum connections to keep warm (prevent auto-pause) idleTimeoutMillis: env.DATABASE_POOL_TIMEOUT * 1000 || 30000, // Use env timeout - connectionTimeoutMillis: 5000, // 5 seconds - query_timeout: 10000, // 10 seconds - statement_timeout: 10000, // 10 seconds + connectionTimeoutMillis: 10000, // 10 seconds (increased for Neon cold starts) + query_timeout: 15000, // 15 seconds (increased for Neon) + statement_timeout: 15000, // 15 seconds (increased for Neon) + + // Keepalive settings to prevent Neon auto-pause + keepAlive: true, + keepAliveInitialDelayMillis: 10000, + + // Application name for monitoring in Neon dashboard + application_name: + dbUrl.searchParams.get("application_name") || "livedash-app", // Connection lifecycle - allowExitOnIdle: true, + allowExitOnIdle: false, // Keep minimum connections alive for Neon }; const adapter = new PrismaPg(poolConfig);