mirror of
https://github.com/kjanat/livedash-node.git
synced 2026-01-16 11:12:11 +01:00
fix: resolve all TypeScript compilation errors and enable production build
- Fixed missing type imports in lib/api/index.ts - Updated Zod error property from 'errors' to 'issues' for compatibility - Added missing lru-cache dependency for performance caching - Fixed LRU Cache generic type constraints for TypeScript compliance - Resolved Map iteration ES5 compatibility issues using Array.from() - Fixed Redis configuration by removing unsupported socket options - Corrected Prisma relationship naming (auditLogs vs securityAuditLogs) - Applied type casting for missing database schema fields - Created missing security types file for enhanced security service - Disabled deprecated ESLint during build (using Biome for linting) - Removed deprecated critters dependency and disabled CSS optimization - Achieved successful production build with all 47 pages generated
This commit is contained in:
@ -5,6 +5,7 @@ This document describes the Content Security Policy (CSP) metrics and violation
|
||||
## Overview
|
||||
|
||||
The CSP Metrics API provides comprehensive monitoring of Content Security Policy violations, including:
|
||||
|
||||
- Real-time violation tracking and metrics
|
||||
- Bypass attempt detection and risk assessment
|
||||
- Policy optimization recommendations
|
||||
@ -22,6 +23,7 @@ POST /api/csp-report
|
||||
```
|
||||
|
||||
#### Request Headers
|
||||
|
||||
- `Content-Type`: `application/csp-report` or `application/json`
|
||||
|
||||
#### Request Body (Automatic from Browser)
|
||||
@ -40,6 +42,7 @@ POST /api/csp-report
|
||||
```
|
||||
|
||||
#### Features
|
||||
|
||||
- **Rate Limiting**: 10 reports per minute per IP
|
||||
- **Risk Assessment**: Automatic classification of violation severity
|
||||
- **Bypass Detection**: Identifies potential CSP bypass attempts
|
||||
@ -55,16 +58,17 @@ GET /api/csp-metrics
|
||||
|
||||
#### Query Parameters
|
||||
|
||||
| Parameter | Type | Description | Default | Example |
|
||||
|-----------|------|-------------|---------|---------|
|
||||
| `timeRange` | string | Time range for metrics | `24h` | `?timeRange=7d` |
|
||||
| `format` | string | Response format | `json` | `?format=csv` |
|
||||
| `groupBy` | string | Group results by field | `hour` | `?groupBy=directive` |
|
||||
| Parameter | Type | Description | Default | Example |
|
||||
| ---------------- | ------- | ------------------------- | ------- | ---------------------- |
|
||||
| `timeRange` | string | Time range for metrics | `24h` | `?timeRange=7d` |
|
||||
| `format` | string | Response format | `json` | `?format=csv` |
|
||||
| `groupBy` | string | Group results by field | `hour` | `?groupBy=directive` |
|
||||
| `includeDetails` | boolean | Include violation details | `false` | `?includeDetails=true` |
|
||||
|
||||
#### Time Range Options
|
||||
|
||||
- `1h` - Last 1 hour
|
||||
- `6h` - Last 6 hours
|
||||
- `6h` - Last 6 hours
|
||||
- `24h` - Last 24 hours (default)
|
||||
- `7d` - Last 7 days
|
||||
- `30d` - Last 30 days
|
||||
@ -72,11 +76,14 @@ GET /api/csp-metrics
|
||||
#### Example Request
|
||||
|
||||
```javascript
|
||||
const response = await fetch('/api/csp-metrics?' + new URLSearchParams({
|
||||
timeRange: '24h',
|
||||
groupBy: 'directive',
|
||||
includeDetails: 'true'
|
||||
}));
|
||||
const response = await fetch(
|
||||
"/api/csp-metrics?" +
|
||||
new URLSearchParams({
|
||||
timeRange: "24h",
|
||||
groupBy: "directive",
|
||||
includeDetails: "true",
|
||||
})
|
||||
);
|
||||
|
||||
const metrics = await response.json();
|
||||
```
|
||||
@ -109,10 +116,7 @@ const metrics = await response.json();
|
||||
"count": 15,
|
||||
"percentage": 33.3,
|
||||
"riskLevel": "medium",
|
||||
"topBlockedUris": [
|
||||
"https://malicious.com/script.js",
|
||||
"inline"
|
||||
]
|
||||
"topBlockedUris": ["https://malicious.com/script.js", "inline"]
|
||||
}
|
||||
],
|
||||
"riskAnalysis": {
|
||||
@ -150,14 +154,10 @@ The monitoring service (`lib/csp-monitoring.ts`) provides advanced violation ana
|
||||
|
||||
```javascript
|
||||
// Automatic processing when violations are reported
|
||||
const result = await cspMonitoring.processViolation(
|
||||
violationReport,
|
||||
clientIP,
|
||||
userAgent
|
||||
);
|
||||
const result = await cspMonitoring.processViolation(violationReport, clientIP, userAgent);
|
||||
|
||||
console.log(result.alertLevel); // low, medium, high, critical
|
||||
console.log(result.shouldAlert); // boolean
|
||||
console.log(result.alertLevel); // low, medium, high, critical
|
||||
console.log(result.shouldAlert); // boolean
|
||||
console.log(result.recommendations); // array of suggestions
|
||||
```
|
||||
|
||||
@ -177,13 +177,13 @@ Automatic detection of common CSP bypass attempts:
|
||||
|
||||
```javascript
|
||||
const bypassPatterns = [
|
||||
/javascript:/i, // javascript: protocol injection
|
||||
/data:text\/html/i, // HTML data URI injection
|
||||
/eval\(/i, // Direct eval() calls
|
||||
/Function\(/i, // Function constructor
|
||||
/setTimeout.*string/i, // Timer string execution
|
||||
/location\s*=/i, // Location manipulation
|
||||
/document\.write/i, // Document.write injection
|
||||
/javascript:/i, // javascript: protocol injection
|
||||
/data:text\/html/i, // HTML data URI injection
|
||||
/eval\(/i, // Direct eval() calls
|
||||
/Function\(/i, // Function constructor
|
||||
/setTimeout.*string/i, // Timer string execution
|
||||
/location\s*=/i, // Location manipulation
|
||||
/document\.write/i, // Document.write injection
|
||||
];
|
||||
```
|
||||
|
||||
@ -200,12 +200,12 @@ Based on violation patterns, the service provides actionable recommendations:
|
||||
|
||||
### Risk Levels
|
||||
|
||||
| Risk Level | Score | Description | Action |
|
||||
|------------|-------|-------------|---------|
|
||||
| Risk Level | Score | Description | Action |
|
||||
| ------------ | ------ | --------------------------------------------- | ----------------------- |
|
||||
| **Critical** | 90-100 | Active bypass attempts, known attack patterns | Immediate investigation |
|
||||
| **High** | 70-89 | Suspicious patterns, potential security risks | Urgent review |
|
||||
| **Medium** | 40-69 | Policy violations, may need attention | Regular monitoring |
|
||||
| **Low** | 0-39 | Minor violations, likely legitimate | Log for trends |
|
||||
| **High** | 70-89 | Suspicious patterns, potential security risks | Urgent review |
|
||||
| **Medium** | 40-69 | Policy violations, may need attention | Regular monitoring |
|
||||
| **Low** | 0-39 | Minor violations, likely legitimate | Log for trends |
|
||||
|
||||
### Alert Conditions
|
||||
|
||||
@ -215,18 +215,18 @@ const alertConditions = {
|
||||
critical: {
|
||||
bypassAttempt: true,
|
||||
unknownExternalDomain: true,
|
||||
suspiciousUserAgent: true
|
||||
suspiciousUserAgent: true,
|
||||
},
|
||||
high: {
|
||||
repeatedViolations: '>5 in 10 minutes',
|
||||
repeatedViolations: ">5 in 10 minutes",
|
||||
scriptInjectionAttempt: true,
|
||||
dataUriWithScript: true
|
||||
dataUriWithScript: true,
|
||||
},
|
||||
medium: {
|
||||
newExternalDomain: true,
|
||||
inlineScriptViolation: true,
|
||||
unknownSource: true
|
||||
}
|
||||
unknownSource: true,
|
||||
},
|
||||
};
|
||||
```
|
||||
|
||||
@ -237,24 +237,23 @@ const alertConditions = {
|
||||
```javascript
|
||||
// Monitor violations in real-time
|
||||
async function monitorViolations() {
|
||||
const metrics = await fetch('/api/csp-metrics?timeRange=1h');
|
||||
const metrics = await fetch("/api/csp-metrics?timeRange=1h");
|
||||
const data = await metrics.json();
|
||||
|
||||
|
||||
if (data.data.summary.highRiskViolations > 0) {
|
||||
console.warn('High-risk CSP violations detected:',
|
||||
data.data.summary.highRiskViolations);
|
||||
|
||||
console.warn("High-risk CSP violations detected:", data.data.summary.highRiskViolations);
|
||||
|
||||
// Get violation details
|
||||
const details = await fetch('/api/csp-metrics?includeDetails=true');
|
||||
const details = await fetch("/api/csp-metrics?includeDetails=true");
|
||||
const violations = await details.json();
|
||||
|
||||
|
||||
violations.data.violations
|
||||
.filter(v => v.riskLevel === 'high')
|
||||
.forEach(violation => {
|
||||
console.error('High-risk violation:', {
|
||||
.filter((v) => v.riskLevel === "high")
|
||||
.forEach((violation) => {
|
||||
console.error("High-risk violation:", {
|
||||
directive: violation.directive,
|
||||
blockedUri: violation.blockedUri,
|
||||
timestamp: violation.timestamp
|
||||
timestamp: violation.timestamp,
|
||||
});
|
||||
});
|
||||
}
|
||||
@ -270,16 +269,16 @@ setInterval(monitorViolations, 5 * 60 * 1000);
|
||||
// Get CSP metrics for security dashboard
|
||||
async function getCSPDashboardData() {
|
||||
const [current, previous] = await Promise.all([
|
||||
fetch('/api/csp-metrics?timeRange=24h').then(r => r.json()),
|
||||
fetch('/api/csp-metrics?timeRange=24h&offset=24h').then(r => r.json())
|
||||
fetch("/api/csp-metrics?timeRange=24h").then((r) => r.json()),
|
||||
fetch("/api/csp-metrics?timeRange=24h&offset=24h").then((r) => r.json()),
|
||||
]);
|
||||
|
||||
|
||||
return {
|
||||
currentViolations: current.data.summary.totalViolations,
|
||||
previousViolations: previous.data.summary.totalViolations,
|
||||
trend: current.data.trends.trendDirection,
|
||||
riskScore: current.data.riskAnalysis.overallRiskScore,
|
||||
recommendations: current.data.riskAnalysis.recommendations.slice(0, 3)
|
||||
recommendations: current.data.riskAnalysis.recommendations.slice(0, 3),
|
||||
};
|
||||
}
|
||||
```
|
||||
@ -288,24 +287,26 @@ async function getCSPDashboardData() {
|
||||
|
||||
```javascript
|
||||
// Export violations for external analysis
|
||||
async function exportViolations(format = 'csv', timeRange = '7d') {
|
||||
async function exportViolations(format = "csv", timeRange = "7d") {
|
||||
const response = await fetch(`/api/csp-metrics?format=${format}&timeRange=${timeRange}`);
|
||||
|
||||
if (format === 'csv') {
|
||||
|
||||
if (format === "csv") {
|
||||
const csvData = await response.text();
|
||||
downloadFile(csvData, `csp-violations-${timeRange}.csv`, 'text/csv');
|
||||
downloadFile(csvData, `csp-violations-${timeRange}.csv`, "text/csv");
|
||||
} else {
|
||||
const jsonData = await response.json();
|
||||
downloadFile(JSON.stringify(jsonData, null, 2),
|
||||
`csp-violations-${timeRange}.json`,
|
||||
'application/json');
|
||||
downloadFile(
|
||||
JSON.stringify(jsonData, null, 2),
|
||||
`csp-violations-${timeRange}.json`,
|
||||
"application/json"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function downloadFile(content, filename, contentType) {
|
||||
const blob = new Blob([content], { type: contentType });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = filename;
|
||||
a.click();
|
||||
@ -318,9 +319,9 @@ function downloadFile(content, filename, contentType) {
|
||||
```javascript
|
||||
// Analyze violations to optimize CSP policy
|
||||
async function optimizeCSPPolicy() {
|
||||
const metrics = await fetch('/api/csp-metrics?timeRange=30d&includeDetails=true');
|
||||
const metrics = await fetch("/api/csp-metrics?timeRange=30d&includeDetails=true");
|
||||
const data = await metrics.json();
|
||||
|
||||
|
||||
// Group violations by directive
|
||||
const violationsByDirective = data.data.violations.reduce((acc, violation) => {
|
||||
if (!acc[violation.directive]) {
|
||||
@ -329,31 +330,31 @@ async function optimizeCSPPolicy() {
|
||||
acc[violation.directive].push(violation);
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
|
||||
// Generate recommendations
|
||||
const recommendations = Object.entries(violationsByDirective).map(([directive, violations]) => {
|
||||
const uniqueDomains = [...new Set(violations.map(v => v.blockedUri))];
|
||||
const legitimateCount = violations.filter(v => v.riskLevel === 'low').length;
|
||||
|
||||
const uniqueDomains = [...new Set(violations.map((v) => v.blockedUri))];
|
||||
const legitimateCount = violations.filter((v) => v.riskLevel === "low").length;
|
||||
|
||||
if (legitimateCount > violations.length * 0.8) {
|
||||
return {
|
||||
directive,
|
||||
action: 'allow',
|
||||
action: "allow",
|
||||
domains: uniqueDomains.slice(0, 5),
|
||||
confidence: 'high'
|
||||
confidence: "high",
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
directive,
|
||||
action: 'investigate',
|
||||
riskDomains: uniqueDomains.filter((_, i) =>
|
||||
violations.find(v => v.blockedUri === uniqueDomains[i])?.riskLevel === 'high'
|
||||
action: "investigate",
|
||||
riskDomains: uniqueDomains.filter(
|
||||
(_, i) => violations.find((v) => v.blockedUri === uniqueDomains[i])?.riskLevel === "high"
|
||||
),
|
||||
confidence: 'medium'
|
||||
confidence: "medium",
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
return recommendations;
|
||||
}
|
||||
```
|
||||
@ -367,10 +368,10 @@ The CSP metrics system requires proper CSP headers with reporting:
|
||||
```javascript
|
||||
// In next.config.js or middleware
|
||||
const cspDirectives = {
|
||||
'default-src': "'self'",
|
||||
'script-src': "'self' 'nonce-{NONCE}'",
|
||||
'report-uri': '/api/csp-report',
|
||||
'report-to': 'csp-endpoint'
|
||||
"default-src": "'self'",
|
||||
"script-src": "'self' 'nonce-{NONCE}'",
|
||||
"report-uri": "/api/csp-report",
|
||||
"report-to": "csp-endpoint",
|
||||
};
|
||||
```
|
||||
|
||||
@ -380,13 +381,13 @@ For modern browsers, configure the Report-To header:
|
||||
|
||||
```javascript
|
||||
const reportToHeader = JSON.stringify({
|
||||
group: 'csp-endpoint',
|
||||
group: "csp-endpoint",
|
||||
max_age: 86400,
|
||||
endpoints: [{ url: '/api/csp-report' }]
|
||||
endpoints: [{ url: "/api/csp-report" }],
|
||||
});
|
||||
|
||||
// Add to response headers
|
||||
headers['Report-To'] = reportToHeader;
|
||||
headers["Report-To"] = reportToHeader;
|
||||
```
|
||||
|
||||
### Environment Configuration
|
||||
@ -403,16 +404,19 @@ CSP_ALERT_THRESHOLD=5 # violations per 10 minutes
|
||||
## Performance Considerations
|
||||
|
||||
### Rate Limiting
|
||||
|
||||
- **10 reports per minute per IP** prevents spam attacks
|
||||
- **Exponential backoff** for repeated violations from same source
|
||||
- **Memory cleanup** removes old violations automatically
|
||||
|
||||
### Memory Management
|
||||
|
||||
- **Violation buffer** limited to 1 hour of data in memory
|
||||
- **Automatic cleanup** runs every 100 requests (1% probability)
|
||||
- **Efficient storage** using Map data structures
|
||||
|
||||
### Database Impact
|
||||
|
||||
- **No persistent storage** for real-time metrics (memory only)
|
||||
- **Optional logging** to database for long-term analysis
|
||||
- **Indexed queries** for historical data retrieval
|
||||
@ -420,16 +424,19 @@ CSP_ALERT_THRESHOLD=5 # violations per 10 minutes
|
||||
## Security Considerations
|
||||
|
||||
### Privacy Protection
|
||||
|
||||
- **IP anonymization** option for GDPR compliance
|
||||
- **User agent sanitization** removes sensitive information
|
||||
- **No personal data** stored in violation reports
|
||||
|
||||
### Rate Limiting Protection
|
||||
|
||||
- **Per-IP limits** prevent DoS attacks on reporting endpoint
|
||||
- **Content-type validation** ensures proper report format
|
||||
- **Request size limits** prevent memory exhaustion
|
||||
|
||||
### False Positive Handling
|
||||
|
||||
- **Learning mode** for new deployments
|
||||
- **Whitelist support** for known legitimate violations
|
||||
- **Risk score adjustment** based on historical patterns
|
||||
@ -439,41 +446,43 @@ CSP_ALERT_THRESHOLD=5 # violations per 10 minutes
|
||||
### Common Issues
|
||||
|
||||
#### High False Positive Rate
|
||||
|
||||
```javascript
|
||||
// Check for legitimate violations being flagged
|
||||
const metrics = await fetch('/api/csp-metrics?includeDetails=true');
|
||||
const metrics = await fetch("/api/csp-metrics?includeDetails=true");
|
||||
const data = await metrics.json();
|
||||
|
||||
const falsePositives = data.data.violations.filter(v =>
|
||||
v.riskLevel === 'high' &&
|
||||
v.blockedUri.includes('legitimate-domain.com')
|
||||
const falsePositives = data.data.violations.filter(
|
||||
(v) => v.riskLevel === "high" && v.blockedUri.includes("legitimate-domain.com")
|
||||
);
|
||||
|
||||
if (falsePositives.length > 0) {
|
||||
console.log('Consider whitelisting:', falsePositives[0].blockedUri);
|
||||
console.log("Consider whitelisting:", falsePositives[0].blockedUri);
|
||||
}
|
||||
```
|
||||
|
||||
#### Missing Violation Reports
|
||||
|
||||
```javascript
|
||||
// Check if CSP headers are properly configured
|
||||
fetch('/').then(response => {
|
||||
const csp = response.headers.get('Content-Security-Policy');
|
||||
if (!csp.includes('report-uri')) {
|
||||
console.error('CSP report-uri directive missing');
|
||||
fetch("/").then((response) => {
|
||||
const csp = response.headers.get("Content-Security-Policy");
|
||||
if (!csp.includes("report-uri")) {
|
||||
console.error("CSP report-uri directive missing");
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
#### Performance Issues
|
||||
|
||||
```javascript
|
||||
// Monitor API response times
|
||||
const start = performance.now();
|
||||
const response = await fetch('/api/csp-metrics');
|
||||
const response = await fetch("/api/csp-metrics");
|
||||
const duration = performance.now() - start;
|
||||
|
||||
if (duration > 2000) {
|
||||
console.warn('CSP metrics API slow response:', duration + 'ms');
|
||||
console.warn("CSP metrics API slow response:", duration + "ms");
|
||||
}
|
||||
```
|
||||
|
||||
@ -486,9 +495,9 @@ if (duration > 2000) {
|
||||
|
||||
## API Reference Summary
|
||||
|
||||
| Endpoint | Method | Purpose | Auth Required |
|
||||
|----------|--------|---------|---------------|
|
||||
| `/api/csp-report` | POST | Receive CSP violation reports | No (public) |
|
||||
| `/api/csp-metrics` | GET | Get violation metrics and analytics | Admin |
|
||||
| Endpoint | Method | Purpose | Auth Required |
|
||||
| ------------------ | ------ | ----------------------------------- | ------------- |
|
||||
| `/api/csp-report` | POST | Receive CSP violation reports | No (public) |
|
||||
| `/api/csp-metrics` | GET | Get violation metrics and analytics | Admin |
|
||||
|
||||
Both APIs are production-ready and provide comprehensive CSP monitoring capabilities for enterprise security requirements.
|
||||
Both APIs are production-ready and provide comprehensive CSP monitoring capabilities for enterprise security requirements.
|
||||
|
||||
@ -6,10 +6,10 @@ This document outlines the comprehensive database performance optimizations impl
|
||||
|
||||
The optimization focuses on the most frequently queried patterns in the application, particularly around:
|
||||
|
||||
- AI processing request tracking and batching
|
||||
- Session analytics and filtering
|
||||
- Security audit log analysis
|
||||
- Multi-tenant data isolation performance
|
||||
- AI processing request tracking and batching
|
||||
- Session analytics and filtering
|
||||
- Security audit log analysis
|
||||
- Multi-tenant data isolation performance
|
||||
|
||||
## Applied Optimizations
|
||||
|
||||
@ -20,20 +20,20 @@ The optimization focuses on the most frequently queried patterns in the applicat
|
||||
|
||||
```sql
|
||||
-- Query pattern: companyId + processingStatus + requestedAt
|
||||
CREATE INDEX "AIProcessingRequest_companyId_processingStatus_requestedAt_idx"
|
||||
CREATE INDEX "AIProcessingRequest_companyId_processingStatus_requestedAt_idx"
|
||||
ON "AIProcessingRequest" ("sessionId", "processingStatus", "requestedAt");
|
||||
|
||||
-- Covering index for batch processing
|
||||
CREATE INDEX "AIProcessingRequest_session_companyId_processingStatus_idx"
|
||||
ON "AIProcessingRequest" ("sessionId")
|
||||
CREATE INDEX "AIProcessingRequest_session_companyId_processingStatus_idx"
|
||||
ON "AIProcessingRequest" ("sessionId")
|
||||
INCLUDE ("processingStatus", "batchId", "requestedAt");
|
||||
```
|
||||
|
||||
**Impact**:
|
||||
|
||||
- ~70% faster batch job queries
|
||||
- Reduced I/O for cost analysis reports
|
||||
- Improved scheduler performance
|
||||
- ~70% faster batch job queries
|
||||
- Reduced I/O for cost analysis reports
|
||||
- Improved scheduler performance
|
||||
|
||||
### 2. Session Analytics Optimizations
|
||||
|
||||
@ -42,21 +42,21 @@ INCLUDE ("processingStatus", "batchId", "requestedAt");
|
||||
|
||||
```sql
|
||||
-- Time-range queries with sentiment filtering
|
||||
CREATE INDEX "Session_companyId_startTime_sentiment_covering_idx"
|
||||
ON "Session" ("companyId", "startTime", "sentiment")
|
||||
CREATE INDEX "Session_companyId_startTime_sentiment_covering_idx"
|
||||
ON "Session" ("companyId", "startTime", "sentiment")
|
||||
INCLUDE ("endTime", "category", "escalated", "messagesSent");
|
||||
|
||||
-- Performance analysis queries
|
||||
CREATE INDEX "Session_companyId_performance_idx"
|
||||
ON "Session" ("companyId", "avgResponseTime", "escalated")
|
||||
CREATE INDEX "Session_companyId_performance_idx"
|
||||
ON "Session" ("companyId", "avgResponseTime", "escalated")
|
||||
INCLUDE ("startTime", "messagesSent");
|
||||
```
|
||||
|
||||
**Impact**:
|
||||
|
||||
- ~85% faster dashboard load times
|
||||
- Efficient date range filtering
|
||||
- Optimized sentiment analysis queries
|
||||
- ~85% faster dashboard load times
|
||||
- Efficient date range filtering
|
||||
- Optimized sentiment analysis queries
|
||||
|
||||
### 3. Security Audit Log Optimizations
|
||||
|
||||
@ -65,21 +65,21 @@ INCLUDE ("startTime", "messagesSent");
|
||||
|
||||
```sql
|
||||
-- Admin security dashboard
|
||||
CREATE INDEX "SecurityAuditLog_companyId_eventType_outcome_timestamp_idx"
|
||||
CREATE INDEX "SecurityAuditLog_companyId_eventType_outcome_timestamp_idx"
|
||||
ON "SecurityAuditLog" ("companyId", "eventType", "outcome", "timestamp");
|
||||
|
||||
-- Threat detection queries
|
||||
CREATE INDEX "SecurityAuditLog_geographic_threat_idx"
|
||||
ON "SecurityAuditLog" ("ipAddress", "country", "timestamp")
|
||||
WHERE "outcome" IN ('FAILURE', 'BLOCKED', 'SUSPICIOUS')
|
||||
CREATE INDEX "SecurityAuditLog_geographic_threat_idx"
|
||||
ON "SecurityAuditLog" ("ipAddress", "country", "timestamp")
|
||||
WHERE "outcome" IN ('FAILURE', 'BLOCKED', 'SUSPICIOUS')
|
||||
INCLUDE ("eventType", "severity", "userId", "companyId");
|
||||
```
|
||||
|
||||
**Impact**:
|
||||
|
||||
- ~90% faster security monitoring
|
||||
- Efficient threat detection
|
||||
- Improved compliance reporting
|
||||
- ~90% faster security monitoring
|
||||
- Efficient threat detection
|
||||
- Improved compliance reporting
|
||||
|
||||
### 4. Message Processing Optimizations
|
||||
|
||||
@ -88,15 +88,15 @@ INCLUDE ("eventType", "severity", "userId", "companyId");
|
||||
|
||||
```sql
|
||||
-- Message timeline with role filtering
|
||||
CREATE INDEX "Message_sessionId_timestamp_role_covering_idx"
|
||||
ON "Message" ("sessionId", "timestamp", "role")
|
||||
CREATE INDEX "Message_sessionId_timestamp_role_covering_idx"
|
||||
ON "Message" ("sessionId", "timestamp", "role")
|
||||
INCLUDE ("content");
|
||||
```
|
||||
|
||||
**Impact**:
|
||||
|
||||
- ~60% faster conversation loading
|
||||
- Reduced memory usage for message queries
|
||||
- ~60% faster conversation loading
|
||||
- Reduced memory usage for message queries
|
||||
|
||||
### 5. Processing Pipeline Optimizations
|
||||
|
||||
@ -105,42 +105,42 @@ INCLUDE ("content");
|
||||
|
||||
```sql
|
||||
-- Processing pipeline monitoring
|
||||
CREATE INDEX "SessionProcessingStatus_stage_status_startedAt_idx"
|
||||
ON "SessionProcessingStatus" ("stage", "status", "startedAt")
|
||||
CREATE INDEX "SessionProcessingStatus_stage_status_startedAt_idx"
|
||||
ON "SessionProcessingStatus" ("stage", "status", "startedAt")
|
||||
INCLUDE ("sessionId", "completedAt", "retryCount");
|
||||
|
||||
-- Error analysis (partial index)
|
||||
CREATE INDEX "SessionProcessingStatus_error_analysis_idx"
|
||||
ON "SessionProcessingStatus" ("status", "stage")
|
||||
WHERE "status" IN ('FAILED', 'RETRY_PENDING')
|
||||
CREATE INDEX "SessionProcessingStatus_error_analysis_idx"
|
||||
ON "SessionProcessingStatus" ("status", "stage")
|
||||
WHERE "status" IN ('FAILED', 'RETRY_PENDING')
|
||||
INCLUDE ("sessionId", "errorMessage", "retryCount", "startedAt");
|
||||
```
|
||||
|
||||
**Impact**:
|
||||
|
||||
- ~75% faster processing monitoring
|
||||
- Efficient error tracking
|
||||
- Improved retry logic performance
|
||||
- ~75% faster processing monitoring
|
||||
- Efficient error tracking
|
||||
- Improved retry logic performance
|
||||
|
||||
## Index Strategy Principles
|
||||
|
||||
### 1. Composite Index Design
|
||||
|
||||
- **Leading column**: Most selective filter (usually companyId for multi-tenancy)
|
||||
- **Secondary columns**: Common WHERE clause filters
|
||||
- **Covering columns**: SELECT list columns via INCLUDE
|
||||
- **Leading column**: Most selective filter (usually companyId for multi-tenancy)
|
||||
- **Secondary columns**: Common WHERE clause filters
|
||||
- **Covering columns**: SELECT list columns via INCLUDE
|
||||
|
||||
### 2. Partial Indexes
|
||||
|
||||
- Used for error analysis and specific status filtering
|
||||
- Reduces index size and maintenance overhead
|
||||
- Improves write performance
|
||||
- Used for error analysis and specific status filtering
|
||||
- Reduces index size and maintenance overhead
|
||||
- Improves write performance
|
||||
|
||||
### 3. Covering Indexes
|
||||
|
||||
- Include frequently accessed columns to avoid table lookups
|
||||
- Reduces I/O for read-heavy operations
|
||||
- Particularly effective for dashboard queries
|
||||
- Include frequently accessed columns to avoid table lookups
|
||||
- Reduces I/O for read-heavy operations
|
||||
- Particularly effective for dashboard queries
|
||||
|
||||
## Query Pattern Analysis
|
||||
|
||||
@ -154,41 +154,41 @@ INCLUDE ("sessionId", "errorMessage", "retryCount", "startedAt");
|
||||
|
||||
### Before vs After Performance
|
||||
|
||||
| Query Type | Before (ms) | After (ms) | Improvement |
|
||||
|------------|-------------|------------|-------------|
|
||||
| Dashboard load | 2,500 | 375 | 85% |
|
||||
| Batch queries | 1,800 | 540 | 70% |
|
||||
| Security monitoring | 3,200 | 320 | 90% |
|
||||
| Message timeline | 800 | 320 | 60% |
|
||||
| Processing status | 1,200 | 300 | 75% |
|
||||
| Query Type | Before (ms) | After (ms) | Improvement |
|
||||
| ------------------- | ----------- | ---------- | ----------- |
|
||||
| Dashboard load | 2,500 | 375 | 85% |
|
||||
| Batch queries | 1,800 | 540 | 70% |
|
||||
| Security monitoring | 3,200 | 320 | 90% |
|
||||
| Message timeline | 800 | 320 | 60% |
|
||||
| Processing status | 1,200 | 300 | 75% |
|
||||
|
||||
## Maintenance Considerations
|
||||
|
||||
### Index Monitoring
|
||||
|
||||
- Monitor index usage with `pg_stat_user_indexes`
|
||||
- Track bloat with `pg_stat_user_tables`
|
||||
- Regular ANALYZE after bulk operations
|
||||
- Monitor index usage with `pg_stat_user_indexes`
|
||||
- Track bloat with `pg_stat_user_tables`
|
||||
- Regular ANALYZE after bulk operations
|
||||
|
||||
### Write Performance Impact
|
||||
|
||||
- Composite indexes add ~15% write overhead
|
||||
- Offset by dramatic read performance gains
|
||||
- Monitored via slow query logs
|
||||
- Composite indexes add ~15% write overhead
|
||||
- Offset by dramatic read performance gains
|
||||
- Monitored via slow query logs
|
||||
|
||||
### Storage Impact
|
||||
|
||||
- Indexes add ~25% to total storage
|
||||
- Covering indexes reduce need for table scans
|
||||
- Partial indexes minimize storage overhead
|
||||
- Indexes add ~25% to total storage
|
||||
- Covering indexes reduce need for table scans
|
||||
- Partial indexes minimize storage overhead
|
||||
|
||||
## Migration Safety
|
||||
|
||||
### CONCURRENTLY Operations
|
||||
|
||||
- All indexes created with `CREATE INDEX CONCURRENTLY`
|
||||
- No table locks during creation
|
||||
- Production-safe deployment
|
||||
- All indexes created with `CREATE INDEX CONCURRENTLY`
|
||||
- No table locks during creation
|
||||
- Production-safe deployment
|
||||
|
||||
### Rollback Strategy
|
||||
|
||||
@ -201,14 +201,14 @@ DROP INDEX CONCURRENTLY "specific_index_name";
|
||||
|
||||
```sql
|
||||
-- Check index usage
|
||||
SELECT schemaname, tablename, attname, n_distinct, correlation
|
||||
FROM pg_stats
|
||||
SELECT schemaname, tablename, attname, n_distinct, correlation
|
||||
FROM pg_stats
|
||||
WHERE tablename IN ('Session', 'AIProcessingRequest', 'SecurityAuditLog');
|
||||
|
||||
-- Monitor query performance
|
||||
SELECT query, mean_exec_time, calls
|
||||
FROM pg_stat_statements
|
||||
ORDER BY mean_exec_time DESC
|
||||
SELECT query, mean_exec_time, calls
|
||||
FROM pg_stat_statements
|
||||
ORDER BY mean_exec_time DESC
|
||||
LIMIT 10;
|
||||
```
|
||||
|
||||
@ -238,18 +238,18 @@ LIMIT 10;
|
||||
|
||||
### Monitoring Strategy
|
||||
|
||||
- Set up automated index usage monitoring
|
||||
- Track slow query evolution
|
||||
- Monitor storage growth patterns
|
||||
- Implement performance alerting
|
||||
- Set up automated index usage monitoring
|
||||
- Track slow query evolution
|
||||
- Monitor storage growth patterns
|
||||
- Implement performance alerting
|
||||
|
||||
## Conclusion
|
||||
|
||||
These database optimizations provide:
|
||||
|
||||
- **70-90% improvement** in query performance
|
||||
- **Reduced server load** through efficient indexing
|
||||
- **Better user experience** with faster dashboards
|
||||
- **Scalable foundation** for future growth
|
||||
- **70-90% improvement** in query performance
|
||||
- **Reduced server load** through efficient indexing
|
||||
- **Better user experience** with faster dashboards
|
||||
- **Scalable foundation** for future growth
|
||||
|
||||
The optimizations are designed to be production-safe and monitoring-friendly, ensuring both immediate performance gains and long-term maintainability.
|
||||
|
||||
@ -15,21 +15,21 @@ Can't reach database server at `ep-tiny-math-a2zsshve-pooler.eu-central-1.aws.ne
|
||||
|
||||
### 1. Neon Connection Limits
|
||||
|
||||
- **Free Tier**: 20 concurrent connections
|
||||
- **Pro Tier**: 100 concurrent connections
|
||||
- **Multiple schedulers** can quickly exhaust connections
|
||||
- **Free Tier**: 20 concurrent connections
|
||||
- **Pro Tier**: 100 concurrent connections
|
||||
- **Multiple schedulers** can quickly exhaust connections
|
||||
|
||||
### 2. Connection Pooling Issues
|
||||
|
||||
- Each scheduler was creating separate PrismaClient instances
|
||||
- No connection reuse between operations
|
||||
- No retry logic for temporary failures
|
||||
- Each scheduler was creating separate PrismaClient instances
|
||||
- No connection reuse between operations
|
||||
- No retry logic for temporary failures
|
||||
|
||||
### 3. Neon-Specific Challenges
|
||||
|
||||
- **Auto-pause**: Databases pause after inactivity
|
||||
- **Cold starts**: First connection after pause takes longer
|
||||
- **Regional latency**: eu-central-1 may have variable latency
|
||||
- **Auto-pause**: Databases pause after inactivity
|
||||
- **Cold starts**: First connection after pause takes longer
|
||||
- **Regional latency**: eu-central-1 may have variable latency
|
||||
|
||||
## Solutions Implemented
|
||||
|
||||
@ -106,9 +106,9 @@ curl -H "Authorization: Bearer YOUR_API_TOKEN" \
|
||||
|
||||
### 2. Neon Dashboard Monitoring
|
||||
|
||||
- Monitor "Active connections" in Neon dashboard
|
||||
- Check for connection spikes during scheduler runs
|
||||
- Review query performance and slow queries
|
||||
- Monitor "Active connections" in Neon dashboard
|
||||
- Check for connection spikes during scheduler runs
|
||||
- Review query performance and slow queries
|
||||
|
||||
### 3. Application Logs
|
||||
|
||||
@ -158,9 +158,9 @@ const prisma = new PrismaClient({
|
||||
|
||||
**Causes:**
|
||||
|
||||
- Neon database auto-paused
|
||||
- Connection limit exceeded
|
||||
- Network issues
|
||||
- Neon database auto-paused
|
||||
- Connection limit exceeded
|
||||
- Network issues
|
||||
|
||||
**Solutions:**
|
||||
|
||||
@ -173,9 +173,9 @@ const prisma = new PrismaClient({
|
||||
|
||||
**Causes:**
|
||||
|
||||
- Idle connection timeout
|
||||
- Neon maintenance
|
||||
- Long-running transactions
|
||||
- Idle connection timeout
|
||||
- Neon maintenance
|
||||
- Long-running transactions
|
||||
|
||||
**Solutions:**
|
||||
|
||||
@ -187,9 +187,9 @@ const prisma = new PrismaClient({
|
||||
|
||||
**Causes:**
|
||||
|
||||
- Blocking database operations
|
||||
- Scheduler overlap
|
||||
- High CPU usage
|
||||
- Blocking database operations
|
||||
- Scheduler overlap
|
||||
- High CPU usage
|
||||
|
||||
**Solutions:**
|
||||
|
||||
@ -230,10 +230,10 @@ SESSION_PROCESSING_INTERVAL="0 */2 * * *"
|
||||
|
||||
## Monitoring Checklist
|
||||
|
||||
- [ ] Check Neon dashboard for connection spikes
|
||||
- [ ] Monitor scheduler execution times
|
||||
- [ ] Review error logs for connection patterns
|
||||
- [ ] Test health endpoint regularly
|
||||
- [ ] Set up alerts for connection failures
|
||||
- [ ] Check Neon dashboard for connection spikes
|
||||
- [ ] Monitor scheduler execution times
|
||||
- [ ] Review error logs for connection patterns
|
||||
- [ ] Test health endpoint regularly
|
||||
- [ ] Set up alerts for connection failures
|
||||
|
||||
With these optimizations, your Neon database connections should be much more stable and efficient!
|
||||
|
||||
@ -22,17 +22,18 @@ Abstract base class providing common functionality:
|
||||
export abstract class BaseSchedulerService extends EventEmitter {
|
||||
// Common scheduler functionality
|
||||
protected abstract executeTask(): Promise<void>;
|
||||
|
||||
async start(): Promise<void>
|
||||
async stop(): Promise<void>
|
||||
pause(): void
|
||||
resume(): void
|
||||
getHealthStatus(): HealthStatus
|
||||
getMetrics(): SchedulerMetrics
|
||||
|
||||
async start(): Promise<void>;
|
||||
async stop(): Promise<void>;
|
||||
pause(): void;
|
||||
resume(): void;
|
||||
getHealthStatus(): HealthStatus;
|
||||
getMetrics(): SchedulerMetrics;
|
||||
}
|
||||
```
|
||||
|
||||
**Features:**
|
||||
|
||||
- Status management (STOPPED, STARTING, RUNNING, PAUSED, ERROR)
|
||||
- Metrics collection (run counts, timing, success/failure rates)
|
||||
- Event emission for monitoring
|
||||
@ -55,6 +56,7 @@ const csvScheduler = new CsvImportSchedulerService({
|
||||
```
|
||||
|
||||
**Features:**
|
||||
|
||||
- Batch processing with configurable concurrency
|
||||
- Duplicate detection
|
||||
- Company-specific error handling
|
||||
@ -85,6 +87,7 @@ await manager.startAll();
|
||||
```
|
||||
|
||||
**Features:**
|
||||
|
||||
- Automatic restart of failed critical schedulers
|
||||
- Health monitoring across all schedulers
|
||||
- Coordinated start/stop operations
|
||||
@ -103,6 +106,7 @@ npx tsx lib/services/schedulers/StandaloneSchedulerRunner.ts --list
|
||||
```
|
||||
|
||||
**Features:**
|
||||
|
||||
- Independent process execution
|
||||
- Environment variable configuration
|
||||
- Graceful shutdown handling
|
||||
@ -116,17 +120,19 @@ All schedulers run within the main Next.js server process:
|
||||
|
||||
```typescript
|
||||
// server.ts
|
||||
import { initializeSchedulers } from './lib/services/schedulers/ServerSchedulerIntegration';
|
||||
import { initializeSchedulers } from "./lib/services/schedulers/ServerSchedulerIntegration";
|
||||
|
||||
await initializeSchedulers();
|
||||
```
|
||||
|
||||
**Pros:**
|
||||
|
||||
- Simple deployment
|
||||
- Lower resource usage
|
||||
- Easy local development
|
||||
|
||||
**Cons:**
|
||||
|
||||
- Limited scalability
|
||||
- Single point of failure
|
||||
- Resource contention
|
||||
@ -142,16 +148,18 @@ npm run dev
|
||||
# Terminal 2: CSV Import Scheduler
|
||||
npm run scheduler:csv-import
|
||||
|
||||
# Terminal 3: Session Processing Scheduler
|
||||
# Terminal 3: Session Processing Scheduler
|
||||
npm run scheduler:session-processing
|
||||
```
|
||||
|
||||
**Pros:**
|
||||
|
||||
- Independent scaling
|
||||
- Fault isolation
|
||||
- Resource optimization per scheduler
|
||||
|
||||
**Cons:**
|
||||
|
||||
- More complex deployment
|
||||
- Higher resource overhead
|
||||
- Inter-process coordination needed
|
||||
@ -162,20 +170,20 @@ Each scheduler runs in separate containers managed by Kubernetes/Docker Swarm:
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml
|
||||
version: '3.8'
|
||||
version: "3.8"
|
||||
services:
|
||||
app:
|
||||
build: .
|
||||
environment:
|
||||
- SCHEDULER_ENABLED=false # Disable in-process schedulers
|
||||
|
||||
- SCHEDULER_ENABLED=false # Disable in-process schedulers
|
||||
|
||||
csv-import-scheduler:
|
||||
build: .
|
||||
command: npx tsx lib/services/schedulers/StandaloneSchedulerRunner.ts --scheduler=csv-import
|
||||
environment:
|
||||
- CSV_IMPORT_INTERVAL=*/10 * * * *
|
||||
- CSV_IMPORT_BATCH_SIZE=10
|
||||
|
||||
|
||||
session-processing-scheduler:
|
||||
build: .
|
||||
command: npx tsx lib/services/schedulers/StandaloneSchedulerRunner.ts --scheduler=session-processing
|
||||
@ -184,12 +192,14 @@ services:
|
||||
```
|
||||
|
||||
**Pros:**
|
||||
|
||||
- Full horizontal scaling
|
||||
- Independent resource allocation
|
||||
- Health monitoring integration
|
||||
- Zero-downtime deployments
|
||||
|
||||
**Cons:**
|
||||
|
||||
- Complex orchestration setup
|
||||
- Network latency considerations
|
||||
- Distributed system challenges
|
||||
@ -213,7 +223,7 @@ CSV_IMPORT_TIMEOUT=300000
|
||||
IMPORT_PROCESSING_INTERVAL="*/2 * * * *"
|
||||
IMPORT_PROCESSING_TIMEOUT=120000
|
||||
|
||||
# Session Processing Scheduler
|
||||
# Session Processing Scheduler
|
||||
SESSION_PROCESSING_INTERVAL="*/5 * * * *"
|
||||
SESSION_PROCESSING_BATCH_SIZE=50
|
||||
|
||||
@ -281,21 +291,27 @@ spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: scheduler
|
||||
image: livedash:latest
|
||||
command: ["npx", "tsx", "lib/services/schedulers/StandaloneSchedulerRunner.ts", "--scheduler=csv-import"]
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/admin/schedulers/health
|
||||
port: 3000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/admin/schedulers/health
|
||||
port: 3000
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
- name: scheduler
|
||||
image: livedash:latest
|
||||
command:
|
||||
[
|
||||
"npx",
|
||||
"tsx",
|
||||
"lib/services/schedulers/StandaloneSchedulerRunner.ts",
|
||||
"--scheduler=csv-import",
|
||||
]
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /api/admin/schedulers/health
|
||||
port: 3000
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /api/admin/schedulers/health
|
||||
port: 3000
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
```
|
||||
|
||||
## Scaling Strategies
|
||||
@ -310,10 +326,10 @@ csv-import-scheduler:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2.0'
|
||||
cpus: "2.0"
|
||||
memory: 2G
|
||||
reservations:
|
||||
cpus: '1.0'
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
```
|
||||
|
||||
@ -328,16 +344,16 @@ kind: Deployment
|
||||
metadata:
|
||||
name: csv-import-scheduler
|
||||
spec:
|
||||
replicas: 3 # Multiple instances
|
||||
replicas: 3 # Multiple instances
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: scheduler
|
||||
env:
|
||||
- name: SCHEDULER_INSTANCE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: scheduler
|
||||
env:
|
||||
- name: SCHEDULER_INSTANCE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
```
|
||||
|
||||
**Note:** Ensure scheduler logic handles multiple instances correctly (e.g., using database locks or partitioning).
|
||||
@ -352,10 +368,10 @@ csv-import-scheduler-us:
|
||||
environment:
|
||||
- REGION=us
|
||||
- CSV_COMPANIES_FILTER=region:us
|
||||
|
||||
|
||||
csv-import-scheduler-eu:
|
||||
environment:
|
||||
- REGION=eu
|
||||
- REGION=eu
|
||||
- CSV_COMPANIES_FILTER=region:eu
|
||||
```
|
||||
|
||||
@ -405,4 +421,4 @@ csv-import-scheduler-eu:
|
||||
2. Add distributed coordination for multi-instance schedulers
|
||||
3. Create Kubernetes operators for automatic scaling
|
||||
4. Implement scheduler-specific metrics and dashboards
|
||||
5. Add scheduler performance optimization tools
|
||||
5. Add scheduler performance optimization tools
|
||||
|
||||
Reference in New Issue
Block a user