kjanat/livedash-node
1
0
Fork 0
mirror of https://github.com/kjanat/livedash-node.git synced 2026-01-16 07:32:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: add rawTranscriptContent field to SessionImport model

Browse Source 
feat: enhance server initialization with environment validation and import processing scheduler

test: add Jest setup for unit tests and mock console methods

test: implement unit tests for environment management and validation

test: create unit tests for transcript fetcher functionality
This commit is contained in:
Max Kowalski
2025-06-27 19:00:22 +02:00
parent 50b230aa9b
commit 5c1ced5900
25 changed files with 3492 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.env.example
View File

@ -15,6 +15,8 @@ OPENAI_API_KEY=your_openai_api_key_here
# Scheduler Configuration
SCHEDULER_ENABLED=false # Enable/disable all schedulers (false for dev, true for production)
CSV_IMPORT_INTERVAL=*/15 * * * * # Cron expression for CSV imports (every 15 minutes)
SESSION_PROCESSING_INTERVAL=0 * * * * # Cron expression for session processing (every hour)
IMPORT_PROCESSING_INTERVAL=*/5 * * * * # Cron expression for processing imports to sessions (every 5 minutes)
IMPORT_PROCESSING_BATCH_SIZE=50 # Number of imports to process at once
SESSION_PROCESSING_INTERVAL=0 * * * * # Cron expression for AI session processing (every hour)
SESSION_PROCESSING_BATCH_SIZE=0 # 0 = unlimited sessions, >0 = specific limit
SESSION_PROCESSING_CONCURRENCY=5 # How many sessions to process in parallel

5
.env.local.example
View File

@ -11,12 +11,15 @@ OPENAI_API_KEY=your_openai_api_key_here
# Scheduler Configuration
SCHEDULER_ENABLED=true # Set to false to disable all schedulers during development
CSV_IMPORT_INTERVAL=*/15 * * * * # Every 15 minutes (cron format)
SESSION_PROCESSING_INTERVAL=0 * * * * # Every hour (cron format)
IMPORT_PROCESSING_INTERVAL=*/5 * * * * # Every 5 minutes (cron format) - converts imports to sessions
IMPORT_PROCESSING_BATCH_SIZE=50 # Number of imports to process at once
SESSION_PROCESSING_INTERVAL=0 * * * * # Every hour (cron format) - AI processing
SESSION_PROCESSING_BATCH_SIZE=0 # 0 = process all sessions, >0 = limit number
SESSION_PROCESSING_CONCURRENCY=5 # Number of sessions to process in parallel
# Example configurations:
# - For development (no schedulers): SCHEDULER_ENABLED=false
# - For testing (every 5 minutes): CSV_IMPORT_INTERVAL=*/5 * * * *
# - For faster import processing: IMPORT_PROCESSING_INTERVAL=*/2 * * * *
# - For limited processing: SESSION_PROCESSING_BATCH_SIZE=10
# - For high concurrency: SESSION_PROCESSING_CONCURRENCY=10

4
app/dashboard/company/page.tsx
View File

@ -77,8 +77,8 @@ export default function CompanySettingsPage() {
return <div className="text-center py-10">Loading settings...</div>;
}
// Check for admin access
if (session?.user?.role !== "admin") {
// Check for ADMIN access
if (session?.user?.role !== "ADMIN") {
return (
<div className="text-center py-10 bg-white rounded-xl shadow p-6">
<h2 className="font-bold text-xl text-red-600 mb-2">Access Denied</h2>

2
app/dashboard/overview/page.tsx
View File

@ -31,7 +31,7 @@ function DashboardContent() {
const [selectedStartDate, setSelectedStartDate] = useState<string>("");
const [selectedEndDate, setSelectedEndDate] = useState<string>("");
const isAuditor = session?.user?.role === "auditor";
const isAuditor = session?.user?.role === "AUDITOR";
// Function to fetch metrics with optional date range
const fetchMetrics = useCallback(async (startDate?: string, endDate?: string) => {

4
app/dashboard/page.tsx
View File

@ -62,7 +62,7 @@ const DashboardPage: FC = () => {
</button>
</div>
{session?.user?.role === "admin" && (
{session?.user?.role === "ADMIN" && (
<div className="bg-gradient-to-br from-purple-50 to-purple-100 p-6 rounded-xl shadow-sm hover:shadow-md transition-shadow">
<h2 className="text-lg font-semibold text-purple-700">
Company Settings
@ -79,7 +79,7 @@ const DashboardPage: FC = () => {
</div>
)}
{session?.user?.role === "admin" && (
{session?.user?.role === "ADMIN" && (
<div className="bg-gradient-to-br from-amber-50 to-amber-100 p-6 rounded-xl shadow-sm hover:shadow-md transition-shadow">
<h2 className="text-lg font-semibold text-amber-700">
User Management

2
app/dashboard/settings.tsx
View File

@ -37,7 +37,7 @@ export default function DashboardSettings({
else setMessage("Failed.");
}
if (session.user.role !== "admin") return null;
if (session.user.role !== "ADMIN") return null;
return (
<div className="bg-white p-6 rounded-xl shadow mb-6">

6
app/dashboard/users.tsx
View File

@ -34,7 +34,7 @@ export default function UserManagement({ session }: UserManagementProps) {
else setMsg("Failed.");
}
if (session.user.role !== "admin") return null;
if (session.user.role !== "ADMIN") return null;
return (
<div className="bg-white p-6 rounded-xl shadow mb-6">
@ -52,8 +52,8 @@ export default function UserManagement({ session }: UserManagementProps) {
onChange={(e) => setRole(e.target.value)}
>
<option value="user">User</option>
<option value="admin">Admin</option>
<option value="auditor">Auditor</option>
<option value="ADMIN">Admin</option>
<option value="AUDITOR">Auditor</option>
</select>
<button
className="bg-blue-600 text-white rounded px-4 py-2 sm:py-0 w-full sm:w-auto"

10
app/dashboard/users/page.tsx
View File

@ -69,7 +69,7 @@ export default function UserManagementPage() {
}
// Check for admin access
if (session?.user?.role !== "admin") {
if (session?.user?.role !== "ADMIN") {
return (
<div className="text-center py-10 bg-white rounded-xl shadow p-6">
<h2 className="font-bold text-xl text-red-600 mb-2">Access Denied</h2>
@ -124,8 +124,8 @@ export default function UserManagementPage() {
onChange={(e) => setRole(e.target.value)}
>
<option value="user">User</option>
<option value="admin">Admin</option>
<option value="auditor">Auditor</option>
<option value="ADMIN">Admin</option>
<option value="AUDITOR">Auditor</option>
</select>
</div>
@ -183,9 +183,9 @@ export default function UserManagementPage() {
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-500">
<span
className={`px-2 inline-flex text-xs leading-5 font-semibold rounded-full ${
user.role === "admin"
user.role === "ADMIN"
? "bg-purple-100 text-purple-800"
: user.role === "auditor"
: user.role === "AUDITOR"
? "bg-blue-100 text-blue-800"
: "bg-green-100 text-green-800"
}`}

4
app/register/page.tsx
View File

@ -7,7 +7,7 @@ export default function RegisterPage() {
const [company, setCompany] = useState<string>("");
const [password, setPassword] = useState<string>("");
const [csvUrl, setCsvUrl] = useState<string>("");
const [role, setRole] = useState<string>("admin"); // Default to admin for company registration
const [role, setRole] = useState<string>("ADMIN"); // Default to ADMIN for company registration
const [error, setError] = useState<string>("");
const router = useRouter();
@ -66,7 +66,7 @@ export default function RegisterPage() {
>
<option value="admin">Admin</option>
<option value="user">User</option>
<option value="auditor">Auditor</option>
<option value="AUDITOR">Auditor</option>
</select>
<button className="bg-blue-600 text-white rounded py-2" type="submit">
Register & Continue

22
jest.config.js Normal file
View File

@ -0,0 +1,22 @@
/** @type {import('jest').Config} */
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
roots: ['<rootDir>/tests'],
testMatch: ['**/__tests__/**/*.ts', '**/?(*.)+(spec|test).ts'],
transform: {
'^.+\\.ts$': 'ts-jest',
},
collectCoverageFrom: [
'lib/**/*.ts',
'!lib/**/*.d.ts',
'!lib/**/*.test.ts',
],
coverageDirectory: 'coverage',
coverageReporters: ['text', 'lcov', 'html'],
setupFilesAfterEnv: ['<rootDir>/tests/setup.ts'],
moduleNameMapping: {
'^@/(.*)$': '<rootDir>/$1',
},
testTimeout: 10000,
};

111
lib/env.ts Normal file
View File

@ -0,0 +1,111 @@
// Centralized environment variable management
import { readFileSync } from "fs";
import { fileURLToPath } from "url";
import { dirname, join } from "path";
// Load environment variables from .env.local
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const envPath = join(__dirname, '..', '.env.local');
// Load .env.local if it exists
try {
const envFile = readFileSync(envPath, 'utf8');
const envVars = envFile.split('\n').filter(line => line.trim() && !line.startsWith('#'));
envVars.forEach(line => {
const [key, ...valueParts] = line.split('=');
if (key && valueParts.length > 0) {
const value = valueParts.join('=').trim();
if (!process.env[key.trim()]) {
process.env[key.trim()] = value;
}
}
});
} catch (error) {
// Silently fail if .env.local doesn't exist
}
/**
* Typed environment variables with defaults
*/
export const env = {
// NextAuth
NEXTAUTH_URL: process.env.NEXTAUTH_URL || 'http://localhost:3000',
NEXTAUTH_SECRET: process.env.NEXTAUTH_SECRET || '',
NODE_ENV: process.env.NODE_ENV || 'development',
// OpenAI
OPENAI_API_KEY: process.env.OPENAI_API_KEY || '',
// Scheduler Configuration
SCHEDULER_ENABLED: process.env.SCHEDULER_ENABLED === 'true',
CSV_IMPORT_INTERVAL: process.env.CSV_IMPORT_INTERVAL || '*/15 * * * *',
IMPORT_PROCESSING_INTERVAL: process.env.IMPORT_PROCESSING_INTERVAL || '*/5 * * * *',
IMPORT_PROCESSING_BATCH_SIZE: parseInt(process.env.IMPORT_PROCESSING_BATCH_SIZE || '50', 10),
SESSION_PROCESSING_INTERVAL: process.env.SESSION_PROCESSING_INTERVAL || '0 * * * *',
SESSION_PROCESSING_BATCH_SIZE: parseInt(process.env.SESSION_PROCESSING_BATCH_SIZE || '0', 10),
SESSION_PROCESSING_CONCURRENCY: parseInt(process.env.SESSION_PROCESSING_CONCURRENCY || '5', 10),
// Server
PORT: parseInt(process.env.PORT || '3000', 10),
} as const;
/**
* Validate required environment variables
*/
export function validateEnv(): { valid: boolean; errors: string[] } {
const errors: string[] = [];
if (!env.NEXTAUTH_SECRET) {
errors.push('NEXTAUTH_SECRET is required');
}
if (!env.OPENAI_API_KEY && env.NODE_ENV === 'production') {
errors.push('OPENAI_API_KEY is required in production');
}
return {
valid: errors.length === 0,
errors,
};
}
/**
* Get scheduler configuration from environment variables
*/
export function getSchedulerConfig() {
return {
enabled: env.SCHEDULER_ENABLED,
csvImport: {
interval: env.CSV_IMPORT_INTERVAL,
},
importProcessing: {
interval: env.IMPORT_PROCESSING_INTERVAL,
batchSize: env.IMPORT_PROCESSING_BATCH_SIZE,
},
sessionProcessing: {
interval: env.SESSION_PROCESSING_INTERVAL,
batchSize: env.SESSION_PROCESSING_BATCH_SIZE,
concurrency: env.SESSION_PROCESSING_CONCURRENCY,
},
};
}
/**
* Log environment configuration (safe for production)
*/
export function logEnvConfig(): void {
console.log('[Environment] Configuration:');
console.log(` NODE_ENV: ${env.NODE_ENV}`);
console.log(` NEXTAUTH_URL: ${env.NEXTAUTH_URL}`);
console.log(` SCHEDULER_ENABLED: ${env.SCHEDULER_ENABLED}`);
console.log(` PORT: ${env.PORT}`);
if (env.SCHEDULER_ENABLED) {
console.log(' Scheduler intervals:');
console.log(` CSV Import: ${env.CSV_IMPORT_INTERVAL}`);
console.log(` Import Processing: ${env.IMPORT_PROCESSING_INTERVAL}`);
console.log(` Session Processing: ${env.SESSION_PROCESSING_INTERVAL}`);
}
}

225
lib/importProcessor.ts Normal file
View File

@ -0,0 +1,225 @@
// SessionImport to Session processor
import { PrismaClient, ImportStatus, SentimentCategory } from "@prisma/client";
import { getSchedulerConfig } from "./env";
import { fetchTranscriptContent, isValidTranscriptUrl } from "./transcriptFetcher";
import cron from "node-cron";
const prisma = new PrismaClient();
/**
* Process a single SessionImport record into a Session record
*/
async function processSingleImport(importRecord: any): Promise<{ success: boolean; error?: string }> {
try {
// Parse dates
const startTime = new Date(importRecord.startTimeRaw);
const endTime = new Date(importRecord.endTimeRaw);
// Validate dates
if (isNaN(startTime.getTime()) || isNaN(endTime.getTime())) {
throw new Error(`Invalid date format: start=${importRecord.startTimeRaw}, end=${importRecord.endTimeRaw}`);
}
// Process sentiment
let sentiment: number | null = null;
let sentimentCategory: SentimentCategory | null = null;
if (importRecord.sentimentRaw) {
const sentimentStr = importRecord.sentimentRaw.toLowerCase();
if (sentimentStr.includes('positive')) {
sentiment = 0.8;
sentimentCategory = SentimentCategory.POSITIVE;
} else if (sentimentStr.includes('negative')) {
sentiment = -0.8;
sentimentCategory = SentimentCategory.NEGATIVE;
} else {
sentiment = 0.0;
sentimentCategory = SentimentCategory.NEUTRAL;
}
}
// Process boolean fields
const escalated = importRecord.escalatedRaw ?
['true', '1', 'yes', 'escalated'].includes(importRecord.escalatedRaw.toLowerCase()) : null;
const forwardedHr = importRecord.forwardedHrRaw ?
['true', '1', 'yes', 'forwarded'].includes(importRecord.forwardedHrRaw.toLowerCase()) : null;
// Keep country code as-is, will be processed by OpenAI later
const country = importRecord.countryCode;
// Fetch transcript content if URL is provided and not already fetched
let transcriptContent = importRecord.rawTranscriptContent;
if (!transcriptContent && importRecord.fullTranscriptUrl && isValidTranscriptUrl(importRecord.fullTranscriptUrl)) {
console.log(`[Import Processor] Fetching transcript for ${importRecord.externalSessionId}...`);
// Get company credentials for transcript fetching
const company = await prisma.company.findUnique({
where: { id: importRecord.companyId },
select: { csvUsername: true, csvPassword: true },
});
const transcriptResult = await fetchTranscriptContent(
importRecord.fullTranscriptUrl,
company?.csvUsername || undefined,
company?.csvPassword || undefined
);
if (transcriptResult.success) {
transcriptContent = transcriptResult.content;
console.log(`[Import Processor] ✓ Fetched transcript for ${importRecord.externalSessionId} (${transcriptContent?.length} chars)`);
// Update the import record with the fetched content
await prisma.sessionImport.update({
where: { id: importRecord.id },
data: { rawTranscriptContent: transcriptContent },
});
} else {
console.log(`[Import Processor] ⚠️ Failed to fetch transcript for ${importRecord.externalSessionId}: ${transcriptResult.error}`);
}
}
// Create or update Session record
const session = await prisma.session.upsert({
where: {
importId: importRecord.id,
},
update: {
startTime,
endTime,
ipAddress: importRecord.ipAddress,
country,
language: importRecord.language,
messagesSent: importRecord.messagesSent,
sentiment,
sentimentCategory,
escalated,
forwardedHr,
fullTranscriptUrl: importRecord.fullTranscriptUrl,
avgResponseTime: importRecord.avgResponseTimeSeconds,
tokens: importRecord.tokens,
tokensEur: importRecord.tokensEur,
category: importRecord.category,
initialMsg: importRecord.initialMessage,
processed: false, // Will be processed later by AI
},
create: {
companyId: importRecord.companyId,
importId: importRecord.id,
startTime,
endTime,
ipAddress: importRecord.ipAddress,
country,
language: importRecord.language,
messagesSent: importRecord.messagesSent,
sentiment,
sentimentCategory,
escalated,
forwardedHr,
fullTranscriptUrl: importRecord.fullTranscriptUrl,
avgResponseTime: importRecord.avgResponseTimeSeconds,
tokens: importRecord.tokens,
tokensEur: importRecord.tokensEur,
category: importRecord.category,
initialMsg: importRecord.initialMessage,
processed: false, // Will be processed later by AI
},
});
// Update import status to DONE
await prisma.sessionImport.update({
where: { id: importRecord.id },
data: {
status: ImportStatus.DONE,
processedAt: new Date(),
errorMsg: null,
},
});
return { success: true };
} catch (error) {
// Update import status to ERROR
await prisma.sessionImport.update({
where: { id: importRecord.id },
data: {
status: ImportStatus.ERROR,
errorMsg: error instanceof Error ? error.message : String(error),
},
});
return {
success: false,
error: error instanceof Error ? error.message : String(error),
};
}
}
/**
* Process queued SessionImport records into Session records
*/
export async function processQueuedImports(batchSize: number = 50): Promise<void> {
console.log('[Import Processor] Starting to process queued imports...');
// Find queued imports
const queuedImports = await prisma.sessionImport.findMany({
where: {
status: ImportStatus.QUEUED,
},
take: batchSize,
orderBy: {
createdAt: 'asc', // Process oldest first
},
});
if (queuedImports.length === 0) {
console.log('[Import Processor] No queued imports found');
return;
}
console.log(`[Import Processor] Processing ${queuedImports.length} queued imports...`);
let successCount = 0;
let errorCount = 0;
// Process each import
for (const importRecord of queuedImports) {
const result = await processSingleImport(importRecord);
if (result.success) {
successCount++;
console.log(`[Import Processor] ✓ Processed import ${importRecord.externalSessionId}`);
} else {
errorCount++;
console.log(`[Import Processor] ✗ Failed to process import ${importRecord.externalSessionId}: ${result.error}`);
}
}
console.log(`[Import Processor] Completed: ${successCount} successful, ${errorCount} failed`);
}
/**
* Start the import processing scheduler
*/
export function startImportProcessingScheduler(): void {
const config = getSchedulerConfig();
if (!config.enabled) {
console.log('[Import Processing Scheduler] Disabled via configuration');
return;
}
// Use a more frequent interval for import processing (every 5 minutes by default)
const interval = process.env.IMPORT_PROCESSING_INTERVAL || '*/5 * * * *';
const batchSize = parseInt(process.env.IMPORT_PROCESSING_BATCH_SIZE || '50', 10);
console.log(`[Import Processing Scheduler] Starting with interval: ${interval}`);
console.log(`[Import Processing Scheduler] Batch size: ${batchSize}`);
cron.schedule(interval, async () => {
try {
await processQueuedImports(batchSize);
} catch (error) {
console.error(`[Import Processing Scheduler] Error: ${error}`);
}
});
}

62
lib/schedulerConfig.ts
View File

@ -1,30 +1,7 @@
// Unified scheduler configuration
import { readFileSync } from "fs";
import { fileURLToPath } from "url";
import { dirname, join } from "path";
// Legacy scheduler configuration - now uses centralized env management
// This file is kept for backward compatibility but delegates to lib/env.ts
// Load environment variables from .env.local
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const envPath = join(__dirname, '..', '.env.local');
// Load .env.local if it exists
try {
const envFile = readFileSync(envPath, 'utf8');
const envVars = envFile.split('\n').filter(line => line.trim() && !line.startsWith('#'));
envVars.forEach(line => {
const [key, ...valueParts] = line.split('=');
if (key && valueParts.length > 0) {
const value = valueParts.join('=').trim();
if (!process.env[key.trim()]) {
process.env[key.trim()] = value;
}
}
});
} catch (error) {
// Silently fail if .env.local doesn't exist
}
import { getSchedulerConfig as getEnvSchedulerConfig, logEnvConfig } from "./env";
export interface SchedulerConfig {
enabled: boolean;
@ -40,43 +17,28 @@ export interface SchedulerConfig {
/**
* Get scheduler configuration from environment variables
* @deprecated Use getSchedulerConfig from lib/env.ts instead
*/
export function getSchedulerConfig(): SchedulerConfig {
const enabled = process.env.SCHEDULER_ENABLED === 'true';
// Default values
const defaults = {
csvImportInterval: '*/15 * * * *', // Every 15 minutes
sessionProcessingInterval: '0 * * * *', // Every hour
sessionProcessingBatchSize: 0, // Unlimited
sessionProcessingConcurrency: 5,
};
const config = getEnvSchedulerConfig();
return {
enabled,
enabled: config.enabled,
csvImport: {
interval: process.env.CSV_IMPORT_INTERVAL || defaults.csvImportInterval,
interval: config.csvImport.interval,
},
sessionProcessing: {
interval: process.env.SESSION_PROCESSING_INTERVAL || defaults.sessionProcessingInterval,
batchSize: parseInt(process.env.SESSION_PROCESSING_BATCH_SIZE || '0', 10) || defaults.sessionProcessingBatchSize,
concurrency: parseInt(process.env.SESSION_PROCESSING_CONCURRENCY || '5', 10) || defaults.sessionProcessingConcurrency,
interval: config.sessionProcessing.interval,
batchSize: config.sessionProcessing.batchSize,
concurrency: config.sessionProcessing.concurrency,
},
};
}
/**
* Log scheduler configuration
* @deprecated Use logEnvConfig from lib/env.ts instead
*/
export function logSchedulerConfig(config: SchedulerConfig): void {
if (!config.enabled) {
console.log('[Scheduler] Schedulers are DISABLED (SCHEDULER_ENABLED=false)');
return;
}
console.log('[Scheduler] Configuration:');
console.log(` CSV Import: ${config.csvImport.interval}`);
console.log(` Session Processing: ${config.sessionProcessing.interval}`);
console.log(` Batch Size: ${config.sessionProcessing.batchSize === 0 ? 'unlimited' : config.sessionProcessing.batchSize}`);
console.log(` Concurrency: ${config.sessionProcessing.concurrency}`);
logEnvConfig();
}

151
lib/transcriptFetcher.ts Normal file
View File

@ -0,0 +1,151 @@
// Transcript fetching utility
import fetch from "node-fetch";
export interface TranscriptFetchResult {
success: boolean;
content?: string;
error?: string;
}
/**
* Fetch transcript content from a URL
* @param url The transcript URL
* @param username Optional username for authentication
* @param password Optional password for authentication
* @returns Promise with fetch result
*/
export async function fetchTranscriptContent(
url: string,
username?: string,
password?: string
): Promise<TranscriptFetchResult> {
try {
if (!url || !url.trim()) {
return {
success: false,
error: 'No transcript URL provided',
};
}
// Prepare authentication header if credentials provided
const authHeader =
username && password
? "Basic " + Buffer.from(`${username}:${password}`).toString("base64")
: undefined;
const headers: Record<string, string> = {
'User-Agent': 'LiveDash-Transcript-Fetcher/1.0',
};
if (authHeader) {
headers.Authorization = authHeader;
}
// Fetch the transcript with timeout
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 30000); // 30 second timeout
const response = await fetch(url, {
method: 'GET',
headers,
signal: controller.signal,
});
clearTimeout(timeoutId);
if (!response.ok) {
return {
success: false,
error: `HTTP ${response.status}: ${response.statusText}`,
};
}
const content = await response.text();
if (!content || content.trim().length === 0) {
return {
success: false,
error: 'Empty transcript content',
};
}
return {
success: true,
content: content.trim(),
};
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
// Handle common network errors
if (errorMessage.includes('ENOTFOUND')) {
return {
success: false,
error: 'Domain not found',
};
}
if (errorMessage.includes('ECONNREFUSED')) {
return {
success: false,
error: 'Connection refused',
};
}
if (errorMessage.includes('timeout')) {
return {
success: false,
error: 'Request timeout',
};
}
return {
success: false,
error: errorMessage,
};
}
}
/**
* Validate if a URL looks like a valid transcript URL
* @param url The URL to validate
* @returns boolean indicating if URL appears valid
*/
export function isValidTranscriptUrl(url: string): boolean {
if (!url || typeof url !== 'string') {
return false;
}
try {
const parsedUrl = new URL(url);
return parsedUrl.protocol === 'http:' || parsedUrl.protocol === 'https:';
} catch {
return false;
}
}
/**
* Extract session ID from transcript content if possible
* This is a helper function that can be enhanced based on transcript format
* @param content The transcript content
* @returns Extracted session ID or null
*/
export function extractSessionIdFromTranscript(content: string): string | null {
if (!content) return null;
// Look for common session ID patterns
const patterns = [
/session[_-]?id[:\s]*([a-zA-Z0-9-]+)/i,
/id[:\s]*([a-zA-Z0-9-]{8,})/i,
/^([a-zA-Z0-9-]{8,})/m, // First line might be session ID
];
for (const pattern of patterns) {
const match = content.match(pattern);
if (match && match[1]) {
return match[1].trim();
}
}
return null;
}

7
package.json
View File

@ -18,6 +18,9 @@
"prisma:push:force": "prisma db push --force-reset",
"prisma:studio": "prisma studio",
"start": "node server.mjs",
"test": "jest",
"test:watch": "jest --watch",
"test:coverage": "jest --coverage",
"lint:md": "markdownlint-cli2 \"**/*.md\" \"!.trunk/**\" \"!.venv/**\" \"!node_modules/**\"",
"lint:md:fix": "markdownlint-cli2 --fix \"**/*.md\" \"!.trunk/**\" \"!.venv/**\" \"!node_modules/**\""
},
@ -54,9 +57,11 @@
"devDependencies": {
"@eslint/eslintrc": "^3.3.1",
"@eslint/js": "^9.27.0",
"@jest/globals": "^30.0.3",
"@playwright/test": "^1.52.0",
"@tailwindcss/postcss": "^4.1.7",
"@types/bcryptjs": "^2.4.2",
"@types/jest": "^30.0.0",
"@types/node": "^22.15.21",
"@types/node-cron": "^3.0.8",
"@types/react": "^19.1.5",
@ -66,12 +71,14 @@
"eslint": "^9.27.0",
"eslint-config-next": "^15.3.2",
"eslint-plugin-prettier": "^5.4.0",
"jest": "^30.0.3",
"markdownlint-cli2": "^0.18.1",
"postcss": "^8.5.3",
"prettier": "^3.5.3",
"prettier-plugin-jinja-template": "^2.1.0",
"prisma": "^6.10.1",
"tailwindcss": "^4.1.7",
"ts-jest": "^29.4.0",
"ts-node": "^10.9.2",
"tsx": "^4.20.3",
"typescript": "^5.0.0"

13
pages/api/admin/refresh-sessions.ts
View File

@ -1,6 +1,7 @@
// API route to refresh (fetch+parse+update) session data for a company
import { NextApiRequest, NextApiResponse } from "next";
import { fetchAndParseCsv } from "../../../lib/csvFetcher";
import { processQueuedImports } from "../../../lib/importProcessor";
import { prisma } from "../../../lib/prisma";
export default async function handler(
@ -113,11 +114,21 @@ export default async function handler(
}
}
// Immediately process the queued imports to create Session records
console.log('[Refresh API] Processing queued imports...');
await processQueuedImports(100); // Process up to 100 imports immediately
// Count how many sessions were created
const sessionCount = await prisma.session.count({
where: { companyId: company.id }
});
res.json({
ok: true,
imported: importedCount,
total: rawSessionData.length,
message: `Successfully imported ${importedCount} session records to SessionImport table`
sessions: sessionCount,
message: `Successfully imported ${importedCount} records and processed them into sessions. Total sessions: ${sessionCount}`
});
} catch (e) {
const error = e instanceof Error ? e.message : "An unknown error occurred";

2
pages/api/admin/trigger-processing.ts
View File

@ -40,7 +40,7 @@ export default async function handler(
return res.status(401).json({ error: "No user found" });
}
// Check if user has admin role
// Check if user has ADMIN role
if (user.role !== "ADMIN") {
return res.status(403).json({ error: "Admin access required" });
}

2
pages/api/dashboard/settings.ts
View File

@ -8,7 +8,7 @@ export default async function handler(
res: NextApiResponse
) {
const session = await getServerSession(req, res, authOptions);
if (!session?.user || session.user.role !== "admin")
if (!session?.user || session.user.role !== "ADMIN")
return res.status(403).json({ error: "Forbidden" });
const user = await prisma.user.findUnique({

2
pages/api/dashboard/users.ts
View File

@ -17,7 +17,7 @@ export default async function handler(
res: NextApiResponse
) {
const session = await getServerSession(req, res, authOptions);
if (!session?.user || session.user.role !== "admin")
if (!session?.user || session.user.role !== "ADMIN")
return res.status(403).json({ error: "Forbidden" });
const user = await prisma.user.findUnique({

2497
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

3
prisma/schema.prisma
View File

@ -155,6 +155,9 @@ model SessionImport {
category String?
initialMessage String?
// ─── Raw transcript content ─────────────────────────
rawTranscriptContent String? // Fetched content from fullTranscriptUrl
// ─── bookkeeping ─────────────────────────────────
status ImportStatus @default(QUEUED)
errorMsg String?

15
server.ts
View File

@ -3,8 +3,9 @@ import { createServer } from "http";
import { parse } from "url";
import next from "next";
import { startCsvImportScheduler } from "./lib/scheduler.js";
import { startImportProcessingScheduler } from "./lib/importProcessor.js";
import { startProcessingScheduler } from "./lib/processingScheduler.js";
import { getSchedulerConfig, logSchedulerConfig } from "./lib/schedulerConfig.js";
import { getSchedulerConfig, logEnvConfig, validateEnv } from "./lib/env.js";
const dev = process.env.NODE_ENV !== "production";
const hostname = "localhost";
@ -15,14 +16,22 @@ const app = next({ dev, hostname, port });
const handle = app.getRequestHandler();
app.prepare().then(() => {
// Get and log scheduler configuration
// Validate and log environment configuration
const envValidation = validateEnv();
if (!envValidation.valid) {
console.error('[Environment] Validation errors:', envValidation.errors);
}
logEnvConfig();
// Get scheduler configuration
const config = getSchedulerConfig();
logSchedulerConfig(config);
// Initialize schedulers based on configuration
if (config.enabled) {
console.log("Initializing schedulers...");
startCsvImportScheduler();
startImportProcessingScheduler();
startProcessingScheduler();
console.log("All schedulers initialized successfully");
}

25
tests/setup.ts Normal file
View File

@ -0,0 +1,25 @@
// Jest test setup
import { jest } from '@jest/globals';
// Mock console methods to reduce noise in tests
global.console = {
...console,
log: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
};
// Set test environment variables
Object.defineProperty(process.env, 'NODE_ENV', {
value: 'test',
writable: true,
configurable: true,
});
process.env.NEXTAUTH_SECRET = 'test-secret';
process.env.NEXTAUTH_URL = 'http://localhost:3000';
// Mock node-fetch for transcript fetcher tests
jest.mock('node-fetch', () => ({
__esModule: true,
default: jest.fn(),
}));

174
tests/unit/env.test.ts Normal file
View File

@ -0,0 +1,174 @@
// Unit tests for environment management
import { describe, it, expect, beforeEach, afterEach } from '@jest/globals';
import { env, validateEnv, getSchedulerConfig } from '../../lib/env';
describe('Environment Management', () => {
let originalEnv: NodeJS.ProcessEnv;
beforeEach(() => {
// Save original environment
originalEnv = { ...process.env };
});
afterEach(() => {
// Restore original environment
process.env = originalEnv;
});
describe('env object', () => {
it('should have default values when environment variables are not set', () => {
// Clear relevant env vars
delete process.env.NEXTAUTH_URL;
delete process.env.SCHEDULER_ENABLED;
delete process.env.PORT;
// Re-import to get fresh env object
jest.resetModules();
const { env: freshEnv } = require('../../lib/env');
expect(freshEnv.NEXTAUTH_URL).toBe('http://localhost:3000');
expect(freshEnv.SCHEDULER_ENABLED).toBe(false);
expect(freshEnv.PORT).toBe(3000);
});
it('should use environment variables when set', () => {
process.env.NEXTAUTH_URL = 'https://example.com';
process.env.SCHEDULER_ENABLED = 'true';
process.env.PORT = '8080';
jest.resetModules();
const { env: freshEnv } = require('../../lib/env');
expect(freshEnv.NEXTAUTH_URL).toBe('https://example.com');
expect(freshEnv.SCHEDULER_ENABLED).toBe(true);
expect(freshEnv.PORT).toBe(8080);
});
it('should parse numeric environment variables correctly', () => {
process.env.IMPORT_PROCESSING_BATCH_SIZE = '100';
process.env.SESSION_PROCESSING_CONCURRENCY = '10';
jest.resetModules();
const { env: freshEnv } = require('../../lib/env');
expect(freshEnv.IMPORT_PROCESSING_BATCH_SIZE).toBe(100);
expect(freshEnv.SESSION_PROCESSING_CONCURRENCY).toBe(10);
});
it('should handle invalid numeric values gracefully', () => {
process.env.IMPORT_PROCESSING_BATCH_SIZE = 'invalid';
process.env.SESSION_PROCESSING_CONCURRENCY = '';
jest.resetModules();
const { env: freshEnv } = require('../../lib/env');
expect(freshEnv.IMPORT_PROCESSING_BATCH_SIZE).toBe(50); // default
expect(freshEnv.SESSION_PROCESSING_CONCURRENCY).toBe(5); // default
});
});
describe('validateEnv', () => {
it('should return valid when all required variables are set', () => {
process.env.NEXTAUTH_SECRET = 'test-secret';
process.env.OPENAI_API_KEY = 'test-key';
Object.defineProperty(process.env, 'NODE_ENV', {
value: 'production',
writable: true,
configurable: true,
});
jest.resetModules();
const { validateEnv: freshValidateEnv } = require('../../lib/env');
const result = freshValidateEnv();
expect(result.valid).toBe(true);
expect(result.errors).toHaveLength(0);
});
it('should return invalid when NEXTAUTH_SECRET is missing', () => {
delete process.env.NEXTAUTH_SECRET;
jest.resetModules();
const { validateEnv: freshValidateEnv } = require('../../lib/env');
const result = freshValidateEnv();
expect(result.valid).toBe(false);
expect(result.errors).toContain('NEXTAUTH_SECRET is required');
});
it('should require OPENAI_API_KEY in production', () => {
process.env.NEXTAUTH_SECRET = 'test-secret';
delete process.env.OPENAI_API_KEY;
Object.defineProperty(process.env, 'NODE_ENV', {
value: 'production',
writable: true,
configurable: true,
});
jest.resetModules();
const { validateEnv: freshValidateEnv } = require('../../lib/env');
const result = freshValidateEnv();
expect(result.valid).toBe(false);
expect(result.errors).toContain('OPENAI_API_KEY is required in production');
});
it('should not require OPENAI_API_KEY in development', () => {
process.env.NEXTAUTH_SECRET = 'test-secret';
delete process.env.OPENAI_API_KEY;
Object.defineProperty(process.env, 'NODE_ENV', {
value: 'development',
writable: true,
configurable: true,
});
jest.resetModules();
const { validateEnv: freshValidateEnv } = require('../../lib/env');
const result = freshValidateEnv();
expect(result.valid).toBe(true);
expect(result.errors).toHaveLength(0);
});
});
describe('getSchedulerConfig', () => {
it('should return correct scheduler configuration', () => {
process.env.SCHEDULER_ENABLED = 'true';
process.env.CSV_IMPORT_INTERVAL = '*/10 * * * *';
process.env.IMPORT_PROCESSING_INTERVAL = '*/3 * * * *';
process.env.IMPORT_PROCESSING_BATCH_SIZE = '25';
process.env.SESSION_PROCESSING_INTERVAL = '0 2 * * *';
process.env.SESSION_PROCESSING_BATCH_SIZE = '100';
process.env.SESSION_PROCESSING_CONCURRENCY = '8';
jest.resetModules();
const { getSchedulerConfig: freshGetSchedulerConfig } = require('../../lib/env');
const config = freshGetSchedulerConfig();
expect(config.enabled).toBe(true);
expect(config.csvImport.interval).toBe('*/10 * * * *');
expect(config.importProcessing.interval).toBe('*/3 * * * *');
expect(config.importProcessing.batchSize).toBe(25);
expect(config.sessionProcessing.interval).toBe('0 2 * * *');
expect(config.sessionProcessing.batchSize).toBe(100);
expect(config.sessionProcessing.concurrency).toBe(8);
});
it('should use defaults when environment variables are not set', () => {
delete process.env.SCHEDULER_ENABLED;
delete process.env.CSV_IMPORT_INTERVAL;
delete process.env.IMPORT_PROCESSING_INTERVAL;
jest.resetModules();
const { getSchedulerConfig: freshGetSchedulerConfig } = require('../../lib/env');
const config = freshGetSchedulerConfig();
expect(config.enabled).toBe(false);
expect(config.csvImport.interval).toBe('*/15 * * * *');
expect(config.importProcessing.interval).toBe('*/5 * * * *');
expect(config.importProcessing.batchSize).toBe(50);
});
});
});

222
tests/unit/transcriptFetcher.test.ts Normal file
View File

@ -0,0 +1,222 @@
// Unit tests for transcript fetcher
import { describe, it, expect, beforeEach, afterEach, jest } from '@jest/globals';
import fetch from 'node-fetch';
import {
fetchTranscriptContent,
isValidTranscriptUrl,
extractSessionIdFromTranscript
} from '../../lib/transcriptFetcher';
// Mock node-fetch
const mockFetch = fetch as jest.MockedFunction<typeof fetch>;
describe('Transcript Fetcher', () => {
beforeEach(() => {
jest.clearAllMocks();
});
describe('fetchTranscriptContent', () => {
it('should successfully fetch transcript content', async () => {
const mockResponse = {
ok: true,
text: jest.fn().mockResolvedValue('Session transcript content'),
};
mockFetch.mockResolvedValue(mockResponse as any);
const result = await fetchTranscriptContent('https://example.com/transcript');
expect(result.success).toBe(true);
expect(result.content).toBe('Session transcript content');
expect(result.error).toBeUndefined();
expect(mockFetch).toHaveBeenCalledWith('https://example.com/transcript', {
method: 'GET',
headers: {
'User-Agent': 'LiveDash-Transcript-Fetcher/1.0',
},
signal: expect.any(AbortSignal),
});
});
it('should handle authentication with username and password', async () => {
const mockResponse = {
ok: true,
text: jest.fn().mockResolvedValue('Authenticated transcript'),
};
mockFetch.mockResolvedValue(mockResponse as any);
const result = await fetchTranscriptContent(
'https://example.com/transcript',
'user123',
'pass456'
);
expect(result.success).toBe(true);
expect(result.content).toBe('Authenticated transcript');
const expectedAuth = 'Basic ' + Buffer.from('user123:pass456').toString('base64');
expect(mockFetch).toHaveBeenCalledWith('https://example.com/transcript', {
method: 'GET',
headers: {
'User-Agent': 'LiveDash-Transcript-Fetcher/1.0',
'Authorization': expectedAuth,
},
signal: expect.any(AbortSignal),
});
});
it('should handle HTTP errors', async () => {
const mockResponse = {
ok: false,
status: 404,
statusText: 'Not Found',
};
mockFetch.mockResolvedValue(mockResponse as any);
const result = await fetchTranscriptContent('https://example.com/transcript');
expect(result.success).toBe(false);
expect(result.error).toBe('HTTP 404: Not Found');
expect(result.content).toBeUndefined();
});
it('should handle empty transcript content', async () => {
const mockResponse = {
ok: true,
text: jest.fn().mockResolvedValue(' '),
};
mockFetch.mockResolvedValue(mockResponse as any);
const result = await fetchTranscriptContent('https://example.com/transcript');
expect(result.success).toBe(false);
expect(result.error).toBe('Empty transcript content');
expect(result.content).toBeUndefined();
});
it('should handle network errors', async () => {
mockFetch.mockRejectedValue(new Error('ENOTFOUND example.com'));
const result = await fetchTranscriptContent('https://example.com/transcript');
expect(result.success).toBe(false);
expect(result.error).toBe('Domain not found');
expect(result.content).toBeUndefined();
});
it('should handle connection refused errors', async () => {
mockFetch.mockRejectedValue(new Error('ECONNREFUSED'));
const result = await fetchTranscriptContent('https://example.com/transcript');
expect(result.success).toBe(false);
expect(result.error).toBe('Connection refused');
expect(result.content).toBeUndefined();
});
it('should handle timeout errors', async () => {
mockFetch.mockRejectedValue(new Error('Request timeout'));
const result = await fetchTranscriptContent('https://example.com/transcript');
expect(result.success).toBe(false);
expect(result.error).toBe('Request timeout');
expect(result.content).toBeUndefined();
});
it('should handle empty URL', async () => {
const result = await fetchTranscriptContent('');
expect(result.success).toBe(false);
expect(result.error).toBe('No transcript URL provided');
expect(result.content).toBeUndefined();
expect(mockFetch).not.toHaveBeenCalled();
});
it('should trim whitespace from content', async () => {
const mockResponse = {
ok: true,
text: jest.fn().mockResolvedValue(' \n Session content \n '),
};
mockFetch.mockResolvedValue(mockResponse as any);
const result = await fetchTranscriptContent('https://example.com/transcript');
expect(result.success).toBe(true);
expect(result.content).toBe('Session content');
});
});
describe('isValidTranscriptUrl', () => {
it('should validate HTTP URLs', () => {
expect(isValidTranscriptUrl('http://example.com/transcript')).toBe(true);
});
it('should validate HTTPS URLs', () => {
expect(isValidTranscriptUrl('https://example.com/transcript')).toBe(true);
});
it('should reject invalid URLs', () => {
expect(isValidTranscriptUrl('not-a-url')).toBe(false);
expect(isValidTranscriptUrl('ftp://example.com')).toBe(false);
expect(isValidTranscriptUrl('')).toBe(false);
expect(isValidTranscriptUrl(null as any)).toBe(false);
expect(isValidTranscriptUrl(undefined as any)).toBe(false);
});
it('should handle malformed URLs', () => {
expect(isValidTranscriptUrl('http://')).toBe(false);
expect(isValidTranscriptUrl('https://')).toBe(false);
expect(isValidTranscriptUrl('://example.com')).toBe(false);
});
});
describe('extractSessionIdFromTranscript', () => {
it('should extract session ID from session_id pattern', () => {
const content = 'session_id: abc123def456\nOther content...';
const result = extractSessionIdFromTranscript(content);
expect(result).toBe('abc123def456');
});
it('should extract session ID from sessionId pattern', () => {
const content = 'sessionId: xyz789\nTranscript data...';
const result = extractSessionIdFromTranscript(content);
expect(result).toBe('xyz789');
});
it('should extract session ID from id pattern', () => {
const content = 'id: session-12345678\nChat log...';
const result = extractSessionIdFromTranscript(content);
expect(result).toBe('session-12345678');
});
it('should extract session ID from first line', () => {
const content = 'abc123def456\nUser: Hello\nBot: Hi there';
const result = extractSessionIdFromTranscript(content);
expect(result).toBe('abc123def456');
});
it('should return null for content without session ID', () => {
const content = 'User: Hello\nBot: Hi there\nUser: How are you?';
const result = extractSessionIdFromTranscript(content);
expect(result).toBe(null);
});
it('should return null for empty content', () => {
expect(extractSessionIdFromTranscript('')).toBe(null);
expect(extractSessionIdFromTranscript(null as any)).toBe(null);
expect(extractSessionIdFromTranscript(undefined as any)).toBe(null);
});
it('should handle case-insensitive patterns', () => {
const content = 'SESSION_ID: ABC123\nContent...';
const result = extractSessionIdFromTranscript(content);
expect(result).toBe('ABC123');
});
it('should extract the first matching pattern', () => {
const content = 'session_id: first123\nid: second456\nMore content...';
const result = extractSessionIdFromTranscript(content);
expect(result).toBe('first123');
});
});
});