mirror of
https://github.com/kjanat/livedash-node.git
synced 2026-01-16 06:12:09 +01:00
Refactor code for improved readability and consistency
- Updated formatting in SessionDetails component for better readability. - Enhanced documentation in scheduler-fixes.md to clarify issues and solutions. - Improved error handling and logging in csvFetcher.js and processingScheduler.js. - Standardized code formatting across various scripts and components for consistency. - Added validation checks for CSV URLs and transcript content to prevent processing errors. - Enhanced logging messages for better tracking of processing status and errors.
This commit is contained in:
Max Kowalski
130
TODO.md
130
TODO.md
@ -2,95 +2,107 @@
|
||||
|
||||
## Dashboard Integration
|
||||
|
||||
- [ ] **Resolve `GeographicMap.tsx` and `ResponseTimeDistribution.tsx` data simulation**
|
||||
- Investigate integrating real data sources with server-side analytics
|
||||
- Replace simulated data mentioned in `docs/dashboard-components.md`
|
||||
- [ ] **Resolve `GeographicMap.tsx` and `ResponseTimeDistribution.tsx` data simulation**
|
||||
- Investigate integrating real data sources with server-side analytics
|
||||
- Replace simulated data mentioned in `docs/dashboard-components.md`
|
||||
|
||||
## Component Specific
|
||||
|
||||
- [ ] **Implement robust emailing of temporary passwords**
|
||||
- File: `pages/api/dashboard/users.ts`
|
||||
- Set up proper email service integration
|
||||
- [ ] **Implement robust emailing of temporary passwords**
|
||||
|
||||
- [x] **Session page improvements** ✅
|
||||
- File: `app/dashboard/sessions/page.tsx`
|
||||
- Implemented pagination, advanced filtering, and sorting
|
||||
- File: `pages/api/dashboard/users.ts`
|
||||
- Set up proper email service integration
|
||||
|
||||
- [x] **Session page improvements** ✅
|
||||
- File: `app/dashboard/sessions/page.tsx`
|
||||
- Implemented pagination, advanced filtering, and sorting
|
||||
|
||||
## File Cleanup
|
||||
|
||||
- [x] **Remove backup files** ✅
|
||||
- Reviewed and removed `.bak` and `.new` files after integration
|
||||
- Cleaned up `GeographicMap.tsx.bak`, `SessionDetails.tsx.bak`, `SessionDetails.tsx.new`
|
||||
- [x] **Remove backup files** ✅
|
||||
- Reviewed and removed `.bak` and `.new` files after integration
|
||||
- Cleaned up `GeographicMap.tsx.bak`, `SessionDetails.tsx.bak`, `SessionDetails.tsx.new`
|
||||
|
||||
## Database Schema Improvements
|
||||
|
||||
- [ ] **Update EndTime field**
|
||||
- Make `endTime` field nullable in Prisma schema to match TypeScript interfaces
|
||||
- [ ] **Update EndTime field**
|
||||
|
||||
- [ ] **Add database indices**
|
||||
- Add appropriate indices to improve query performance
|
||||
- Focus on dashboard metrics and session listing queries
|
||||
- Make `endTime` field nullable in Prisma schema to match TypeScript interfaces
|
||||
|
||||
- [ ] **Implement production email service**
|
||||
- Replace console logging in `lib/sendEmail.ts`
|
||||
- Consider providers: Nodemailer, SendGrid, AWS SES
|
||||
- [ ] **Add database indices**
|
||||
|
||||
- Add appropriate indices to improve query performance
|
||||
- Focus on dashboard metrics and session listing queries
|
||||
|
||||
- [ ] **Implement production email service**
|
||||
- Replace console logging in `lib/sendEmail.ts`
|
||||
- Consider providers: Nodemailer, SendGrid, AWS SES
|
||||
|
||||
## General Enhancements & Features
|
||||
|
||||
- [ ] **Real-time updates**
|
||||
- Implement for dashboard and session list
|
||||
- Consider WebSockets or Server-Sent Events
|
||||
- [ ] **Real-time updates**
|
||||
|
||||
- [ ] **Data export functionality**
|
||||
- Allow users (especially admins) to export session data
|
||||
- Support CSV format initially
|
||||
- Implement for dashboard and session list
|
||||
- Consider WebSockets or Server-Sent Events
|
||||
|
||||
- [ ] **Customizable dashboard**
|
||||
- Allow users to customize dashboard view
|
||||
- Let users choose which metrics/charts are most important
|
||||
- [ ] **Data export functionality**
|
||||
|
||||
- Allow users (especially admins) to export session data
|
||||
- Support CSV format initially
|
||||
|
||||
- [ ] **Customizable dashboard**
|
||||
- Allow users to customize dashboard view
|
||||
- Let users choose which metrics/charts are most important
|
||||
|
||||
## Testing & Quality Assurance
|
||||
|
||||
- [ ] **Comprehensive testing suite**
|
||||
- [ ] Unit tests for utility functions and API logic
|
||||
- [ ] Integration tests for API endpoints with database
|
||||
- [ ] End-to-end tests for user flows (Playwright or Cypress)
|
||||
- [ ] **Comprehensive testing suite**
|
||||
|
||||
- [ ] **Error monitoring and logging**
|
||||
- Integrate robust error monitoring service (Sentry)
|
||||
- Enhance server-side logging
|
||||
- [ ] Unit tests for utility functions and API logic
|
||||
- [ ] Integration tests for API endpoints with database
|
||||
- [ ] End-to-end tests for user flows (Playwright or Cypress)
|
||||
|
||||
- [ ] **Accessibility improvements**
|
||||
- Review application against WCAG guidelines
|
||||
- Improve keyboard navigation and screen reader compatibility
|
||||
- Check color contrast ratios
|
||||
- [ ] **Error monitoring and logging**
|
||||
|
||||
- Integrate robust error monitoring service (Sentry)
|
||||
- Enhance server-side logging
|
||||
|
||||
- [ ] **Accessibility improvements**
|
||||
- Review application against WCAG guidelines
|
||||
- Improve keyboard navigation and screen reader compatibility
|
||||
- Check color contrast ratios
|
||||
|
||||
## Security Enhancements
|
||||
|
||||
- [x] **Password reset functionality** ✅
|
||||
- Implemented secure password reset mechanism
|
||||
- Files: `app/forgot-password/page.tsx`, `app/reset-password/page.tsx`, `pages/api/forgot-password.ts`, `pages/api/reset-password.ts`
|
||||
- [x] **Password reset functionality** ✅
|
||||
|
||||
- [ ] **Two-Factor Authentication (2FA)**
|
||||
- Consider adding 2FA, especially for admin accounts
|
||||
- Implemented secure password reset mechanism
|
||||
- Files: `app/forgot-password/page.tsx`, `app/reset-password/page.tsx`, `pages/api/forgot-password.ts`, `pages/api/reset-password.ts`
|
||||
|
||||
- [ ] **Input validation and sanitization**
|
||||
- Review all user inputs (API request bodies, query parameters)
|
||||
- Ensure proper validation and sanitization
|
||||
- [ ] **Two-Factor Authentication (2FA)**
|
||||
|
||||
- Consider adding 2FA, especially for admin accounts
|
||||
|
||||
- [ ] **Input validation and sanitization**
|
||||
- Review all user inputs (API request bodies, query parameters)
|
||||
- Ensure proper validation and sanitization
|
||||
|
||||
## Code Quality & Development
|
||||
|
||||
- [ ] **Code review process**
|
||||
- Enforce code reviews for all changes
|
||||
- [ ] **Code review process**
|
||||
|
||||
- [ ] **Environment configuration**
|
||||
- Ensure secure management of environment-specific configurations
|
||||
- Enforce code reviews for all changes
|
||||
|
||||
- [ ] **Dependency management**
|
||||
- Periodically review dependencies for vulnerabilities
|
||||
- Keep dependencies updated
|
||||
- [ ] **Environment configuration**
|
||||
|
||||
- [ ] **Documentation updates**
|
||||
- [ ] Ensure `docs/dashboard-components.md` reflects actual implementations
|
||||
- [ ] Verify "Dashboard Enhancements" are consistently applied
|
||||
- [ ] Update documentation for improved layout and visual hierarchies
|
||||
- Ensure secure management of environment-specific configurations
|
||||
|
||||
- [ ] **Dependency management**
|
||||
|
||||
- Periodically review dependencies for vulnerabilities
|
||||
- Keep dependencies updated
|
||||
|
||||
- [ ] **Documentation updates**
|
||||
- [ ] Ensure `docs/dashboard-components.md` reflects actual implementations
|
||||
- [ ] Verify "Dashboard Enhancements" are consistently applied
|
||||
- [ ] Update documentation for improved layout and visual hierarchies
|
||||
|
||||
@ -30,18 +30,18 @@ export default function MessageViewer({ messages }: MessageViewerProps) {
|
||||
<div
|
||||
key={message.id}
|
||||
className={`flex ${
|
||||
message.role.toLowerCase() === 'user'
|
||||
? 'justify-end'
|
||||
: 'justify-start'
|
||||
message.role.toLowerCase() === "user"
|
||||
? "justify-end"
|
||||
: "justify-start"
|
||||
}`}
|
||||
>
|
||||
<div
|
||||
className={`max-w-xs lg:max-w-md px-4 py-2 rounded-lg ${
|
||||
message.role.toLowerCase() === 'user'
|
||||
? 'bg-blue-500 text-white'
|
||||
: message.role.toLowerCase() === 'assistant'
|
||||
? 'bg-gray-200 text-gray-800'
|
||||
: 'bg-yellow-100 text-yellow-800'
|
||||
message.role.toLowerCase() === "user"
|
||||
? "bg-blue-500 text-white"
|
||||
: message.role.toLowerCase() === "assistant"
|
||||
? "bg-gray-200 text-gray-800"
|
||||
: "bg-yellow-100 text-yellow-800"
|
||||
}`}
|
||||
>
|
||||
<div className="flex items-center justify-between mb-1">
|
||||
@ -66,7 +66,8 @@ export default function MessageViewer({ messages }: MessageViewerProps) {
|
||||
First message: {new Date(messages[0].timestamp).toLocaleString()}
|
||||
</span>
|
||||
<span>
|
||||
Last message: {new Date(messages[messages.length - 1].timestamp).toLocaleString()}
|
||||
Last message:{" "}
|
||||
{new Date(messages[messages.length - 1].timestamp).toLocaleString()}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -161,7 +161,9 @@ export default function SessionDetails({ session }: SessionDetailsProps) {
|
||||
{session.ipAddress && (
|
||||
<div className="flex justify-between border-b pb-2">
|
||||
<span className="text-gray-600">IP Address:</span>
|
||||
<span className="font-medium font-mono text-sm">{session.ipAddress}</span>
|
||||
<span className="font-medium font-mono text-sm">
|
||||
{session.ipAddress}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
|
||||
@ -3,25 +3,31 @@
|
||||
## Issues Identified and Resolved
|
||||
|
||||
### 1. Invalid Company Configuration
|
||||
|
||||
**Problem**: Company `26fc3d34-c074-4556-85bd-9a66fafc0e08` had an invalid CSV URL (`https://example.com/data.csv`) with no authentication credentials.
|
||||
|
||||
**Solution**:
|
||||
**Solution**:
|
||||
|
||||
- Added validation in `fetchAndStoreSessionsForAllCompanies()` to skip companies with example/invalid URLs
|
||||
- Removed the invalid company record from the database using `fix_companies.js`
|
||||
|
||||
### 2. Transcript Fetching Errors
|
||||
|
||||
**Problem**: Multiple "Error fetching transcript: Unauthorized" messages were flooding the logs when individual transcript files couldn't be accessed.
|
||||
|
||||
**Solution**:
|
||||
|
||||
- Improved error handling in `fetchTranscriptContent()` function
|
||||
- Added probabilistic logging (only ~10% of errors logged) to prevent log spam
|
||||
- Added timeout (10 seconds) for transcript fetching
|
||||
- Made transcript fetching failures non-blocking (sessions are still created without transcript content)
|
||||
|
||||
### 3. CSV Fetching Errors
|
||||
|
||||
**Problem**: "Failed to fetch CSV: Not Found" errors for companies with invalid URLs.
|
||||
|
||||
**Solution**:
|
||||
|
||||
- Added URL validation to skip companies with `example.com` URLs
|
||||
- Improved error logging to be more descriptive
|
||||
|
||||
@ -35,6 +41,7 @@
|
||||
## Remaining Companies
|
||||
|
||||
After cleanup, only valid companies remain:
|
||||
|
||||
- **Demo Company** (`790b9233-d369-451f-b92c-f4dceb42b649`)
|
||||
- CSV URL: `https://proto.notso.ai/jumbo/chats`
|
||||
- Has valid authentication credentials
|
||||
@ -43,6 +50,7 @@ After cleanup, only valid companies remain:
|
||||
## Files Modified
|
||||
|
||||
1. **lib/csvFetcher.js**
|
||||
|
||||
- Added company URL validation
|
||||
- Improved transcript fetching error handling
|
||||
- Reduced error log verbosity
|
||||
|
||||
@ -410,15 +410,19 @@ async function fetchTranscriptContent(url, username, password) {
|
||||
|
||||
if (!response.ok) {
|
||||
// Only log error once per batch, not for every transcript
|
||||
if (Math.random() < 0.1) { // Log ~10% of errors to avoid spam
|
||||
console.warn(`[CSV] Transcript fetch failed for ${url}: ${response.status} ${response.statusText}`);
|
||||
if (Math.random() < 0.1) {
|
||||
// Log ~10% of errors to avoid spam
|
||||
console.warn(
|
||||
`[CSV] Transcript fetch failed for ${url}: ${response.status} ${response.statusText}`
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
return await response.text();
|
||||
} catch (error) {
|
||||
// Only log error once per batch, not for every transcript
|
||||
if (Math.random() < 0.1) { // Log ~10% of errors to avoid spam
|
||||
if (Math.random() < 0.1) {
|
||||
// Log ~10% of errors to avoid spam
|
||||
console.warn(`[CSV] Transcript fetch error for ${url}:`, error.message);
|
||||
}
|
||||
return null;
|
||||
@ -505,13 +509,20 @@ export async function fetchAndStoreSessionsForAllCompanies() {
|
||||
|
||||
for (const company of companies) {
|
||||
if (!company.csvUrl) {
|
||||
console.log(`[Scheduler] Skipping company ${company.id} - no CSV URL configured`);
|
||||
console.log(
|
||||
`[Scheduler] Skipping company ${company.id} - no CSV URL configured`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip companies with invalid/example URLs
|
||||
if (company.csvUrl.includes('example.com') || company.csvUrl === 'https://example.com/data.csv') {
|
||||
console.log(`[Scheduler] Skipping company ${company.id} - invalid/example CSV URL: ${company.csvUrl}`);
|
||||
if (
|
||||
company.csvUrl.includes("example.com") ||
|
||||
company.csvUrl === "https://example.com/data.csv"
|
||||
) {
|
||||
console.log(
|
||||
`[Scheduler] Skipping company ${company.id} - invalid/example CSV URL: ${company.csvUrl}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -581,11 +592,17 @@ export async function fetchAndStoreSessionsForAllCompanies() {
|
||||
country: session.country || null,
|
||||
language: session.language || null,
|
||||
messagesSent:
|
||||
typeof session.messagesSent === "number" ? session.messagesSent : 0,
|
||||
typeof session.messagesSent === "number"
|
||||
? session.messagesSent
|
||||
: 0,
|
||||
sentiment:
|
||||
typeof session.sentiment === "number" ? session.sentiment : null,
|
||||
typeof session.sentiment === "number"
|
||||
? session.sentiment
|
||||
: null,
|
||||
escalated:
|
||||
typeof session.escalated === "boolean" ? session.escalated : null,
|
||||
typeof session.escalated === "boolean"
|
||||
? session.escalated
|
||||
: null,
|
||||
forwardedHr:
|
||||
typeof session.forwardedHr === "boolean"
|
||||
? session.forwardedHr
|
||||
@ -596,9 +613,12 @@ export async function fetchAndStoreSessionsForAllCompanies() {
|
||||
typeof session.avgResponseTime === "number"
|
||||
? session.avgResponseTime
|
||||
: null,
|
||||
tokens: typeof session.tokens === "number" ? session.tokens : null,
|
||||
tokens:
|
||||
typeof session.tokens === "number" ? session.tokens : null,
|
||||
tokensEur:
|
||||
typeof session.tokensEur === "number" ? session.tokensEur : null,
|
||||
typeof session.tokensEur === "number"
|
||||
? session.tokensEur
|
||||
: null,
|
||||
category: session.category || null,
|
||||
initialMsg: session.initialMsg || null,
|
||||
},
|
||||
@ -607,9 +627,14 @@ export async function fetchAndStoreSessionsForAllCompanies() {
|
||||
addedCount++;
|
||||
}
|
||||
|
||||
console.log(`[Scheduler] Added ${addedCount} new sessions for company ${company.id}`);
|
||||
console.log(
|
||||
`[Scheduler] Added ${addedCount} new sessions for company ${company.id}`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(`[Scheduler] Error processing company ${company.id}:`, error);
|
||||
console.error(
|
||||
`[Scheduler] Error processing company ${company.id}:`,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@ -126,7 +126,9 @@ function validateOpenAIResponse(data) {
|
||||
|
||||
// Validate field types
|
||||
if (typeof data.language !== "string" || !/^[a-z]{2}$/.test(data.language)) {
|
||||
throw new Error("Invalid language format. Expected ISO 639-1 code (e.g., 'en')");
|
||||
throw new Error(
|
||||
"Invalid language format. Expected ISO 639-1 code (e.g., 'en')"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.messages_sent !== "number" || data.messages_sent < 0) {
|
||||
@ -134,7 +136,9 @@ function validateOpenAIResponse(data) {
|
||||
}
|
||||
|
||||
if (!["positive", "neutral", "negative"].includes(data.sentiment)) {
|
||||
throw new Error("Invalid sentiment. Expected 'positive', 'neutral', or 'negative'");
|
||||
throw new Error(
|
||||
"Invalid sentiment. Expected 'positive', 'neutral', or 'negative'"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.escalated !== "boolean") {
|
||||
@ -162,15 +166,23 @@ function validateOpenAIResponse(data) {
|
||||
];
|
||||
|
||||
if (!validCategories.includes(data.category)) {
|
||||
throw new Error(`Invalid category. Expected one of: ${validCategories.join(", ")}`);
|
||||
throw new Error(
|
||||
`Invalid category. Expected one of: ${validCategories.join(", ")}`
|
||||
);
|
||||
}
|
||||
|
||||
if (!Array.isArray(data.questions)) {
|
||||
throw new Error("Invalid questions. Expected array of strings");
|
||||
}
|
||||
|
||||
if (typeof data.summary !== "string" || data.summary.length < 10 || data.summary.length > 300) {
|
||||
throw new Error("Invalid summary. Expected string between 10-300 characters");
|
||||
if (
|
||||
typeof data.summary !== "string" ||
|
||||
data.summary.length < 10 ||
|
||||
data.summary.length > 300
|
||||
) {
|
||||
throw new Error(
|
||||
"Invalid summary. Expected string between 10-300 characters"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.session_id !== "string") {
|
||||
@ -182,7 +194,9 @@ function validateOpenAIResponse(data) {
|
||||
* Process unprocessed sessions
|
||||
*/
|
||||
export async function processUnprocessedSessions() {
|
||||
process.stdout.write("[ProcessingScheduler] Starting to process unprocessed sessions...\n");
|
||||
process.stdout.write(
|
||||
"[ProcessingScheduler] Starting to process unprocessed sessions...\n"
|
||||
);
|
||||
|
||||
// Find sessions that have messages but haven't been processed
|
||||
const sessionsToProcess = await prisma.session.findMany({
|
||||
@ -193,43 +207,58 @@ export async function processUnprocessedSessions() {
|
||||
},
|
||||
include: {
|
||||
messages: {
|
||||
orderBy: { order: 'asc' }
|
||||
}
|
||||
orderBy: { order: "asc" },
|
||||
},
|
||||
},
|
||||
take: 10, // Process in batches to avoid overloading the system
|
||||
});
|
||||
|
||||
// Filter to only sessions that have messages
|
||||
const sessionsWithMessages = sessionsToProcess.filter(session => session.messages.length > 0);
|
||||
const sessionsWithMessages = sessionsToProcess.filter(
|
||||
(session) => session.messages.length > 0
|
||||
);
|
||||
|
||||
if (sessionsWithMessages.length === 0) {
|
||||
process.stdout.write("[ProcessingScheduler] No sessions found requiring processing.\n");
|
||||
process.stdout.write(
|
||||
"[ProcessingScheduler] No sessions found requiring processing.\n"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
process.stdout.write(`[ProcessingScheduler] Found ${sessionsWithMessages.length} sessions to process.\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Found ${sessionsWithMessages.length} sessions to process.\n`
|
||||
);
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
for (const session of sessionsWithMessages) {
|
||||
if (session.messages.length === 0) {
|
||||
process.stderr.write(`[ProcessingScheduler] Session ${session.id} has no messages, skipping.\n`);
|
||||
process.stderr.write(
|
||||
`[ProcessingScheduler] Session ${session.id} has no messages, skipping.\n`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
process.stdout.write(`[ProcessingScheduler] Processing messages for session ${session.id}...\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Processing messages for session ${session.id}...\n`
|
||||
);
|
||||
try {
|
||||
// Convert messages back to transcript format for OpenAI processing
|
||||
const transcript = session.messages.map(msg =>
|
||||
`[${new Date(msg.timestamp).toLocaleString('en-GB', {
|
||||
day: '2-digit',
|
||||
month: '2-digit',
|
||||
year: 'numeric',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
second: '2-digit'
|
||||
}).replace(',', '')}] ${msg.role}: ${msg.content}`
|
||||
).join('\n');
|
||||
const transcript = session.messages
|
||||
.map(
|
||||
(msg) =>
|
||||
`[${new Date(msg.timestamp)
|
||||
.toLocaleString("en-GB", {
|
||||
day: "2-digit",
|
||||
month: "2-digit",
|
||||
year: "numeric",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
})
|
||||
.replace(",", "")}] ${msg.role}: ${msg.content}`
|
||||
)
|
||||
.join("\n");
|
||||
|
||||
const processedData = await processTranscriptWithOpenAI(
|
||||
session.id,
|
||||
@ -260,17 +289,25 @@ export async function processUnprocessedSessions() {
|
||||
},
|
||||
});
|
||||
|
||||
process.stdout.write(`[ProcessingScheduler] Successfully processed session ${session.id}.\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Successfully processed session ${session.id}.\n`
|
||||
);
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[ProcessingScheduler] Error processing session ${session.id}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[ProcessingScheduler] Error processing session ${session.id}: ${error}\n`
|
||||
);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
process.stdout.write("[ProcessingScheduler] Session processing complete.\n");
|
||||
process.stdout.write(`[ProcessingScheduler] Successfully processed: ${successCount} sessions.\n`);
|
||||
process.stdout.write(`[ProcessingScheduler] Failed to process: ${errorCount} sessions.\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Successfully processed: ${successCount} sessions.\n`
|
||||
);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Failed to process: ${errorCount} sessions.\n`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -282,9 +319,13 @@ export function startProcessingScheduler() {
|
||||
try {
|
||||
await processUnprocessedSessions();
|
||||
} catch (error) {
|
||||
process.stderr.write(`[ProcessingScheduler] Error in scheduler: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[ProcessingScheduler] Error in scheduler: ${error}\n`
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
process.stdout.write("[ProcessingScheduler] Started processing scheduler (runs hourly).\n");
|
||||
process.stdout.write(
|
||||
"[ProcessingScheduler] Started processing scheduler (runs hourly).\n"
|
||||
);
|
||||
}
|
||||
|
||||
@ -103,7 +103,7 @@ async function processTranscriptWithOpenAI(
|
||||
throw new Error(`OpenAI API error: ${response.status} - ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json() as any;
|
||||
const data = (await response.json()) as any;
|
||||
const processedData = JSON.parse(data.choices[0].message.content);
|
||||
|
||||
// Validate the response against our expected schema
|
||||
@ -120,7 +120,9 @@ async function processTranscriptWithOpenAI(
|
||||
* Validates the OpenAI response against our expected schema
|
||||
* @param data The data to validate
|
||||
*/
|
||||
function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData {
|
||||
function validateOpenAIResponse(
|
||||
data: any
|
||||
): asserts data is OpenAIProcessedData {
|
||||
// Check required fields
|
||||
const requiredFields = [
|
||||
"language",
|
||||
@ -142,7 +144,9 @@ function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData
|
||||
|
||||
// Validate field types
|
||||
if (typeof data.language !== "string" || !/^[a-z]{2}$/.test(data.language)) {
|
||||
throw new Error("Invalid language format. Expected ISO 639-1 code (e.g., 'en')");
|
||||
throw new Error(
|
||||
"Invalid language format. Expected ISO 639-1 code (e.g., 'en')"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.messages_sent !== "number" || data.messages_sent < 0) {
|
||||
@ -150,7 +154,9 @@ function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData
|
||||
}
|
||||
|
||||
if (!["positive", "neutral", "negative"].includes(data.sentiment)) {
|
||||
throw new Error("Invalid sentiment. Expected 'positive', 'neutral', or 'negative'");
|
||||
throw new Error(
|
||||
"Invalid sentiment. Expected 'positive', 'neutral', or 'negative'"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.escalated !== "boolean") {
|
||||
@ -178,15 +184,23 @@ function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData
|
||||
];
|
||||
|
||||
if (!validCategories.includes(data.category)) {
|
||||
throw new Error(`Invalid category. Expected one of: ${validCategories.join(", ")}`);
|
||||
throw new Error(
|
||||
`Invalid category. Expected one of: ${validCategories.join(", ")}`
|
||||
);
|
||||
}
|
||||
|
||||
if (!Array.isArray(data.questions)) {
|
||||
throw new Error("Invalid questions. Expected array of strings");
|
||||
}
|
||||
|
||||
if (typeof data.summary !== "string" || data.summary.length < 10 || data.summary.length > 300) {
|
||||
throw new Error("Invalid summary. Expected string between 10-300 characters");
|
||||
if (
|
||||
typeof data.summary !== "string" ||
|
||||
data.summary.length < 10 ||
|
||||
data.summary.length > 300
|
||||
) {
|
||||
throw new Error(
|
||||
"Invalid summary. Expected string between 10-300 characters"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.session_id !== "string") {
|
||||
@ -198,7 +212,9 @@ function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData
|
||||
* Process unprocessed sessions
|
||||
*/
|
||||
async function processUnprocessedSessions() {
|
||||
process.stdout.write("[ProcessingScheduler] Starting to process unprocessed sessions...\n");
|
||||
process.stdout.write(
|
||||
"[ProcessingScheduler] Starting to process unprocessed sessions...\n"
|
||||
);
|
||||
|
||||
// Find sessions that have transcript content but haven't been processed
|
||||
const sessionsToProcess = await prisma.session.findMany({
|
||||
@ -217,22 +233,30 @@ async function processUnprocessedSessions() {
|
||||
});
|
||||
|
||||
if (sessionsToProcess.length === 0) {
|
||||
process.stdout.write("[ProcessingScheduler] No sessions found requiring processing.\n");
|
||||
process.stdout.write(
|
||||
"[ProcessingScheduler] No sessions found requiring processing.\n"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
process.stdout.write(`[ProcessingScheduler] Found ${sessionsToProcess.length} sessions to process.\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Found ${sessionsToProcess.length} sessions to process.\n`
|
||||
);
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
for (const session of sessionsToProcess) {
|
||||
if (!session.transcriptContent) {
|
||||
// Should not happen due to query, but good for type safety
|
||||
process.stderr.write(`[ProcessingScheduler] Session ${session.id} has no transcript content, skipping.\n`);
|
||||
process.stderr.write(
|
||||
`[ProcessingScheduler] Session ${session.id} has no transcript content, skipping.\n`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
process.stdout.write(`[ProcessingScheduler] Processing transcript for session ${session.id}...\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Processing transcript for session ${session.id}...\n`
|
||||
);
|
||||
try {
|
||||
const processedData = await processTranscriptWithOpenAI(
|
||||
session.id,
|
||||
@ -263,17 +287,25 @@ async function processUnprocessedSessions() {
|
||||
},
|
||||
});
|
||||
|
||||
process.stdout.write(`[ProcessingScheduler] Successfully processed session ${session.id}.\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Successfully processed session ${session.id}.\n`
|
||||
);
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[ProcessingScheduler] Error processing session ${session.id}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[ProcessingScheduler] Error processing session ${session.id}: ${error}\n`
|
||||
);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
process.stdout.write("[ProcessingScheduler] Session processing complete.\n");
|
||||
process.stdout.write(`[ProcessingScheduler] Successfully processed: ${successCount} sessions.\n`);
|
||||
process.stdout.write(`[ProcessingScheduler] Failed to process: ${errorCount} sessions.\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Successfully processed: ${successCount} sessions.\n`
|
||||
);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Failed to process: ${errorCount} sessions.\n`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -285,9 +317,13 @@ export function startProcessingScheduler() {
|
||||
try {
|
||||
await processUnprocessedSessions();
|
||||
} catch (error) {
|
||||
process.stderr.write(`[ProcessingScheduler] Error in scheduler: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[ProcessingScheduler] Error in scheduler: ${error}\n`
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
process.stdout.write("[ProcessingScheduler] Started processing scheduler (runs hourly).\n");
|
||||
process.stdout.write(
|
||||
"[ProcessingScheduler] Started processing scheduler (runs hourly).\n"
|
||||
);
|
||||
}
|
||||
|
||||
@ -31,5 +31,7 @@ export function startScheduler() {
|
||||
}
|
||||
});
|
||||
|
||||
console.log("[Scheduler] Started session refresh scheduler (runs every 15 minutes).");
|
||||
console.log(
|
||||
"[Scheduler] Started session refresh scheduler (runs every 15 minutes)."
|
||||
);
|
||||
}
|
||||
|
||||
@ -10,17 +10,20 @@ const prisma = new PrismaClient();
|
||||
*/
|
||||
export function parseChatLogToJSON(logString) {
|
||||
// Convert to string if it's not already
|
||||
const stringData = typeof logString === 'string' ? logString : String(logString);
|
||||
const stringData =
|
||||
typeof logString === "string" ? logString : String(logString);
|
||||
|
||||
// Split by lines and filter out empty lines
|
||||
const lines = stringData.split('\n').filter(line => line.trim() !== '');
|
||||
const lines = stringData.split("\n").filter((line) => line.trim() !== "");
|
||||
|
||||
const messages = [];
|
||||
let currentMessage = null;
|
||||
|
||||
for (const line of lines) {
|
||||
// Check if line starts with a timestamp pattern [DD.MM.YYYY HH:MM:SS]
|
||||
const timestampMatch = line.match(/^\[(\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}:\d{2})\] (.+?): (.*)$/);
|
||||
const timestampMatch = line.match(
|
||||
/^\[(\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}:\d{2})\] (.+?): (.*)$/
|
||||
);
|
||||
|
||||
if (timestampMatch) {
|
||||
// If we have a previous message, push it to the array
|
||||
@ -32,9 +35,9 @@ export function parseChatLogToJSON(logString) {
|
||||
const [, timestamp, sender, content] = timestampMatch;
|
||||
|
||||
// Convert DD.MM.YYYY HH:MM:SS to ISO format
|
||||
const [datePart, timePart] = timestamp.split(' ');
|
||||
const [day, month, year] = datePart.split('.');
|
||||
const [hour, minute, second] = timePart.split(':');
|
||||
const [datePart, timePart] = timestamp.split(" ");
|
||||
const [day, month, year] = datePart.split(".");
|
||||
const [hour, minute, second] = timePart.split(":");
|
||||
|
||||
const dateObject = new Date(year, month - 1, day, hour, minute, second);
|
||||
|
||||
@ -42,11 +45,11 @@ export function parseChatLogToJSON(logString) {
|
||||
currentMessage = {
|
||||
timestamp: dateObject.toISOString(),
|
||||
role: sender,
|
||||
content: content
|
||||
content: content,
|
||||
};
|
||||
} else if (currentMessage) {
|
||||
// This is a continuation of the previous message (multiline)
|
||||
currentMessage.content += '\n' + line;
|
||||
currentMessage.content += "\n" + line;
|
||||
}
|
||||
}
|
||||
|
||||
@ -67,7 +70,7 @@ export function parseChatLogToJSON(logString) {
|
||||
// This puts "User" before "Assistant" when timestamps are the same
|
||||
return b.role.localeCompare(a.role);
|
||||
}),
|
||||
totalMessages: messages.length
|
||||
totalMessages: messages.length,
|
||||
};
|
||||
}
|
||||
|
||||
@ -80,7 +83,7 @@ export async function storeMessagesForSession(sessionId, messages) {
|
||||
try {
|
||||
// First, delete any existing messages for this session
|
||||
await prisma.message.deleteMany({
|
||||
where: { sessionId }
|
||||
where: { sessionId },
|
||||
});
|
||||
|
||||
// Then insert the new messages
|
||||
@ -89,19 +92,23 @@ export async function storeMessagesForSession(sessionId, messages) {
|
||||
timestamp: new Date(message.timestamp),
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
order: index
|
||||
order: index,
|
||||
}));
|
||||
|
||||
if (messageData.length > 0) {
|
||||
await prisma.message.createMany({
|
||||
data: messageData
|
||||
data: messageData,
|
||||
});
|
||||
}
|
||||
|
||||
process.stdout.write(`[TranscriptParser] Stored ${messageData.length} messages for session ${sessionId}\n`);
|
||||
process.stdout.write(
|
||||
`[TranscriptParser] Stored ${messageData.length} messages for session ${sessionId}\n`
|
||||
);
|
||||
return messageData.length;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Error storing messages for session ${sessionId}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Error storing messages for session ${sessionId}: ${error}\n`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@ -112,9 +119,12 @@ export async function storeMessagesForSession(sessionId, messages) {
|
||||
* @param {string} transcriptContent - Raw transcript content
|
||||
* @returns {Promise<Object>} Processing result with message count
|
||||
*/
|
||||
export async function processTranscriptForSession(sessionId, transcriptContent) {
|
||||
if (!transcriptContent || transcriptContent.trim() === '') {
|
||||
throw new Error('No transcript content provided');
|
||||
export async function processTranscriptForSession(
|
||||
sessionId,
|
||||
transcriptContent
|
||||
) {
|
||||
if (!transcriptContent || transcriptContent.trim() === "") {
|
||||
throw new Error("No transcript content provided");
|
||||
}
|
||||
|
||||
try {
|
||||
@ -122,16 +132,21 @@ export async function processTranscriptForSession(sessionId, transcriptContent)
|
||||
const parsed = parseChatLogToJSON(transcriptContent);
|
||||
|
||||
// Store messages in database
|
||||
const messageCount = await storeMessagesForSession(sessionId, parsed.messages);
|
||||
const messageCount = await storeMessagesForSession(
|
||||
sessionId,
|
||||
parsed.messages
|
||||
);
|
||||
|
||||
return {
|
||||
sessionId,
|
||||
messageCount,
|
||||
totalMessages: parsed.totalMessages,
|
||||
success: true
|
||||
success: true,
|
||||
};
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Error processing transcript for session ${sessionId}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Error processing transcript for session ${sessionId}: ${error}\n`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@ -140,7 +155,9 @@ export async function processTranscriptForSession(sessionId, transcriptContent)
|
||||
* Processes transcripts for all sessions that have transcript content but no parsed messages
|
||||
*/
|
||||
export async function processAllUnparsedTranscripts() {
|
||||
process.stdout.write("[TranscriptParser] Starting to process unparsed transcripts...\n");
|
||||
process.stdout.write(
|
||||
"[TranscriptParser] Starting to process unparsed transcripts...\n"
|
||||
);
|
||||
|
||||
try {
|
||||
// Find sessions with transcript content but no messages
|
||||
@ -149,42 +166,58 @@ export async function processAllUnparsedTranscripts() {
|
||||
AND: [
|
||||
{ transcriptContent: { not: null } },
|
||||
{ transcriptContent: { not: "" } },
|
||||
]
|
||||
],
|
||||
},
|
||||
include: {
|
||||
messages: true
|
||||
}
|
||||
messages: true,
|
||||
},
|
||||
});
|
||||
|
||||
// Filter to only sessions without messages
|
||||
const unparsedSessions = sessionsToProcess.filter(session => session.messages.length === 0);
|
||||
const unparsedSessions = sessionsToProcess.filter(
|
||||
(session) => session.messages.length === 0
|
||||
);
|
||||
|
||||
if (unparsedSessions.length === 0) {
|
||||
process.stdout.write("[TranscriptParser] No unparsed transcripts found.\n");
|
||||
process.stdout.write(
|
||||
"[TranscriptParser] No unparsed transcripts found.\n"
|
||||
);
|
||||
return { processed: 0, errors: 0 };
|
||||
}
|
||||
|
||||
process.stdout.write(`[TranscriptParser] Found ${unparsedSessions.length} sessions with unparsed transcripts.\n`);
|
||||
process.stdout.write(
|
||||
`[TranscriptParser] Found ${unparsedSessions.length} sessions with unparsed transcripts.\n`
|
||||
);
|
||||
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
for (const session of unparsedSessions) {
|
||||
try {
|
||||
const result = await processTranscriptForSession(session.id, session.transcriptContent);
|
||||
process.stdout.write(`[TranscriptParser] Processed session ${session.id}: ${result.messageCount} messages\n`);
|
||||
const result = await processTranscriptForSession(
|
||||
session.id,
|
||||
session.transcriptContent
|
||||
);
|
||||
process.stdout.write(
|
||||
`[TranscriptParser] Processed session ${session.id}: ${result.messageCount} messages\n`
|
||||
);
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Failed to process session ${session.id}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Failed to process session ${session.id}: ${error}\n`
|
||||
);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
process.stdout.write(`[TranscriptParser] Completed processing. Success: ${successCount}, Errors: ${errorCount}\n`);
|
||||
process.stdout.write(
|
||||
`[TranscriptParser] Completed processing. Success: ${successCount}, Errors: ${errorCount}\n`
|
||||
);
|
||||
return { processed: successCount, errors: errorCount };
|
||||
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Error in processAllUnparsedTranscripts: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Error in processAllUnparsedTranscripts: ${error}\n`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@ -198,12 +231,14 @@ export async function getMessagesForSession(sessionId) {
|
||||
try {
|
||||
const messages = await prisma.message.findMany({
|
||||
where: { sessionId },
|
||||
orderBy: { order: 'asc' }
|
||||
orderBy: { order: "asc" },
|
||||
});
|
||||
|
||||
return messages;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Error getting messages for session ${sessionId}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Error getting messages for session ${sessionId}: ${error}\n`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
@ -21,9 +21,9 @@ export default async function handler(
|
||||
where: { id },
|
||||
include: {
|
||||
messages: {
|
||||
orderBy: { order: 'asc' }
|
||||
}
|
||||
}
|
||||
orderBy: { order: "asc" },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!prismaSession) {
|
||||
@ -63,15 +63,16 @@ export default async function handler(
|
||||
processed: prismaSession.processed ?? null, // New field
|
||||
questions: prismaSession.questions ?? null, // New field
|
||||
summary: prismaSession.summary ?? null, // New field
|
||||
messages: prismaSession.messages?.map(msg => ({
|
||||
id: msg.id,
|
||||
sessionId: msg.sessionId,
|
||||
timestamp: new Date(msg.timestamp),
|
||||
role: msg.role,
|
||||
content: msg.content,
|
||||
order: msg.order,
|
||||
createdAt: new Date(msg.createdAt)
|
||||
})) ?? [], // New field - parsed messages
|
||||
messages:
|
||||
prismaSession.messages?.map((msg) => ({
|
||||
id: msg.id,
|
||||
sessionId: msg.sessionId,
|
||||
timestamp: new Date(msg.timestamp),
|
||||
role: msg.role,
|
||||
content: msg.content,
|
||||
order: msg.order,
|
||||
createdAt: new Date(msg.createdAt),
|
||||
})) ?? [], // New field - parsed messages
|
||||
};
|
||||
|
||||
return res.status(200).json({ session });
|
||||
|
||||
@ -48,7 +48,7 @@ async function triggerProcessingScheduler() {
|
||||
});
|
||||
|
||||
console.log(`Found ${sessionsToProcess.length} sessions to process:`);
|
||||
sessionsToProcess.forEach(session => {
|
||||
sessionsToProcess.forEach((session) => {
|
||||
console.log(`- Session ${session.id}: processed=${session.processed}`);
|
||||
});
|
||||
|
||||
@ -58,7 +58,9 @@ async function triggerProcessingScheduler() {
|
||||
}
|
||||
|
||||
// Import and run the processing function
|
||||
const { processUnprocessedSessions } = await import("../lib/processingScheduler.js");
|
||||
const { processUnprocessedSessions } = await import(
|
||||
"../lib/processingScheduler.js"
|
||||
);
|
||||
await processUnprocessedSessions();
|
||||
|
||||
console.log("✅ Processing scheduler completed");
|
||||
@ -74,7 +76,9 @@ async function triggerTranscriptParsing() {
|
||||
console.log("=== Manual Transcript Parsing Trigger ===");
|
||||
try {
|
||||
const result = await processAllUnparsedTranscripts();
|
||||
console.log(`✅ Transcript parsing completed: ${result.processed} processed, ${result.errors} errors`);
|
||||
console.log(
|
||||
`✅ Transcript parsing completed: ${result.processed} processed, ${result.errors} errors`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error("❌ Transcript parsing failed:", error);
|
||||
}
|
||||
@ -89,25 +93,22 @@ async function showProcessingStatus() {
|
||||
try {
|
||||
const totalSessions = await prisma.session.count();
|
||||
const processedSessions = await prisma.session.count({
|
||||
where: { processed: true }
|
||||
where: { processed: true },
|
||||
});
|
||||
const unprocessedSessions = await prisma.session.count({
|
||||
where: { processed: { not: true } }
|
||||
where: { processed: { not: true } },
|
||||
});
|
||||
const withMessages = await prisma.session.count({
|
||||
where: {
|
||||
messages: {
|
||||
some: {}
|
||||
}
|
||||
}
|
||||
some: {},
|
||||
},
|
||||
},
|
||||
});
|
||||
const readyForProcessing = await prisma.session.count({
|
||||
where: {
|
||||
AND: [
|
||||
{ messages: { some: {} } },
|
||||
{ processed: { not: true } }
|
||||
]
|
||||
}
|
||||
AND: [{ messages: { some: {} } }, { processed: { not: true } }],
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`📊 Total sessions: ${totalSessions}`);
|
||||
@ -121,24 +122,22 @@ async function showProcessingStatus() {
|
||||
console.log("\n📋 Sample unprocessed sessions:");
|
||||
const samples = await prisma.session.findMany({
|
||||
where: {
|
||||
AND: [
|
||||
{ messages: { some: {} } },
|
||||
{ processed: { not: true } }
|
||||
]
|
||||
AND: [{ messages: { some: {} } }, { processed: { not: true } }],
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
processed: true,
|
||||
startTime: true,
|
||||
},
|
||||
take: 3
|
||||
take: 3,
|
||||
});
|
||||
|
||||
samples.forEach(session => {
|
||||
console.log(`- ${session.id} (${session.startTime.toISOString()}) - processed: ${session.processed}`);
|
||||
samples.forEach((session) => {
|
||||
console.log(
|
||||
`- ${session.id} (${session.startTime.toISOString()}) - processed: ${session.processed}`
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error("❌ Failed to get processing status:", error);
|
||||
}
|
||||
@ -148,24 +147,24 @@ async function showProcessingStatus() {
|
||||
const command = process.argv[2];
|
||||
|
||||
switch (command) {
|
||||
case 'refresh':
|
||||
case "refresh":
|
||||
await triggerSessionRefresh();
|
||||
break;
|
||||
case 'process':
|
||||
case "process":
|
||||
await triggerProcessingScheduler();
|
||||
break;
|
||||
case 'parse':
|
||||
case "parse":
|
||||
await triggerTranscriptParsing();
|
||||
break;
|
||||
case 'status':
|
||||
case "status":
|
||||
await showProcessingStatus();
|
||||
break;
|
||||
case 'both':
|
||||
case "both":
|
||||
await triggerSessionRefresh();
|
||||
console.log("\n" + "=".repeat(50) + "\n");
|
||||
await triggerProcessingScheduler();
|
||||
break;
|
||||
case 'all':
|
||||
case "all":
|
||||
await triggerSessionRefresh();
|
||||
console.log("\n" + "=".repeat(50) + "\n");
|
||||
await triggerTranscriptParsing();
|
||||
@ -175,9 +174,13 @@ switch (command) {
|
||||
default:
|
||||
console.log("Usage: node scripts/manual-triggers.js [command]");
|
||||
console.log("Commands:");
|
||||
console.log(" refresh - Trigger session refresh (fetch new sessions from CSV)");
|
||||
console.log(
|
||||
" refresh - Trigger session refresh (fetch new sessions from CSV)"
|
||||
);
|
||||
console.log(" parse - Parse transcripts into structured messages");
|
||||
console.log(" process - Trigger processing scheduler (process unprocessed sessions)");
|
||||
console.log(
|
||||
" process - Trigger processing scheduler (process unprocessed sessions)"
|
||||
);
|
||||
console.log(" status - Show current processing status");
|
||||
console.log(" both - Run both refresh and processing");
|
||||
console.log(" all - Run refresh, parse, and processing in sequence");
|
||||
|
||||
@ -125,7 +125,9 @@ function validateOpenAIResponse(data) {
|
||||
|
||||
// Validate field types
|
||||
if (typeof data.language !== "string" || !/^[a-z]{2}$/.test(data.language)) {
|
||||
throw new Error("Invalid language format. Expected ISO 639-1 code (e.g., 'en')");
|
||||
throw new Error(
|
||||
"Invalid language format. Expected ISO 639-1 code (e.g., 'en')"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.messages_sent !== "number" || data.messages_sent < 0) {
|
||||
@ -133,7 +135,9 @@ function validateOpenAIResponse(data) {
|
||||
}
|
||||
|
||||
if (!["positive", "neutral", "negative"].includes(data.sentiment)) {
|
||||
throw new Error("Invalid sentiment. Expected 'positive', 'neutral', or 'negative'");
|
||||
throw new Error(
|
||||
"Invalid sentiment. Expected 'positive', 'neutral', or 'negative'"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.escalated !== "boolean") {
|
||||
@ -161,15 +165,23 @@ function validateOpenAIResponse(data) {
|
||||
];
|
||||
|
||||
if (!validCategories.includes(data.category)) {
|
||||
throw new Error(`Invalid category. Expected one of: ${validCategories.join(", ")}`);
|
||||
throw new Error(
|
||||
`Invalid category. Expected one of: ${validCategories.join(", ")}`
|
||||
);
|
||||
}
|
||||
|
||||
if (!Array.isArray(data.questions)) {
|
||||
throw new Error("Invalid questions. Expected array of strings");
|
||||
}
|
||||
|
||||
if (typeof data.summary !== "string" || data.summary.length < 10 || data.summary.length > 300) {
|
||||
throw new Error("Invalid summary. Expected string between 10-300 characters");
|
||||
if (
|
||||
typeof data.summary !== "string" ||
|
||||
data.summary.length < 10 ||
|
||||
data.summary.length > 300
|
||||
) {
|
||||
throw new Error(
|
||||
"Invalid summary. Expected string between 10-300 characters"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.session_id !== "string") {
|
||||
@ -210,7 +222,9 @@ async function processUnprocessedSessions() {
|
||||
for (const session of sessionsToProcess) {
|
||||
if (!session.transcriptContent) {
|
||||
// Should not happen due to query, but good for type safety
|
||||
console.warn(`Session ${session.id} has no transcript content, skipping.`);
|
||||
console.warn(
|
||||
`Session ${session.id} has no transcript content, skipping.`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@ -101,7 +101,7 @@ async function processTranscriptWithOpenAI(
|
||||
throw new Error(`OpenAI API error: ${response.status} - ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json() as any;
|
||||
const data = (await response.json()) as any;
|
||||
const processedData = JSON.parse(data.choices[0].message.content);
|
||||
|
||||
// Validate the response against our expected schema
|
||||
@ -118,7 +118,9 @@ async function processTranscriptWithOpenAI(
|
||||
* Validates the OpenAI response against our expected schema
|
||||
* @param data The data to validate
|
||||
*/
|
||||
function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData {
|
||||
function validateOpenAIResponse(
|
||||
data: any
|
||||
): asserts data is OpenAIProcessedData {
|
||||
// Check required fields
|
||||
const requiredFields = [
|
||||
"language",
|
||||
@ -140,7 +142,9 @@ function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData
|
||||
|
||||
// Validate field types
|
||||
if (typeof data.language !== "string" || !/^[a-z]{2}$/.test(data.language)) {
|
||||
throw new Error("Invalid language format. Expected ISO 639-1 code (e.g., 'en')");
|
||||
throw new Error(
|
||||
"Invalid language format. Expected ISO 639-1 code (e.g., 'en')"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.messages_sent !== "number" || data.messages_sent < 0) {
|
||||
@ -148,7 +152,9 @@ function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData
|
||||
}
|
||||
|
||||
if (!["positive", "neutral", "negative"].includes(data.sentiment)) {
|
||||
throw new Error("Invalid sentiment. Expected 'positive', 'neutral', or 'negative'");
|
||||
throw new Error(
|
||||
"Invalid sentiment. Expected 'positive', 'neutral', or 'negative'"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.escalated !== "boolean") {
|
||||
@ -176,15 +182,23 @@ function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData
|
||||
];
|
||||
|
||||
if (!validCategories.includes(data.category)) {
|
||||
throw new Error(`Invalid category. Expected one of: ${validCategories.join(", ")}`);
|
||||
throw new Error(
|
||||
`Invalid category. Expected one of: ${validCategories.join(", ")}`
|
||||
);
|
||||
}
|
||||
|
||||
if (!Array.isArray(data.questions)) {
|
||||
throw new Error("Invalid questions. Expected array of strings");
|
||||
}
|
||||
|
||||
if (typeof data.summary !== "string" || data.summary.length < 10 || data.summary.length > 300) {
|
||||
throw new Error("Invalid summary. Expected string between 10-300 characters");
|
||||
if (
|
||||
typeof data.summary !== "string" ||
|
||||
data.summary.length < 10 ||
|
||||
data.summary.length > 300
|
||||
) {
|
||||
throw new Error(
|
||||
"Invalid summary. Expected string between 10-300 characters"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.session_id !== "string") {
|
||||
@ -225,7 +239,9 @@ async function processUnprocessedSessions() {
|
||||
for (const session of sessionsToProcess) {
|
||||
if (!session.transcriptContent) {
|
||||
// Should not happen due to query, but good for type safety
|
||||
console.warn(`Session ${session.id} has no transcript content, skipping.`);
|
||||
console.warn(
|
||||
`Session ${session.id} has no transcript content, skipping.`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
22
server.js
22
server.js
@ -1,12 +1,12 @@
|
||||
// Custom Next.js server with scheduler initialization
|
||||
const { createServer } = require('http');
|
||||
const { parse } = require('url');
|
||||
const next = require('next');
|
||||
const { startScheduler } = require('./lib/scheduler');
|
||||
const { startProcessingScheduler } = require('./lib/processingScheduler');
|
||||
const { createServer } = require("http");
|
||||
const { parse } = require("url");
|
||||
const next = require("next");
|
||||
const { startScheduler } = require("./lib/scheduler");
|
||||
const { startProcessingScheduler } = require("./lib/processingScheduler");
|
||||
|
||||
const dev = process.env.NODE_ENV !== 'production';
|
||||
const hostname = 'localhost';
|
||||
const dev = process.env.NODE_ENV !== "production";
|
||||
const hostname = "localhost";
|
||||
const port = process.env.PORT || 3000;
|
||||
|
||||
// Initialize Next.js
|
||||
@ -15,10 +15,10 @@ const handle = app.getRequestHandler();
|
||||
|
||||
app.prepare().then(() => {
|
||||
// Initialize schedulers when the server starts
|
||||
console.log('Starting schedulers...');
|
||||
console.log("Starting schedulers...");
|
||||
startScheduler();
|
||||
startProcessingScheduler();
|
||||
console.log('All schedulers initialized successfully');
|
||||
console.log("All schedulers initialized successfully");
|
||||
|
||||
createServer(async (req, res) => {
|
||||
try {
|
||||
@ -28,9 +28,9 @@ app.prepare().then(() => {
|
||||
// Let Next.js handle the request
|
||||
await handle(req, res, parsedUrl);
|
||||
} catch (err) {
|
||||
console.error('Error occurred handling', req.url, err);
|
||||
console.error("Error occurred handling", req.url, err);
|
||||
res.statusCode = 500;
|
||||
res.end('Internal Server Error');
|
||||
res.end("Internal Server Error");
|
||||
}
|
||||
}).listen(port, (err) => {
|
||||
if (err) throw err;
|
||||
|
||||
28
server.mjs
28
server.mjs
@ -1,15 +1,15 @@
|
||||
// Custom Next.js server with scheduler initialization
|
||||
import { createServer } from 'http';
|
||||
import { parse } from 'url';
|
||||
import next from 'next';
|
||||
import { createServer } from "http";
|
||||
import { parse } from "url";
|
||||
import next from "next";
|
||||
|
||||
// We'll need to dynamically import these after they're compiled
|
||||
let startScheduler;
|
||||
let startProcessingScheduler;
|
||||
|
||||
const dev = process.env.NODE_ENV !== 'production';
|
||||
const hostname = 'localhost';
|
||||
const port = parseInt(process.env.PORT || '3000', 10);
|
||||
const dev = process.env.NODE_ENV !== "production";
|
||||
const hostname = "localhost";
|
||||
const port = parseInt(process.env.PORT || "3000", 10);
|
||||
|
||||
// Initialize Next.js
|
||||
const app = next({ dev, hostname, port });
|
||||
@ -18,37 +18,37 @@ const handle = app.getRequestHandler();
|
||||
async function init() {
|
||||
try {
|
||||
// Dynamically import the schedulers
|
||||
const scheduler = await import('./lib/scheduler.js');
|
||||
const processingScheduler = await import('./lib/processingScheduler.js');
|
||||
const scheduler = await import("./lib/scheduler.js");
|
||||
const processingScheduler = await import("./lib/processingScheduler.js");
|
||||
|
||||
startScheduler = scheduler.startScheduler;
|
||||
startProcessingScheduler = processingScheduler.startProcessingScheduler;
|
||||
|
||||
app.prepare().then(() => {
|
||||
// Initialize schedulers when the server starts
|
||||
console.log('Starting schedulers...');
|
||||
console.log("Starting schedulers...");
|
||||
startScheduler();
|
||||
startProcessingScheduler();
|
||||
console.log('All schedulers initialized successfully');
|
||||
console.log("All schedulers initialized successfully");
|
||||
|
||||
createServer(async (req, res) => {
|
||||
try {
|
||||
// Parse the URL
|
||||
const parsedUrl = parse(req.url || '', true);
|
||||
const parsedUrl = parse(req.url || "", true);
|
||||
|
||||
// Let Next.js handle the request
|
||||
await handle(req, res, parsedUrl);
|
||||
} catch (err) {
|
||||
console.error('Error occurred handling', req.url, err);
|
||||
console.error("Error occurred handling", req.url, err);
|
||||
res.statusCode = 500;
|
||||
res.end('Internal Server Error');
|
||||
res.end("Internal Server Error");
|
||||
}
|
||||
}).listen(port, () => {
|
||||
console.log(`> Ready on http://${hostname}:${port}`);
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Failed to initialize server:', error);
|
||||
console.error("Failed to initialize server:", error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
26
server.ts
26
server.ts
@ -1,13 +1,13 @@
|
||||
// Custom Next.js server with scheduler initialization
|
||||
import { createServer } from 'http';
|
||||
import { parse } from 'url';
|
||||
import next from 'next';
|
||||
import { startScheduler } from './lib/scheduler.js';
|
||||
import { startProcessingScheduler } from './lib/processingScheduler.js';
|
||||
import { createServer } from "http";
|
||||
import { parse } from "url";
|
||||
import next from "next";
|
||||
import { startScheduler } from "./lib/scheduler.js";
|
||||
import { startProcessingScheduler } from "./lib/processingScheduler.js";
|
||||
|
||||
const dev = process.env.NODE_ENV !== 'production';
|
||||
const hostname = 'localhost';
|
||||
const port = parseInt(process.env.PORT || '3000', 10);
|
||||
const dev = process.env.NODE_ENV !== "production";
|
||||
const hostname = "localhost";
|
||||
const port = parseInt(process.env.PORT || "3000", 10);
|
||||
|
||||
// Initialize Next.js
|
||||
const app = next({ dev, hostname, port });
|
||||
@ -15,22 +15,22 @@ const handle = app.getRequestHandler();
|
||||
|
||||
app.prepare().then(() => {
|
||||
// Initialize schedulers when the server starts
|
||||
console.log('Starting schedulers...');
|
||||
console.log("Starting schedulers...");
|
||||
startScheduler();
|
||||
startProcessingScheduler();
|
||||
console.log('All schedulers initialized successfully');
|
||||
console.log("All schedulers initialized successfully");
|
||||
|
||||
createServer(async (req, res) => {
|
||||
try {
|
||||
// Parse the URL
|
||||
const parsedUrl = parse(req.url || '', true);
|
||||
const parsedUrl = parse(req.url || "", true);
|
||||
|
||||
// Let Next.js handle the request
|
||||
await handle(req, res, parsedUrl);
|
||||
} catch (err) {
|
||||
console.error('Error occurred handling', req.url, err);
|
||||
console.error("Error occurred handling", req.url, err);
|
||||
res.statusCode = 500;
|
||||
res.end('Internal Server Error');
|
||||
res.end("Internal Server Error");
|
||||
}
|
||||
}).listen(port, () => {
|
||||
console.log(`> Ready on http://${hostname}:${port}`);
|
||||
|
||||
Reference in New Issue
Block a user