mirror of
https://github.com/kjanat/livedash-node.git
synced 2026-01-16 07:52:10 +01:00
Refactor code for improved readability and consistency
- Updated formatting in SessionDetails component for better readability. - Enhanced documentation in scheduler-fixes.md to clarify issues and solutions. - Improved error handling and logging in csvFetcher.js and processingScheduler.js. - Standardized code formatting across various scripts and components for consistency. - Added validation checks for CSV URLs and transcript content to prevent processing errors. - Enhanced logging messages for better tracking of processing status and errors.
This commit is contained in:
Max Kowalski
@ -410,15 +410,19 @@ async function fetchTranscriptContent(url, username, password) {
|
||||
|
||||
if (!response.ok) {
|
||||
// Only log error once per batch, not for every transcript
|
||||
if (Math.random() < 0.1) { // Log ~10% of errors to avoid spam
|
||||
console.warn(`[CSV] Transcript fetch failed for ${url}: ${response.status} ${response.statusText}`);
|
||||
if (Math.random() < 0.1) {
|
||||
// Log ~10% of errors to avoid spam
|
||||
console.warn(
|
||||
`[CSV] Transcript fetch failed for ${url}: ${response.status} ${response.statusText}`
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
return await response.text();
|
||||
} catch (error) {
|
||||
// Only log error once per batch, not for every transcript
|
||||
if (Math.random() < 0.1) { // Log ~10% of errors to avoid spam
|
||||
if (Math.random() < 0.1) {
|
||||
// Log ~10% of errors to avoid spam
|
||||
console.warn(`[CSV] Transcript fetch error for ${url}:`, error.message);
|
||||
}
|
||||
return null;
|
||||
@ -505,13 +509,20 @@ export async function fetchAndStoreSessionsForAllCompanies() {
|
||||
|
||||
for (const company of companies) {
|
||||
if (!company.csvUrl) {
|
||||
console.log(`[Scheduler] Skipping company ${company.id} - no CSV URL configured`);
|
||||
console.log(
|
||||
`[Scheduler] Skipping company ${company.id} - no CSV URL configured`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip companies with invalid/example URLs
|
||||
if (company.csvUrl.includes('example.com') || company.csvUrl === 'https://example.com/data.csv') {
|
||||
console.log(`[Scheduler] Skipping company ${company.id} - invalid/example CSV URL: ${company.csvUrl}`);
|
||||
if (
|
||||
company.csvUrl.includes("example.com") ||
|
||||
company.csvUrl === "https://example.com/data.csv"
|
||||
) {
|
||||
console.log(
|
||||
`[Scheduler] Skipping company ${company.id} - invalid/example CSV URL: ${company.csvUrl}`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -581,11 +592,17 @@ export async function fetchAndStoreSessionsForAllCompanies() {
|
||||
country: session.country || null,
|
||||
language: session.language || null,
|
||||
messagesSent:
|
||||
typeof session.messagesSent === "number" ? session.messagesSent : 0,
|
||||
typeof session.messagesSent === "number"
|
||||
? session.messagesSent
|
||||
: 0,
|
||||
sentiment:
|
||||
typeof session.sentiment === "number" ? session.sentiment : null,
|
||||
typeof session.sentiment === "number"
|
||||
? session.sentiment
|
||||
: null,
|
||||
escalated:
|
||||
typeof session.escalated === "boolean" ? session.escalated : null,
|
||||
typeof session.escalated === "boolean"
|
||||
? session.escalated
|
||||
: null,
|
||||
forwardedHr:
|
||||
typeof session.forwardedHr === "boolean"
|
||||
? session.forwardedHr
|
||||
@ -596,9 +613,12 @@ export async function fetchAndStoreSessionsForAllCompanies() {
|
||||
typeof session.avgResponseTime === "number"
|
||||
? session.avgResponseTime
|
||||
: null,
|
||||
tokens: typeof session.tokens === "number" ? session.tokens : null,
|
||||
tokens:
|
||||
typeof session.tokens === "number" ? session.tokens : null,
|
||||
tokensEur:
|
||||
typeof session.tokensEur === "number" ? session.tokensEur : null,
|
||||
typeof session.tokensEur === "number"
|
||||
? session.tokensEur
|
||||
: null,
|
||||
category: session.category || null,
|
||||
initialMsg: session.initialMsg || null,
|
||||
},
|
||||
@ -607,9 +627,14 @@ export async function fetchAndStoreSessionsForAllCompanies() {
|
||||
addedCount++;
|
||||
}
|
||||
|
||||
console.log(`[Scheduler] Added ${addedCount} new sessions for company ${company.id}`);
|
||||
console.log(
|
||||
`[Scheduler] Added ${addedCount} new sessions for company ${company.id}`
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(`[Scheduler] Error processing company ${company.id}:`, error);
|
||||
console.error(
|
||||
`[Scheduler] Error processing company ${company.id}:`,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@ -126,7 +126,9 @@ function validateOpenAIResponse(data) {
|
||||
|
||||
// Validate field types
|
||||
if (typeof data.language !== "string" || !/^[a-z]{2}$/.test(data.language)) {
|
||||
throw new Error("Invalid language format. Expected ISO 639-1 code (e.g., 'en')");
|
||||
throw new Error(
|
||||
"Invalid language format. Expected ISO 639-1 code (e.g., 'en')"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.messages_sent !== "number" || data.messages_sent < 0) {
|
||||
@ -134,7 +136,9 @@ function validateOpenAIResponse(data) {
|
||||
}
|
||||
|
||||
if (!["positive", "neutral", "negative"].includes(data.sentiment)) {
|
||||
throw new Error("Invalid sentiment. Expected 'positive', 'neutral', or 'negative'");
|
||||
throw new Error(
|
||||
"Invalid sentiment. Expected 'positive', 'neutral', or 'negative'"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.escalated !== "boolean") {
|
||||
@ -162,15 +166,23 @@ function validateOpenAIResponse(data) {
|
||||
];
|
||||
|
||||
if (!validCategories.includes(data.category)) {
|
||||
throw new Error(`Invalid category. Expected one of: ${validCategories.join(", ")}`);
|
||||
throw new Error(
|
||||
`Invalid category. Expected one of: ${validCategories.join(", ")}`
|
||||
);
|
||||
}
|
||||
|
||||
if (!Array.isArray(data.questions)) {
|
||||
throw new Error("Invalid questions. Expected array of strings");
|
||||
}
|
||||
|
||||
if (typeof data.summary !== "string" || data.summary.length < 10 || data.summary.length > 300) {
|
||||
throw new Error("Invalid summary. Expected string between 10-300 characters");
|
||||
if (
|
||||
typeof data.summary !== "string" ||
|
||||
data.summary.length < 10 ||
|
||||
data.summary.length > 300
|
||||
) {
|
||||
throw new Error(
|
||||
"Invalid summary. Expected string between 10-300 characters"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.session_id !== "string") {
|
||||
@ -182,7 +194,9 @@ function validateOpenAIResponse(data) {
|
||||
* Process unprocessed sessions
|
||||
*/
|
||||
export async function processUnprocessedSessions() {
|
||||
process.stdout.write("[ProcessingScheduler] Starting to process unprocessed sessions...\n");
|
||||
process.stdout.write(
|
||||
"[ProcessingScheduler] Starting to process unprocessed sessions...\n"
|
||||
);
|
||||
|
||||
// Find sessions that have messages but haven't been processed
|
||||
const sessionsToProcess = await prisma.session.findMany({
|
||||
@ -193,43 +207,58 @@ export async function processUnprocessedSessions() {
|
||||
},
|
||||
include: {
|
||||
messages: {
|
||||
orderBy: { order: 'asc' }
|
||||
}
|
||||
orderBy: { order: "asc" },
|
||||
},
|
||||
},
|
||||
take: 10, // Process in batches to avoid overloading the system
|
||||
});
|
||||
|
||||
// Filter to only sessions that have messages
|
||||
const sessionsWithMessages = sessionsToProcess.filter(session => session.messages.length > 0);
|
||||
const sessionsWithMessages = sessionsToProcess.filter(
|
||||
(session) => session.messages.length > 0
|
||||
);
|
||||
|
||||
if (sessionsWithMessages.length === 0) {
|
||||
process.stdout.write("[ProcessingScheduler] No sessions found requiring processing.\n");
|
||||
process.stdout.write(
|
||||
"[ProcessingScheduler] No sessions found requiring processing.\n"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
process.stdout.write(`[ProcessingScheduler] Found ${sessionsWithMessages.length} sessions to process.\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Found ${sessionsWithMessages.length} sessions to process.\n`
|
||||
);
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
for (const session of sessionsWithMessages) {
|
||||
if (session.messages.length === 0) {
|
||||
process.stderr.write(`[ProcessingScheduler] Session ${session.id} has no messages, skipping.\n`);
|
||||
process.stderr.write(
|
||||
`[ProcessingScheduler] Session ${session.id} has no messages, skipping.\n`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
process.stdout.write(`[ProcessingScheduler] Processing messages for session ${session.id}...\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Processing messages for session ${session.id}...\n`
|
||||
);
|
||||
try {
|
||||
// Convert messages back to transcript format for OpenAI processing
|
||||
const transcript = session.messages.map(msg =>
|
||||
`[${new Date(msg.timestamp).toLocaleString('en-GB', {
|
||||
day: '2-digit',
|
||||
month: '2-digit',
|
||||
year: 'numeric',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
second: '2-digit'
|
||||
}).replace(',', '')}] ${msg.role}: ${msg.content}`
|
||||
).join('\n');
|
||||
const transcript = session.messages
|
||||
.map(
|
||||
(msg) =>
|
||||
`[${new Date(msg.timestamp)
|
||||
.toLocaleString("en-GB", {
|
||||
day: "2-digit",
|
||||
month: "2-digit",
|
||||
year: "numeric",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
})
|
||||
.replace(",", "")}] ${msg.role}: ${msg.content}`
|
||||
)
|
||||
.join("\n");
|
||||
|
||||
const processedData = await processTranscriptWithOpenAI(
|
||||
session.id,
|
||||
@ -260,17 +289,25 @@ export async function processUnprocessedSessions() {
|
||||
},
|
||||
});
|
||||
|
||||
process.stdout.write(`[ProcessingScheduler] Successfully processed session ${session.id}.\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Successfully processed session ${session.id}.\n`
|
||||
);
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[ProcessingScheduler] Error processing session ${session.id}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[ProcessingScheduler] Error processing session ${session.id}: ${error}\n`
|
||||
);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
process.stdout.write("[ProcessingScheduler] Session processing complete.\n");
|
||||
process.stdout.write(`[ProcessingScheduler] Successfully processed: ${successCount} sessions.\n`);
|
||||
process.stdout.write(`[ProcessingScheduler] Failed to process: ${errorCount} sessions.\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Successfully processed: ${successCount} sessions.\n`
|
||||
);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Failed to process: ${errorCount} sessions.\n`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -282,9 +319,13 @@ export function startProcessingScheduler() {
|
||||
try {
|
||||
await processUnprocessedSessions();
|
||||
} catch (error) {
|
||||
process.stderr.write(`[ProcessingScheduler] Error in scheduler: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[ProcessingScheduler] Error in scheduler: ${error}\n`
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
process.stdout.write("[ProcessingScheduler] Started processing scheduler (runs hourly).\n");
|
||||
process.stdout.write(
|
||||
"[ProcessingScheduler] Started processing scheduler (runs hourly).\n"
|
||||
);
|
||||
}
|
||||
|
||||
@ -103,7 +103,7 @@ async function processTranscriptWithOpenAI(
|
||||
throw new Error(`OpenAI API error: ${response.status} - ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json() as any;
|
||||
const data = (await response.json()) as any;
|
||||
const processedData = JSON.parse(data.choices[0].message.content);
|
||||
|
||||
// Validate the response against our expected schema
|
||||
@ -120,7 +120,9 @@ async function processTranscriptWithOpenAI(
|
||||
* Validates the OpenAI response against our expected schema
|
||||
* @param data The data to validate
|
||||
*/
|
||||
function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData {
|
||||
function validateOpenAIResponse(
|
||||
data: any
|
||||
): asserts data is OpenAIProcessedData {
|
||||
// Check required fields
|
||||
const requiredFields = [
|
||||
"language",
|
||||
@ -142,7 +144,9 @@ function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData
|
||||
|
||||
// Validate field types
|
||||
if (typeof data.language !== "string" || !/^[a-z]{2}$/.test(data.language)) {
|
||||
throw new Error("Invalid language format. Expected ISO 639-1 code (e.g., 'en')");
|
||||
throw new Error(
|
||||
"Invalid language format. Expected ISO 639-1 code (e.g., 'en')"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.messages_sent !== "number" || data.messages_sent < 0) {
|
||||
@ -150,7 +154,9 @@ function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData
|
||||
}
|
||||
|
||||
if (!["positive", "neutral", "negative"].includes(data.sentiment)) {
|
||||
throw new Error("Invalid sentiment. Expected 'positive', 'neutral', or 'negative'");
|
||||
throw new Error(
|
||||
"Invalid sentiment. Expected 'positive', 'neutral', or 'negative'"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.escalated !== "boolean") {
|
||||
@ -178,15 +184,23 @@ function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData
|
||||
];
|
||||
|
||||
if (!validCategories.includes(data.category)) {
|
||||
throw new Error(`Invalid category. Expected one of: ${validCategories.join(", ")}`);
|
||||
throw new Error(
|
||||
`Invalid category. Expected one of: ${validCategories.join(", ")}`
|
||||
);
|
||||
}
|
||||
|
||||
if (!Array.isArray(data.questions)) {
|
||||
throw new Error("Invalid questions. Expected array of strings");
|
||||
}
|
||||
|
||||
if (typeof data.summary !== "string" || data.summary.length < 10 || data.summary.length > 300) {
|
||||
throw new Error("Invalid summary. Expected string between 10-300 characters");
|
||||
if (
|
||||
typeof data.summary !== "string" ||
|
||||
data.summary.length < 10 ||
|
||||
data.summary.length > 300
|
||||
) {
|
||||
throw new Error(
|
||||
"Invalid summary. Expected string between 10-300 characters"
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof data.session_id !== "string") {
|
||||
@ -198,7 +212,9 @@ function validateOpenAIResponse(data: any): asserts data is OpenAIProcessedData
|
||||
* Process unprocessed sessions
|
||||
*/
|
||||
async function processUnprocessedSessions() {
|
||||
process.stdout.write("[ProcessingScheduler] Starting to process unprocessed sessions...\n");
|
||||
process.stdout.write(
|
||||
"[ProcessingScheduler] Starting to process unprocessed sessions...\n"
|
||||
);
|
||||
|
||||
// Find sessions that have transcript content but haven't been processed
|
||||
const sessionsToProcess = await prisma.session.findMany({
|
||||
@ -217,22 +233,30 @@ async function processUnprocessedSessions() {
|
||||
});
|
||||
|
||||
if (sessionsToProcess.length === 0) {
|
||||
process.stdout.write("[ProcessingScheduler] No sessions found requiring processing.\n");
|
||||
process.stdout.write(
|
||||
"[ProcessingScheduler] No sessions found requiring processing.\n"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
process.stdout.write(`[ProcessingScheduler] Found ${sessionsToProcess.length} sessions to process.\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Found ${sessionsToProcess.length} sessions to process.\n`
|
||||
);
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
for (const session of sessionsToProcess) {
|
||||
if (!session.transcriptContent) {
|
||||
// Should not happen due to query, but good for type safety
|
||||
process.stderr.write(`[ProcessingScheduler] Session ${session.id} has no transcript content, skipping.\n`);
|
||||
process.stderr.write(
|
||||
`[ProcessingScheduler] Session ${session.id} has no transcript content, skipping.\n`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
process.stdout.write(`[ProcessingScheduler] Processing transcript for session ${session.id}...\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Processing transcript for session ${session.id}...\n`
|
||||
);
|
||||
try {
|
||||
const processedData = await processTranscriptWithOpenAI(
|
||||
session.id,
|
||||
@ -263,17 +287,25 @@ async function processUnprocessedSessions() {
|
||||
},
|
||||
});
|
||||
|
||||
process.stdout.write(`[ProcessingScheduler] Successfully processed session ${session.id}.\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Successfully processed session ${session.id}.\n`
|
||||
);
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[ProcessingScheduler] Error processing session ${session.id}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[ProcessingScheduler] Error processing session ${session.id}: ${error}\n`
|
||||
);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
process.stdout.write("[ProcessingScheduler] Session processing complete.\n");
|
||||
process.stdout.write(`[ProcessingScheduler] Successfully processed: ${successCount} sessions.\n`);
|
||||
process.stdout.write(`[ProcessingScheduler] Failed to process: ${errorCount} sessions.\n`);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Successfully processed: ${successCount} sessions.\n`
|
||||
);
|
||||
process.stdout.write(
|
||||
`[ProcessingScheduler] Failed to process: ${errorCount} sessions.\n`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -285,9 +317,13 @@ export function startProcessingScheduler() {
|
||||
try {
|
||||
await processUnprocessedSessions();
|
||||
} catch (error) {
|
||||
process.stderr.write(`[ProcessingScheduler] Error in scheduler: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[ProcessingScheduler] Error in scheduler: ${error}\n`
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
process.stdout.write("[ProcessingScheduler] Started processing scheduler (runs hourly).\n");
|
||||
process.stdout.write(
|
||||
"[ProcessingScheduler] Started processing scheduler (runs hourly).\n"
|
||||
);
|
||||
}
|
||||
|
||||
@ -31,5 +31,7 @@ export function startScheduler() {
|
||||
}
|
||||
});
|
||||
|
||||
console.log("[Scheduler] Started session refresh scheduler (runs every 15 minutes).");
|
||||
console.log(
|
||||
"[Scheduler] Started session refresh scheduler (runs every 15 minutes)."
|
||||
);
|
||||
}
|
||||
|
||||
@ -10,17 +10,20 @@ const prisma = new PrismaClient();
|
||||
*/
|
||||
export function parseChatLogToJSON(logString) {
|
||||
// Convert to string if it's not already
|
||||
const stringData = typeof logString === 'string' ? logString : String(logString);
|
||||
const stringData =
|
||||
typeof logString === "string" ? logString : String(logString);
|
||||
|
||||
// Split by lines and filter out empty lines
|
||||
const lines = stringData.split('\n').filter(line => line.trim() !== '');
|
||||
const lines = stringData.split("\n").filter((line) => line.trim() !== "");
|
||||
|
||||
const messages = [];
|
||||
let currentMessage = null;
|
||||
|
||||
for (const line of lines) {
|
||||
// Check if line starts with a timestamp pattern [DD.MM.YYYY HH:MM:SS]
|
||||
const timestampMatch = line.match(/^\[(\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}:\d{2})\] (.+?): (.*)$/);
|
||||
const timestampMatch = line.match(
|
||||
/^\[(\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}:\d{2})\] (.+?): (.*)$/
|
||||
);
|
||||
|
||||
if (timestampMatch) {
|
||||
// If we have a previous message, push it to the array
|
||||
@ -32,9 +35,9 @@ export function parseChatLogToJSON(logString) {
|
||||
const [, timestamp, sender, content] = timestampMatch;
|
||||
|
||||
// Convert DD.MM.YYYY HH:MM:SS to ISO format
|
||||
const [datePart, timePart] = timestamp.split(' ');
|
||||
const [day, month, year] = datePart.split('.');
|
||||
const [hour, minute, second] = timePart.split(':');
|
||||
const [datePart, timePart] = timestamp.split(" ");
|
||||
const [day, month, year] = datePart.split(".");
|
||||
const [hour, minute, second] = timePart.split(":");
|
||||
|
||||
const dateObject = new Date(year, month - 1, day, hour, minute, second);
|
||||
|
||||
@ -42,11 +45,11 @@ export function parseChatLogToJSON(logString) {
|
||||
currentMessage = {
|
||||
timestamp: dateObject.toISOString(),
|
||||
role: sender,
|
||||
content: content
|
||||
content: content,
|
||||
};
|
||||
} else if (currentMessage) {
|
||||
// This is a continuation of the previous message (multiline)
|
||||
currentMessage.content += '\n' + line;
|
||||
currentMessage.content += "\n" + line;
|
||||
}
|
||||
}
|
||||
|
||||
@ -67,7 +70,7 @@ export function parseChatLogToJSON(logString) {
|
||||
// This puts "User" before "Assistant" when timestamps are the same
|
||||
return b.role.localeCompare(a.role);
|
||||
}),
|
||||
totalMessages: messages.length
|
||||
totalMessages: messages.length,
|
||||
};
|
||||
}
|
||||
|
||||
@ -80,7 +83,7 @@ export async function storeMessagesForSession(sessionId, messages) {
|
||||
try {
|
||||
// First, delete any existing messages for this session
|
||||
await prisma.message.deleteMany({
|
||||
where: { sessionId }
|
||||
where: { sessionId },
|
||||
});
|
||||
|
||||
// Then insert the new messages
|
||||
@ -89,19 +92,23 @@ export async function storeMessagesForSession(sessionId, messages) {
|
||||
timestamp: new Date(message.timestamp),
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
order: index
|
||||
order: index,
|
||||
}));
|
||||
|
||||
if (messageData.length > 0) {
|
||||
await prisma.message.createMany({
|
||||
data: messageData
|
||||
data: messageData,
|
||||
});
|
||||
}
|
||||
|
||||
process.stdout.write(`[TranscriptParser] Stored ${messageData.length} messages for session ${sessionId}\n`);
|
||||
process.stdout.write(
|
||||
`[TranscriptParser] Stored ${messageData.length} messages for session ${sessionId}\n`
|
||||
);
|
||||
return messageData.length;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Error storing messages for session ${sessionId}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Error storing messages for session ${sessionId}: ${error}\n`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@ -112,9 +119,12 @@ export async function storeMessagesForSession(sessionId, messages) {
|
||||
* @param {string} transcriptContent - Raw transcript content
|
||||
* @returns {Promise<Object>} Processing result with message count
|
||||
*/
|
||||
export async function processTranscriptForSession(sessionId, transcriptContent) {
|
||||
if (!transcriptContent || transcriptContent.trim() === '') {
|
||||
throw new Error('No transcript content provided');
|
||||
export async function processTranscriptForSession(
|
||||
sessionId,
|
||||
transcriptContent
|
||||
) {
|
||||
if (!transcriptContent || transcriptContent.trim() === "") {
|
||||
throw new Error("No transcript content provided");
|
||||
}
|
||||
|
||||
try {
|
||||
@ -122,16 +132,21 @@ export async function processTranscriptForSession(sessionId, transcriptContent)
|
||||
const parsed = parseChatLogToJSON(transcriptContent);
|
||||
|
||||
// Store messages in database
|
||||
const messageCount = await storeMessagesForSession(sessionId, parsed.messages);
|
||||
const messageCount = await storeMessagesForSession(
|
||||
sessionId,
|
||||
parsed.messages
|
||||
);
|
||||
|
||||
return {
|
||||
sessionId,
|
||||
messageCount,
|
||||
totalMessages: parsed.totalMessages,
|
||||
success: true
|
||||
success: true,
|
||||
};
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Error processing transcript for session ${sessionId}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Error processing transcript for session ${sessionId}: ${error}\n`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@ -140,7 +155,9 @@ export async function processTranscriptForSession(sessionId, transcriptContent)
|
||||
* Processes transcripts for all sessions that have transcript content but no parsed messages
|
||||
*/
|
||||
export async function processAllUnparsedTranscripts() {
|
||||
process.stdout.write("[TranscriptParser] Starting to process unparsed transcripts...\n");
|
||||
process.stdout.write(
|
||||
"[TranscriptParser] Starting to process unparsed transcripts...\n"
|
||||
);
|
||||
|
||||
try {
|
||||
// Find sessions with transcript content but no messages
|
||||
@ -149,42 +166,58 @@ export async function processAllUnparsedTranscripts() {
|
||||
AND: [
|
||||
{ transcriptContent: { not: null } },
|
||||
{ transcriptContent: { not: "" } },
|
||||
]
|
||||
],
|
||||
},
|
||||
include: {
|
||||
messages: true
|
||||
}
|
||||
messages: true,
|
||||
},
|
||||
});
|
||||
|
||||
// Filter to only sessions without messages
|
||||
const unparsedSessions = sessionsToProcess.filter(session => session.messages.length === 0);
|
||||
const unparsedSessions = sessionsToProcess.filter(
|
||||
(session) => session.messages.length === 0
|
||||
);
|
||||
|
||||
if (unparsedSessions.length === 0) {
|
||||
process.stdout.write("[TranscriptParser] No unparsed transcripts found.\n");
|
||||
process.stdout.write(
|
||||
"[TranscriptParser] No unparsed transcripts found.\n"
|
||||
);
|
||||
return { processed: 0, errors: 0 };
|
||||
}
|
||||
|
||||
process.stdout.write(`[TranscriptParser] Found ${unparsedSessions.length} sessions with unparsed transcripts.\n`);
|
||||
process.stdout.write(
|
||||
`[TranscriptParser] Found ${unparsedSessions.length} sessions with unparsed transcripts.\n`
|
||||
);
|
||||
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
for (const session of unparsedSessions) {
|
||||
try {
|
||||
const result = await processTranscriptForSession(session.id, session.transcriptContent);
|
||||
process.stdout.write(`[TranscriptParser] Processed session ${session.id}: ${result.messageCount} messages\n`);
|
||||
const result = await processTranscriptForSession(
|
||||
session.id,
|
||||
session.transcriptContent
|
||||
);
|
||||
process.stdout.write(
|
||||
`[TranscriptParser] Processed session ${session.id}: ${result.messageCount} messages\n`
|
||||
);
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Failed to process session ${session.id}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Failed to process session ${session.id}: ${error}\n`
|
||||
);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
process.stdout.write(`[TranscriptParser] Completed processing. Success: ${successCount}, Errors: ${errorCount}\n`);
|
||||
process.stdout.write(
|
||||
`[TranscriptParser] Completed processing. Success: ${successCount}, Errors: ${errorCount}\n`
|
||||
);
|
||||
return { processed: successCount, errors: errorCount };
|
||||
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Error in processAllUnparsedTranscripts: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Error in processAllUnparsedTranscripts: ${error}\n`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@ -198,12 +231,14 @@ export async function getMessagesForSession(sessionId) {
|
||||
try {
|
||||
const messages = await prisma.message.findMany({
|
||||
where: { sessionId },
|
||||
orderBy: { order: 'asc' }
|
||||
orderBy: { order: "asc" },
|
||||
});
|
||||
|
||||
return messages;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Error getting messages for session ${sessionId}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Error getting messages for session ${sessionId}: ${error}\n`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user