mirror of
https://github.com/kjanat/livedash-node.git
synced 2026-01-16 09:52:09 +01:00
Refactor code for improved readability and consistency
- Updated formatting in SessionDetails component for better readability. - Enhanced documentation in scheduler-fixes.md to clarify issues and solutions. - Improved error handling and logging in csvFetcher.js and processingScheduler.js. - Standardized code formatting across various scripts and components for consistency. - Added validation checks for CSV URLs and transcript content to prevent processing errors. - Enhanced logging messages for better tracking of processing status and errors.
This commit is contained in:
Max Kowalski
@ -10,17 +10,20 @@ const prisma = new PrismaClient();
|
||||
*/
|
||||
export function parseChatLogToJSON(logString) {
|
||||
// Convert to string if it's not already
|
||||
const stringData = typeof logString === 'string' ? logString : String(logString);
|
||||
const stringData =
|
||||
typeof logString === "string" ? logString : String(logString);
|
||||
|
||||
// Split by lines and filter out empty lines
|
||||
const lines = stringData.split('\n').filter(line => line.trim() !== '');
|
||||
const lines = stringData.split("\n").filter((line) => line.trim() !== "");
|
||||
|
||||
const messages = [];
|
||||
let currentMessage = null;
|
||||
|
||||
for (const line of lines) {
|
||||
// Check if line starts with a timestamp pattern [DD.MM.YYYY HH:MM:SS]
|
||||
const timestampMatch = line.match(/^\[(\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}:\d{2})\] (.+?): (.*)$/);
|
||||
const timestampMatch = line.match(
|
||||
/^\[(\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}:\d{2})\] (.+?): (.*)$/
|
||||
);
|
||||
|
||||
if (timestampMatch) {
|
||||
// If we have a previous message, push it to the array
|
||||
@ -32,9 +35,9 @@ export function parseChatLogToJSON(logString) {
|
||||
const [, timestamp, sender, content] = timestampMatch;
|
||||
|
||||
// Convert DD.MM.YYYY HH:MM:SS to ISO format
|
||||
const [datePart, timePart] = timestamp.split(' ');
|
||||
const [day, month, year] = datePart.split('.');
|
||||
const [hour, minute, second] = timePart.split(':');
|
||||
const [datePart, timePart] = timestamp.split(" ");
|
||||
const [day, month, year] = datePart.split(".");
|
||||
const [hour, minute, second] = timePart.split(":");
|
||||
|
||||
const dateObject = new Date(year, month - 1, day, hour, minute, second);
|
||||
|
||||
@ -42,11 +45,11 @@ export function parseChatLogToJSON(logString) {
|
||||
currentMessage = {
|
||||
timestamp: dateObject.toISOString(),
|
||||
role: sender,
|
||||
content: content
|
||||
content: content,
|
||||
};
|
||||
} else if (currentMessage) {
|
||||
// This is a continuation of the previous message (multiline)
|
||||
currentMessage.content += '\n' + line;
|
||||
currentMessage.content += "\n" + line;
|
||||
}
|
||||
}
|
||||
|
||||
@ -67,7 +70,7 @@ export function parseChatLogToJSON(logString) {
|
||||
// This puts "User" before "Assistant" when timestamps are the same
|
||||
return b.role.localeCompare(a.role);
|
||||
}),
|
||||
totalMessages: messages.length
|
||||
totalMessages: messages.length,
|
||||
};
|
||||
}
|
||||
|
||||
@ -80,7 +83,7 @@ export async function storeMessagesForSession(sessionId, messages) {
|
||||
try {
|
||||
// First, delete any existing messages for this session
|
||||
await prisma.message.deleteMany({
|
||||
where: { sessionId }
|
||||
where: { sessionId },
|
||||
});
|
||||
|
||||
// Then insert the new messages
|
||||
@ -89,19 +92,23 @@ export async function storeMessagesForSession(sessionId, messages) {
|
||||
timestamp: new Date(message.timestamp),
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
order: index
|
||||
order: index,
|
||||
}));
|
||||
|
||||
if (messageData.length > 0) {
|
||||
await prisma.message.createMany({
|
||||
data: messageData
|
||||
data: messageData,
|
||||
});
|
||||
}
|
||||
|
||||
process.stdout.write(`[TranscriptParser] Stored ${messageData.length} messages for session ${sessionId}\n`);
|
||||
process.stdout.write(
|
||||
`[TranscriptParser] Stored ${messageData.length} messages for session ${sessionId}\n`
|
||||
);
|
||||
return messageData.length;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Error storing messages for session ${sessionId}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Error storing messages for session ${sessionId}: ${error}\n`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@ -112,9 +119,12 @@ export async function storeMessagesForSession(sessionId, messages) {
|
||||
* @param {string} transcriptContent - Raw transcript content
|
||||
* @returns {Promise<Object>} Processing result with message count
|
||||
*/
|
||||
export async function processTranscriptForSession(sessionId, transcriptContent) {
|
||||
if (!transcriptContent || transcriptContent.trim() === '') {
|
||||
throw new Error('No transcript content provided');
|
||||
export async function processTranscriptForSession(
|
||||
sessionId,
|
||||
transcriptContent
|
||||
) {
|
||||
if (!transcriptContent || transcriptContent.trim() === "") {
|
||||
throw new Error("No transcript content provided");
|
||||
}
|
||||
|
||||
try {
|
||||
@ -122,16 +132,21 @@ export async function processTranscriptForSession(sessionId, transcriptContent)
|
||||
const parsed = parseChatLogToJSON(transcriptContent);
|
||||
|
||||
// Store messages in database
|
||||
const messageCount = await storeMessagesForSession(sessionId, parsed.messages);
|
||||
const messageCount = await storeMessagesForSession(
|
||||
sessionId,
|
||||
parsed.messages
|
||||
);
|
||||
|
||||
return {
|
||||
sessionId,
|
||||
messageCount,
|
||||
totalMessages: parsed.totalMessages,
|
||||
success: true
|
||||
success: true,
|
||||
};
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Error processing transcript for session ${sessionId}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Error processing transcript for session ${sessionId}: ${error}\n`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@ -140,7 +155,9 @@ export async function processTranscriptForSession(sessionId, transcriptContent)
|
||||
* Processes transcripts for all sessions that have transcript content but no parsed messages
|
||||
*/
|
||||
export async function processAllUnparsedTranscripts() {
|
||||
process.stdout.write("[TranscriptParser] Starting to process unparsed transcripts...\n");
|
||||
process.stdout.write(
|
||||
"[TranscriptParser] Starting to process unparsed transcripts...\n"
|
||||
);
|
||||
|
||||
try {
|
||||
// Find sessions with transcript content but no messages
|
||||
@ -149,42 +166,58 @@ export async function processAllUnparsedTranscripts() {
|
||||
AND: [
|
||||
{ transcriptContent: { not: null } },
|
||||
{ transcriptContent: { not: "" } },
|
||||
]
|
||||
],
|
||||
},
|
||||
include: {
|
||||
messages: true
|
||||
}
|
||||
messages: true,
|
||||
},
|
||||
});
|
||||
|
||||
// Filter to only sessions without messages
|
||||
const unparsedSessions = sessionsToProcess.filter(session => session.messages.length === 0);
|
||||
const unparsedSessions = sessionsToProcess.filter(
|
||||
(session) => session.messages.length === 0
|
||||
);
|
||||
|
||||
if (unparsedSessions.length === 0) {
|
||||
process.stdout.write("[TranscriptParser] No unparsed transcripts found.\n");
|
||||
process.stdout.write(
|
||||
"[TranscriptParser] No unparsed transcripts found.\n"
|
||||
);
|
||||
return { processed: 0, errors: 0 };
|
||||
}
|
||||
|
||||
process.stdout.write(`[TranscriptParser] Found ${unparsedSessions.length} sessions with unparsed transcripts.\n`);
|
||||
process.stdout.write(
|
||||
`[TranscriptParser] Found ${unparsedSessions.length} sessions with unparsed transcripts.\n`
|
||||
);
|
||||
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
for (const session of unparsedSessions) {
|
||||
try {
|
||||
const result = await processTranscriptForSession(session.id, session.transcriptContent);
|
||||
process.stdout.write(`[TranscriptParser] Processed session ${session.id}: ${result.messageCount} messages\n`);
|
||||
const result = await processTranscriptForSession(
|
||||
session.id,
|
||||
session.transcriptContent
|
||||
);
|
||||
process.stdout.write(
|
||||
`[TranscriptParser] Processed session ${session.id}: ${result.messageCount} messages\n`
|
||||
);
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Failed to process session ${session.id}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Failed to process session ${session.id}: ${error}\n`
|
||||
);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
process.stdout.write(`[TranscriptParser] Completed processing. Success: ${successCount}, Errors: ${errorCount}\n`);
|
||||
process.stdout.write(
|
||||
`[TranscriptParser] Completed processing. Success: ${successCount}, Errors: ${errorCount}\n`
|
||||
);
|
||||
return { processed: successCount, errors: errorCount };
|
||||
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Error in processAllUnparsedTranscripts: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Error in processAllUnparsedTranscripts: ${error}\n`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@ -198,12 +231,14 @@ export async function getMessagesForSession(sessionId) {
|
||||
try {
|
||||
const messages = await prisma.message.findMany({
|
||||
where: { sessionId },
|
||||
orderBy: { order: 'asc' }
|
||||
orderBy: { order: "asc" },
|
||||
});
|
||||
|
||||
return messages;
|
||||
} catch (error) {
|
||||
process.stderr.write(`[TranscriptParser] Error getting messages for session ${sessionId}: ${error}\n`);
|
||||
process.stderr.write(
|
||||
`[TranscriptParser] Error getting messages for session ${sessionId}: ${error}\n`
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user