livedash-node/lib/transcriptParser.js

// Transcript parser utility - converts raw transcript text to structured messages
import { PrismaClient } from "@prisma/client";

const prisma = new PrismaClient();

/**
 * Parses chat log string to JSON format with individual messages
 * @param {string} logString - Raw transcript content
 * @returns {Object} Parsed data with messages array and metadata
 */
export function parseChatLogToJSON(logString) {
  // Convert to string if it's not already
  const stringData = typeof logString === 'string' ? logString : String(logString);

  // Split by lines and filter out empty lines
  const lines = stringData.split('\n').filter(line => line.trim() !== '');

  const messages = [];
  let currentMessage = null;

  for (const line of lines) {
    // Check if line starts with a timestamp pattern [DD.MM.YYYY HH:MM:SS]
    const timestampMatch = line.match(/^\[(\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}:\d{2})\] (.+?): (.*)$/);

    if (timestampMatch) {
      // If we have a previous message, push it to the array
      if (currentMessage) {
        messages.push(currentMessage);
      }

      // Parse the timestamp
      const [, timestamp, sender, content] = timestampMatch;

      // Convert DD.MM.YYYY HH:MM:SS to ISO format
      const [datePart, timePart] = timestamp.split(' ');
      const [day, month, year] = datePart.split('.');
      const [hour, minute, second] = timePart.split(':');

      const dateObject = new Date(year, month - 1, day, hour, minute, second);

      // Create new message object
      currentMessage = {
        timestamp: dateObject.toISOString(),
        role: sender,
        content: content
      };
    } else if (currentMessage) {
      // This is a continuation of the previous message (multiline)
      currentMessage.content += '\n' + line;
    }
  }

  // Don't forget the last message
  if (currentMessage) {
    messages.push(currentMessage);
  }

  return {
    messages: messages.sort((a, b) => {
      // First sort by timestamp (ascending)
      const timeComparison = new Date(a.timestamp) - new Date(b.timestamp);
      if (timeComparison !== 0) {
        return timeComparison;
      }

      // If timestamps are equal, sort by role (descending)
      // This puts "User" before "Assistant" when timestamps are the same
      return b.role.localeCompare(a.role);
    }),
    totalMessages: messages.length
  };
}

/**
 * Stores parsed messages in the database for a session
 * @param {string} sessionId - The session ID
 * @param {Array} messages - Array of parsed message objects
 */
export async function storeMessagesForSession(sessionId, messages) {
  try {
    // First, delete any existing messages for this session
    await prisma.message.deleteMany({
      where: { sessionId }
    });

    // Then insert the new messages
    const messageData = messages.map((message, index) => ({
      sessionId,
      timestamp: new Date(message.timestamp),
      role: message.role,
      content: message.content,
      order: index
    }));

    if (messageData.length > 0) {
      await prisma.message.createMany({
        data: messageData
      });
    }

    process.stdout.write(`[TranscriptParser] Stored ${messageData.length} messages for session ${sessionId}\n`);
    return messageData.length;
  } catch (error) {
    process.stderr.write(`[TranscriptParser] Error storing messages for session ${sessionId}: ${error}\n`);
    throw error;
  }
}

/**
 * Processes and stores transcript for a single session
 * @param {string} sessionId - The session ID
 * @param {string} transcriptContent - Raw transcript content
 * @returns {Promise<Object>} Processing result with message count
 */
export async function processTranscriptForSession(sessionId, transcriptContent) {
  if (!transcriptContent || transcriptContent.trim() === '') {
    throw new Error('No transcript content provided');
  }

  try {
    // Parse the transcript
    const parsed = parseChatLogToJSON(transcriptContent);

    // Store messages in database
    const messageCount = await storeMessagesForSession(sessionId, parsed.messages);

    return {
      sessionId,
      messageCount,
      totalMessages: parsed.totalMessages,
      success: true
    };
  } catch (error) {
    process.stderr.write(`[TranscriptParser] Error processing transcript for session ${sessionId}: ${error}\n`);
    throw error;
  }
}

/**
 * Processes transcripts for all sessions that have transcript content but no parsed messages
 */
export async function processAllUnparsedTranscripts() {
  process.stdout.write("[TranscriptParser] Starting to process unparsed transcripts...\n");

  try {
    // Find sessions with transcript content but no messages
    const sessionsToProcess = await prisma.session.findMany({
      where: {
        AND: [
          { transcriptContent: { not: null } },
          { transcriptContent: { not: "" } },
        ]
      },
      include: {
        messages: true
      }
    });

    // Filter to only sessions without messages
    const unparsedSessions = sessionsToProcess.filter(session => session.messages.length === 0);

    if (unparsedSessions.length === 0) {
      process.stdout.write("[TranscriptParser] No unparsed transcripts found.\n");
      return { processed: 0, errors: 0 };
    }

    process.stdout.write(`[TranscriptParser] Found ${unparsedSessions.length} sessions with unparsed transcripts.\n`);

    let successCount = 0;
    let errorCount = 0;

    for (const session of unparsedSessions) {
      try {
        const result = await processTranscriptForSession(session.id, session.transcriptContent);
        process.stdout.write(`[TranscriptParser] Processed session ${session.id}: ${result.messageCount} messages\n`);
        successCount++;
      } catch (error) {
        process.stderr.write(`[TranscriptParser] Failed to process session ${session.id}: ${error}\n`);
        errorCount++;
      }
    }

    process.stdout.write(`[TranscriptParser] Completed processing. Success: ${successCount}, Errors: ${errorCount}\n`);
    return { processed: successCount, errors: errorCount };

  } catch (error) {
    process.stderr.write(`[TranscriptParser] Error in processAllUnparsedTranscripts: ${error}\n`);
    throw error;
  }
}

/**
 * Gets parsed messages for a session
 * @param {string} sessionId - The session ID
 * @returns {Promise<Array>} Array of message objects
 */
export async function getMessagesForSession(sessionId) {
  try {
    const messages = await prisma.message.findMany({
      where: { sessionId },
      orderBy: { order: 'asc' }
    });

    return messages;
  } catch (error) {
    process.stderr.write(`[TranscriptParser] Error getting messages for session ${sessionId}: ${error}\n`);
    throw error;
  }
}