feat: Enhance session processing and metrics

- Updated session processing commands in documentation for clarity.
- Removed transcript content fetching from session processing, allowing on-demand retrieval.
- Improved session metrics calculations and added new metrics for dashboard.
- Refactored processing scheduler to handle sessions in parallel with concurrency limits.
- Added manual trigger API for processing unprocessed sessions with admin checks.
- Implemented scripts for fetching and parsing transcripts, checking transcript content, and testing processing status.
- Updated Prisma schema to enforce default values for processed sessions.
- Added error handling and logging improvements throughout the processing workflow.

pages/api/admin/refresh-sessions.ts

@@ -119,15 +119,8 @@ export default async function handler(
           ? session.endTime
           : new Date();
 
-      // Fetch transcript content if URL is available
-      let transcriptContent: string | null = null;
-      if (session.fullTranscriptUrl) {
-        transcriptContent = await fetchTranscriptContent(
-          session.fullTranscriptUrl,
-          company.csvUsername as string | undefined,
-          company.csvPassword as string | undefined
-        );
-      }
+      // Note: transcriptContent field was removed from schema
+      // Transcript content can be fetched on-demand from fullTranscriptUrl
 
       // Check if the session already exists
       const existingSession = await prisma.session.findUnique({
@@ -160,7 +153,6 @@ export default async function handler(
               ? session.forwardedHr
               : null,
           fullTranscriptUrl: session.fullTranscriptUrl || null,
-          transcriptContent: transcriptContent, // Add the transcript content
           avgResponseTime:
             typeof session.avgResponseTime === "number"
               ? session.avgResponseTime

pages/api/admin/trigger-processing.ts

@@ -0,0 +1,103 @@
+import { NextApiRequest, NextApiResponse } from "next";
+import { getServerSession } from "next-auth";
+import { authOptions } from "../auth/[...nextauth]";
+import { prisma } from "../../../lib/prisma";
+import { processUnprocessedSessions } from "../../../lib/processingScheduler";
+
+interface SessionUser {
+  email: string;
+  name?: string;
+}
+
+interface SessionData {
+  user: SessionUser;
+}
+
+export default async function handler(
+  req: NextApiRequest,
+  res: NextApiResponse
+) {
+  if (req.method !== "POST") {
+    return res.status(405).json({ error: "Method not allowed" });
+  }
+
+  const session = (await getServerSession(
+    req,
+    res,
+    authOptions
+  )) as SessionData | null;
+
+  if (!session?.user) {
+    return res.status(401).json({ error: "Not logged in" });
+  }
+
+  const user = await prisma.user.findUnique({
+    where: { email: session.user.email },
+    include: { company: true },
+  });
+
+  if (!user) {
+    return res.status(401).json({ error: "No user found" });
+  }
+
+  // Check if user has admin role
+  if (user.role !== "admin") {
+    return res.status(403).json({ error: "Admin access required" });
+  }
+
+  try {
+    // Get optional parameters from request body
+    const { batchSize, maxConcurrency } = req.body;
+
+    // Validate parameters
+    const validatedBatchSize = batchSize && batchSize > 0 ? parseInt(batchSize) : null;
+    const validatedMaxConcurrency = maxConcurrency && maxConcurrency > 0 ? parseInt(maxConcurrency) : 5;
+
+    // Check how many unprocessed sessions exist
+    const unprocessedCount = await prisma.session.count({
+      where: {
+        companyId: user.companyId,
+        processed: false,
+        messages: { some: {} }, // Must have messages
+      },
+    });
+
+    if (unprocessedCount === 0) {
+      return res.json({
+        success: true,
+        message: "No unprocessed sessions found",
+        unprocessedCount: 0,
+        processedCount: 0,
+      });
+    }
+
+    // Start processing (this will run asynchronously)
+    const startTime = Date.now();
+
+    // Note: We're calling the function but not awaiting it to avoid timeout
+    // The processing will continue in the background
+    processUnprocessedSessions(validatedBatchSize, validatedMaxConcurrency)
+      .then(() => {
+        console.log(`[Manual Trigger] Processing completed for company ${user.companyId}`);
+      })
+      .catch((error) => {
+        console.error(`[Manual Trigger] Processing failed for company ${user.companyId}:`, error);
+      });
+
+    return res.json({
+      success: true,
+      message: `Started processing ${unprocessedCount} unprocessed sessions`,
+      unprocessedCount,
+      batchSize: validatedBatchSize || unprocessedCount,
+      maxConcurrency: validatedMaxConcurrency,
+      startedAt: new Date().toISOString(),
+    });
+
+  } catch (error) {
+    console.error("[Manual Trigger] Error:", error);
+    return res.status(500).json({
+      error: "Failed to trigger processing",
+      details: error instanceof Error ? error.message : String(error),
+    });
+  }
+}

pages/api/dashboard/metrics.ts

@@ -35,10 +35,13 @@ export default async function handler(
 
   // Get date range from query parameters
   const { startDate, endDate } = req.query;
-  
-  // Build where clause with optional date filtering
-  const whereClause: any = { companyId: user.companyId };
-  
+
+  // Build where clause with optional date filtering and only processed sessions
+  const whereClause: any = {
+    companyId: user.companyId,
+    processed: true, // Only show processed sessions in dashboard
+  };
+
   if (startDate && endDate) {
     whereClause.startTime = {
       gte: new Date(startDate as string),